diff --git a/.agents/skills/openwrt-package-update/SKILL.md b/.agents/skills/openwrt-package-update/SKILL.md new file mode 100644 index 000000000..7f9164525 --- /dev/null +++ b/.agents/skills/openwrt-package-update/SKILL.md @@ -0,0 +1,60 @@ +--- +name: openwrt-package-update +description: > + Use when updating any forked OpenWrt package in a NethSecurity workspace from + the upstream openwrt/packages feed. Covers the full update cycle: version bump, + merging upstream file changes, updating dependent ns-api handlers and migration + scripts, UCI overlay defaults, and correlated documentation. Triggers on updating + a package by name (adblock, mwan3, banip, rsyslog, snort3, keepalived, + python-jinja2, python-semver, and similar forks), syncing with upstream, or any + request to align a local package fork with openwrt/packages — even if upstream + is not mentioned explicitly. +compatibility: Requires git and curl. Works in a NethSecurity workspace with build.conf.defaults present. +metadata: + domain: nethsecurity-packages + type: package-update +allowed-tools: Bash(git:*) Bash(curl:*) Bash(diff:*) Bash(tar:*) Bash(grep:*) Bash(sed:*) Bash(awk:*) Read Write +--- + +## What I do + +Update a forked upstream OpenWrt package from the openwrt/packages feed. Auto-discovers the package path, compares the current version against the upstream target, extracts snapshots for comparison, applies upstream changes, propagates impact to ns-api handlers and migration scripts, and updates correlated documentation. + +## Quick start + +1. **Setup**: `bash scripts/setup-package.sh ` — auto-discovers upstream path, finds version commits, extracts old/new snapshots into `assets/_old/` and `assets/_new/` +2. **Compare**: `bash scripts/diff-package.sh ` — shows what changed upstream +3. **Merge**: Read both snapshots, apply upstream changes to local files in `packages//`; update PKG_VERSION and PKG_RELEASE in the Makefile +4. **Cross-package**: Grep for any changed config keys or function names in ns-api, migration scripts, and files/ overlay; update as needed +5. **Documentation**: Update `docs/` and `packages/ns-api/README.md` if user-facing behavior changed +6. **Verify**: Run `ruff check` on any changed Python files; build the package inside the container + +For per-step detail on any of the above, read [references/WORKFLOW.md](references/WORKFLOW.md). + +## How it works + +The skill uses three helper scripts: + +- **get-target-commit.sh** — reads `build.conf.defaults` for OWRT_VERSION, fetches `feeds.conf.default` from that tag, extracts the openwrt/packages commit hash +- **setup-package.sh** — auto-discovers package upstream path, uses git pickaxe to find the commit matching current PKG_VERSION then PKG_RELEASE, clones/fetches openwrt/packages bare repo into `assets/.openwrt-packages-repo/` (persistent cache), extracts both versions into named snapshots +- **diff-package.sh** — convenience wrapper: `diff -rN assets/_old/ assets/_new/` + +## Snapshot structure + +Each snapshot has the exact upstream package files at root level: +``` +assets/adblock_old/ +├── Makefile +├── files/ +│ ├── adblock.sh +│ ├── adblock.init +│ ├── adblock.conf +│ └── ... +``` + +## Edge cases + +- **mwan3**: Deep fork — apply upstream changes conservatively, one by one +- **snort3**: Has local patches — verify patches still apply after update +- **Version not found**: If script fails, local version is very old; check `assets/.openwrt-packages-repo/` git history +- **Empty diff**: Versions match; only update PKG_VERSION/PKG_RELEASE if tracking newer release diff --git a/.agents/skills/openwrt-package-update/assets/.gitignore b/.agents/skills/openwrt-package-update/assets/.gitignore new file mode 100644 index 000000000..72e8ffc0d --- /dev/null +++ b/.agents/skills/openwrt-package-update/assets/.gitignore @@ -0,0 +1 @@ +* diff --git a/.agents/skills/openwrt-package-update/references/WORKFLOW.md b/.agents/skills/openwrt-package-update/references/WORKFLOW.md new file mode 100644 index 000000000..ec4af820c --- /dev/null +++ b/.agents/skills/openwrt-package-update/references/WORKFLOW.md @@ -0,0 +1,88 @@ +# Detailed Workflow: Updating Upstream Packages + +## 1. Identify the package + +Confirm the package exists and is in scope. Read `packages//Makefile` to understand the current version: +- Note PKG_VERSION and PKG_RELEASE +- Check if `packages//patches/` exists (indicates local patches) +- Scan `files/` directory to understand what configuration files and scripts are included + +## 2. Setup snapshots + +Run the setup script: +```bash +bash scripts/setup-package.sh +``` + +This script will: +- Auto-discover the upstream path in openwrt/packages (e.g., `net/adblock`) +- Use git pickaxe to find the commit that introduced the current PKG_VERSION +- Then find the commit that set the current PKG_RELEASE +- Extract both versions into `assets/_old/` and `assets/_new/` +- Output summary with commit hashes and paths + +## 3. Compare and classify + +Examine both snapshots using your file reading tools. Run: +```bash +bash scripts/diff-package.sh +``` + +Classify each change: +- **Makefile**: version bumps, new dependencies, build flags +- **files/*.sh** / **files/*.init**: behavioral changes to shell scripts +- **files/*.conf**: UCI configuration schema changes (new/removed/renamed options) +- **files/*.sources**, **files/*.categories**, data files: content-only updates +- **New files** added / **files removed** + +## 4. Apply upstream changes + +For each changed file: +- If the local copy is identical to old version or has no NethSecurity-specific changes → take upstream as-is +- If the local copy has customizations → merge carefully: + - Use the diff tool to understand what changed upstream + - Manually integrate the upstream changes into the local file + - Preserve any NethSecurity-specific logic or configuration + +Always: +- Update PKG_VERSION and PKG_RELEASE in `packages//Makefile` +- If `packages//patches/` exists, re-verify each patch still applies cleanly with the new version +- Test build: `make package/feeds/nethsecurity//compile V=sc` inside the build container + +## 5. Cross-package impact detection + +For each changed UCI option name, function name, or config key in the diff: +- Grep the workspace for references: + ```bash + grep -r "" packages/ files/ docs/ --include="*.py" --include="*.sh" --include="*.json" --include="*.conf" --include="*.md" + ``` +- Grep `packages/ns-migration/` for migration scripts that reference old option names +- Grep `packages/ns-api/` for API scripts that read/write this package's configuration +- Present findings to user before making changes + +## 6. Apply cross-package fixes (after user confirmation) + +Update affected files: +- **ns-api scripts**: if UCI schema changed, update handlers in `packages/ns-api/files/ns.` +- **Migration scripts**: if old config options are removed, add migration logic to `packages/ns-migration/files/` +- **files/ overlay defaults**: if default values changed, update `files/etc/uci-defaults/` or `files/etc/config/` +- **Documentation**: update `docs/` if user-facing behavior changed + +## Edge cases + +**mwan3**: This is a deep fork with significant local modifications. Apply upstream changes one by one, preferring manual review for any behavioral change. Verify the result against firewall rules and failover behavior. + +**snort3, openvpn-easy-rsa**: These packages have local patches (see `packages//patches/`). After updating upstream files, re-run `quilt refresh` or regenerate patches. If patches no longer apply cleanly, update them to match the new upstream. + +**Empty diff**: If upstream and local versions are the same, only update PKG_VERSION/PKG_RELEASE in the Makefile if tracking a newer release. Otherwise, the package is up-to-date. + +**Version not found**: If the script cannot find the current PKG_VERSION in upstream history, it means the local version is very old or from a different source. Check git history in the bare repo (`assets/.openwrt-packages-repo/`) to understand the version trajectory. + +## Helpful commands + +Inside the skill directory: +- `bash scripts/get-target-commit.sh` — print the target packages feed commit hash from feeds.conf.default +- `bash scripts/setup-package.sh ` — extract old/new snapshots +- `bash scripts/diff-package.sh ` — show recursive diff +- `find assets/_old/ -type f` — list all files in old snapshot +- `cat assets/_old/Makefile | grep PKG_` — check old version details diff --git a/.agents/skills/openwrt-package-update/scripts/diff-package.sh b/.agents/skills/openwrt-package-update/scripts/diff-package.sh new file mode 100755 index 000000000..a32cc4ee8 --- /dev/null +++ b/.agents/skills/openwrt-package-update/scripts/diff-package.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# Diff old and new package snapshots +# +# Usage: diff-package.sh +# +# Outputs a recursive diff between assets/_old/ and assets/_new/ +# + +set -euo pipefail + +PACKAGE_NAME="${1:-}" + +if [[ -z "$PACKAGE_NAME" ]]; then + echo "Usage: diff-package.sh " >&2 + exit 1 +fi + +# Find skill root +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SKILL_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +ASSETS_DIR="$SKILL_ROOT/assets" + +OLD_DIR="$ASSETS_DIR/${PACKAGE_NAME}_old" +NEW_DIR="$ASSETS_DIR/${PACKAGE_NAME}_new" + +if [[ ! -d "$OLD_DIR" ]]; then + echo "Error: $OLD_DIR not found. Run setup-package.sh first." >&2 + exit 1 +fi + +if [[ ! -d "$NEW_DIR" ]]; then + echo "Error: $NEW_DIR not found. Run setup-package.sh first." >&2 + exit 1 +fi + +# Run recursive diff, showing added/removed/modified files +diff -rN "$OLD_DIR" "$NEW_DIR" || true diff --git a/.agents/skills/openwrt-package-update/scripts/get-target-commit.sh b/.agents/skills/openwrt-package-update/scripts/get-target-commit.sh new file mode 100755 index 000000000..687a5ee67 --- /dev/null +++ b/.agents/skills/openwrt-package-update/scripts/get-target-commit.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Get the target OpenWrt packages feed commit hash from feeds.conf.default +# at the version specified in build.conf.defaults +# +# Outputs the commit hash to stdout, or exits with error if fetch/parse fails. +# + +set -euo pipefail + +# Find workspace root +WORKSPACE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && git rev-parse --show-toplevel 2>/dev/null || echo ".") + +# Read OWRT_VERSION from build.conf.defaults +if [[ ! -f "$WORKSPACE_ROOT/build.conf.defaults" ]]; then + echo "Error: build.conf.defaults not found at $WORKSPACE_ROOT" >&2 + exit 1 +fi + +OWRT_VERSION=$(grep '^OWRT_VERSION=' "$WORKSPACE_ROOT/build.conf.defaults" | cut -d'=' -f2 | tr -d ' ') + +if [[ -z "$OWRT_VERSION" ]]; then + echo "Error: OWRT_VERSION not found or empty in build.conf.defaults" >&2 + exit 1 +fi + +# Fetch feeds.conf.default from OpenWrt at the specified tag +FEEDS_URL="https://raw.githubusercontent.com/openwrt/openwrt/$OWRT_VERSION/feeds.conf.default" + +FEEDS_CONTENT=$(curl -fsSL "$FEEDS_URL" 2>/dev/null || { + echo "Error: Failed to fetch $FEEDS_URL" >&2 + exit 1 +}) + +# Parse the packages line to extract the commit hash (after the ^) +# Expected format: src-git packages https://git.openwrt.org/feed/packages.git^ +PACKAGES_COMMIT=$(echo "$FEEDS_CONTENT" | grep '^src-git packages' | sed 's/.*\^//g' | head -1) + +if [[ -z "$PACKAGES_COMMIT" ]]; then + echo "Error: Could not parse packages commit hash from feeds.conf.default" >&2 + exit 1 +fi + +echo "$PACKAGES_COMMIT" diff --git a/.agents/skills/openwrt-package-update/scripts/setup-package.sh b/.agents/skills/openwrt-package-update/scripts/setup-package.sh new file mode 100755 index 000000000..0ce5bf135 --- /dev/null +++ b/.agents/skills/openwrt-package-update/scripts/setup-package.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# +# Setup old and new package snapshots for comparison +# +# Usage: setup-package.sh +# +# Outputs: +# - Creates assets/_old/ with the current pinned upstream version +# - Creates assets/_new/ with the target upstream version (from feeds.conf.default) +# - Prints to stdout: package name, CURRENT_COMMIT, TARGET_COMMIT, upstream path +# + +set -euo pipefail + +PACKAGE_NAME="${1:-}" + +if [[ -z "$PACKAGE_NAME" ]]; then + echo "Usage: setup-package.sh " >&2 + exit 1 +fi + +# Find workspace root (convert to absolute path) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORKSPACE_ROOT=$(cd "$SCRIPT_DIR/../../../.." && git rev-parse --show-toplevel 2>/dev/null || pwd) +SKILL_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +ASSETS_DIR="$SKILL_ROOT/assets" + +# Ensure assets dir exists +mkdir -p "$ASSETS_DIR" + +# Read current package version from local Makefile +LOCAL_MAKEFILE="$WORKSPACE_ROOT/packages/$PACKAGE_NAME/Makefile" +if [[ ! -f "$LOCAL_MAKEFILE" ]]; then + echo "Error: $LOCAL_MAKEFILE not found" >&2 + exit 1 +fi + +PKG_VERSION=$(grep '^PKG_VERSION:=' "$LOCAL_MAKEFILE" | cut -d'=' -f2 | tr -d ' ') +PKG_RELEASE=$(grep '^PKG_RELEASE:=' "$LOCAL_MAKEFILE" | cut -d'=' -f2 | tr -d ' ') + +if [[ -z "$PKG_VERSION" ]]; then + echo "Error: PKG_VERSION not found in $LOCAL_MAKEFILE" >&2 + exit 1 +fi + +echo "Package: $PACKAGE_NAME (version $PKG_VERSION-$PKG_RELEASE)" >&2 + +# Get target commit from feeds.conf.default +TARGET_COMMIT=$("$SKILL_ROOT/scripts/get-target-commit.sh") +echo "Target commit: $TARGET_COMMIT" >&2 + +# Clone/fetch openwrt/packages bare repo +BARE_REPO="$ASSETS_DIR/.openwrt-packages-repo" + +if [[ ! -d "$BARE_REPO" ]]; then + echo "Cloning openwrt/packages (bare repo)..." >&2 + git clone --bare --filter=blob:none https://github.com/openwrt/packages.git "$BARE_REPO" 2>/dev/null || { + echo "Error: Failed to clone openwrt/packages" >&2 + exit 1 + } +else + echo "Fetching openwrt/packages..." >&2 + cd "$BARE_REPO" + git fetch origin 2>/dev/null || { + echo "Error: Failed to fetch openwrt/packages" >&2 + exit 1 + } + cd - > /dev/null +fi + +# Auto-discover the upstream path by searching for /Makefile +# Look in the target commit first to find the exact path +echo "Discovering upstream package path..." >&2 +UPSTREAM_PATH=$(cd "$BARE_REPO" && git ls-tree -r --name-only "$TARGET_COMMIT" | grep -E "^[^/]+/$PACKAGE_NAME/Makefile$" | sed 's|/Makefile$||' | head -1) + +if [[ -z "$UPSTREAM_PATH" ]]; then + echo "Error: Could not find $PACKAGE_NAME in upstream openwrt/packages at $TARGET_COMMIT" >&2 + exit 1 +fi + +echo "Upstream path: $UPSTREAM_PATH" >&2 + +# Find the commit that introduced the current version +# Step 1: Find when PKG_VERSION was introduced using pickaxe +# Step 2: Find when PKG_RELEASE was set using pickaxe +echo "Finding commit that matches local version ($PKG_VERSION-$PKG_RELEASE)..." >&2 + +CURRENT_COMMIT="" + +# Step 1: Find commits that introduced PKG_VERSION +echo " Step 1: Finding commits with PKG_VERSION:=$PKG_VERSION..." >&2 +version_commits=$(cd "$BARE_REPO" && git log -S "PKG_VERSION:=$PKG_VERSION" --all --format="%H" -- "$UPSTREAM_PATH/Makefile" 2>/dev/null) + +if [[ -z "$version_commits" ]]; then + echo "Error: Could not find PKG_VERSION:=$PKG_VERSION in git history" >&2 + exit 1 +fi + +# Get the oldest commit where PKG_VERSION was introduced (last in the list when searching backwards) +version_commit=$(echo "$version_commits" | tail -1) +echo " PKG_VERSION introduced at: $version_commit" >&2 + +# Step 2: From that commit forward, find when PKG_RELEASE was set +# We search commits from version_commit onwards for PKG_RELEASE change +echo " Step 2: Finding commits with PKG_RELEASE:=$PKG_RELEASE after version introduction..." >&2 +release_commits=$(cd "$BARE_REPO" && git log -S "PKG_RELEASE:=$PKG_RELEASE" --all --format="%H" -- "$UPSTREAM_PATH/Makefile" 2>/dev/null) + +if [[ -n "$release_commits" ]]; then + # Find the first commit in release_commits that is an ancestor of or after version_commit + # Also verify both PKG_VERSION and PKG_RELEASE are present + while IFS= read -r commit; do + makefile_content=$(cd "$BARE_REPO" && git show "$commit:$UPSTREAM_PATH/Makefile" 2>/dev/null || echo "") + + if echo "$makefile_content" | grep -q "^PKG_VERSION:=$PKG_VERSION" && \ + echo "$makefile_content" | grep -q "^PKG_RELEASE:=$PKG_RELEASE"; then + CURRENT_COMMIT="$commit" + break + fi + done <<< "$release_commits" +fi + +if [[ -z "$CURRENT_COMMIT" ]]; then + echo "Error: Could not find a commit matching $PKG_VERSION-$PKG_RELEASE in upstream history" >&2 + echo "Available version range (last 20 commits):" >&2 + cd "$BARE_REPO" && git log --all --oneline -20 -- "$UPSTREAM_PATH/Makefile" >&2 + exit 1 +fi + +echo "Current commit: $CURRENT_COMMIT" >&2 + +# Extract the old version (current commit) +OLD_DIR="$ASSETS_DIR/${PACKAGE_NAME}_old" +rm -rf "$OLD_DIR" +mkdir -p "$OLD_DIR" + +echo "Extracting old version to $OLD_DIR..." >&2 +strip_level=$(echo "$UPSTREAM_PATH" | awk -F/ '{print NF}') +cd "$BARE_REPO" && git archive "$CURRENT_COMMIT" "$UPSTREAM_PATH/" | tar -x --strip-components=$strip_level -C "$OLD_DIR" 2>/dev/null || { + echo "Error: Failed to extract old version" >&2 + exit 1 +} + +# Extract the new version (target commit) +NEW_DIR="$ASSETS_DIR/${PACKAGE_NAME}_new" +rm -rf "$NEW_DIR" +mkdir -p "$NEW_DIR" + +echo "Extracting new version to $NEW_DIR..." >&2 +cd "$BARE_REPO" && git archive "$TARGET_COMMIT" "$UPSTREAM_PATH/" | tar -x --strip-components=$strip_level -C "$NEW_DIR" 2>/dev/null || { + echo "Error: Failed to extract new version" >&2 + exit 1 +} + +# Output summary to stdout +echo "" +echo "===== SETUP COMPLETE =====" +echo "Package: $PACKAGE_NAME" +echo "Current upstream commit: $CURRENT_COMMIT" +echo "Target upstream commit: $TARGET_COMMIT" +echo "Upstream path: $UPSTREAM_PATH" +echo "Old version snapshot: $OLD_DIR" +echo "New version snapshot: $NEW_DIR" diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index eae1f39af..c52f863b8 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -11,10 +11,8 @@ on: - 'files/**' - 'packages/**' - 'patches/**' - - 'build.conf.example' + - 'build.conf.defaults' - 'build-nethsec.sh' - tags: - - '*' pull_request: paths: - 'builder/**' @@ -22,7 +20,7 @@ on: - 'files/**' - 'packages/**' - 'patches/**' - - 'build.conf.example' + - 'build.conf.defaults' - 'build-nethsec.sh' jobs: @@ -33,8 +31,8 @@ jobs: NETHSECURITY_VERSION: ${{ steps.build_vars.outputs.NETHSECURITY_VERSION }} REPO_CHANNEL: ${{ steps.build_vars.outputs.REPO_CHANNEL }} env: - USIGN_PUB_KEY: ${{ secrets.USIGN_PUB_KEY }} - USIGN_PRIV_KEY: ${{ secrets.USIGN_PRIV_KEY }} + APK_PUB_KEY: ${{ secrets.APK_PUB_KEY }} + APK_PRIV_KEY: ${{ secrets.APK_PRIV_KEY }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} steps: @@ -42,32 +40,34 @@ jobs: - name: Generate build variables id: build_vars run: | - # export OWRT_VERSION from build.conf.example - echo "OWRT_VERSION=$(grep -oP 'OWRT_VERSION=\K.*' build.conf.example)" >> $GITHUB_OUTPUT + # export OWRT_VERSION from build.conf.defaults + echo "OWRT_VERSION=$(grep -oP 'OWRT_VERSION=\K.*' build.conf.defaults)" >> $GITHUB_OUTPUT - # export TARGET from build.conf.example - echo "TARGET=$(grep -oP 'TARGET=\K.*' build.conf.example)" >> $GITHUB_OUTPUT + # export TARGET from build.conf.defaults + echo "TARGET=$(grep -oP 'TARGET=\K.*' build.conf.defaults)" >> $GITHUB_OUTPUT - # export NETHSECURITY_VERSION from build - echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.example)" >> $GITHUB_OUTPUT - - # When pushing a tag, set REPO_CHANNEL to stable - if [[ "${{ github.ref }}" == refs/tags/* ]]; then - echo "REPO_CHANNEL=stable" >> $GITHUB_OUTPUT - # save NETHSECURITY_VERSION to env - echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.example)" >> $GITHUB_OUTPUT + # export base NETHSECURITY_VERSION from build.conf.defaults + BASE_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.defaults) + COMMIT_HASH=$(git rev-parse --short HEAD) + TIMESTAMP=$(date +'%Y%m%d%H%M%S') + echo "NETHSECURITY_VERSION=${BASE_VERSION}" >> $GITHUB_OUTPUT # When pushing to main branch, set REPO_CHANNEL to dev - elif [[ "${{ github.ref }}" == refs/heads/main ]]; then + if [[ "${{ github.ref }}" == refs/heads/main ]]; then echo "REPO_CHANNEL=dev" >> $GITHUB_OUTPUT - # save NETHSECURITY_VERSION to env and append -dev to it - echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.example)-dev+$(git rev-parse --short HEAD).$(date +'%Y%m%d%H%M%S')" >> $GITHUB_OUTPUT + echo "BUILD_SEMVER_SUFFIX=-dev.${TIMESTAMP}.${COMMIT_HASH}" >> $GITHUB_OUTPUT + + # For pull requests and workflow_dispatch, add timestamp suffix + elif [[ "${{ github.event_name }}" == 'pull_request' ]]; then + BRANCH_NAME="${{ github.head_ref }}" + echo "REPO_CHANNEL=${BRANCH_NAME}" >> $GITHUB_OUTPUT + echo "BUILD_SEMVER_SUFFIX=-${BRANCH_NAME}.${TIMESTAMP}.${COMMIT_HASH}" >> $GITHUB_OUTPUT + + elif [[ "${{ github.event_name }}" == 'workflow_dispatch' ]]; then + BRANCH_NAME=$(echo "${{ github.ref }}" | sed 's|refs/heads/||') + echo "REPO_CHANNEL=${BRANCH_NAME}" >> $GITHUB_OUTPUT + echo "BUILD_SEMVER_SUFFIX=-.${TIMESTAMP}.${COMMIT_HASH}" >> $GITHUB_OUTPUT - # Otherwise, get the branch name of the PR pushing if REPO_CHANNEL is not set - elif [[ "${{ github.event_name }}" == 'pull_request' && ! -v REPO_CHANNEL ]]; then - echo "REPO_CHANNEL=${{ github.head_ref }}" >> $GITHUB_OUTPUT - # save NETHSECURITY_VERSION to env and append last commit hash to it - echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.example)-${{ github.head_ref }}+$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT fi - name: Build the image env: @@ -75,12 +75,14 @@ jobs: NETHSECURITY_VERSION: ${{ steps.build_vars.outputs.NETHSECURITY_VERSION }} REPO_CHANNEL: ${{ steps.build_vars.outputs.REPO_CHANNEL }} TARGET: ${{ steps.build_vars.outputs.TARGET }} + BUILD_SEMVER_SUFFIX: ${{ steps.build_vars.outputs.BUILD_SEMVER_SUFFIX }} run: ./build-nethsec.sh - name: Update latest_release file run: | - # Create release file pointing to 8-VERSION - echo "${{ steps.build_vars.outputs.NETHSECURITY_VERSION }}" > latest_release - echo "::notice title='Image published':: ${{ steps.build_vars.outputs.NETHSECURITY_VERSION }}" + # Create release file with the full display version (base + suffix) + FULL_VERSION="${{ steps.build_vars.outputs.NETHSECURITY_VERSION }}${{ steps.build_vars.outputs.BUILD_SEMVER_SUFFIX }}" + echo "${FULL_VERSION}" > latest_release + echo "::notice title='Image published':: ${FULL_VERSION}" - uses: actions/upload-artifact@v7 name: Upload image with: @@ -117,8 +119,14 @@ jobs: RCLONE_CONFIG_REPO_ACCESS_KEY_ID: ${{ secrets.DO_SPACE_ACCESS_KEY }} RCLONE_CONFIG_REPO_SECRET_ACCESS_KEY: ${{ secrets.DO_SPACE_SECRET_KEY }} run: | + # Sync binaries to channel/version/ rclone sync bin/ repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/${{ steps.build_vars.outputs.NETHSECURITY_VERSION }} --progress --create-empty-src-dirs - rclone copy latest_release repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/ --progress --create-empty-src-dirs + + # Place latest_release inside the version directory + rclone copy latest_release repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/${{ steps.build_vars.outputs.NETHSECURITY_VERSION }}/ --progress + + # Also place latest_release at channel root as fallback + rclone copy latest_release repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/ --progress tools: name: 'Run tools' diff --git a/.gitignore b/.gitignore index f54bf6bc0..40665e2e7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ -key-build* -build.conf.override +private-key.pem +public-key.pem /bin build-logs build.conf netify-flow-actions netify-agent-stats-plugin +scripts/netifyd-apks diff --git a/AGENTS.md b/AGENTS.md index 6425615b1..2b4437365 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,6 +12,7 @@ This project has domain-specific skills available. You MUST activate the relevan - `ns-api` — **ACTIVATE** when writing, modifying, or reviewing Python RPCD API scripts. Triggers: creating or updating `ns.*` RPCD API endpoints, handling UCI configuration changes, managing pre/post-commit hooks, defining ACL permissions, documenting methods in OpenAPI 3.1.0, or when user mentions ns-api, API endpoints, hooks, or references `/usr/libexec/rpcd/ns.` files. Covers stdin/stdout JSON protocol, error handling, naming conventions, code style, and spec file updates. - `openwrt-package` — **ACTIVATE** when creating or modifying OpenWrt `ns-*` packages. Triggers: building new packages for NethSecurity, managing package dependencies, patching upstream feeds, modifying Makefiles, or when user mentions Makefile, package structure, config fragments, or upstream patches. Covers naming conventions, required Makefile fields, architecture selection, external version management, and patch workflows. +- `openwrt-package-update` — **ACTIVATE** when updating forked OpenWrt packages from the upstream feed (adblock, mwan3, banip, etc.). Triggers: updating non-ns- packages, comparing local forks against openwrt/packages, merging upstream improvements, or when user mentions upstream package updates. Auto-discovers packages; extracts old/new snapshots for side-by-side comparison; guides cross-package impact detection. - `python-nethsecurity` — **ACTIVATE** when writing or modifying Python scripts for NethSecurity packages. Triggers: creating new Python scripts, configuring package build systems, writing utilities, or when user mentions Python code in packages/ns-* or references Python scripts in the NethSecurity package tree. Covers shebang, license headers, extension handling, ruff compliance, available modules, and UCI commit conventions. --- diff --git a/build-nethsec.sh b/build-nethsec.sh index 9e570ea92..000cb2100 100755 --- a/build-nethsec.sh +++ b/build-nethsec.sh @@ -7,10 +7,16 @@ set -e -# Source build files if it exists +# Source versioned defaults first set -o allexport +if [ -f build.conf.defaults ]; then + echo "Loading build.conf.defaults..." + . ./build.conf.defaults +fi + +# Source local overrides second (can override anything) if [ -f build.conf ]; then - echo "Loading build.conf file..." + echo "Loading build.conf (local overrides)..." . ./build.conf fi set +o allexport @@ -20,10 +26,11 @@ OWRT_VERSION=${OWRT_VERSION:?Missing OWRT_VERSION environment variable} NETHSECURITY_VERSION=${NETHSECURITY_VERSION:?Missing NETHSECURITY_VERSION environment variable} REPO_CHANNEL=${REPO_CHANNEL:-dev} TARGET=${TARGET:-x86_64} +BUILD_SEMVER_SUFFIX=${BUILD_SEMVER_SUFFIX:-} -if [ -f "./key-build" ] && [ -f "./key-build.pub" ]; then - USIGN_PRIV_KEY="$(cat ./key-build)" - USIGN_PUB_KEY="$(cat ./key-build.pub)" +if [ -f "./private-key.pem" ] && [ -f "./public-key.pem" ]; then + APK_PRIV_KEY="$(cat ./private-key.pem)" + APK_PUB_KEY="$(cat ./public-key.pem)" fi @@ -32,20 +39,20 @@ podman build \ --layers \ --file builder/Containerfile \ --tag nethsecurity-next \ - --target builder \ --jobs 0 \ --build-arg OWRT_VERSION="$OWRT_VERSION" \ --build-arg REPO_CHANNEL="$REPO_CHANNEL" \ --build-arg TARGET="$TARGET" \ --build-arg NETHSECURITY_VERSION="$NETHSECURITY_VERSION" \ + --build-arg BUILD_SEMVER_SUFFIX="$BUILD_SEMVER_SUFFIX" \ . set +e status=0 podman run \ - --env USIGN_PRIV_KEY="$USIGN_PRIV_KEY" \ - --env USIGN_PUB_KEY="$USIGN_PUB_KEY" \ + --env APK_PRIV_KEY="$APK_PRIV_KEY" \ + --env APK_PUB_KEY="$APK_PUB_KEY" \ --name nethsecurity-builder \ --interactive \ --tty \ diff --git a/build.conf.defaults b/build.conf.defaults new file mode 100644 index 000000000..15e0198cd --- /dev/null +++ b/build.conf.defaults @@ -0,0 +1,4 @@ +OWRT_VERSION=v25.12.3 +NETHSECURITY_VERSION=8.8.0 +TARGET=x86_64 +REPO_CHANNEL=dev diff --git a/build.conf.example b/build.conf.example deleted file mode 100644 index 0f05f1e39..000000000 --- a/build.conf.example +++ /dev/null @@ -1,4 +0,0 @@ -OWRT_VERSION=v24.10.5 -NETHSECURITY_VERSION=8.7.2 -TARGET=x86_64 -REPO_CHANNEL=dev diff --git a/builder/Containerfile b/builder/Containerfile index 351dda0ef..7d44137ae 100644 --- a/builder/Containerfile +++ b/builder/Containerfile @@ -1,53 +1,59 @@ # -# Copyright (C) 2025 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # -# 2025-06-30 -FROM debian:12.11 AS base -ARG SOURCE_DATE_EPOCH=1751241600 +FROM debian:13.4 AS base RUN apt-get update \ && apt-get install --yes --no-install-recommends --no-install-suggests \ + # openwrt build dependencies https://openwrt.org/docs/guide-developer/toolchain/install-buildsystem#debianubuntumint bison \ build-essential \ - ca-certificates \ clang \ - cmake \ - curl \ file \ flex \ g++ \ + g++-multilib \ gawk \ gcc-multilib \ gettext \ git \ libncurses5-dev \ libssl-dev \ - python3-distutils \ + python3-setuptools \ rsync \ - sudo \ + swig \ unzip \ - vim \ wget \ - zlib1g-dev - -FROM base AS usign_build -RUN git clone --depth 1 https://git.openwrt.org/project/usign.git /tmp/usign \ - && cd /tmp/usign \ - && cmake . \ - && make + zlib1g-dev \ + # other dependencies + ca-certificates \ + cmake \ + curl \ + less \ + quilt \ + sudo \ + vim -FROM base AS builder RUN groupadd -g 1000 'buildbot' \ && useradd -m -s '/bin/bash' -u 1000 -g 1000 'buildbot' \ - && echo 'buildbot ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/buildbot + && echo 'buildbot ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/buildbot \ + && echo "QUILT_DIFF_ARGS=--no-timestamps --no-index -p ab --color=auto" \ + "QUILT_REFRESH_ARGS=--no-timestamps --no-index -p ab" \ + "QUILT_SERIES_ARGS=--color=auto" \ + "QUILT_PATCH_OPTS=--unified" \ + "QUILT_DIFF_OPTS=-p" \ + "EDITOR=vim" \ + > /home/buildbot/.quiltrc USER buildbot WORKDIR /home/buildbot # OpenWRT repo ARG OWRT_VERSION RUN git clone --branch "${OWRT_VERSION}" https://github.com/openwrt/openwrt.git WORKDIR /home/buildbot/openwrt -RUN sed -i '/telephony/d' feeds.conf.default +RUN sed -i '/telephony/d' feeds.conf.default \ + && sed -i '/routing/d' feeds.conf.default \ + && sed -i '/video/d' feeds.conf.default RUN ./scripts/feeds update -a COPY --chmod=777 builder/apply-patches.sh /usr/local/bin/apply-patches COPY --chown=buildbot:buildbot patches patches @@ -63,19 +69,19 @@ COPY --chown=buildbot:buildbot config config ARG REPO_CHANNEL ARG TARGET ARG NETHSECURITY_VERSION +ARG BUILD_SEMVER_SUFFIX COPY --chmod=777 builder/configure-build.sh /usr/local/bin/configure-build RUN /usr/local/bin/configure-build COPY --chmod=777 builder/entrypoint.sh /usr/local/bin/entrypoint.sh ENTRYPOINT [ "/usr/local/bin/entrypoint.sh" ] -COPY --from=usign_build /tmp/usign/usign /usr/local/bin/usign RUN mkdir -p \ - .ccache \ - build_dir \ - dl \ - download \ - staging_dir + .ccache \ + build_dir \ + dl \ + download \ + staging_dir VOLUME "/home/buildbot/openwrt/.ccache" \ - "/home/buildbot/openwrt/build_dir" \ - "/home/buildbot/openwrt/dl" \ - "/home/buildbot/openwrt/download" \ - "/home/buildbot/openwrt/staging_dir" + "/home/buildbot/openwrt/build_dir" \ + "/home/buildbot/openwrt/dl" \ + "/home/buildbot/openwrt/download" \ + "/home/buildbot/openwrt/staging_dir" diff --git a/builder/configure-build.sh b/builder/configure-build.sh index 8f1cf189a..f21b34fe1 100644 --- a/builder/configure-build.sh +++ b/builder/configure-build.sh @@ -12,6 +12,13 @@ nethsecurity_version=${NETHSECURITY_VERSION:?Missing NETHSECURITY_VERSION enviro repo_channel=${REPO_CHANNEL:?Missing REPO_CHANNEL environment variable} target=${TARGET:?Missing TARGET environment variable} owrt_version=${OWRT_VERSION:?Missing OWRT_VERSION environment variable} +build_semver_suffix=${BUILD_SEMVER_SUFFIX:-} + +if [ -n "$build_semver_suffix" ]; then + image_version="${nethsecurity_version}${build_semver_suffix}" +else + image_version="${nethsecurity_version}" +fi # For each file inside the config directory, cat the content into a .config file for file in config/*.conf; do @@ -31,7 +38,7 @@ CONFIG_VERSION_DIST="NethSecurity" CONFIG_VERSION_HOME_URL="https://github.com/nethserver/nethsecurity" CONFIG_VERSION_MANUFACTURER="Nethesis" CONFIG_VERSION_MANUFACTURER_URL="https://www.nethesis.it" -CONFIG_VERSION_NUMBER="${nethsecurity_version}" +CONFIG_VERSION_NUMBER="${image_version}" CONFIG_VERSION_CODE="${owrt_version}" CONFIG_VERSION_PRODUCT="NethSecurity" CONFIG_VERSION_REPO="https://updates.nethsecurity.nethserver.org/${repo_channel}/${nethsecurity_version}" diff --git a/builder/entrypoint.sh b/builder/entrypoint.sh index ff3b2a27f..09f4f5409 100644 --- a/builder/entrypoint.sh +++ b/builder/entrypoint.sh @@ -7,12 +7,9 @@ set -e -if [ -n "$USIGN_PUB_KEY" ] && [ -n "$USIGN_PRIV_KEY" ]; then - echo "$USIGN_PUB_KEY" > /home/buildbot/openwrt/key-build.pub - echo "$USIGN_PRIV_KEY" > /home/buildbot/openwrt/key-build -else - echo "No signing keys found. Generating dummy keys..." - usign -G -s ./key-build -p ./key-build.pub -c "Local build key" +if [ -n "$APK_PRIV_KEY" ] && [ -n "$APK_PUB_KEY" ]; then + echo "$APK_PRIV_KEY" > /home/buildbot/openwrt/private-key.pem + echo "$APK_PUB_KEY" > /home/buildbot/openwrt/public-key.pem fi # if command $1 is a file or a executable, run it diff --git a/config/avahi.conf b/config/avahi.conf new file mode 100644 index 000000000..c06fb1e5c --- /dev/null +++ b/config/avahi.conf @@ -0,0 +1 @@ +CONFIG_PACKAGE_avahi-nodbus-daemon=m diff --git a/config/monitoring.conf b/config/monitoring.conf new file mode 100644 index 000000000..f5e4fa381 --- /dev/null +++ b/config/monitoring.conf @@ -0,0 +1,3 @@ +CONFIG_PACKAGE_victoria-metrics=y +CONFIG_PACKAGE_victoria-logs=m +CONFIG_PACKAGE_telegraf=y diff --git a/docs/build/index.md b/docs/build/index.md index 1afbbc4dc..a0c4e9ea9 100644 --- a/docs/build/index.md +++ b/docs/build/index.md @@ -39,7 +39,15 @@ By default, the CI will build the `x86_64` target. To build a different target, To build locally, it's recommended to populate the `build.conf` file with the options you want to use for the build. This file is ignored by Git and should not be committed to the repository. -You can use the `build.conf.example` file as a starting point. Refer to [Environment variables](#environment-variables) for more details on the available options. +The `build.conf.defaults` file contains the versioned defaults and is always tracked by Git. + +You can create a local `build.conf` override that inherits from `build.conf.defaults`: +```bash +cp build.conf.defaults build.conf +# Edit build.conf to override any variables as needed +``` + +Refer to [Environment variables](#environment-variables) for more details on the available options. To build images locally on your machine, make sure these minimum requirements are met: @@ -65,23 +73,30 @@ During the start-up, the container will download netifyd plugins if configuratio ### Environment variables -The `build-nethsec.sh` script behavior can be changed by giving the following environment variables or setting them inside the `build.conf` file: +The `build-nethsec.sh` script behavior can be changed by setting environment variables or by populating the `build.conf` file (git-ignored, local overrides only). + +**Variable loading order:** +1. `build.conf.defaults` (versioned, always loaded first — contains canonical defaults) +2. `build.conf` (git-ignored, optional — can override any variable) +3. Environment variables set before calling `./build-nethsec.sh` (highest priority) + +**Available variables:** - `OWRT_VERSION`: specify the OpenWrt version to build, it can be either a TAG or a branch in the [GitHub OpenWRT repo](https://github.com/openwrt/openwrt); **required** - `NETHSECURITY_VERSION`: specify what to call the NethSecurity image; **required** - `TARGET`: specify the target to build; if not set default is `x86_64` - `REPO_CHANNEL`: specify the channel to publish the image to; if not set default is `dev` -- `USIGN_PUB_KEY` and `USIGN_PRIV_KEY`: see [package signing section](#package-signing) - with the given keys +- `BUILD_SEMVER_SUFFIX`: optional semver suffix appended to the image version only (not the distfeed URL). Use pre-release format (`-rc.1`, `-beta.2`) or metadata format (`+hotfix.1`, `+testing`) or both (`-rc.1+fix.1`). +- `APK_PUB_KEY` and `APK_PRIV_KEY`: see [package signing section](#package-signing) -The `USIGN_PUB_KEY`, `USIGN_PRIV_KEY` variables are always set as secrets inside the CI pipeline, but +The `APK_PUB_KEY`, `APK_PRIV_KEY` variables are always set as secrets inside the CI pipeline, but for [security reasons](https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#accessing-secrets) they are not accessible when building pull requests from forks. ### Build locally for a release If you need to build some packages locally for a release, make sure the following environment variables are set: -- `USIGN_PUB_KEY` and `USIGN_PRIV_KEY`: refer to the [package signing section](#package-signing) for more info +- `APK_PUB_KEY` and `APK_PRIV_KEY`: refer to the [package signing section](#package-signing) for more info Then execute the build as described in the [Build locally](#build-locally) section. @@ -133,7 +148,8 @@ Development version example: ## Upstream version change -To change the OpenWrt version used by NethSecurity, you can just replace the `OWRT_VERSION` variable inside the `build.conf.example` file with the new OpenWrt version. +To change the OpenWrt version used by NethSecurity, update the `OWRT_VERSION` variable inside the `build.conf.defaults` file (versioned, always tracked by Git). +This ensures all developers and CI get the same default version. ## Release new image checklist @@ -256,26 +272,26 @@ To replace an upstream package just create a new package with the same name insi ### Package signing -All packages are signed with the following public key generated with [OpenBSD signify](nethsecurity-pub.key). - -Public key fingerprint: `7640d16662de3b89` +Packages are signed using an EC prime256v1 key pair (PEM format) via the APK package manager. -Public key content: +To generate a new signing key pair: ``` -untrusted comment: NethSecurity sign key -RWR2QNFmYt47ieK7g/zEPwgk+MN8bHsA2vFnPThSpnLZ48L7sh6wxB/f +openssl ecparam -name prime256v1 -genkey -noout -out private-key.pem +openssl ec -in private-key.pem -pubout -out public-key.pem ``` -To sign the packages, just execute the `build-nethsec.sh` script with the following environment variables: -- `USIGN_PUB_KEY` -- `USIGN_PRIV_KEY` +To sign the packages, execute the `build-nethsec.sh` script with the following environment variables: +- `APK_PUB_KEY` +- `APK_PRIV_KEY` Usage example: ``` -USIGN_PUB_KEY=$(cat nethsecurity-pub.key) USIGN_PRIV_KEY=$(cat nethsecurity-priv.key) ./build-nethsec.sh +APK_PUB_KEY=$(cat public-key.pem) APK_PRIV_KEY=$(cat private-key.pem) ./build-nethsec.sh ``` -Or you can have the keys as two files named `key-build` and `key-build.pub` in the root of the repository. They will be automatically used by the build script. +Or you can place the keys as two files named `private-key.pem` and `public-key.pem` in the root of the repository. They will be automatically used by the build script. + +If no keys are provided, OpenWrt will auto-generate a throwaway key pair at build time. Builds executed inside CI will sign the packages with the correct key. diff --git a/files/bin/ipcalc.sh b/files/bin/ipcalc.sh deleted file mode 100755 index e8c7a07df..000000000 --- a/files/bin/ipcalc.sh +++ /dev/null @@ -1,142 +0,0 @@ -#!/bin/sh - -. /lib/functions/ipv4.sh - -PROG="$(basename "$0")" - -# wrapper to convert an integer to an address, unless we're using -# decimal output format. -# hook for library function -_ip2str() { - local var="$1" n="$2" - assert_uint32 "$n" || exit 1 - - if [ "$decimal" -ne 0 ]; then - export -- "$var=$n" - elif [ "$hexadecimal" -ne 0 ]; then - export -- "$var=$(printf "%x" "$n")" - else - ip2str "$@" - fi -} - -usage() { - echo "Usage: $PROG [ -d | -x ] address/prefix [ start limit ]" >&2 - exit 1 -} - -decimal=0 -hexadecimal=0 -if [ "$1" = "-d" ]; then - decimal=1 - shift -elif [ "$1" = "-x" ]; then - hexadecimal=1 - shift -fi - -if [ $# -eq 0 ]; then - usage -fi - -case "$1" in -*/*.*) - # data is n.n.n.n/m.m.m.m format, like on a Cisco router - str2ip ipaddr "${1%/*}" || exit 1 - str2ip netmask "${1#*/}" || exit 1 - netmask2prefix prefix "$netmask" || exit 1 - shift - ;; -*/*) - # more modern prefix notation of n.n.n.n/p - str2ip ipaddr "${1%/*}" || exit 1 - prefix="${1#*/}" - assert_uint32 "$prefix" || exit 1 - if [ "$prefix" -gt 32 ]; then - printf "Prefix out of range (%s)\n" "$prefix" >&2 - exit 1 - fi - prefix2netmask netmask "$prefix" || exit 1 - shift - ;; -*) - # address and netmask as two separate arguments - str2ip ipaddr "$1" || exit 1 - str2ip netmask "$2" || exit 1 - netmask2prefix prefix "$netmask" || exit 1 - shift 2 - ;; -esac - -# we either have no arguments left, or we have a range start and length -if [ $# -ne 0 ] && [ $# -ne 2 ]; then - usage -fi - -# complement of the netmask, i.e. the hostmask -hostmask=$((netmask ^ 0xffffffff)) -network=$((ipaddr & netmask)) -broadcast=$((network | hostmask)) -count=$((hostmask + 1)) - -_ip2str IP "$ipaddr" -_ip2str NETMASK "$netmask" -_ip2str NETWORK "$network" - -echo "IP=$IP" -echo "NETMASK=$NETMASK" -# don't include this-network or broadcast addresses -if [ "$prefix" -le 30 ]; then - _ip2str BROADCAST "$broadcast" - echo "BROADCAST=$BROADCAST" -fi -echo "NETWORK=$NETWORK" -echo "PREFIX=$prefix" -echo "COUNT=$count" - -# if there's no range, we're done -[ $# -eq 0 ] && exit 0 -[ -z "$1$2" ] && exit 0 - -if [ "$prefix" -le 30 ]; then - lower=$((network + 1)) -else - lower="$network" -fi - -start="$1" -assert_uint32 "$start" || exit 1 -start=$((network | (start & hostmask))) -[ "$start" -lt "$lower" ] && start="$lower" -[ "$start" -eq "$ipaddr" ] && start=$((start + 1)) - -if [ "$prefix" -le 30 ]; then - upper=$(((network | hostmask) - 1)) -elif [ "$prefix" -eq 31 ]; then - upper=$((network | hostmask)) -else - upper="$network" -fi - -range="$2" -assert_uint32 "$range" || exit 1 -end=$((start + range - 1)) -[ "$end" -gt "$upper" ] && end="$upper" -[ "$end" -eq "$ipaddr" ] && end=$((end - 1)) - -if [ "$start" -gt "$end" ]; then - echo "network ($NETWORK/$prefix) too small" >&2 - exit 1 -fi - -_ip2str START "$start" -_ip2str END "$end" - -if [ "$start" -le "$ipaddr" ] && [ "$ipaddr" -le "$end" ]; then - echo "warning: address $IP inside range $START..$END" >&2 -fi - -echo "START=$START" -echo "END=$END" - -exit 0 diff --git a/files/etc/profile.d/busybox-history-file.sh b/files/etc/profile.d/busybox-history-file.sh new file mode 100644 index 000000000..9f2c9ab5f --- /dev/null +++ b/files/etc/profile.d/busybox-history-file.sh @@ -0,0 +1 @@ +export HISTFILE=~/.bash_history diff --git a/packages/adblock/Makefile b/packages/adblock/Makefile index 9cfeef42c..0ef88b82a 100644 --- a/packages/adblock/Makefile +++ b/packages/adblock/Makefile @@ -1,13 +1,13 @@ -# -# Copyright (c) 2015-2023 Dirk Brenken (dev@brenken.org) +# dns based ad/abuse domain blocking +# Copyright (c) 2015-2026 Dirk Brenken (dev@brenken.org) # This is free software, licensed under the GNU General Public License v3. # include $(TOPDIR)/rules.mk PKG_NAME:=adblock -PKG_VERSION:=4.1.5 -PKG_RELEASE:=9 +PKG_VERSION:=4.5.5 +PKG_RELEASE:=3 PKG_LICENSE:=GPL-3.0-or-later PKG_MAINTAINER:=Dirk Brenken @@ -16,22 +16,23 @@ include $(INCLUDE_DIR)/package.mk define Package/adblock SECTION:=net CATEGORY:=Network - TITLE:=Powerful adblock script to block ad/abuse domains by using DNS - DEPENDS:=+jshn +jsonfilter +coreutils +coreutils-sort +ca-bundle +opkg + TITLE:=adblock blocks ad/abuse domains by using DNS + DEPENDS:=+jshn +jsonfilter +firewall4 +coreutils +coreutils-sort +gawk +ca-bundle +rpcd +rpcd-mod-rpcsys PKGARCH:=all endef define Package/adblock/description -Powerful adblock solution to block ad/abuse domains via dnsmasq, unbound, named or kresd. -The script supports many domain blacklist sites plus manual black- and whitelist overrides. +adblock blocks ad/abuse domains via dnsmasq, unbound, named, smartdns or kresd. +adblock consumes a minimum of memory, is very fast and supports many domain blocklist sites plus local block- and allowlist overrides. Please see https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md for further information. endef define Package/adblock/conffiles /etc/config/adblock -/etc/adblock/adblock.whitelist -/etc/adblock/adblock.blacklist +/etc/adblock/adblock.allowlist +/etc/adblock/adblock.blocklist +/etc/adblock/adblock.custom.feeds endef define Build/Prepare @@ -55,11 +56,15 @@ define Package/adblock/install $(INSTALL_DIR) $(1)/etc/adblock $(INSTALL_BIN) ./files/adblock.mail $(1)/etc/adblock - $(INSTALL_CONF) ./files/adblock.blacklist $(1)/etc/adblock - $(INSTALL_CONF) ./files/adblock.whitelist $(1)/etc/adblock + $(INSTALL_CONF) ./files/adblock.allowlist $(1)/etc/adblock + $(INSTALL_CONF) ./files/adblock.blocklist $(1)/etc/adblock $(INSTALL_CONF) ./files/adblock.categories $(1)/etc/adblock - $(INSTALL_CONF) ./files/adblock.sources $(1)/etc/adblock - gzip -9n $(1)/etc/adblock/adblock.sources + $(INSTALL_CONF) ./files/adblock.feeds $(1)/etc/adblock + $(INSTALL_CONF) ./files/adblock.custom.feeds $(1)/etc/adblock + + $(INSTALL_DIR) $(1)/etc/uci-defaults + $(INSTALL_BIN) ./files/95-adblock-housekeeping $(1)/etc/uci-defaults + $(INSTALL_BIN) ./files/99_adblock_migrate_lists.sh $(1)/etc/uci-defaults/99_adblock_migrate_lists endef $(eval $(call BuildPackage,adblock)) diff --git a/packages/adblock/files/95-adblock-housekeeping b/packages/adblock/files/95-adblock-housekeeping new file mode 100755 index 000000000..3cef57d79 --- /dev/null +++ b/packages/adblock/files/95-adblock-housekeeping @@ -0,0 +1,114 @@ +#!/bin/sh +# Copyright (c) 2015-2026 Dirk Brenken (dev@brenken.org) +# This is free software, licensed under the GNU General Public License v3. + +# (s)hellcheck exceptions +# shellcheck disable=all + +export LC_ALL=C +export PATH="/usr/sbin:/usr/bin:/sbin:/bin" + +config="adblock" +old_options="adb_sources adb_forcedns adb_fetchutil adb_hag_sources adb_hst_sources adb_stb_sources adb_utc_sources \ + adb_maxqueue adb_backup adb_dnsfilereset adb_tmpbase adb_mailcnt adb_safesearchmod adb_srcfile adb_srcarc adb_nice \ + adb_hag_feed adb_jaildir adb_dnsdenyip adb_dnsallowip adb_zonelist adb_portlist adb_dnsforce adb_replisten" + +for option in ${old_options}; do + inplace="0" + if uci -q get ${config}.global.${option} >/dev/null 2>&1; then + old_values="$(uci -q get ${config}.global.${option})" + for value in ${old_values}; do + case "${option}" in + "adb_sources") + if ! uci -q get ${config}.global.adb_feed | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_feed="${value}" + fi + ;; + "adb_hag_sources") + if ! uci -q get ${config}.global.adb_hag_feed | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_hag_feed="${value}" + fi + ;; + "adb_hst_sources") + if ! uci -q get ${config}.global.adb_hst_feed | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_hst_feed="${value}" + fi + ;; + "adb_stb_sources") + if ! uci -q get ${config}.global.adb_stb_feed | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_stb_feed="${value}" + fi + ;; + "adb_utc_sources") + if ! uci -q get ${config}.global.adb_utc_feed | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_utc_feed="${value}" + fi + ;; + "adb_fetchutil") + uci -q set ${config}.global.adb_fetchcmd="${value}" + ;; + "adb_tmpbase") + uci -q set ${config}.global.adb_basedir="${value}" + ;; + "adb_nice") + uci -q set ${config}.global.adb_nicelimit="${value}" + ;; + "adb_hag_feed") + inplace="1" + if ! printf '%s' "${value}" | grep -qE "^(wildcard/|domains/)"; then + uci -q del_list ${config}.global.adb_hag_feed="${value}" + uci -q add_list ${config}.global.adb_hag_feed="wildcard/${value}" + fi + ;; + "adb_forcedns" | "adb_dnsforce") + uci -q set ${config}.global.adb_nftforce="${value}" + ;; + "adb_zonelist") + if ! uci -q get ${config}.global.adb_nftdevforce | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_nftdevforce="${value}" + fi + ;; + "adb_portlist") + if ! uci -q get ${config}.global.adb_nftportforce | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_nftportforce="${value}" + fi + ;; + "adb_replisten") + if ! uci -q get ${config}.global.adb_repport | grep -q "${value}"; then + uci -q add_list ${config}.global.adb_repport="${value}" + fi + ;; + esac + done + [ "${inplace}" = "0" ] && uci -q delete ${config}.global.${option} + fi +done + +# NethSecurity: migrate adb_bypass to ns_tsdns_bypass +# +if uci -q get ${config}.global.adb_bypass >/dev/null 2>&1; then + old_bypass="$(uci -q get ${config}.global.adb_bypass)" + for value in ${old_bypass}; do + if ! uci -q get ${config}.global.ns_tsdns_bypass | grep -q "${value}"; then + uci -q add_list ${config}.global.ns_tsdns_bypass="${value}" + fi + done + uci -q delete ${config}.global.adb_bypass +fi + +[ -n "$(uci -q changes ${config})" ] && uci -q commit ${config} + +# remove former adblock-related firewall sections (adblock_* redirects and tsdns_bypass ipset) +# +fwcfg="$(uci -qNX show "firewall" | awk 'BEGIN{FS="[.=]"};/adblock_/{if(zone==$2){next}else{ORS=" ";zone=$2;print zone}}')" +for section in ${fwcfg}; do + uci -q delete firewall."${section}" +done +if uci -q get firewall.tsdns_bypass >/dev/null 2>&1; then + uci -q delete firewall.tsdns_bypass +fi +if [ -n "$(uci -q changes firewall)" ]; then + uci -q commit firewall + /etc/init.d/firewall reload +fi +exit 0 diff --git a/packages/adblock/files/99_adblock_migrate_lists.sh b/packages/adblock/files/99_adblock_migrate_lists.sh new file mode 100644 index 000000000..ceea7796d --- /dev/null +++ b/packages/adblock/files/99_adblock_migrate_lists.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Migrate local allow/block list files to staged UCI storage. +# Skip once the dedicated section is already present. +uci -q get adblock.ns_lists >/dev/null 2>&1 && exit 0 + +uci set adblock.ns_lists=ns_lists + +for type in allowlist blocklist; do + file="/etc/adblock/adblock.${type}" + [ -f "${file}" ] || continue + + while IFS= read -r line || [ -n "${line}" ]; do + [ -z "${line}" ] && continue + uci add_list "adblock.ns_lists.${type}=${line}" + done < "${file}" +done + +uci commit adblock diff --git a/packages/adblock/files/README.md b/packages/adblock/files/README.md index 688bdb7cc..725aade52 100644 --- a/packages/adblock/files/README.md +++ b/packages/adblock/files/README.md @@ -2,54 +2,41 @@ # DNS based ad/abuse domain blocking + ## Description -A lot of people already use adblocker plugins within their desktop browsers, but what if you are using your (smart) phone, tablet, watch or any other (wlan) gadget!? Getting rid of annoying ads, trackers and other abuse sites (like facebook) is simple: block them with your router. When the DNS server on your router receives DNS requests, you will sort out queries that ask for the resource records of ad servers and return a simple 'NXDOMAIN'. This is nothing but **N**on-e**X**istent Internet or Intranet domain name, if domain name is unable to resolved using the DNS server, a condition called the 'NXDOMAIN' occurred. +A lot of people already use adblocker plugins within their desktop browsers, but what if you are using your (smart) phone, tablet, watch or any other (wlan) gadget!? Getting rid of annoying ads, trackers and other abuse sites (like facebook) is simple: block them with your router. +When the DNS server on your router receives DNS requests, you will sort out queries that ask for the resource records of ad servers and return a simple 'NXDOMAIN'. This is nothing but **N**on-e**X**istent Internet or Intranet domain name, if a domain name cannot be resolved using the DNS server, a condition called the 'NXDOMAIN' occurred. + + ## Main Features -* Support of the following fully pre-configured domain blocklist sources (free for private usage, for commercial use please check their individual licenses) +* Support of the following fully pre-configured domain blocklist feeds (free for private usage, for commercial use please check their individual licenses) -| Source | Enabled | Size | Focus | Information | +| Feed | Enabled | Size | Focus | Information | | :------------------ | :-----: | :--- | :--------------- | :-------------------------------------------------------------------------------- | -| adaway | x | S | mobile | [Link](https://github.com/AdAway/adaway.github.io) | -| adguard | x | L | general | [Link](https://adguard.com) | -| adguard_tracking | | S | tracking | [Link](https://github.com/AdguardTeam/cname-trackers) | +| 1Hosts | | VAR | compilation | [Link](https://github.com/badmojr/1Hosts) | +| adguard | x | L | general | [Link](https://adguard.com) | +| adguard_tracking | x | L | tracking | [Link](https://github.com/AdguardTeam/cname-trackers) | | android_tracking | | S | tracking | [Link](https://github.com/Perflyst/PiHoleBlocklist) | | andryou | | L | compilation | [Link](https://gitlab.com/andryou/block/-/blob/master/readme.md) | | anti_ad | | L | compilation | [Link](https://github.com/privacy-protection-tools/anti-AD/blob/master/README.md) | -| antipopads | | L | compilation | [Link](https://github.com/AdroitAdorKhan/antipopads-re) | | anudeep | | M | compilation | [Link](https://github.com/anudeepND/blacklist) | | bitcoin | | S | mining | [Link](https://github.com/hoshsadiq/adblock-nocoin-list) | +| certpl | x | L | phishing | [Link](https://cert.pl/en/warning-list/) | | cpbl | | XL | compilation | [Link](https://github.com/bongochong/CombinedPrivacyBlockLists) | -| disconnect | x | S | general | [Link](https://disconnect.me) | +| disconnect | | S | general | [Link](https://disconnect.me) | +| divested | | XXL | compilation | [Link](https://divested.dev/pages/dnsbl) | | doh_blocklist | | S | doh_server | [Link](https://github.com/dibdot/DoH-IP-blocklists) | -| easylist | | M | compilation | [Link](https://easylist.to) | -| easyprivacy | | M | tracking | [Link](https://easylist.to) | | firetv_tracking | | S | tracking | [Link](https://github.com/Perflyst/PiHoleBlocklist) | | games_tracking | | S | tracking | [Link](https://www.gameindustry.eu) | +| hagezi | | VAR | compilation | [Link](https://github.com/hagezi/dns-blocklists) | | hblock | | XL | compilation | [Link](https://hblock.molinero.dev) | -| lightswitch05 | | XL | compilation | [Link](https://github.com/lightswitch05/hosts) | -| notracking | | XL | tracking | [Link](https://github.com/notracking/hosts-blocklists) | +| ipfire_dbl | | VAR | compilation | [Link](https://www.ipfire.org/dbl) | | oisd_big | | XXL | general | [Link](https://oisd.nl) | | oisd_nsfw | | XXL | porn | [Link](https://oisd.nl) | +| oisd_nsfw_small | | M | porn | [Link](https://oisd.nl) | | oisd_small | | L | general | [Link](https://oisd.nl) | -| openphish | | S | phishing | [Link](https://openphish.com) | | phishing_army | | S | phishing | [Link](https://phishing.army) | -| reg_cn | | S | reg_china | [Link](https://easylist.to) | -| reg_cz | | S | reg_czech+slovak | [Link](https://easylist.to) | -| reg_de | | S | reg_germany | [Link](https://easylist.to) | -| reg_es | | S | reg_espania | [Link](https://easylist.to) | -| reg_fi | | S | reg_finland | [Link](https://github.com/finnish-easylist-addition) | -| reg_fr | | M | reg_france | [Link](https://forums.lanik.us/viewforum.php?f=91) | -| reg_id | | S | reg_indonesia | [Link](https://easylist.to) | -| reg_it | | S | reg_italy | [Link](https://easylist.to) | -| reg_jp | | S | reg_japan | [Link](https://github.com/k2jp/abp-japanese-filters) | -| reg_kr | | S | reg_korea | [Link](https://github.com/List-KR/List-KR) | -| reg_nl | | S | reg_netherlands | [Link](https://easylist.to) | -| reg_pl | | M | reg_poland | [Link](https://kadantiscam.netlify.com) | -| reg_ro | | S | reg_romania | [Link](https://easylist.to) | -| reg_ru | | S | reg_russia | [Link](https://easylist.to) | -| reg_se | | S | reg_sweden | [Link](https://github.com/lassekongo83/Frellwits-filter-lists) | -| reg_vn | | S | reg_vietnam | [Link](https://bigdargon.github.io/hostsVN) | | smarttv_tracking | | S | tracking | [Link](https://github.com/Perflyst/PiHoleBlocklist) | | spam404 | | S | general | [Link](https://github.com/Dawsey21) | | stevenblack | | VAR | compilation | [Link](https://github.com/StevenBlack/hosts) | @@ -57,68 +44,76 @@ A lot of people already use adblocker plugins within their desktop browsers, but | utcapitole | | VAR | general | [Link](https://dsi.ut-capitole.fr/blacklists/index_en.php) | | wally3k | | S | compilation | [Link](https://firebog.net/about) | | whocares | | M | general | [Link](https://someonewhocares.org) | -| winhelp | | S | general | [Link](https://winhelp2002.mvps.org) | | winspy | | S | win_telemetry | [Link](https://github.com/crazy-max/WindowsSpyBlocker) | -| yoyo | x | S | general | [Link](https://pgl.yoyo.org/adservers) | - -* List of supported and fully pre-configured adblock sources, already active sources are pre-selected. - To avoid OOM errors, please do not select too many lists! - List size information with the respective domain ranges as follows: - • S (-10k), M (10k-30k) and L (30k-80k) should work for 128 MByte devices, - • XL (80k-200k) should work for 256-512 MByte devices, - • XXL (200k-) needs more RAM and Multicore support, e.g. x86 or raspberry devices. - • VAR (50k-500k) variable size depending on the selection. +| yoyo | | S | general | [Link](https://pgl.yoyo.org/adservers) | + +* List of supported and fully pre-configured adblock sources, already active sources are pre-selected. + To avoid OOM errors, please do not select too many lists! + List size information with the respective domain ranges as follows: + • S (-10k), M (10k-30k) and L (30k-80k) should work for 128 MByte devices + • XL (80k-200k) should work for 256-512 MByte devices + • XXL (200k-) needs more RAM and Multicore support, e.g. x86 or raspberry devices + • VAR (50k-900k) variable size depending on the selection * Zero-conf like automatic installation & setup, usually no manual changes needed * Simple but yet powerful adblock engine: adblock does not use error prone external iptables rulesets, http pixel server instances and things like that -* Supports five different DNS backend formats: dnsmasq, unbound, named (bind), kresd or raw (e.g. used by dnscrypt-proxy) -* Supports four different SSL-enabled download utilities: uclient-fetch, wget, curl or aria2c -* Supports SafeSearch for google, bing, duckduckgo, yandex, youtube and pixabay -* Supports RPZ-trigger 'RPZ-CLIENT-IP' to always allow/deny certain DNS clients based on their IP address (currently only supported by bind dns backend) +* Supports six different DNS backend formats: dnsmasq, unbound, named (bind), kresd, smartdns or raw (e.g. used by dnscrypt-proxy) +* Supports three different SSL-enabled download utilities: uclient-fetch, full wget or curl +* Supports SafeSearch for google, bing, brave, duckduckgo, yandex, youtube and pixabay * Fast downloads & list processing as they are handled in parallel running background jobs with multicore support +* The download engine supports ETAG headers to download only updated feeds * Supports a wide range of router modes, even AP modes are supported * Full IPv4 and IPv6 support * Provides top level domain compression ('tld compression'), this feature removes thousands of needless host entries from the blocklist and lowers the memory footprint for the DNS backend -* Provides a 'DNS File Reset', where the generated DNS blocklist file will be purged after DNS backend loading to save storage space -* Source parsing by fast & flexible regex rulesets, all rules and source information are placed in an external/compredd JSON file ('/etc/adblock/adblock.sources.gz') +* Provides a 'DNS Blocklist Shift', where the generated final DNS blocklist is moved to the backup directory and only a soft link to this file is set in memory. As long as your backup directory is located on an external drive, you should activate this option to save valuable RAM. +* Feed parsing by a very fast & secure domain validator, all domain rules and feed information are placed in an external JSON file ('/etc/adblock/adblock.feeds') * Overall duplicate removal in generated blocklist file 'adb_list.overall' -* Additional local blacklist for manual overrides, located in '/etc/adblock/adblock.blacklist' -* Additional local whitelist for manual overrides, located in '/etc/adblock/adblock.whitelist' -* Quality checks during blocklist update to ensure a reliable DNS backend service +* Additional local allowlist for manual overrides, located in '/etc/adblock/adblock.allowlist' (only exact matches). +* Additional local blocklist for manual overrides, located in '/etc/adblock/adblock.blocklist' +* Implements firewall‑based DNS Control to force DNS interfaces/ports and to redirect to external unfiltered/filtered DNS server +* Includes firewall‑based Remote DNS Allow, a CGI-Interface to allow certain MACs temporary bypass the local adblock DNS +* Supports firewall‑based temporary DNS Bridging, to ensure a Zero‑Downtime during adblock-related DNS Restarts +* Connection checks during blocklist update to ensure a reliable DNS backend service * Minimal status & error logging to syslog, enable debug logging to receive more output -* Procd based init system support ('start', 'stop', 'restart', 'reload', 'enable', 'disable', 'running', 'status', 'suspend', 'resume', 'query', 'report', 'list', 'timer') +* Procd based init system support ('start', 'stop', 'restart', 'reload', 'enable', 'disable', 'running', 'status', 'suspend', 'resume', 'search', 'report') * Auto-Startup via procd network interface trigger or via classic time based startup -* Suspend & Resume adblock temporarily without blocklist reloading +* Suspend & Resume adblock temporarily without blocklist re-processing * Provides comprehensive runtime information -* Provides a detailed DNS Query Report with DNS related information about client requests, top (blocked) domains and more -* Provides a powerful query function to quickly find blocked (sub-)domains, e.g. for whitelisting -* Provides an easily configurable blocklist update scheduler called 'Refresh Timer' -* Includes an option to generate an additional, restrictive 'adb_list.jail' to block access to all domains except those listed in the whitelist file. You can use this restrictive blocklist manually e.g. for guest wifi or kidsafe configurations -* Includes an option to force DNS requests to the local resolver +* Provides a detailed DNS Report with DNS related information about client requests, top (blocked) domains and more +* Provides a powerful search function to quickly find blocked (sub-)domains, e.g. to allow certain domains +* Implements a jail mode - only domains on the allowlist are permitted, all other DNS requests are rejected * Automatic blocklist backup & restore, these backups will be used in case of download errors and during startup -* Send notification E-Mails in case of a processing error or if the overall domain count is ≤ 0 -* Add new adblock sources on your own, see example below +* Send notification E-Mails, see example configuration below +* Add new adblock feeds on your own with the 'Custom Feed Editor' in LuCI or via CLI, see example below * Strong LuCI support, all relevant options are exposed to the web frontend + ## Prerequisites -* [OpenWrt](https://openwrt.org), tested with the stable release series and with the latest rolling snapshot releases. - Please note: Devices with less than 128 MByte RAM are _not_ supported! -* A usual setup with an enabled DNS backend at minimum - dumb AP modes without a working DNS backend are _not_ supported -* A download utility with SSL support: 'wget', 'uclient-fetch' with one of the 'libustream-*' ssl libraries, 'aria2c' or 'curl' is required +* **[OpenWrt](https://openwrt.org)**, latest stable release or a development snapshot +* A usual setup with a working DNS backend +* A download utility with SSL support: 'wget', 'uclient-fetch' with one of the 'libustream-*' ssl libraries or 'curl' is required * A certificate store such as 'ca-bundle' or 'ca-certificates', as adblock checks the validity of the SSL certificates of all download sites by default -* Optional E-Mail notification support: for E-Mail notifications you need to install the additional 'msmtp' package -* Optional DNS Query Report support: for DNS reporting you need to install the additional package 'tcpdump-mini' or 'tcpdump' -* Optional support for gnu awk as alternative to the busybox default, install the additional package 'gawk' +* For E-Mail notifications you need to install and setup the additional 'msmtp' package +* For DNS reporting you need to install the additional package 'tcpdump-mini' or 'tcpdump' +**Please note:** +* Devices with less than 128MB of RAM are **_not_** supported +* For performance reasons, adblock depends on gnu sort and gawk +* Before update from former adblock releases please make a backup of your local allow- and blocklists. In the latest adblock these lists have been renamed to '/etc/adblock/adblock.allowlist' and '/etc/adblock/adblock.blocklist'. There is no automatic content transition to the new files. +* The uci configuration of adblock is automatically migrated during package installation via the uci-defaults mechanism using a housekeeping script + + ## Installation & Usage -* Update your local opkg repository (_opkg update_) -* Install 'adblock' (_opkg install adblock_). The adblock service is enabled by default -* Install the LuCI companion package 'luci-app-adblock' (_opkg install luci-app-adblock_) +* Make a backup and update your local opkg/apk repository +* Install the LuCI companion package 'luci-app-adblock' which also installs the main 'adblock' package as a dependency +* Enable the adblock system service (System -> Startup) and enable adblock itself (adblock -> General Settings) * It's strongly recommended to use the LuCI frontend to easily configure all aspects of adblock, the application is located in LuCI under the 'Services' menu -* Update from a former adblock version is easy. During the update a backup is made of the old configuration '/etc/config/adblock-backup' and replaced by the new config - that's all +* It's also strongly recommended to configure a ‘Startup Trigger Interface’ to ensure automatic adblock startup on WAN-ifup events during boot or reboot of your router -## Adblock CLI Options -* All important adblock functions are accessible via CLI as well. -

+
+## Adblock CLI interface
+* The most important adblock functions are accessible via CLI as well.
+
+```
 ~# /etc/init.d/adblock
 Syntax: /etc/init.d/adblock [command]
 
@@ -132,73 +127,91 @@ Available commands:
 	enabled         Check if service is started on boot
 	suspend         Suspend adblock processing
 	resume          Resume adblock processing
-	query           <domain> Query active blocklists and backups for a specific domain
-	report          [<search>] Print DNS statistics with an optional search parameter
-	list            [<add>|<add_utc>|<add_eng>|<add_stb>|<remove>|<remove_utc>|<remove_eng>|<remove_stb>] <source(s)> List/Edit available sources
-	timer           [<add> <tasks> <hour> [<minute>] [<weekday>]]|[<remove> <line no.>] List/Edit cron update intervals
-	version         Print version information
+	search           Search active blocklists and backups for a specific domain
+	report          [|||] Print DNS statistics
 	running         Check if service is running
 	status          Service status
 	trace           Start with syscall trace
-
+ info Dump procd service info +``` + ## Adblock Config Options * Usually the auto pre-configured adblock setup works quite well and no manual overrides are needed -| Option | Default | Description/Valid Values | -| :----------------- | :--------------------------------- | :--------------------------------------------------------------------------------------------- | -| adb_enabled | 1, enabled | set to 0 to disable the adblock service | -| adb_srcarc | -, /etc/adblock/adblock.sources.gz | full path to the used adblock source archive | -| adb_srcfile | -, /tmp/adb_sources.json | full path to the used adblock source file, which has a higher precedence than the archive file | -| adb_dns | -, auto-detected | 'dnsmasq', 'unbound', 'named', 'kresd' or 'raw' | -| adb_fetchutil | -, auto-detected | 'uclient-fetch', 'wget', 'curl' or 'aria2c' | -| adb_fetchparm | -, auto-detected | manually override the config options for the selected download utility | -| adb_fetchinsecure | 0, disabled | don't check SSL server certificates during download | -| adb_trigger | -, not set | trigger network interface or 'not set' to use a time-based startup | -| adb_triggerdelay | 2 | additional trigger delay in seconds before adblock processing begins | -| adb_debug | 0, disabled | set to 1 to enable the debug output | -| adb_nice | 0, standard prio. | valid nice level range 0-19 of the adblock processes | -| adb_forcedns | 0, disabled | set to 1 to force DNS requests to the local resolver | -| adb_bypass | -, not set | list of IP addresses excluded from `adb_forcedns` option | -| adb_dnsdir | -, auto-detected | path for the generated blocklist file 'adb_list.overall' | -| adb_dnstimeout | 10 | timeout in seconds to wait for a successful DNS backend restart | -| adb_dnsinstance | 0, first instance | set to the relevant dns backend instance used by adblock (dnsmasq only) | -| adb_dnsflush | 0, disabled | set to 1 to flush the DNS Cache before & after adblock processing | -| adb_dnsallow | -, not set | set to 1 to disable selective DNS whitelisting (RPZ-PASSTHRU) | -| adb_lookupdomain | example.com | external domain to check for a successful DNS backend restart or 'false' to disable this check | -| adb_portlist | 53 853 5353 | space separated list of firewall ports which should be redirected locally | -| adb_report | 0, disabled | set to 1 to enable the background tcpdump gathering process for reporting | -| adb_reportdir | /tmp | path for DNS related report files | -| adb_repiface | -, auto-detected | name of the reporting interface or 'any' used by tcpdump | -| adb_replisten | 53 | space separated list of reporting port(s) used by tcpdump | -| adb_repchunkcnt | 5 | report chunk count used by tcpdump | -| adb_repchunksize | 1 | report chunk size used by tcpdump in MB | -| adb_represolve | 0, disabled | resolve reporting IP addresses using reverse DNS (PTR) lookups | -| adb_backup | 1, enabled | set to 0 to disable the backup function | -| adb_backupdir | /tmp | path for adblock backups | -| adb_tmpbase | /tmp | path for all adblock related runtime operations, e.g. downloading, sorting, merging etc. | -| adb_safesearch | 0, disabled | set to 1 to enforce SafeSearch for google, bing, duckduckgo, yandex, youtube and pixabay | -| adb_safesearchlist | -, not set | Limit SafeSearch to certain provider (see above) | -| adb_safesearchmod | 0, disabled | set to 1 to enable moderate SafeSearch filters for youtube | -| adb_mail | 0, disabled | set to 1 to enable notification E-Mails in case of a processing errors | -| adb_mailreceiver | -, not set | receiver address for adblock notification E-Mails | -| adb_mailsender | no-reply@adblock | sender address for adblock notification E-Mails | -| adb_mailtopic | adblock notification | topic for adblock notification E-Mails | -| adb_mailprofile | adb_notify | mail profile used in 'msmtp' for adblock notification E-Mails | -| adb_mailcnt | 0 | minimum domain count to trigger E-Mail notifications | -| adb_jail | 0 | set to 1 to enable the additional, restrictive 'adb_list.jail' creation | -| adb_jaildir | /tmp | path for the generated jail list | +| Option | Default | Description/Valid Values | +| :------------------- | :--------------------------------- | :------------------------------------------------------------------------------------------------- | +| adb_enabled | 1, enabled | set to 0 to disable the adblock service | +| adb_feedfile | /etc/adblock/adblock.feeds | full path to the used adblock feed file | +| adb_dns | -, auto-detected | 'dnsmasq', 'unbound', 'named', 'kresd', 'smartdns' or 'raw' | +| adb_cores | -, auto-detected | limit the cpu cores used by adblock to save RAM | +| adb_fetchcmd | -, auto-detected | 'uclient-fetch', 'wget' or 'curl' | +| adb_fetchparm | -, auto-detected | manually override the config options for the selected download utility | +| adb_fetchinsecure | 0, disabled | don't check SSL server certificates during download | +| adb_trigger | -, not set | trigger network interface or 'not set' to use a time-based startup | +| adb_triggerdelay | 5 | additional trigger delay in seconds before adblock processing begins | +| adb_debug | 0, disabled | set to 1 to enable the debug output | +| adb_nicelimit | 0, standard prio. | valid nice level range 0-19 of the adblock processes | +| adb_dnsshift | 0, disabled | shift the blocklist to the backup directory and only set a soft link to this file in memory | +| adb_dnsdir | -, auto-detected | path for the generated blocklist file 'adb_list.overall' | +| adb_dnstimeout | 20 | timeout in seconds to wait for a successful DNS backend restart | +| adb_dnsinstance | 0, first instance | set the relevant dnsmasq backend instance used by adblock | +| adb_dnsflush | 0, disabled | set to 1 to flush the DNS Cache before & after adblock processing | +| adb_lookupdomain | localhost | domain to check for a successful DNS backend restart | +| adb_report | 0, disabled | set to 1 to enable the background tcpdump gathering process for reporting | +| adb_map | 0, disabled | enable a GeoIP Map with blocked domains | +| adb_reportdir | /tmp/adblock-report | path for DNS related report files | +| adb_repiface | -, auto-detected | name of the reporting interface or 'any' used by tcpdump | +| adb_repport | 53 | list of reporting port(s) used by tcpdump | +| adb_repchunkcnt | 5 | report chunk count used by tcpdump | +| adb_repchunksize | 1 | report chunk size used by tcpdump in MB | +| adb_represolve | 0, disabled | resolve reporting IP addresses using reverse DNS (PTR) lookups | +| adb_tld | 1, enabled | set to 0 to disable the top level domain compression (tld) function | +| adb_basedir | /tmp | path for all adblock related runtime operations, e.g. downloading, sorting, merging etc. | +| adb_backupdir | /tmp/adblock-backup | path for adblock backups | +| adb_safesearch | 0, disabled | enforce SafeSearch for google, bing, brave, duckduckgo, yandex, youtube and pixabay | +| adb_safesearchlist | -, not set | limit SafeSearch to certain provider (see above) | +| adb_mail | 0, disabled | set to 1 to enable notification E-Mails in case of a processing errors | +| adb_mailreceiver | -, not set | receiver address for adblock notification E-Mails | +| adb_mailsender | no-reply@adblock | sender address for adblock notification E-Mails | +| adb_mailtopic | adblock notification | topic for adblock notification E-Mails | +| adb_mailprofile | adb_notify | mail profile used in 'msmtp' for adblock notification E-Mails | +| adb_jail | 0 | jail mode - only domains on the allowlist are permitted, all other DNS requests are rejected | +| adb_nftforce | 0, disabled | redirect all local DNS queries from specified LAN zones to the local DNS resolver | +| adb_nftdevforce | -, not set | firewall LAN Devices/VLANs that should be forced locally | +| adb_nftportforce | -, not set | firewall ports that should be forced locally | +| adb_nftallow | 0, disabled | routes MACs or interfaces to an unfiltered external DNS resolver, bypassing local adblock | +| adb_nftmacallow | -, not set | listed MAC addresses will always use the configured unfiltered DNS server | +| adb_nftdevallow | -, not set | entire interfaces or VLANs will be routed to the unfiltered DNS server | +| adb_allowdnsv4 | -, not set | external IPv4 DNS resolver applied to MACs and interfaces using the unfiltered DNS policy | +| adb_allowdnsv6 | -, not set | external IPv6 DNS resolver applied to MACs and interfaces using the unfiltered DNS policy | +| adb_nftremote | 0, disabled | routes MACs to an unfiltered external DNS resolver, bypassing local adblock | +| adb_nftmacremote | -, not set | Allows listed MACs to remotely access an unfiltered external DNS resolver, bypassing local adblock | +| adb_nftremotetimeout | 15 | Time limit in minutes for remote DNS access of the listed MAC addresses | +| adb_remotednsv4 | -, not set | external IPv4 DNS resolver applied to MACs using the unfiltered remote DNS policy | +| adb_remotednsv6 | -, not set | external IPv6 DNS resolver applied to MACs using the unfiltered remote DNS policy | +| adb_nftblock | 0, disabled | routes MACs or interfaces to a filtered external DNS resolver, bypassing local adblock | +| adb_nftmacblock | -, not set | listed MAC addresses will always use the configured filtered DNS server | +| adb_nftdevblock | -, not set | entire interfaces or VLANs will be routed to the filtered DNS server | +| adb_blockdnsv4 | -, not set | external IPv4 DNS resolver applied to MACs and interfaces using the filtered DNS policy | +| adb_blockdnsv6 | -, not set | external IPv6 DNS resolver applied to MACs and interfaces using the filtered DNS policy | +| adb_nftbridge | -, not set | enables a temporary DNS bridge to an external DNS resolver during local DNS restarts | +| adb_bridgednsv4 | -, not set | external IPv4 DNS resolver used during bridging | +| adb_bridgednsv6 | -, not set | external IPv6 DNS resolver used during bridging | + ## Examples -**Change the DNS backend to 'unbound':** -No further configuration is needed, adblock deposits the final blocklist 'adb_list.overall' in '/var/lib/unbound' by default. + +**Change the DNS backend to 'unbound':** +No further configuration is needed, adblock deposits the final blocklist 'adb_list.overall' in '/var/lib/unbound' by default. To preserve the DNS cache after adblock processing please install the additional package 'unbound-control'. -**Change the DNS backend to 'bind':** -Adblock deposits the final blocklist 'adb_list.overall' in '/var/lib/bind' by default. -To preserve the DNS cache after adblock processing please install the additional package 'bind-rdnc'. +**Change the DNS backend to 'bind':** +Adblock deposits the final blocklist 'adb_list.overall' in '/var/lib/bind' by default. +To preserve the DNS cache after adblock processing please install the additional package 'bind-rndc'. To use the blocklist please modify '/etc/bind/named.conf': -

+
+```
 in the 'options' namespace add:
   response-policy { zone "rpz"; };
 
@@ -209,26 +222,108 @@ and at the end of the file add:
     allow-query { none; };
     allow-transfer { none; };
   };
-
+``` + +**Change the DNS backend to 'kresd':** +Adblock deposits the final blocklist 'adb_list.overall' in '/tmp/kresd', no further configuration needed. + +**Change the DNS backend to 'smartdns':** +No further configuration is needed, adblock deposits the final blocklist 'adb_list.overall' in '/tmp/smartdns' by default. + +**Service status output:** +In LuCI you'll see the realtime status in the 'Runtime' section on the overview page. +To get the status in the CLI, just call _/etc/init.d/adblock status_ or _/etc/init.d/adblock status\_service_: + +```sh +root@blackhole:~# /etc/init.d/adblock status +::: adblock runtime information + + adblock_status : enabled + + frontend_ver : 4.5.2-r4 + + backend_ver : 4.5.2-r4 + + blocked_domains : 888 135 + + active_feeds : 1hosts, adguard, adguard_tracking, bitcoin, certpl, doh_blocklist, hagezi, ipfire_dbl, phishing_army, smarttv_tracking, stevenblack, winspy + + dns_backend : unbound (1.24.2-r1), /mnt/data/adblock/backup, 346.57 MB + + run_ifaces : trigger: wan, report: br-lan + + run_information : base: /mnt/data/adblock, dns: /var/lib/unbound, backup: /mnt/data/adblock/backup, report: /mnt/data/adblock/report, error: /dev/null + + run_flags : shift: ✔, custom feed: ✘, ext. DNS (std/prot/remote/bridge): ✘/✔/✔/✔, force: ✔, flush: ✘, tld: ✔, search: ✘, report: ✔, mail: ✔, jail: ✘, debug: ✘ + + last_run : mode: reload, 2026-03-12T19:08:41+01:00, duration: 0m 57s, 1337.01 MB available + + system_info : cores: 4, fetch: curl, Bananapi BPI-R3, mediatek/filogic, OpenWrt SNAPSHOT (r33360-ab0872a734) +``` + + +## Best practice and tweaks + +**Recommendation for low memory systems** +adblock keeps all working data in RAM to avoid unnecessary flash wear. On devices with only 128–256 MB RAM, you can reduce memory pressure with the following optimizations: +* Use external storage: Set adb_basedir, adb_backupdir and adb_reportdir to a USB drive or SSD to offload temporary and persistent data +* Limit CPU parallelism: Set adb_cores=1 to reduce peak memory usage during feed processing +* Enable blocklist shifting: Activate adb_dnsshift to store the generated blocklist on external storage and keep only a symlink in RAM +* Use firewall‑based DNS redirection: Route DNS queries via nftables to external filtered DNS resolvers and keep only a minimal local blocklist active + +**Sensible choice of blocklists** +The following feeds are just my personal recommendation as an initial setup: +* 'adguard', 'adguard_tracking' and 'certpl' + +In total, this feed selection blocks about 280K domains. It may also be useful to include compilations like hagezi, stevenblack or oisd. +Please note: don't just blindly activate too many feeds at once, sooner or later this will lead to OOM conditions. + +**DNS reporting, enable the GeoIP Map** +adblock includes a powerful reporting tool on the DNS Report tab which shows the latest DNS statistics generated by tcpdump. To get the latest statistics always press the "Refresh" button. +In addition to a tabular overview adblock reporting includes a GeoIP map in a modal popup window/iframe that shows the geolocation of your own uplink addresses (in green) and the locations of blocked domains in red. To enable the GeoIP Map set the following option in "Advanced Report Settings" config tab: set 'adb_map' to '1' to include the external components listed below and activate the GeoIP map. + +To make this work, adblock uses the following external components: +* [Leaflet](https://leafletjs.com/) is a lightweight open-source JavaScript library for interactive maps +* [OpenStreetMap](https://www.openstreetmap.org/) provides the map data under an open-source license +* [CARTO basemap styles](https://github.com/CartoDB/basemap-styles) based on [OpenMapTiles](https://openmaptiles.org/schema) +* The free and quite fast [IP Geolocation API](https://ip-api.com/) to resolve the required IP/geolocation information (max. 45 blocked Domains per request) + +**External adblock test** +In addition to the built‑in DNS reporting and GeoIP map, adblock users can verify the effectiveness of their configuration with an external test page. The [Adblock Test](https://adblock.turtlecute.org/) provides a simple way to check whether your current adblock setup is working as expected. It loads a series of test elements (ads, trackers, and other resources) and reports whether they are successfully blocked by your configuration. + +The test runs entirely in the browser and does not require additional configuration. For best results, open the page in the same environment where adblock is active and review the results displayed. + +**Firewall‑Based DNS Control** +adblock provides several advanced firewall‑integrated features that allow you to enforce DNS policies directly at the network layer. These mechanisms operate independently of the local DNS resolver and ensure that DNS traffic follows your filtering rules, even when clients attempt to bypass them. +* Unfiltered external DNS Routing: routes DNS queries from selected devices or interfaces to an external unfiltered DNS resolver +* Filtered external DNS Routing: routes DNS queries from selected devices or interfaces to an external filtered DNS resolver +* Force DNS: blocks or redirects all external DNS traffic to ensure that clients use the local resolver + +The DNS routing allows you to apply external DNS (unfiltered and/or filtered) to specific devices or entire network segments. DNS queries from these targets are transparently redirected to a chosen external resolver (IPv4 and/or IPv6): +* MAC‑based targeting for individual devices +* Interface/VLAN targeting for entire segments +* Separate IPv4/IPv6 resolver selection +* Transparent DNS redirection without client‑side configuration +This mode is ideal for guest networks, IoT devices, or environments where certain clients require stricter/lesser DNS filtering. + +force DNS ensures that all DNS traffic on your network by specific devices or entire network segments is processed by the local resolver. Any attempt to use external DNS servers is blocked or redirected. +* Blocks external DNS on port 53 and redirects DNS queries to the local resolver when appropriate +* Also prevents DNS bypassing by clients with hardcoded DNS settings on other ports, e.g. on port 853 +This mode guarantees that adblock’s filtering pipeline is always applied. + +adblock's firewall rules are based on nftables in a separate isolated nftables table (inet adblock) and chains (prerouting), with MAC addresses stored in a nftables set. The configuration is carried out centrally in LuCI on the ‘Firewall Settings’ tab in adblock. + +**Remote DNS Allow (Temporary MAC‑Based Bypass)** +This additional firewall feature lets selected client devices temporarily bypass local DNS blocking and use an external, unfiltered DNS resolver. It is designed for situations where a device needs short‑term access to content normally blocked by the adblock rules. -**Change the DNS backend to 'kresd':** -Adblock deposits the final blocklist 'adb_list.overall' in '/etc/kresd', no further configuration needed. -Please note: The knot-resolver (kresd) is only available on Turris devices and does not support the SafeSearch functionality yet. +A lightweight CGI endpoint handles the workflow: +* The client opens the URL, e.g. https://\cgi-bin/adblock (preferably transferred via QR code shown in LuCI) +* The script automatically detects the device’s MAC address +* If the MAC is authorized, the script displays the current status: + * Not in the nftables set → option to request a temporary allow (“Renew”) + * Already active → shows remaining timeout +* When renewing, the CGI adds the MAC to an nftables Set with a per‑entry timeout -**Use restrictive jail modes:** -You can enable a restrictive 'adb_list.jail' to block access to all domains except those listed in the whitelist file. Usually this list will be generated as an additional list for guest or kidsafe configurations (for a separate dns server instance). If the jail directory points to your primary dns directory, adblock enables the restrictive jail mode automatically (jail mode only). +The CGI interface is mobile‑friendly and includes a LuCI‑style loading spinner during the renew process, giving immediate visual feedback while the nftables entry is created. All operations are atomic and safe even when multiple devices renew access in parallel. -**Manually override the download options:** -By default adblock uses the following pre-configured download options: -* aria2c: --timeout=20 --allow-overwrite=true --auto-file-renaming=false --log-level=warn --dir=/ -o -* curl: --connect-timeout 20 --silent --show-error --location -o -* uclient-fetch: --timeout=20 -O -* wget: --no-cache --no-cookies --max-redirect=0 --timeout=20 -O +**Temporary DNS Bridging (Zero‑Downtime during DNS Restarts)** +Adblock can optionally enable a temporary DNS bridging mode to avoid DNS downtime during DNS backend restarts. +When this feature is enabled, all DNS queries from LAN clients are briefly redirected to an external fallback resolver until the local DNS backend becomes available again. This ensures that DNS resolution continues to work seamlessly for all clients, even while adblock reloads blocklists or restarts the DNS service. Just set the options 'adb_nftbridge', 'adb_bridgednsv4' and 'adb_bridgednsv6' accordingly. -To override the default set 'adb_fetchparm' manually to your needs. +**Jail mode (allowlist-only):** +Enforces a strict allowlist‑only DNS policy in which only domains listed in the allowlist file are resolved, while every other query is rejected. This mode is intended for highly restrictive environments and depends on a carefully maintained allowlist, typically managed manually. -**Enable E-Mail notification via 'msmtp':** -To use the email notification you have to install & configure the package 'msmtp'. +**Enable E-Mail notification via 'msmtp':** +To use the email notification you have to install & configure the package 'msmtp'. Modify the file '/etc/msmtprc': 

 [...]
@@ -246,70 +341,102 @@ from            dev.adblock@gmail.com
 user            dev.adblock
 password        xxx
-Finally enable E-Mail support and add a valid E-Mail receiver address in LuCI. +Finally enable E-Mail support, add a valid E-Mail receiver address in LuCI and setup an appropriate cron job. -**Service status output:** -In LuCI you'll see the realtime status in the 'Runtime' section on the overview page. -To get the status in the CLI, just call _/etc/init.d/adblock status_ or _/etc/init.d/adblock status\_service_: -

-~#@blackhole:~# /etc/init.d/adblock status
-::: adblock runtime information
-  + adblock_status  : enabled
-  + adblock_version : 4.1.4
-  + blocked_domains : 268355
-  + active_sources  : adaway, adguard, adguard_tracking, android_tracking, bitcoin, disconnect, firetv_tracking, games_t
-                      racking, hblock, oisd_basic, phishing_army, smarttv_tracking, stopforumspam, wally3k, winspy, yoyo
-  + dns_backend     : unbound (unbound-control), /var/lib/unbound
-  + run_utils       : download: /usr/bin/curl, sort: /usr/libexec/sort-coreutils, awk: /bin/busybox
-  + run_ifaces      : trigger: wan, report: br-lan
-  + run_directories : base: /tmp, backup: /mnt/data/adblock-Backup, report: /mnt/data/adblock-Report, jail: /tmp
-  + run_flags       : backup: ✔, flush: ✘, force: ✔, search: ✘, report: ✔, mail: ✔, jail: ✘
-  + last_run        : restart, 3m 17s, 249/73/68, 2022-09-10T13:43:07+02:00
-  + system          : ASUS RT-AX53U, OpenWrt SNAPSHOT r20535-2ca5602864
-
-The 'last\_run' line includes the used start type, the run duration, the memory footprint after DNS backend loading (total/free/available) and the date/time of the last run. +**Automatic adblock feed updates and E-Mail reports** +For a regular, automatic update of the used feeds or other regular adblock tasks set up a cron job. In LuCI you find the cron settings under 'System' => 'Scheduled Tasks'. On the command line the cron file is located at '/etc/crontabs/root': -**Edit, add new adblock sources:** -The adblock blocklist sources are stored in an external, compressed JSON file '/etc/adblock/adblock.sources.gz'. -This file is directly parsed in LuCI and accessible via CLI, just call _/etc/init.d/adblock list_: -

-/etc/init.d/adblock list
-::: Available adblock sources
-:::
-    Name                 Enabled   Size   Focus               Info URL
-    ------------------------------------------------------------------
-  + adaway               x         S      mobile              https://adaway.org
-  + adguard              x         L      general             https://adguard.com
-  + andryou              x         L      compilation         https://gitlab.com/andryou/block/-/blob/master/readme.md
-  + bitcoin              x         S      mining              https://github.com/hoshsadiq/adblock-nocoin-list
-  + disconnect           x         S      general             https://disconnect.me
-  + dshield                        XL     general             https://www.dshield.org
-[...]
-  + winhelp                        S      general             http://winhelp2002.mvps.org
-  + winspy               x         S      win_telemetry       https://github.com/crazy-max/WindowsSpyBlocker
-  + yoyo                 x         S      general             https://pgl.yoyo.org
-
+Example 1 +```sh +# update the adblock feeds every morning at 4 o'clock +00 04 * * * /etc/init.d/adblock reload +``` -To add new or edit existing sources extract the compressed JSON file _gunzip /etc/adblock/adblock.sources.gz_. -A valid JSON source object contains the following required information, e.g.: -

+Example 2
+```sh
+# update the adblock feeds every hour
+0 */1 * * * /etc/init.d/adblock reload
+```
+
+Example 3
+```sh
+# send an adblock E-Mail report every morning at 3 o'clock
+00 03 * * * /etc/init.d/adblock report mail
+```
+
+**Change/add adblock feeds**
+The adblock blocklist feeds are stored in an external JSON file '/etc/adblock/adblock.feeds'. All custom changes should be stored in an external JSON file '/etc/adblock/adblock.custom.feeds' (empty by default). It's recommended to use the LuCI based Custom Feed Editor to make changes to this file.
+A valid JSON source object contains the following information, e.g.:
+
+```json
 	[...]
-	"adaway": {
-		"url": "https://raw.githubusercontent.com/AdAway/adaway.github.io/master/hosts.txt",
-		"rule": "/^127\\.0\\.0\\.1[[:space:]]+([[:alnum:]_-]+\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}",
-		"size": "S",
-		"focus": "mobile",
-		"descurl": "https://github.com/AdAway/adaway.github.io"
+	"stevenblack": {
+		"url": "https://raw.githubusercontent.com/StevenBlack/hosts/master/",
+		"rule": "feed 0.0.0.0 2",
+		"size": "VAR",
+		"descr": "compilation"
 	},
 	[...]
-
-Add an unique object name, make the required changes to 'url', 'rule', 'size' and 'descurl' and finally compress the changed JSON file _gzip /etc/adblock/adblock.sources_ to use the new source object in adblock. -Please note: if you're going to add new sources on your own, please make a copy of the default file and work with that copy further on, cause the default will be overwritten with every adblock update. To reference your copy set the option 'adb\_srcarc' which points by default to '/etc/adblock/adblock.sources.gz' -Please note: when adblock starts, it looks for the uncompressed 'adb\_srcfile', only if this file is not found the archive 'adb\_srcarc' is unpacked once and then the uncompressed file is used +``` + +Add a unique feed name (no spaces, no special chars) and make the required changes: adapt at least the URL, check/change the rule, the size and the description for a new feed. +The rule consist of max. 4 individual, space separated parameters: +1. type: always 'feed' (required) +2. prefix: an optional search term (a string literal, no regex) to identify valid domain list entries, e.g. '0.0.0.0' +3. column: the domain column within the feed file, e.g. '2' (required) +4. separator: an optional field separator, default is the character class '[[:space:]]' + +**Enable debug mode** +Adblock provides an optional debug mode that writes diagnostic information to the system log and captures internal error output in a dedicated error logfile - by default located in the adblock base directory as '/tmp/adb_error.log'. The log file is automatically cleared at the beginning of each run. Under normal conditions, all error messages are discarded to keep regular runs clean and silent. To enable debug mode, set the option 'adb_debug' to '1'. When enabled, the script produces significantly more log output to assist with troubleshooting. ## Support Please join the adblock discussion in this [forum thread](https://forum.openwrt.org/t/adblock-support-thread/507) or contact me by mail -Have fun! +## Removal +Stop all adblock related services with _/etc/init.d/adblock stop_ and remove the adblock package if necessary. + +## Donations +You like this project - is there a way to donate? Generally speaking "No" - I have a well-paying full-time job and my OpenWrt projects are just a hobby of mine in my spare time. + +If you still insist to donate some bucks ... +* I would be happy if you put your money in kind into other, social projects in your area, e.g. a children's hospice +* Let's meet and invite me for a coffee if you are in my area, the “Markgräfler Land” in southern Germany or in Switzerland (Basel) +* Send your money to my [PayPal account](https://www.paypal.me/DirkBrenken) and I will collect your donations over the year to support various social projects in my area + +No matter what you decide - thank you very much for your support! + +Have fun! Dirk +--- + +## NethSecurity Integration + +NethSecurity ships adblock as the DNS-blocking engine for the Threat Shield DNS feature. + +### Threat Shield DNS + +The Threat Shield DNS integration is controlled by the `ts_enabled` option in `adblock.global`. When enabled, `ts-dns` populates `/etc/adblock/adblock.custom.feeds` with Nethesis enterprise feeds (if a subscription is active) and community free feeds, which adblock reads automatically. + +Relevant UCI options set by NethSecurity (in addition to standard adblock options): + +| Option | Default | Description | +| :--- | :--- | :--- | +| `ts_enabled` | `0` | Set to `1` by NethSecurity to activate Threat Shield DNS mode | +| `ns_tsdns_bypass` | -, not set | List of IP addresses or subnets excluded from `adb_nftforce` DNS redirection | + +### IP-based DNS bypass + +The standard adblock `adb_nftforce` feature forces DNS queries from specified LAN devices/VLANs through the local resolver. NethSecurity extends this with an **IP-based bypass** list (`ns_tsdns_bypass`): any source IP or subnet in that list is exempt from DNS redirection, even when DNS enforcement is active. + +To add a bypass address or subnet: +```sh +uci add_list adblock.global.ns_tsdns_bypass=192.168.100.2 +uci commit adblock +/etc/init.d/adblock restart +``` + +The bypass rules are injected into the `inet adblock pre-routing` chain (adblock's own nftables table) as `return` rules that take effect before the DNS redirect rules. No fw4/firewall rules are involved. + +### Disabling the CGI remote allow page +The upstream `adblock.cgi` CGI endpoint is **not installed** in NethSecurity. All adblock management is handled through the NethSecurity API (`ns.threatshield`). diff --git a/packages/adblock/files/adblock.blacklist b/packages/adblock/files/adblock.allowlist similarity index 100% rename from packages/adblock/files/adblock.blacklist rename to packages/adblock/files/adblock.allowlist diff --git a/packages/adblock/files/adblock.whitelist b/packages/adblock/files/adblock.blocklist similarity index 100% rename from packages/adblock/files/adblock.whitelist rename to packages/adblock/files/adblock.blocklist diff --git a/packages/adblock/files/adblock.categories b/packages/adblock/files/adblock.categories index 1d1118837..fad1e4abc 100644 --- a/packages/adblock/files/adblock.categories +++ b/packages/adblock/files/adblock.categories @@ -1,21 +1,100 @@ -stb;fakenews;alternates/fakenews/hosts -stb;fakenews-gambling;alternates/fakenews-gambling/hosts -stb;fakenews-gambling-porn;alternates/fakenews-gambling-porn/hosts -stb;fakenews-gambling-porn-social;alternates/fakenews-porn-social/hosts -stb;fakenews-gambling-social;alternates/fakenews-gambling-social/hosts -stb;fakenews-porn;alternates/fakenews-porn/hosts -stb;fakenews-porn-social;alternates/fakenews-porn-social/hosts -stb;fakenews-social;alternates/fakenews-social/hosts -stb;gambling;alternates/gambling/hosts -stb;gambling-porn;alternates/gambling-porn/hosts -stb;gambling-porn-social;alternates/gambling-porn-social/hosts -stb;gambling-social;alternates/gambling-social/hosts -stb;porn;alternates/porn/hosts -stb;porn-social;alternates/porn-social/hosts -stb;social;alternates/social/hosts +hag;multi-light;wildcard/light-onlydomains.txt +hag;multi-normal;wildcard/multi-onlydomains.txt +hag;multi-pro;wildcard/pro-onlydomains.txt +hag;multi-pro.mini;wildcard/pro.mini-onlydomains.txt +hag;multi-pro.plus;wildcard/pro.plus-onlydomains.txt +hag;multi-pro.plus.mini;wildcard/pro.plus.mini-onlydomains.txt +hag;multi-ultimate;wildcard/ultimate-onlydomains.txt +hag;multi-ultimate.mini;wildcard/ultimate.mini-onlydomains.txt +hag;threat-intelligence;wildcard/tif-onlydomains.txt +hag;threat-intelligence.medium;wildcard/tif.medium-onlydomains.txt +hag;threat-intelligence.mini;wildcard/tif.mini-onlydomains.txt +hag;anti.piracy;wildcard/anti.piracy-onlydomains.txt +hag;blocklist-referral;wildcard/blocklist-referral-onlydomains.txt +hag;doh;wildcard/doh-onlydomains.txt +hag;doh-vpn-proxy-bypass;wildcard/doh-vpn-proxy-bypass-onlydomains.txt +hag;dyndns;wildcard/dyndns-onlydomains.txt +hag;fake;wildcard/fake-onlydomains.txt +hag;gambling;wildcard/gambling-onlydomains.txt +hag;gambling.medium;wildcard/gambling.medium-onlydomains.txt +hag;gambling.mini;wildcard/gambling.mini-onlydomains.txt +hag;hoster;wildcard/hoster-onlydomains.txt +hag;nsfw;wildcard/nsfw-onlydomains.txt +hag;tracker.amazon;wildcard/native.amazon-onlydomains.txt +hag;tracker.apple;wildcard/native.apple-onlydomains.txt +hag;tracker.huawei;wildcard/native.huawei-onlydomains.txt +hag;tracker.lgwebos;wildcard/native.lgwebos-onlydomains.txt +hag;tracker.oppo-realme;wildcard/native.oppo-realme-onlydomains.txt +hag;tracker.roku;wildcard/native.roku-onlydomains.txt +hag;tracker.samsung;wildcard/native.samsung-onlydomains.txt +hag;tracker.tiktok;wildcard/native.tiktok-onlydomains.txt +hag;tracker.tiktok.extended;wildcard/native.tiktok.extended-onlydomains.txt +hag;tracker.vivo;wildcard/native.vivo-onlydomains.txt +hag;tracker.winoffice;wildcard/native.winoffice-onlydomains.txt +hag;tracker.xiaomi;wildcard/native.xiaomi-onlydomains.txt +hag;nosafesearch;wildcard/nosafesearch-onlydomains.txt +hag;popupads;wildcard/popupads-onlydomains.txt +hag;urlshortener;wildcard/urlshortener-onlydomains.txt +hag;abusetlds;wildcard/spam-tlds-onlydomains.txt +hag;social;wildcard/social-onlydomains.txt +hag;dga-7days;domains/dga7.txt +hag;dga-14days;domains/dga14.txt +hag;dga-30days;domains/dga30.txt +hag;nrd-7days;domains/nrd7.txt +hag;nrd-14days;domains/nrd14-8.txt +hag;nrd-21days;domains/nrd21-15.txt +hag;nrd-28days;domains/nrd28-22.txt +hag;nrd-35days;domains/nrd35-29.txt +hst;lite;Lite/domains.wildcards +hst;xtra;Xtra/domains.wildcards +ipf;ads;ads/domains.txt +ipf;dating;dating/domains.txt +ipf;doh;doh/domains.txt +ipf;gambling;gambling/domains.txt +ipf;games;games/domains.txt +ipf;malware;malware/domains.txt +ipf;phishing;phishing/domains.txt +ipf;piracy;piracy/domains.txt +ipf;porn;porn/domains.txt +ipf;shopping;shopping/domains.txt +ipf;smart-tv;smart-tv/domains.txt +ipf;social;social/domains.txt +ipf;streaming;streaming/domains.txt +ipf;violence;violence/domains.txt stb;standard;hosts +stb;standard-fakenews;alternates/fakenews/hosts +stb;standard-fakenews-gambling;alternates/fakenews-gambling/hosts +stb;standard-fakenews-gambling-porn;alternates/fakenews-gambling-porn/hosts +stb;standard-fakenews-gambling-porn-social;alternates/fakenews-gambling-porn-social/hosts +stb;standard-fakenews-gambling-social;alternates/fakenews-gambling-social/hosts +stb;standard-fakenews-porn;alternates/fakenews-porn/hosts +stb;standard-fakenews-porn-social;alternates/fakenews-porn-social/hosts +stb;standard-fakenews-social;alternates/fakenews-social/hosts +stb;standard-gambling;alternates/gambling/hosts +stb;standard-gambling-porn;alternates/gambling-porn/hosts +stb;standard-gambling-porn-social;alternates/gambling-porn-social/hosts +stb;standard-gambling-social;alternates/gambling-social/hosts +stb;standard-porn;alternates/porn/hosts +stb;standard-porn-social;alternates/porn-social/hosts +stb;standard-social;alternates/social/hosts +stb;fakenews;alternates/fakenews-only/hosts +stb;fakenews-gambling;alternates/fakenews-gambling-only/hosts +stb;fakenews-gambling-porn;alternates/fakenews-gambling-porn-only/hosts +stb;fakenews-gambling-porn-social;alternates/fakenews-gambling-porn-social-only/hosts +stb;fakenews-gambling-social;alternates/fakenews-gambling-social-only/hosts +stb;fakenews-porn;alternates/fakenews-porn-only/hosts +stb;fakenews-porn-social;alternates/fakenews-porn-social-only/hosts +stb;fakenews-social;alternates/fakenews-social-only/hosts +stb;gambling;alternates/gambling-only/hosts +stb;gambling-porn;alternates/gambling-porn-only/hosts +stb;gambling-porn-social;alternates/gambling-porn-social-only/hosts +stb;gambling-social;alternates/gambling-social-only/hosts +stb;porn;alternates/porn-only/hosts +stb;porn-social;alternates/porn-social-only/hosts +stb;social;alternates/social-only/hosts utc;adult utc;agressif +utc;ai utc;arjel utc;associations_religieuses utc;astrology @@ -36,9 +115,11 @@ utc;dialer utc;doh utc;download utc;drogue +utc;dynamic-dns utc;educational_games utc;examen_pix utc;exceptions_liste_bu +utc;fakenews utc;filehosting utc;financial utc;forums @@ -61,6 +142,7 @@ utc;radio utc;reaffected utc;redirector utc;remote-control +utc;residential-proxies utc;sect utc;sexual_education utc;shopping @@ -73,7 +155,9 @@ utc;strict_redirector utc;strong_redirector utc;translation utc;tricheur +utc;tricheur_pix utc;update utc;vpn utc;warez +utc;webhosting utc;webmail diff --git a/packages/adblock/files/adblock.conf b/packages/adblock/files/adblock.conf index 8b50b4211..37fd67aa6 100644 --- a/packages/adblock/files/adblock.conf +++ b/packages/adblock/files/adblock.conf @@ -2,13 +2,11 @@ config adblock 'global' option adb_enabled '0' option adb_debug '0' - option adb_forcedns '0' + option adb_dnsforce '0' + option adb_dnsshift '0' option adb_safesearch '0' - option adb_dnsfilereset '0' option adb_mail '0' option adb_report '0' - option adb_backup '1' - list adb_sources 'adaway' - list adb_sources 'adguard' - list adb_sources 'disconnect' - list adb_sources 'yoyo' + list adb_feed 'adguard' + list adb_feed 'adguard_tracking' + list adb_feed 'certpl' diff --git a/packages/adblock/files/adblock.custom.feeds b/packages/adblock/files/adblock.custom.feeds new file mode 100644 index 000000000..e69de29bb diff --git a/packages/adblock/files/adblock.feeds b/packages/adblock/files/adblock.feeds new file mode 100644 index 000000000..34a437261 --- /dev/null +++ b/packages/adblock/files/adblock.feeds @@ -0,0 +1,194 @@ +{ + "1hosts": { + "url": "https://raw.githubusercontent.com/badmojr/1Hosts/master/", + "rule": "feed 1", + "size": "VAR", + "descr": "compilation" + }, + "adguard": { + "url": "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt", + "rule": "feed || 3 [|^]", + "size": "L", + "descr": "general" + }, + "adguard_tracking": { + "url": "https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/data/combined_disguised_trackers_justdomains.txt", + "rule": "feed 1", + "size": "L", + "descr": "tracking" + }, + "android_tracking": { + "url": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/android-tracking.txt", + "rule": "feed 1", + "size": "S", + "descr": "tracking" + }, + "andryou": { + "url": "https://gitlab.com/andryou/block/raw/master/kouhai-compressed-domains", + "rule": "feed 1", + "size": "L", + "descr": "compilation" + }, + "anti_ad": { + "url": "https://raw.githubusercontent.com/privacy-protection-tools/anti-AD/master/anti-ad-domains.txt", + "rule": "feed 1", + "size": "L", + "descr": "compilation" + }, + "anudeep": { + "url": "https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt", + "rule": "feed 0.0.0.0 2", + "size": "M", + "descr": "compilation" + }, + "bitcoin": { + "url": "https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt", + "rule": "feed 0.0.0.0 2", + "size": "S", + "descr": "mining" + }, + "certpl": { + "url": "https://hole.cert.pl/domains/v2/domains.txt", + "rule": "feed 1", + "size": "L", + "descr": "phishing" + }, + "cpbl": { + "url": "https://raw.githubusercontent.com/bongochong/CombinedPrivacyBlockLists/master/NoFormatting/cpbl-ctld.txt", + "rule": "feed 1", + "size": "XL", + "descr": "compilation" + }, + "disconnect": { + "url": "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt", + "rule": "feed 1", + "size": "S", + "descr": "general" + }, + "divested": { + "url": "https://divested.dev/hosts-domains-wildcards", + "rule": "feed 1", + "size": "XXL", + "descr": "compilation" + }, + "doh_blocklist": { + "url": "https://raw.githubusercontent.com/dibdot/DoH-IP-blocklists/master/doh-domains_overall.txt", + "rule": "feed 1", + "size": "S", + "descr": "doh_server" + }, + "firetv_tracking": { + "url": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/AmazonFireTV.txt", + "rule": "feed 1", + "size": "S", + "descr": "tracking" + }, + "games_tracking": { + "url": "https://raw.githubusercontent.com/KodoPengin/GameIndustry-hosts-Template/master/Main-Template/hosts", + "rule": "feed 0.0.0.0 2", + "size": "S", + "descr": "tracking" + }, + "hagezi": { + "url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/", + "rule": "feed 1", + "size": "VAR", + "descr": "compilation" + }, + "hblock": { + "url": "https://hblock.molinero.dev/hosts_domains.txt", + "rule": "feed 1", + "size": "XL", + "descr": "compilation" + }, + "ipfire_dbl": { + "url": "https://dbl.ipfire.org/lists/", + "rule": "feed 1", + "size": "VAR", + "descr": "compilation" + }, + "oisd_big": { + "url": "https://big.oisd.nl/domainswild2", + "rule": "feed 1", + "size": "XXL", + "descr": "general" + }, + "oisd_nsfw": { + "url": "https://nsfw.oisd.nl/domainswild2", + "rule": "feed 1", + "size": "XXL", + "descr": "porn" + }, + "oisd_nsfw_small": { + "url": "https://nsfw-small.oisd.nl/domainswild2", + "rule": "feed 1", + "size": "M", + "descr": "porn" + }, + "oisd_small": { + "url": "https://small.oisd.nl/domainswild2", + "rule": "feed 1", + "size": "L", + "descr": "general" + }, + "phishing_army": { + "url": "https://phishing.army/download/phishing_army_blocklist_extended.txt", + "rule": "feed 1", + "size": "S", + "descr": "phishing" + }, + "smarttv_tracking": { + "url": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV.txt", + "rule": "feed 1", + "size": "S", + "descr": "tracking" + }, + "spam404": { + "url": "https://raw.githubusercontent.com/Dawsey21/Lists/master/main-blacklist.txt", + "rule": "feed 1", + "size": "S", + "descr": "general" + }, + "stevenblack": { + "url": "https://raw.githubusercontent.com/StevenBlack/hosts/master/", + "rule": "feed 0.0.0.0 2", + "size": "VAR", + "descr": "compilation" + }, + "stopforumspam": { + "url": "https://www.stopforumspam.com/downloads/toxic_domains_whole.txt", + "rule": "feed 1", + "size": "S", + "descr": "spam" + }, + "utcapitole": { + "url": "https://dsi.ut-capitole.fr/blacklists/download/blacklists.tar.gz", + "rule": "feed 1", + "size": "VAR", + "descr": "general" + }, + "wally3k": { + "url": "https://v.firebog.net/hosts/static/w3kbl.txt", + "rule": "feed 1", + "size": "S", + "descr": "compilation" + }, + "whocares": { + "url": "https://someonewhocares.org/hosts/hosts", + "rule": "feed 127.0.0.1 2", + "size": "M", + "descr": "general" + }, + "winspy": { + "url": "https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt", + "rule": "feed 0.0.0.0 2", + "size": "S", + "descr": "win_telemetry" + }, + "yoyo": { + "url": "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=nohtml&showintro=0&mimetype=plaintext", + "rule": "feed 1", + "size": "S", + "descr": "general" + } +} diff --git a/packages/adblock/files/adblock.init b/packages/adblock/files/adblock.init index 44e6228f8..8bdba396a 100755 --- a/packages/adblock/files/adblock.init +++ b/packages/adblock/files/adblock.init @@ -1,8 +1,8 @@ #!/bin/sh /etc/rc.common -# Copyright (c) 2015-2022 Dirk Brenken (dev@brenken.org) +# Copyright (c) 2015-2026 Dirk Brenken (dev@brenken.org) # This is free software, licensed under the GNU General Public License v3. -# disable (s)hellcheck in release +# (s)hellcheck exceptions # shellcheck disable=all START=30 @@ -10,25 +10,69 @@ USE_PROCD=1 extra_command "suspend" "Suspend adblock processing" extra_command "resume" "Resume adblock processing" -extra_command "query" " Query active blocklists and backups for a specific domain" -extra_command "report" "[[|||] [] [] []] Print DNS statistics with an optional search parameter" -extra_command "list" "[|||||||] List/Edit available sources" -extra_command "timer" "[ [] []]|[ ] List/Edit cron update intervals" +extra_command "search" " Search active blocklists and backups for a specific domain" +extra_command "report" "[|||] Print DNS statistics" adb_init="/etc/init.d/adblock" adb_script="/usr/bin/adblock.sh" -adb_pidfile="/var/run/adblock.pid" - -if [ -s "${adb_pidfile}" ] && { [ "${action}" = "start" ] || [ "${action}" = "stop" ] || - [ "${action}" = "restart" ] || [ "${action}" = "reload" ] || [ "${action}" = "report" ] || - [ "${action}" = "suspend" ] || [ "${action}" = "resume" ] || [ "${action}" = "query" ] || - { [ "${action}" = "list" ] && [ -n "${1}" ]; }; }; then - return 0 +adb_rundir="/var/run/adblock" +adb_pidfile="/var/run/adblock/adblock.pid" + +if [ -z "${IPKG_INSTROOT}" ]; then + + # ensure runtime directory exists + # + [ ! -d "${adb_rundir}" ] && mkdir -p "${adb_rundir}" + + # check for running instance and handle boot trigger + # + case "${action}" in + "boot") + "${adb_init}" running && exit 0 + ;; + esac + + # reset pidfile if no/stale process is found, + # otherwise exit with error to prevent multiple instances + # + if [ -s "${adb_pidfile}" ]; then + pid="$(cat "${adb_pidfile}" 2>/dev/null)" + if [ -n "${pid}" ] && kill -0 "${pid}" 2>/dev/null; then + case "${action}" in + "start" | "stop" | "restart" | "reload" | "report" | "suspend" | "resume" | "search") + exit 1 + ;; + esac + else + : >"${adb_pidfile}" + fi + fi fi boot() { - [ -s "${adb_pidfile}" ] && : >"${adb_pidfile}" - rc_procd start_service + rc_procd start_service boot +} + +f_append_local_list_entry() { + local entry="${1}" + local file="${2}" + + printf '%s\n' "${entry}" >> "${file}" +} + +f_write_local_lists() { + local allowlist_file='/etc/adblock/adblock.allowlist' + local blocklist_file='/etc/adblock/adblock.blocklist' + + uci -q get adblock.ns_lists >/dev/null 2>&1 || return 0 + + config_load adblock + + : > "${allowlist_file}" + config_list_foreach ns_lists allowlist f_append_local_list_entry "${allowlist_file}" + + : > "${blocklist_file}" + config_list_foreach ns_lists blocklist f_append_local_list_entry "${blocklist_file}" } start_service() { @@ -37,30 +81,32 @@ start_service() { if [ "${action}" = "boot" ]; then [ -n "$(uci_get adblock global adb_trigger)" ] && return 0 fi + f_write_local_lists procd_open_instance "adblock" - procd_set_param command "${adb_script}" "${@}" + procd_set_param command "${adb_script}" "${@:-"${action}"}" procd_set_param pidfile "${adb_pidfile}" - procd_set_param nice "$(uci_get adblock global adb_nice "0")" - procd_set_param stdout 1 + procd_set_param nice "$(uci_get adblock global adb_nicelimit "0")" + procd_set_param stdout 0 procd_set_param stderr 1 procd_close_instance fi } +restart() { + /usr/sbin/ts-dns # configure threat shield dns, if needed + stop_service "restart" + rc_procd start_service restart +} + reload_service() { /usr/sbin/ts-dns # configure threat shield dns, if needed + f_write_local_lists + ${adb_script} nft-reload rc_procd start_service reload - /etc/init.d/firewall restart } stop_service() { - rc_procd "${adb_script}" stop -} - -restart() { - /usr/sbin/ts-dns # configure threat shield dns, if needed - rc_procd start_service restart - /etc/init.d/firewall restart + [ -z "${1}" ] && rc_procd "${adb_script}" stop } suspend() { @@ -71,204 +117,50 @@ resume() { rc_procd start_service resume } -query() { - rc_procd "${adb_script}" query "${1}" +search() { + rc_procd "${adb_script}" search "${1}" } report() { rc_procd "${adb_script}" report "${1:-"cli"}" "${2}" "${3}" "${4}" } -list() { - local src_archive src_file src_enabled enabled name utc_list size focus descurl action="${1}" - - if [ "${action%_*}" = "add" ] || [ "${action%_*}" = "remove" ]; then - shift - for name in "${@}"; do - case "${action}" in - "add") - if ! uci_get adblock global adb_sources | grep -q "${name}"; then - uci_add_list adblock global adb_sources "${name}" - printf "%s\n" "::: adblock source '${name}' added to config" - fi - ;; - "remove") - if uci_get adblock global adb_sources | grep -q "${name}"; then - uci_remove_list adblock global adb_sources "${name}" - printf "%s\n" "::: adblock source '${name}' removed from config" - fi - ;; - "add_utc") - if ! uci_get adblock global adb_utc_sources | grep -q "${name}"; then - uci_add_list adblock global adb_utc_sources "${name}" - printf "%s\n" "::: adblock utcapitole '${name}' added to config" - fi - ;; - "remove_utc") - if uci_get adblock global adb_utc_sources | grep -q "${name}"; then - uci_remove_list adblock global adb_utc_sources "${name}" - printf "%s\n" "::: adblock utcapitole '${name}' removed from config" - fi - ;; - "add_eng") - if ! uci_get adblock global adb_eng_sources | grep -q "${name}"; then - uci_add_list adblock global adb_eng_sources "${name}" - printf "%s\n" "::: adblock energized '${name}' added to config" - fi - ;; - "remove_eng") - if uci_get adblock global adb_eng_sources | grep -q "${name}"; then - uci_remove_list adblock global adb_eng_sources "${name}" - printf "%s\n" "::: adblock energized '${name}' removed from config" - fi - ;; - "add_stb") - if ! uci_get adblock global adb_stb_sources | grep -q "${name}"; then - uci_add_list adblock global adb_stb_sources "${name}" - printf "%s\n" "::: adblock stevenblack '${name}' added to config" - fi - ;; - "remove_stb") - if uci_get adblock global adb_stb_sources | grep -q "${name}"; then - uci_remove_list adblock global adb_stb_sources "${name}" - printf "%s\n" "::: adblock stevenblack '${name}' removed from config" - fi - ;; - esac - done - [ -n "$(uci -q changes adblock)" ] && { uci_commit adblock; "${adb_init}" start; } - else - src_archive="$(uci_get adblock global adb_srcarc "/etc/adblock/adblock.sources.gz")" - src_file="$(uci_get adblock global adb_srcfile "/tmp/adb_sources.json")" - src_enabled="$(uci -q show adblock.global.adb_sources)" - [ -r "${src_archive}" ] && zcat "${src_archive}" >"${src_file}" || printf "%s\n" "::: adblock source archive '${src_archive}' not found" - - if [ -r "${src_file}" ]; then - src_enabled="${src_enabled#*=}" - src_enabled="${src_enabled//\'}" - printf "%s\n" "::: Available adblock sources" - printf "%s\n" ":::" - printf "%-25s%-10s%-7s%-21s%s\n" " Name" "Enabled" "Size" "Focus" "Info URL" - printf "%s\n" " -------------------------------------------------------------------" - json_load_file "${src_file}" - json_get_keys keylist - for key in ${keylist}; do - json_select "${key}" - json_get_var size "size" - json_get_var focus "focus" - json_get_var descurl "descurl" - json_get_var url "url" - json_get_var rule "rule" - if [ -n "${url}" ] && [ -n "${rule}" ]; then - if printf "%s" "${src_enabled}" | grep -q "${key}"; then - enabled="x" - else - enabled=" " - fi - src_enabled="${src_enabled/${key}}" - printf " + %-21s%-10s%-7s%-21s%s\n" "${key:0:20}" "${enabled}" "${size:0:3}" "${focus:0:20}" "${descurl:0:50}" - else - src_enabled="${src_enabled} ${key}" - fi - json_select .. - done - utc_list="$(uci_get adblock global adb_utc_sources "-")" - eng_list="$(uci_get adblock global adb_eng_sources "-")" - stb_list="$(uci_get adblock global adb_stb_sources "-")" - printf "%s\n" " ---------------------------------------------------------------------------" - printf " * %s\n" "Configured utcapitole categories: ${utc_list// /, }" - printf " * %s\n" "Configured energized variants: ${eng_list// /, }" - printf " * %s\n" "Configured stevenblack variants: ${stb_list// /, }" - - if [ -n "${src_enabled// }" ]; then - printf "%s\n" " ---------------------------------------------------------------------------" - printf "%s\n" " Sources with invalid configuration" - printf "%s\n" " ---------------------------------------------------------------------------" - for key in ${src_enabled}; do - printf " - %s\n" "${key:0:20}" - done - fi - else - printf "%s\n" "::: adblock source file '${src_file}' not found" - fi - fi -} - status() { status_service } status_service() { - local key keylist value idxval values type rtfile - - rtfile="$(uci_get adblock global adb_rtfile "/tmp/adb_runtime.json")" + local key keylist type value values - json_load_file "${rtfile}" >/dev/null 2>&1 + json_init + json_load_file "/var/run/adblock/adblock.runtime.json" >/dev/null 2>&1 json_get_keys keylist if [ -n "${keylist}" ]; then - printf "%s\n" "::: adblock runtime information" + printf '%s\n' "::: adblock runtime information" for key in ${keylist}; do - json_get_var value "${key}" >/dev/null 2>&1 - if [ "${key%_*}" = "active" ]; then - printf " + %-15s : " "${key}" - json_select "${key}" >/dev/null 2>&1 - values="" - index="1" - while json_get_type type "${index}" && [ "${type}" = "object" ]; do - json_get_values idxval "${index}" >/dev/null 2>&1 - if [ "${index}" = "1" ]; then - values="${idxval}" - else - values="${values}, ${idxval}" - fi - index="$((index + 1))" - done - values="$(printf "%s" "${values}" | awk '{NR=1;max=98;if(length($0)>max+1)while($0){if(NR==1){print substr($0,1,max)}else{printf"%-22s%s\n","",substr($0,1,max)}{$0=substr($0,max+1);NR=NR+1}}else print}')" - printf "%s\n" "${values:-"-"}" - json_select ".." + json_get_type type "${key}" >/dev/null 2>&1 + if [ "${type}" = "array" ]; then + json_get_values values "${key}" >/dev/null 2>&1 + value="${values}" else - printf " + %-15s : %s\n" "${key}" "${value:-"-"}" + json_get_var value "${key}" >/dev/null 2>&1 fi + printf ' + %-15s : %s\n' "${key}" "${value:-"-"}" done else - printf "%s\n" "::: no adblock runtime information available" - fi -} - -timer() { - local cron_file cron_content cron_lineno action="${1:-"list"}" cron_tasks="${2}" hour="${3}" minute="${4:-0}" weekday="${5:-"*"}" - - cron_file="/etc/crontabs/root" - - if [ -s "${cron_file}" ] && [ "${action}" = "list" ]; then - awk '{print NR "> " $0}' "${cron_file}" - elif [ -x "/etc/init.d/cron" ] && [ "${action}" = "add" ]; then - hour="${hour//[[:alpha:]]/}" - minute="${minute//[[:alpha:]]/}" - if [ -n "${cron_tasks}" ] && [ -n "${hour}" ] && [ -n "${minute}" ] && [ -n "${weekday}" ] && - [ "${hour}" -ge 0 ] && [ "${hour}" -le 23 ] && - [ "${minute}" -ge 0 ] && [ "${minute}" -le 59 ]; then - printf "%02d %02d %s\n" "${minute}" "${hour}" "* * ${weekday} ${adb_init} ${cron_tasks}" >>"${cron_file}" - /etc/init.d/cron restart - fi - elif [ -x "/etc/init.d/cron" ] && [ -s "${cron_file}" ] && [ "${action}" = "remove" ]; then - cron_tasks="${cron_tasks//[[:alpha:]]/}" - cron_lineno="$(awk 'END{print NR}' "${cron_file}")" - cron_content="$(awk '{print $0}' "${cron_file}")" - if [ "${cron_tasks:-"0"}" -le "${cron_lineno:-"1"}" ] && [ -n "${cron_content}" ]; then - printf "%s\n" "${cron_content}" | awk "NR!~/^${cron_tasks}$/" >"${cron_file}" - /etc/init.d/cron restart - fi + printf '%s\n' "::: no adblock runtime information available" fi } service_triggers() { - local iface delay + local iface delay trigger - iface="$(uci_get adblock global adb_trigger)" delay="$(uci_get adblock global adb_triggerdelay "5")" - PROCD_RELOAD_DELAY="$((delay * 1000))" + trigger="$(uci_get adblock global adb_trigger)" - [ -n "${iface}" ] && procd_add_interface_trigger "interface.*.up" "${iface}" "${adb_init}" "start" - procd_add_reload_trigger "adblock" + PROCD_RELOAD_DELAY="$((delay * 1000))" + for iface in ${trigger}; do + procd_add_interface_trigger "interface.*.up" "${iface}" "${adb_init}" start + done + procd_add_reload_trigger adblock } diff --git a/packages/adblock/files/adblock.mail b/packages/adblock/files/adblock.mail index 67fc011aa..98f4bb1cc 100755 --- a/packages/adblock/files/adblock.mail +++ b/packages/adblock/files/adblock.mail @@ -1,6 +1,6 @@ #!/bin/sh # send mail script for adblock notifications -# Copyright (c) 2015-2022 Dirk Brenken (dev@brenken.org) +# Copyright (c) 2015-2026 Dirk Brenken (dev@brenken.org) # This is free software, licensed under the GNU General Public License v3. # Please note: you have to manually install and configure the package 'msmtp' before using this script @@ -8,70 +8,66 @@ # set (s)hellcheck exceptions # shellcheck disable=all -LC_ALL=C -PATH="/usr/sbin:/usr/bin:/sbin:/bin" - -[ -r "/lib/functions.sh" ] && . "/lib/functions.sh" +[ -r "/usr/bin/adblock.sh" ] && . "/usr/bin/adblock.sh" "mail" adb_debug="$(uci_get adblock global adb_debug "0")" adb_mailsender="$(uci_get adblock global adb_mailsender "no-reply@adblock")" adb_mailreceiver="$(uci_get adblock global adb_mailreceiver)" adb_mailtopic="$(uci_get adblock global adb_mailtopic "adblock notification")" adb_mailprofile="$(uci_get adblock global adb_mailprofile "adb_notify")" -adb_ver="${1}" -adb_mail="$(command -v msmtp)" -adb_logger="$(command -v logger)" -adb_logread="$(command -v logread)" -adb_rc="1" - -f_log() { - local class="${1}" log_msg="${2}" - - if [ -x "${adb_logger}" ]; then - "${adb_logger}" -p "${class}" -t "adblock-${adb_ver}[${$}]" "${log_msg}" - else - printf "%s %s %s\n" "${class}" "adblock-${adb_ver}[${$}]" "${log_msg}" - fi -} -if [ -z "${adb_mailreceiver}" ]; then - f_log "err" "please set the mail receiver with the 'adb_mailreceiver' option" - exit ${adb_rc} -fi +[ -z "${adb_mailreceiver}" ] && f_log "info" "please set the mail receiver with the 'adb_mailreceiver' option" [ "${adb_debug}" = "1" ] && debug="--debug" -adb_mailhead="From: ${adb_mailsender}\nTo: ${adb_mailreceiver}\nSubject: ${adb_mailtopic}\nReply-to: ${adb_mailsender}\nMime-Version: 1.0\nContent-Type: text/html;charset=utf-8\nContent-Disposition: inline\n\n" - # info preparation # sys_info="$( strings /etc/banner 2>/dev/null - ubus call system board | sed -e 's/\"release\": {//' | sed -e 's/^[ \t]*//' | sed -e 's/[{}\",]//g' | sed -e 's/[ ]/ \t/' | sed '/^$/d' 2>/dev/null + "${adb_ubuscmd}" call system board | "${adb_awkcmd}" 'BEGIN{FS="[{}\"]"}{if($2=="kernel"||$2=="hostname"||$2=="system"||$2=="model"||$2=="description")printf " + %-12s: %s\n",$2,$4}' 2>/dev/null )" adb_info="$(/etc/init.d/adblock status 2>/dev/null)" -rep_info="${2}" -if [ -x "${adb_logread}" ]; then - log_info="$("${adb_logread}" -l 100 -e "adblock-" | awk '{NR=1;max=120;if(length($0)>max+1)while($0){if(NR==1){print substr($0,1,max)}else{print substr($0,1,max)}{$0=substr($0,max+1);NR=NR+1}}else print}')" +rep_info="${1}" +if [ -x "${adb_logreadcmd}" ]; then + log_info="$("${adb_logreadcmd}" -l 100 -e "adblock-" 2>/dev/null)" fi # mail body # -adb_mailtext="
"
-adb_mailtext="${adb_mailtext}\n++\n++ System Information ++\n++\n${sys_info}"
-adb_mailtext="${adb_mailtext}\n\n++\n++ Adblock Information ++\n++\n${adb_info}"
-if [ -n "${rep_info}" ]; then
-	adb_mailtext="${adb_mailtext}\n\n++\n++ Report Information ++\n++\n${rep_info}"
-fi
-adb_mailtext="${adb_mailtext}\n\n++\n++ Logfile Information ++\n++\n${log_info}"
-adb_mailtext="${adb_mailtext}
" +adb_mailtext="$( + printf '%s\n' "
"
+	printf '\n%s\n' "++
+++ System Information ++
+++"
+	printf '%s\n' "${sys_info:-"-"}"
+	printf '\n%s\n' "++
+++ Adblock Information ++
+++"
+	printf '%s\n' "${adb_info:-"-"}"
+	if [ -n "${rep_info}" ]; then
+		printf '\n%s\n' "++
+++ Report Information ++
+++"
+		printf '%s\n' "${rep_info}"
+	fi
+	printf '\n%s\n' "++
+++ Logfile Information ++
+++"
+	printf '%s\n' "${log_info:-"-"}"
+	printf '%s\n' "
" +)" # send mail # -if [ -x "${adb_mail}" ]; then - printf "%b" "${adb_mailhead}${adb_mailtext}" 2>/dev/null | "${adb_mail}" ${debug} -a "${adb_mailprofile}" "${adb_mailreceiver}" >/dev/null 2>&1 - adb_rc=${?} - f_log "info" "mail sent to '${adb_mailreceiver}' with rc '${adb_rc}'" +if [ -x "${adb_mailcmd}" ]; then + adb_mailhead="From: ${adb_mailsender}\nTo: ${adb_mailreceiver}\nSubject: ${adb_mailtopic}\nReply-to: ${adb_mailsender}\nMime-Version: 1.0\nContent-Type: text/html;charset=utf-8\nContent-Disposition: inline\n\n" + printf '%b' "${adb_mailhead}${adb_mailtext}" 2>/dev/null | "${adb_mailcmd}" ${debug} -a "${adb_mailprofile}" "${adb_mailreceiver}" 2>>"${adb_errorlog}" + mail_rc="${?}" + if [ "${mail_rc}" = "0" ]; then + f_log "info" "mail successfully sent to '${adb_mailreceiver}'" + else + f_log "info" "failed to send mail to '${adb_mailreceiver}' with rc '${mail_rc}'" + fi else - f_log "err" "msmtp mail daemon not found" + f_log "info" "msmtp mail daemon not found" fi -exit ${adb_rc} +exit 0 diff --git a/packages/adblock/files/adblock.sh b/packages/adblock/files/adblock.sh index fe6fc10fe..155c6d713 100755 --- a/packages/adblock/files/adblock.sh +++ b/packages/adblock/files/adblock.sh @@ -1,9 +1,9 @@ #!/bin/sh # dns based ad/abuse domain blocking -# Copyright (c) 2015-2023 Dirk Brenken (dev@brenken.org) +# Copyright (c) 2015-2026 Dirk Brenken (dev@brenken.org) # This is free software, licensed under the GNU General Public License v3. -# disable (s)hellcheck in release +# (s)hellcheck exceptions # shellcheck disable=all # set initial defaults @@ -11,125 +11,179 @@ export LC_ALL=C export PATH="/usr/sbin:/usr/bin:/sbin:/bin" -adb_ver="4.1.5" adb_enabled="0" adb_debug="0" -adb_forcedns="0" +adb_nftforce="0" +adb_nftdevforce="" +adb_nftportforce="" +ns_tsdns_bypass="" +adb_nftallow="0" +adb_nftmacallow="" +adb_nftdevallow="" +adb_nftblock="0" +adb_nftmacblock="" +adb_nftdevblock="" +adb_nftremote="0" +adb_nftremotetimeout="15" +adb_nftmacremote="" +adb_nftbridge="0" +adb_allowdnsv4="" +adb_allowdnsv6="" +adb_remotednsv4="" +adb_remotednsv6="" +adb_blockdnsv4="" +adb_blockdnsv6="" +adb_bridgednsv4="" +adb_bridgednsv6="" +adb_dnsshift="0" adb_dnsflush="0" -adb_dnstimeout="20" +adb_dnstimeout="30" adb_safesearch="0" -adb_safesearchlist="" -adb_safesearchmod="0" adb_report="0" adb_trigger="" -adb_triggerdelay="0" -adb_backup="1" +adb_triggerdelay="5" adb_mail="0" -adb_mailcnt="0" adb_jail="0" +adb_map="0" +adb_tld="1" adb_dns="" -adb_dnsprefix="adb_list" -adb_locallist="blacklist whitelist iplist" -adb_tmpbase="/tmp" -adb_backupdir="${adb_tmpbase}/adblock-Backup" -adb_reportdir="${adb_tmpbase}/adblock-Report" -adb_jaildir="/tmp" -adb_pidfile="/var/run/adblock.pid" -adb_blacklist="/etc/adblock/adblock.blacklist" -adb_whitelist="/etc/adblock/adblock.whitelist" +adb_dnspid="" +adb_locallist="allowlist blocklist" +adb_basedir="/tmp" +adb_finaldir="" +adb_backupdir="/tmp/adblock-backup" +adb_reportdir="/tmp/adblock-report" +adb_rundir="/var/run/adblock" +adb_pidfile="${adb_rundir}/adblock.pid" +adb_rtfile="${adb_rundir}/adblock.runtime.json" +adb_etaglock="${adb_rundir}/adblock.etag.lock" +adb_allowlist="/etc/adblock/adblock.allowlist" +adb_blocklist="/etc/adblock/adblock.blocklist" adb_mailservice="/etc/adblock/adblock.mail" -adb_dnsfile="${adb_dnsprefix}.overall" -adb_dnsjail="${adb_dnsprefix}.jail" -adb_srcarc="/etc/adblock/adblock.sources.gz" -adb_srcfile="${adb_tmpbase}/adb_sources.json" -adb_rtfile="${adb_tmpbase}/adb_runtime.json" -adb_loggercmd="$(command -v logger)" -adb_dumpcmd="$(command -v tcpdump)" -adb_lookupcmd="$(command -v nslookup)" -adb_fetchutil="" +adb_dnsfile="adb_list.overall" +adb_feedfile="/etc/adblock/adblock.feeds" +adb_customfeedfile="/etc/adblock/adblock.custom.feeds" +adb_errorlog="/dev/null" +adb_fetchcmd="" adb_fetchinsecure="" -adb_zonelist="" -adb_portlist="" +adb_fetchretry="5" +adb_fetchparm="" +adb_etagparm="" +adb_geoparm="" +adb_geourl="http://ip-api.com/json" adb_repiface="" -adb_replisten="53" +adb_repport="53" adb_repchunkcnt="5" adb_repchunksize="1" adb_represolve="0" -adb_lookupdomain="example.com" -adb_action="${1:-"start"}" +adb_lookupdomain="localhost" +adb_action="${1}" adb_packages="" -adb_sources="" adb_cnt="" -# load & check adblock environment +# helper function to find a command in the system and check if it's executable, +# if not try alternative command or log an error if not found # -f_load() { - local bg_pid iface port ports cpu core +f_cmd() { + local cmd pri_cmd="${1}" sec_cmd="${2}" + + cmd="$(command -v "${pri_cmd}" 2>/dev/null)" + if [ -z "${cmd}" ]; then + if [ -n "${sec_cmd}" ]; then + [ "${sec_cmd}" = "optional" ] && return + cmd="$(command -v "${sec_cmd}" 2>/dev/null)" + fi + if [ -n "${cmd}" ]; then + printf '%s' "${cmd}" + else + f_log "emerg" "command '${pri_cmd:-"-"}'/'${sec_cmd:-"-"}' not found" + fi + else + printf '%s' "${cmd}" + fi +} - adb_sysver="$(ubus -S call system board 2>/dev/null | jsonfilter -q -e '@.model' -e '@.release.description' | - "${adb_awk}" 'BEGIN{RS="";FS="\n"}{printf "%s, %s",$1,$2}')" - adb_memory="$("${adb_awk}" '/^MemTotal|^MemFree|^MemAvailable/{ORS="/"; print int($2/1000)}' "/proc/meminfo" 2>/dev/null | - "${adb_awk}" '{print substr($0,1,length($0)-1)}')" +# load adblock environment +# +f_load() { + local cnt bg_pid port filter tcpdump_filter cpu + # load adblock config and set debug log file + # f_conf + if [ "${adb_debug}" = "1" ] && [ -d "${adb_basedir}" ]; then + adb_errorlog="${adb_basedir}/adb_error.log" + else + adb_errorlog="/dev/null" + fi - cpu="$(grep -c '^processor' /proc/cpuinfo 2>/dev/null)" - core="$(grep -cm1 '^core id' /proc/cpuinfo 2>/dev/null)" - [ "${cpu}" = "0" ] && cpu="1" - [ "${core}" = "0" ] && core="1" - adb_cores="$((cpu * core))" + # fetch installed packages and system information + # + adb_packages="$("${adb_ubuscmd}" -S call rpc-sys packagelist '{ "all": true }' 2>>"${adb_errorlog}")" + adb_bver="$(printf '%s' "${adb_packages}" | "${adb_jsoncmd}" -ql1 -e '@.packages.adblock')" + adb_fver="$(printf '%s' "${adb_packages}" | "${adb_jsoncmd}" -ql1 -e '@.packages["luci-app-adblock"]')" + adb_sysver="$("${adb_ubuscmd}" -S call system board 2>>"${adb_errorlog}" | + "${adb_jsoncmd}" -ql1 -e '@.model' -e '@.release.target' -e '@.release.distribution' -e '@.release.version' -e '@.release.revision' | + "${adb_awkcmd}" 'BEGIN{RS="";FS="\n"}{printf "%s, %s, %s %s (%s)",$1,$2,$3,$4,$5}')" + + # detect cpu cores for parallel processing + # + if [ -z "${adb_cores}" ]; then + cpu="$("${adb_grepcmd}" -cm16 '^processor' /proc/cpuinfo 2>>"${adb_errorlog}")" + [ "${cpu}" = "0" ] && cpu="1" + adb_cores="${cpu}" + fi - if [ "${adb_action}" != "report" ]; then + # load dns backend and fetch utility + # + if [ "${adb_action}" != "report" ] && [ "${adb_action}" != "mail" ]; then f_dns f_fetch fi - if [ "${adb_enabled}" = "0" ]; then - f_extconf - f_temp - f_rmdns - f_jsnup "disabled" - f_log "info" "adblock is currently disabled, please set the config option 'adb_enabled' to '1' to use this service" - exit 0 - fi - + # check if reporting is enabled and tcpdump is available + # if [ "${adb_report}" = "1" ] && [ ! -x "${adb_dumpcmd}" ]; then - f_log "info" "Please install the package 'tcpdump' or 'tcpdump-mini' to use the reporting feature" - elif [ "${adb_report}" = "0" ] && [ "${adb_action}" = "report" ]; then - f_log "info" "Please enable the 'DNS Report' option to use the reporting feature" - exit 0 - fi - - bg_pid="$(pgrep -f "^${adb_dumpcmd}.*adb_report\\.pcap$" | "${adb_awk}" '{ORS=" "; print $1}')" - if [ -x "${adb_dumpcmd}" ] && { [ "${adb_report}" = "0" ] || { [ -n "${bg_pid}" ] && { [ "${adb_action}" = "stop" ] || [ "${adb_action}" = "restart" ]; }; }; }; then - if [ -n "${bg_pid}" ]; then - kill -HUP "${bg_pid}" 2>/dev/null - while kill -0 "${bg_pid}" 2>/dev/null; do - sleep 1 - done - unset bg_pid + f_log "info" "please install the package 'tcpdump' or 'tcpdump-mini' to use the reporting feature" + elif [ -x "${adb_dumpcmd}" ]; then + bg_pid="$("${adb_pgrepcmd}" -nf "${adb_reportdir}/adb_report.pcap")" + if [ -n "${bg_pid}" ] && { [ "${adb_report}" = "0" ] || [ "${adb_action}" = "stop" ] || [ "${adb_action}" = "restart" ]; }; then + if kill -HUP "${bg_pid}" 2>>"${adb_errorlog}"; then + for cnt in 1 2 3; do + kill -0 "${bg_pid}" >/dev/null 2>&1 || break + sleep 1 + done + fi + bg_pid="$("${adb_pgrepcmd}" -nf "${adb_reportdir}/adb_report.pcap")" + "${adb_rmcmd}" -f "${adb_reportdir}"/adb_report.pcap* fi - fi - if [ -x "${adb_dumpcmd}" ] && [ "${adb_report}" = "1" ] && [ -z "${bg_pid}" ] && [ "${adb_action}" != "report" ] && [ "${adb_action}" != "stop" ]; then - for port in ${adb_replisten}; do - [ -z "${ports}" ] && ports="port ${port}" || ports="${ports} or port ${port}" - done - if [ -z "${adb_repiface}" ]; then - network_get_device iface "lan" - [ -z "${iface}" ] && network_get_physdev iface "lan" - [ -n "${iface}" ] && adb_repiface="${iface}" - [ -n "${adb_repiface}" ] && { uci_set adblock global adb_repiface "${adb_repiface}"; f_uci "adblock"; } - fi - if [ -n "${adb_reportdir}" ] && [ ! -d "${adb_reportdir}" ]; then - mkdir -p "${adb_reportdir}" - f_log "info" "report directory '${adb_reportdir}' created" - fi - if [ -n "${adb_repiface}" ] && [ -d "${adb_reportdir}" ]; then - ("${adb_dumpcmd}" -nn -p -s0 -l -i ${adb_repiface} ${ports} -C${adb_repchunksize} -W${adb_repchunkcnt} -w "${adb_reportdir}/adb_report.pcap" >/dev/null 2>&1 &) - bg_pid="$(pgrep -f "^${adb_dumpcmd}.*adb_report\\.pcap$" | "${adb_awk}" '{ORS=" "; print $1}')" - else - f_log "info" "Please set the name of the reporting network device 'adb_repiface' manually" + if [ "${adb_report}" = "1" ] && [ -z "${bg_pid}" ] && [ "${adb_action}" != "report" ] && [ "${adb_action}" != "stop" ]; then + [ ! -d "${adb_reportdir}" ] && mkdir -p "${adb_reportdir}" + if [ -z "${adb_repiface}" ]; then + network_get_device adb_repiface "lan" + [ -z "${adb_repiface}" ] && network_get_physdev adb_repiface "lan" + uci_set adblock global adb_repiface "${adb_repiface:-"any"}" + f_uci "adblock" + fi + for port in ${adb_repport}; do + [ -n "${filter}" ] && filter="${filter} or " + filter="${filter}(udp port ${port}) or (tcp port ${port})" + done + tcpdump_filter="(${filter}) and greater 28" + if [ -n "${adb_repiface}" ] && [ -d "${adb_reportdir}" ]; then + ( + "${adb_dumpcmd}" --immediate-mode -nn -p -s0 -i "${adb_repiface}" \ + "${tcpdump_filter}" \ + -C "${adb_repchunksize}" -W "${adb_repchunkcnt}" \ + -w "${adb_reportdir}/adb_report.pcap" >/dev/null 2>&1 & + ) + sleep 1 + bg_pid="$("${adb_pgrepcmd}" -nf "${adb_reportdir}/adb_report.pcap")" + f_log "info" "tcpdump background process started for interface: ${adb_repiface}, port: ${adb_repport}, dir: ${adb_reportdir}, pid: ${bg_pid}" + else + f_log "info" "please set the reporting interface 'adb_repiface' and reporting directory 'adb_reportdir' manually" + fi fi fi } @@ -137,514 +191,591 @@ f_load() { # check & set environment # f_env() { - adb_starttime="$(date "+%s")" - f_log "info" "adblock instance started ::: action: ${adb_action}, priority: ${adb_nice:-"0"}, pid: ${$}" - f_jsnup "running" + : >"${adb_errorlog}" + read -r adb_starttime _ <"/proc/uptime" + adb_starttime="${adb_starttime%.*}" + f_log "info" "adblock instance started ::: action: ${adb_action}, priority: ${adb_nicelimit:-"0"}, pid: ${$}" + f_jsnup "processing" f_extconf f_temp - - if [ "${adb_dnsflush}" = "1" ] || [ "${adb_memory##*/}" -lt "64" ]; then - printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" - f_dnsup - fi - - if [ ! -r "${adb_srcfile}" ]; then - if [ -r "${adb_srcarc}" ]; then - zcat "${adb_srcarc}" >"${adb_srcfile}" + f_nftadd + json_init + if [ -s "${adb_customfeedfile}" ]; then + if json_load_file "${adb_customfeedfile}" >/dev/null 2>&1; then + return else - f_log "err" "adblock source archive not found" + f_log "info" "can't load adblock custom feed file" fi fi - if [ -r "${adb_srcfile}" ] && [ "${adb_action}" != "report" ]; then - json_init - json_load_file "${adb_srcfile}" + if [ -s "${adb_feedfile}" ] && json_load_file "${adb_feedfile}" >/dev/null 2>&1; then + return else - f_log "err" "adblock source file not found" + f_log "err" "can't load adblock feed file" fi } # load adblock config # f_conf() { - local cnt="0" cnt_max="10" - - [ ! -r "/etc/config/adblock" ] && f_log "err" "no valid adblock config found, please re-install the package via opkg with the '--force-reinstall --force-maintainer' options" - config_cb() { option_cb() { - local option="${1}" - local value="${2}" - eval "${option}=\"${value}\"" + local option="${1}" value="${2//\"/\\\"}" + + case "${option}" in + *[!a-zA-Z0-9_]*) ;; + + *) + eval "${option}=\"\${value}\"" + ;; + esac } list_cb() { - local option="${1}" - local value="${2}" - if [ "${option}" = "adb_sources" ]; then - eval "${option}=\"$(printf "%s" "${adb_sources}") ${value}\"" - elif [ "${option}" = "adb_eng_sources" ]; then - eval "${option}=\"$(printf "%s" "${adb_eng_sources}") ${value}\"" - elif [ "${option}" = "adb_stb_sources" ]; then - eval "${option}=\"$(printf "%s" "${adb_stb_sources}") ${value}\"" - elif [ "${option}" = "adb_utc_sources" ]; then - eval "${option}=\"$(printf "%s" "${adb_utc_sources}") ${value}\"" - elif [ "${option}" = "adb_denyip" ]; then - eval "${option}=\"$(printf "%s" "${adb_denyip}") ${value}\"" - elif [ "${option}" = "adb_allowip" ]; then - eval "${option}=\"$(printf "%s" "${adb_allowip}") ${value}\"" - elif [ "${option}" = "adb_safesearchlist" ]; then - eval "${option}=\"$(printf "%s" "${adb_safesearchlist}") ${value}\"" - elif [ "${option}" = "adb_zonelist" ]; then - eval "${option}=\"$(printf "%s" "${adb_zonelist}") ${value}\"" - elif [ "${option}" = "adb_portlist" ]; then - eval "${option}=\"$(printf "%s" "${adb_portlist}") ${value}\"" - elif [ "${option}" = "adb_bypass" ]; then - eval "${option}=\"$(printf "%s" "${adb_bypass}") ${value}\"" - fi + local append option="${1}" value="${2//\"/\\\"}" + + case "${option}" in + *[!a-zA-Z0-9_]*) ;; + + *) + eval "append=\"\${${option}}\"" + if [ -n "${append}" ]; then + eval "${option}=\"\${append} \${value}\"" + else + eval "${option}=\"\${value}\"" + fi + ;; + esac } } config_load adblock - - if [ -z "${adb_fetchutil}" ] || [ -z "${adb_dns}" ]; then - while [ -z "${adb_packages}" ] && [ "${cnt}" -le "${cnt_max}" ]; do - adb_packages="$(opkg list-installed 2>/dev/null)" - cnt="$((cnt + 1))" - sleep 1 - done - [ -z "${adb_packages}" ] && f_log "err" "local opkg package repository is not available, please set 'adb_fetchutil' and 'adb_dns' manually" - fi } -# status helper function +# domain validation # -f_char() { - local result input="${1}" +f_chkdom() { + local type prefix column separator + + case "${1}" in + "feed" | "local") + type="${1}" + case "${2}" in + [0-9]) + prefix="" + column="${2}" + separator="${3:-[[:space:]]+}" + ;; + *) + prefix="${2}" + column="${3}" + separator="${4:-[[:space:]]+}" + ;; + esac + ;; + "google") + type="${1}" + prefix="" + column="${2}" + separator="${3:-[[:space:]]+}" + ;; + esac - if [ "${input}" = "1" ]; then - result="✔" - else - result="✘" - fi - printf "%s" "${result}" + "${adb_awkcmd}" -v type="${type}" -v pre="${prefix}" -v col="${column}" -v chk="${adb_lookupdomain}" -F "${separator}" ' + { + domain = $col + # remove carriage returns and trim the input + gsub(/\r|^[[:space:]]+|[[:space:]]+$/, "", domain) + # add www. for google safe search + if (type=="google" && domain ~ /^\.+/) { sub(/^\.+/, "", domain); domain="www."domain } + # check optional search prefix + if (pre != "" && index($0, pre) != 1) next + # skip empty lines, comments and special domains + if (domain == "" || domain ~ /^(#|localhost|loopback)/ || index(domain, chk) == 1) next + # no domain with trailing dot + if (substr(domain, length(domain), 1) == ".") next + # check total length (253 characters) + if (length(domain) > 253) next + n = split(domain, L, ".") + valid = 1 + for (i = 1; i <= n; i++) { + l = L[i] + len = length(l) + # label length 1–63 + if (len < 1 || len > 63) { valid = 0; break } + # no leading/trailing hyphen + if (l ~ /^-/ || l ~ /-$/) { valid = 0; break } + # ASCII + hyphen + if (l !~ /^[A-Za-z0-9-]+$/) { valid = 0; break } + } + # TLD must start with a letter or "xn--" + if (valid && L[n] !~ /^[A-Za-z]/ && L[n] !~ /^xn--/) valid = 0 + if (valid) print tolower(domain) + }' + + f_log "debug" "f_chkdom ::: name: ${src_name}, type: ${type}, prefix: ${prefix:-"-"}, column: ${column:-"-"}, separator: ${separator:-"-"}" } # load dns backend config # f_dns() { - local util utils dns_up cnt="0" + local dns dns_list dns_section dns_info free_mem dir + + free_mem="$("${adb_awkcmd}" '/^MemAvailable/{printf "%s",int($2/1000)}' "/proc/meminfo" 2>>"${adb_errorlog}")" + if [ "${adb_action}" = "boot" ] && [ -z "${adb_trigger}" ]; then + sleep ${adb_triggerdelay:-"5"} + fi if [ -z "${adb_dns}" ]; then - utils="knot-resolver bind unbound dnsmasq raw" - for util in ${utils}; do - if [ "${util}" = "raw" ] || printf "%s" "${adb_packages}" | grep -q "^${util}"; then - if [ "${util}" = "knot-resolver" ]; then - util="kresd" - elif [ "${util}" = "bind" ]; then - util="named" - fi - if [ "${util}" = "raw" ] || [ -x "$(command -v "${util}")" ]; then - adb_dns="${util}" - uci_set adblock global adb_dns "${util}" + dns_list="knot-resolver bind-server unbound-daemon smartdns dnsmasq-full dnsmasq-dhcpv6 dnsmasq" + for dns in ${dns_list}; do + if printf '%s' "${adb_packages}" | "${adb_jsoncmd}" -ql1 -e "@.packages[\"${dns}\"]" >/dev/null 2>&1; then + case "${dns}" in + "knot-resolver") + dns="kresd" + ;; + "bind-server") + dns="named" + ;; + "unbound-daemon") + dns="unbound" + ;; + "dnsmasq-full" | "dnsmasq-dhcpv6") + dns="dnsmasq" + ;; + esac + + if command -v "${dns}" >/dev/null 2>&1; then + adb_dns="${dns}" + uci_set adblock global adb_dns "${dns}" f_uci "adblock" break fi fi done - elif [ "${adb_dns}" != "raw" ] && [ ! -x "$(command -v "${adb_dns}")" ]; then - unset adb_dns fi - if [ -n "${adb_dns}" ]; then - case "${adb_dns}" in - "dnsmasq") - adb_dnscachecmd="-" - adb_dnsinstance="${adb_dnsinstance:-"0"}" - adb_dnsuser="${adb_dnsuser:-"dnsmasq"}" - adb_dnsdir="${adb_dnsdir:-"/tmp/dnsmasq.d"}" - adb_dnsheader="${adb_dnsheader:-""}" - adb_dnsdeny="${adb_dnsdeny:-"${adb_awk} '{print \"local=/\"\$0\"/\"}'"}" - adb_dnsallow="${adb_dnsallow:-"${adb_awk} '{print \"local=/\"\$0\"/#\"}'"}" - adb_dnssafesearch="${adb_dnssafesearch:-"${adb_awk} -v item=\"\$item\" '{print \"address=/\"\$0\"/\"item\"\"}'"}" - adb_dnsstop="${adb_dnsstop:-"address=/#/"}" - ;; - "unbound") - adb_dnscachecmd="$(command -v unbound-control || printf "%s" "-")" - adb_dnsinstance="${adb_dnsinstance:-"0"}" - adb_dnsuser="${adb_dnsuser:-"unbound"}" - adb_dnsdir="${adb_dnsdir:-"/var/lib/unbound"}" - adb_dnsheader="${adb_dnsheader:-""}" - adb_dnsdeny="${adb_dnsdeny:-"${adb_awk} '{print \"local-zone: \\042\"\$0\"\\042 always_nxdomain\"}'"}" - adb_dnsallow="${adb_dnsallow:-"${adb_awk} '{print \"local-zone: \\042\"\$0\"\\042 always_transparent\"}'"}" - adb_dnssafesearch="${adb_dnssafesearch:-"${adb_awk} -v item=\"\$item\" '{type=\"AAAA\";if(match(item,/^([0-9]{1,3}\.){3}[0-9]{1,3}$/)){type=\"A\"}}{print \"local-data: \\042\"\$0\" \"type\" \"item\"\\042\"}'"}" - adb_dnsstop="${adb_dnsstop:-"local-zone: \".\" always_nxdomain"}" - ;; - "named") - adb_dnscachecmd="$(command -v rndc || printf "%s" "-")" - adb_dnsinstance="${adb_dnsinstance:-"0"}" - adb_dnsuser="${adb_dnsuser:-"bind"}" - adb_dnsdir="${adb_dnsdir:-"/var/lib/bind"}" - adb_dnsheader="${adb_dnsheader:-"\$TTL 2h\n@ IN SOA localhost. root.localhost. (1 6h 1h 1w 2h)\n IN NS localhost.\n"}" - adb_dnsdeny="${adb_dnsdeny:-"${adb_awk} '{print \"\"\$0\" CNAME .\\n*.\"\$0\" CNAME .\"}'"}" - adb_dnsallow="${adb_dnsallow:-"${adb_awk} '{print \"\"\$0\" CNAME rpz-passthru.\\n*.\"\$0\" CNAME rpz-passthru.\"}'"}" - adb_dnsdenyip="${adb_dnsdenyip:-"${adb_awk} '{print \"\"\$0\".rpz-client-ip CNAME .\"}'"}" - adb_dnsallowip="${adb_dnsallowip:-"${adb_awk} '{print \"\"\$0\".rpz-client-ip CNAME rpz-passthru.\"}'"}" - adb_dnssafesearch="${adb_dnssafesearch:-"${adb_awk} -v item=\"\$item\" '{print \"\"\$0\" CNAME \"item\".\\n*.\"\$0\" CNAME \"item\".\"}'"}" - adb_dnsstop="${adb_dnsstop:-"* CNAME ."}" - ;; - "kresd") - adb_dnscachecmd="-" - adb_dnsinstance="${adb_dnsinstance:-"0"}" - adb_dnsuser="${adb_dnsuser:-"root"}" - adb_dnsdir="${adb_dnsdir:-"/etc/kresd"}" - adb_dnsheader="${adb_dnsheader:-"\$TTL 2h\n@ IN SOA localhost. root.localhost. (1 6h 1h 1w 2h)\n"}" - adb_dnsdeny="${adb_dnsdeny:-"${adb_awk} '{print \"\"\$0\" CNAME .\\n*.\"\$0\" CNAME .\"}'"}" - adb_dnsallow="${adb_dnsallow:-"${adb_awk} '{print \"\"\$0\" CNAME rpz-passthru.\\n*.\"\$0\" CNAME rpz-passthru.\"}'"}" - adb_dnssafesearch="${adb_dnssafesearch:-"${adb_awk} -v item=\"\$item\" '{type=\"AAAA\";if(match(item,/^([0-9]{1,3}\.){3}[0-9]{1,3}$/)){type=\"A\"}}{print \"\"\$0\" \"type\" \"item\"\"}'"}" - adb_dnsstop="${adb_dnsstop:-"* CNAME ."}" - ;; - "raw") - adb_dnscachecmd="-" - adb_dnsinstance="${adb_dnsinstance:-"0"}" - adb_dnsuser="${adb_dnsuser:-"root"}" - adb_dnsdir="${adb_dnsdir:-"/tmp"}" - adb_dnsheader="${adb_dnsheader:-""}" - adb_dnsdeny="${adb_dnsdeny:-"0"}" - adb_dnsallow="${adb_dnsallow:-"1"}" - adb_dnssafesearch="${adb_dnssafesearch:-"0"}" - adb_dnsstop="${adb_dnsstop:-"0"}" - ;; - esac - fi - - if [ "${adb_dns}" != "raw" ] && { [ -z "${adb_dns}" ] || [ ! -x "$(command -v "${adb_dns}")" ]; }; then + if [ "${adb_dns}" != "raw" ] && ! command -v "${adb_dns}" >/dev/null 2>&1; then f_log "err" "dns backend not found, please set 'adb_dns' manually" fi - if [ "${adb_dns}" != "raw" ] && { [ "${adb_dnsdir}" = "${adb_tmpbase}" ] || [ "${adb_dnsdir}" = "${adb_backupdir}" ] || [ "${adb_dnsdir}" = "${adb_reportdir}" ]; }; then - f_log "err" "dns directory '${adb_dnsdir}' has been misconfigured, it must not point to the 'adb_tmpbase', 'adb_backupdir', 'adb_reportdir'" - fi - - if [ "${adb_action}" = "start" ] && [ -z "${adb_trigger}" ]; then - sleep ${adb_triggerdelay} - fi - - if [ "${adb_dns}" != "raw" ] && [ "${adb_action}" != "stop" ]; then - while [ "${cnt}" -le 30 ]; do - dns_up="$(ubus -S call service list "{\"name\":\"${adb_dns}\"}" 2>/dev/null | jsonfilter -l1 -e "@[\"${adb_dns}\"].instances.*.running" 2>/dev/null)" - if [ "${dns_up}" = "true" ]; then - break - fi - sleep 1 - cnt="$((cnt + 1))" - done - fi - - if [ "${adb_action}" != "stop" ]; then - if [ -n "${adb_dnsdir}" ] && [ ! -d "${adb_dnsdir}" ]; then - if mkdir -p "${adb_dnsdir}"; then - f_log "info" "dns backend directory '${adb_dnsdir}' created" + case "${adb_dns}" in + "dnsmasq") + adb_dnscachecmd="" + adb_dnsinstance="${adb_dnsinstance:-"0"}" + adb_dnsuser="dnsmasq" + adb_dnsdir="${adb_dnsdir:-""}" + if [ -z "${adb_dnsdir}" ]; then + dns_section="$("${adb_ubuscmd}" -S call uci get "{\"config\":\"dhcp\", \"section\":\"@dnsmasq[${adb_dnsinstance}]\", \"type\":\"dnsmasq\"}" 2>>"${adb_errorlog}")" + dns_info="$(printf '%s' "${dns_section}" | "${adb_jsoncmd}" -l1 -e '@.values["confdir"]')" + if [ -n "${dns_info}" ]; then + adb_dnsdir="${dns_info}" else - f_log "err" "dns backend directory '${adb_dnsdir}' could not be created" + dns_info="$(printf '%s' "${dns_section}" | "${adb_jsoncmd}" -l1 -e '@.values[".name"]')" + [ -n "${dns_info}" ] && adb_dnsdir="/tmp/dnsmasq.${dns_info}.d" fi fi - [ ! -f "${adb_dnsdir}/${adb_dnsfile}" ] && printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" + adb_dnsheader="" + adb_dnsdeny="1" + adb_dnsallow="1" + adb_dnssafesearch="1" + adb_dnsstop="address=/#/\nlocal=/#/" + f_dnsdeny() { + "${adb_awkcmd}" '{print "local=/"$0"/"}' "${@}" + } + f_dnsallow() { + "${adb_awkcmd}" '{print "local=/"$0"/#"}' "${@}" + } + f_dnssafesearch() { + local item="${1}" - if [ "${dns_up}" != "true" ]; then - if ! f_dnsup 4; then - f_log "err" "dns backend '${adb_dns}' not running or executable" - fi - fi + shift + "${adb_awkcmd}" -v item="${item}" '{print "address=/"$0"/"item"";print "local=/"$0"/"}' "${@}" + } + ;; + "unbound") + adb_dnscachecmd="$(f_cmd unbound-control optional)" + adb_dnsinstance="" + adb_dnsuser="unbound" + adb_dnsdir="${adb_dnsdir:-"/var/lib/unbound"}" + adb_dnsheader="" + adb_dnsdeny="1" + adb_dnsallow="1" + adb_dnssafesearch="1" + adb_dnsstop="local-zone: \".\" always_nxdomain" + f_dnsdeny() { + "${adb_awkcmd}" '{print "local-zone: \042"$0"\042 always_nxdomain"}' "${@}" + } + f_dnsallow() { + "${adb_awkcmd}" '{print "local-zone: \042"$0"\042 always_transparent"}' "${@}" + } + f_dnssafesearch() { + local item="${1}" - if [ "${adb_backup}" = "1" ] && [ -n "${adb_backupdir}" ] && [ ! -d "${adb_backupdir}" ]; then - if mkdir -p "${adb_backupdir}"; then - f_log "info" "backup directory '${adb_backupdir}' created" - else - f_log "err" "backup directory '${adb_backupdir}' could not be created" - fi - fi + shift + "${adb_awkcmd}" -v item="${item}" \ + '{type="AAAA";if(match(item,/^([0-9]{1,3}\.){3}[0-9]{1,3}$/)){type="A"}}{print "local-data: \042"$0" "type" "item"\042"}' "${@}" + } + ;; + "named") + adb_dnscachecmd="$(f_cmd rndc optional)" + adb_dnsinstance="" + adb_dnsuser="bind" + adb_dnsdir="${adb_dnsdir:-"/var/lib/bind"}" + adb_dnsheader="\$TTL 2h\n@ IN SOA localhost. root.localhost. (1 6h 1h 1w 2h)\n IN NS localhost.\n" + adb_dnsdeny="1" + adb_dnsallow="1" + adb_dnssafesearch="1" + adb_dnsstop="* CNAME ." + f_dnsdeny() { + "${adb_awkcmd}" '{print ""$0" CNAME .\n*."$0" CNAME ."}' "${@}" + } + f_dnsallow() { + "${adb_awkcmd}" '{print ""$0" CNAME rpz-passthru.\n*."$0" CNAME rpz-passthru."}' "${@}" + } + f_dnssafesearch() { + local item="${1}" - if [ -n "${adb_jaildir}" ] && [ ! -d "${adb_jaildir}" ]; then - if mkdir -p "${adb_jaildir}"; then - f_log "info" "jail directory '${adb_jaildir}' created" - else - f_log "err" "jail directory '${adb_jaildir}' could not be created" - fi + shift + "${adb_awkcmd}" -v item="${item}" '{print ""$0" CNAME "item".\n*."$0" CNAME "item"."}' "${@}" + } + ;; + "kresd") + adb_dnscachecmd="" + adb_dnsinstance="" + adb_dnsuser="root" + adb_dnsdir="${adb_dnsdir:-"/tmp/kresd"}" + adb_dnsheader="\$TTL 2h\n@ IN SOA localhost. root.localhost. (1 6h 1h 1w 2h)\n" + adb_dnsdeny="1" + adb_dnsallow="1" + adb_dnssafesearch="1" + adb_dnsstop="* CNAME ." + f_dnsdeny() { + "${adb_awkcmd}" '{print ""$0" CNAME .\n*."$0" CNAME ."}' "${@}" + } + f_dnsallow() { + "${adb_awkcmd}" '{print ""$0" CNAME rpz-passthru.\n*."$0" CNAME rpz-passthru."}' "${@}" + } + f_dnssafesearch() { + local item="${1}" + + shift + "${adb_awkcmd}" -v item="${item}" '{print ""$0" CNAME "item".\n*."$0" CNAME "item"."}' "${@}" + } + ;; + "smartdns") + adb_dnscachecmd="" + adb_dnsinstance="${adb_dnsinstance:-"0"}" + adb_dnsuser="root" + adb_dnsdir="${adb_dnsdir:-"/tmp/smartdns"}" + adb_dnsheader="" + adb_dnsdeny="1" + adb_dnsallow="1" + adb_dnssafesearch="1" + adb_dnsstop="address #" + f_dnsdeny() { + "${adb_awkcmd}" '{print "address /"$0"/#"}' "${@}" + } + f_dnsallow() { + "${adb_awkcmd}" '{print "address /"$0"/-"}' "${@}" + } + f_dnssafesearch() { + local item="${1}" + + shift + "${adb_awkcmd}" -v item="${item}" '{print "cname /"$0"/"item""}' "${@}" + } + ;; + "raw") + adb_dnscachecmd="" + adb_dnsinstance="" + adb_dnsuser="root" + adb_dnsdir="${adb_dnsdir:-"/tmp"}" + adb_dnsheader="" + adb_dnsdeny="" + adb_dnsallow="" + adb_dnssafesearch="" + adb_dnsstop="" + ;; + esac + + # determine final dns file directory based on dns shifting + # + if [ "${adb_dnsshift}" = "0" ]; then + adb_finaldir="${adb_dnsdir}" + [ -L "${adb_dnsdir}/${adb_dnsfile}" ] && "${adb_rmcmd}" -f "${adb_dnsdir}/${adb_dnsfile}" + else + adb_finaldir="${adb_backupdir}" + fi + + # create dns file with header if it doesn't exist or dns flushing is enabled, also create backup and final directories if they don't exist + # + if [ "${adb_action}" != "stop" ]; then + for dir in "${adb_dnsdir:-"/tmp"}" "${adb_backupdir:-"/tmp"}"; do + [ ! -d "${dir}" ] && mkdir -p "${dir}" + done + if [ "${adb_dnsflush}" = "1" ] || [ "${free_mem:-"0"}" -lt "64" ]; then + printf '%b' "${adb_dnsheader}" >"${adb_finaldir}/${adb_dnsfile}" + f_dnsup + elif [ ! -f "${adb_finaldir}/${adb_dnsfile}" ]; then + printf '%b' "${adb_dnsheader}" >"${adb_finaldir}/${adb_dnsfile}" fi fi - f_log "debug" "f_dns ::: dns: ${adb_dns}, dns_dir: ${adb_dnsdir}, dns_file: ${adb_dnsfile}, dns_user: ${adb_dnsuser}, dns_instance: ${adb_dnsinstance}, backup: ${adb_backup}, backup_dir: ${adb_backupdir}, jail_dir: ${adb_jaildir}" + + # check if adblock is enabled + # + if [ "${adb_enabled}" = "0" ]; then + f_extconf + f_temp + f_nftremove + f_rmdns + f_jsnup "disabled" + f_log "info" "adblock is currently disabled, please set the config option 'adb_enabled' to '1' to use this service" + exit 0 + fi + + f_log "debug" "f_dns ::: dns: ${adb_dns}, dns_instance: ${adb_dnsinstance:-"-"}, dns_user: ${adb_dnsuser}, dns_dir: ${adb_dnsdir}, backup_dir: ${adb_backupdir}, final_dir: ${adb_finaldir}" } # load fetch utility # f_fetch() { - local util utils insecure cnt="0" - - if [ -z "${adb_fetchutil}" ]; then - utils="aria2c curl wget uclient-fetch" - for util in ${utils}; do - if { [ "${util}" = "uclient-fetch" ] && printf "%s" "${adb_packages}" | grep -q "^libustream-"; } || - { [ "${util}" = "wget" ] && printf "%s" "${adb_packages}" | grep -q "^wget -"; } || - [ "${util}" = "curl" ] || [ "${util}" = "aria2c" ]; then - if [ -x "$(command -v "${util}")" ]; then - adb_fetchutil="${util}" - uci_set adblock global adb_fetchutil "${util}" + local fetch fetch_list insecure update="0" + + adb_fetchcmd="$(command -v "${adb_fetchcmd}" 2>/dev/null)" + if [ -z "${adb_fetchcmd}" ]; then + fetch_list="curl wget-ssl libustream-openssl libustream-wolfssl libustream-mbedtls" + for fetch in ${fetch_list}; do + case "${adb_packages}" in *"\"${fetch}\""*) + case "${fetch}" in + "wget-ssl") + fetch="wget" + ;; + "libustream-openssl" | "libustream-wolfssl" | "libustream-mbedtls") + fetch="uclient-fetch" + ;; + esac + adb_fetchcmd="$(command -v "${fetch}" 2>/dev/null)" + if [ -n "${adb_fetchcmd}" ]; then + update="1" + uci_set adblock global adb_fetchcmd "${fetch}" f_uci "adblock" break fi - fi + ;; + esac done - elif [ ! -x "$(command -v "${adb_fetchutil}")" ]; then - unset adb_fetchutil fi - case "${adb_fetchutil}" in - "aria2c") - [ "${adb_fetchinsecure}" = "1" ] && insecure="--check-certificate=false" - adb_fetchparm="${adb_fetchparm:-"${insecure} --timeout=20 --allow-overwrite=true --auto-file-renaming=false --log-level=warn --dir=/ -o"}" - ;; - "curl") - [ "${adb_fetchinsecure}" = "1" ] && insecure="--insecure" - adb_fetchparm="${adb_fetchparm:-"${insecure} --connect-timeout 20 --fail --silent --show-error --location -o"}" - ;; - "uclient-fetch") - [ "${adb_fetchinsecure}" = "1" ] && insecure="--no-check-certificate" - adb_fetchparm="${adb_fetchparm:-"${insecure} --timeout=20 -O"}" - ;; - "wget") - [ "${adb_fetchinsecure}" = "1" ] && insecure="--no-check-certificate" - adb_fetchparm="${adb_fetchparm:-"${insecure} --no-cache --no-cookies --max-redirect=0 --timeout=20 -O"}" - ;; + + [ -z "${adb_fetchcmd}" ] && f_log "err" "download utility with SSL support not found, please set 'adb_fetchcmd' manually" + + case "${adb_fetchcmd##*/}" in + "curl") + [ "${adb_fetchinsecure}" = "1" ] && insecure="--insecure" + adb_fetchparm="${adb_fetchparm:-"${insecure} --connect-timeout 20 --retry-delay 10 --retry $((adb_fetchretry - 1)) --retry-max-time $(((adb_fetchretry - 1) * 20)) --retry-all-errors --fail --silent --show-error --location -o"}" + adb_etagparm="--connect-timeout 5 --silent --location --head" + adb_geoparm="--connect-timeout 5 --silent --location" + ;; + "wget") + [ "${adb_fetchinsecure}" = "1" ] && insecure="--no-check-certificate" + adb_fetchparm="${adb_fetchparm:-"${insecure} --no-cache --no-cookies --timeout=20 --waitretry=10 --tries=${adb_fetchretry} --retry-connrefused -O"}" + adb_etagparm="--timeout=5 --spider --server-response" + adb_geoparm="--timeout=5 --quiet -O-" + ;; + "uclient-fetch") + [ "${adb_fetchinsecure}" = "1" ] && insecure="--no-check-certificate" + adb_fetchparm="${adb_fetchparm:-"${insecure} --timeout=20 -O"}" + adb_geoparm="--timeout=5 --quiet -O-" + ;; esac - if [ -n "${adb_fetchutil}" ] && [ -n "${adb_fetchparm}" ]; then - adb_fetchutil="$(command -v "${adb_fetchutil}")" - else - f_log "err" "download utility with SSL support not found, please install 'uclient-fetch' with a 'libustream-*' variant or another download utility like 'wget', 'curl' or 'aria2'" - fi - f_log "debug" "f_fetch ::: fetch_util: ${adb_fetchutil:-"-"}, fetch_parm: ${adb_fetchparm:-"-"}" + + f_log "debug" "f_fetch ::: update: ${update}, cmd: ${adb_fetchcmd:-"-"}, parm: ${adb_fetchparm:-"-"}, etag_parm: ${adb_etagparm:-"-"}, geo_parm: ${adb_geoparm:-"-"}" } # create temporary files, directories and set dependent options # f_temp() { - if [ -d "${adb_tmpbase}" ]; then - adb_tmpdir="$(mktemp -p "${adb_tmpbase}" -d)" + if [ -d "${adb_basedir}" ]; then + adb_tmpdir="$(mktemp -p "${adb_basedir}" -d)" adb_tmpload="$(mktemp -p "${adb_tmpdir}" -tu)" adb_tmpfile="$(mktemp -p "${adb_tmpdir}" -tu)" adb_srtopts="--temporary-directory=${adb_tmpdir} --compress-program=gzip --parallel=${adb_cores}" else - f_log "err" "the temp base directory '${adb_tmpbase}' does not exist/is not mounted yet, please create the directory or raise the 'adb_triggerdelay' to defer the adblock start" + f_log "err" "the base directory '${adb_basedir}' does not exist/is not mounted yet, please create the directory or raise the 'adb_triggerdelay' to defer the adblock start" fi - [ ! -s "${adb_pidfile}" ] && printf "%s" "${$}" >"${adb_pidfile}" - f_log "debug" "f_temp ::: tmp_base: ${adb_tmpbase:-"-"}, tmp_dir: ${adb_tmpdir:-"-"}, sort_options: ${adb_srtopts}, pid_file: ${adb_pidfile:-"-"}" + [ ! -s "${adb_pidfile}" ] && printf '%s' "${$}" >"${adb_pidfile}" } # remove temporary files and directories # f_rmtemp() { - [ -d "${adb_tmpdir}" ] && rm -rf "${adb_tmpdir}" - rm -f "${adb_srcfile}" + [ -f "${adb_errorlog}" ] && [ ! -s "${adb_errorlog}" ] && "${adb_rmcmd}" -f "${adb_errorlog}" + [ -d "${adb_tmpdir}" ] && "${adb_rmcmd}" -rf "${adb_tmpdir}" : >"${adb_pidfile}" - f_log "debug" "f_rmtemp ::: tmp_dir: ${adb_tmpdir:-"-"}, src_file: ${adb_srcfile:-"-"}, pid_file: ${adb_pidfile:-"-"}" } # remove dns related files # f_rmdns() { - local status - - status="$(ubus -S call service list '{"name":"adblock"}' 2>/dev/null | jsonfilter -l1 -e '@["adblock"].instances.*.running' 2>/dev/null)" - if [ "${adb_dns}" = "raw" ] || { [ -n "${adb_dns}" ] && [ -n "${status}" ]; }; then - : >"${adb_rtfile}" - [ "${adb_backup}" = "1" ] && rm -f "${adb_backupdir}/${adb_dnsprefix}".*.gz - printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" - f_dnsup 4 + if [ -n "${adb_finaldir}" ]; then + printf '%b' "${adb_dnsheader}" >"${adb_finaldir}/${adb_dnsfile}" + f_dnsup fi f_rmtemp - f_log "debug" "f_rmdns ::: dns: ${adb_dns}, status: ${status:-"-"}, dns_dir: ${adb_dnsdir}, dns_file: ${adb_dnsfile}, rt_file: ${adb_rtfile}, backup_dir: ${adb_backupdir:-"-"}" + if [ -d "${adb_backupdir}" ] && { [ "${adb_action}" = "stop" ] || [ "${adb_enabled}" = "0" ]; }; then + "${adb_findcmd}" "${adb_backupdir}" -maxdepth 1 -type f -name '*.gz' -exec "${adb_rmcmd}" -f {} + + fi } # commit uci changes # f_uci() { - local change config="${1}" - - if [ -n "${config}" ]; then - change="$(uci -q changes "${config}" | "${adb_awk}" '{ORS=" "; print $0}')" - if [ -n "${change}" ]; then - uci_commit "${config}" - case "${config}" in - "firewall") - "/etc/init.d/firewall" reload >/dev/null 2>&1 - ;; - "resolver") - printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" - f_count - f_jsnup "running" - "/etc/init.d/${adb_dns}" reload >/dev/null 2>&1 - ;; - esac + local config="${1}" + + if [ -n "$(uci -q changes "${config}")" ]; then + uci_commit "${config}" + if [ "${config}" = "resolver" ]; then + printf '%b' "${adb_dnsheader}" >"${adb_finaldir}/${adb_dnsfile}" + adb_cnt="0" + f_jsnup "processing" + "/etc/init.d/${adb_dns}" reload >/dev/null 2>&1 fi - f_log "debug" "f_uci ::: config: ${config}, change: ${change}" fi } # get list counter # f_count() { - local file mode="${1}" name="${2}" + local files mode="${1}" file="${2}" var="${3}" adb_cnt="0" - case "${mode}" in - "iplist") - [ -s "${adb_tmpdir}/tmp.add.${name}" ] && adb_cnt="$(wc -l 2>/dev/null <"${adb_tmpdir}/tmp.add.${name}")" - ;; - "blacklist") - [ -s "${adb_tmpfile}.${name}" ] && adb_cnt="$(wc -l 2>/dev/null <"${adb_tmpfile}.${name}")" - ;; - "whitelist") - [ -s "${adb_tmpdir}/tmp.raw.${name}" ] && { adb_cnt="$(wc -l 2>/dev/null <"${adb_tmpdir}/tmp.raw.${name}")"; rm -f "${adb_tmpdir}/tmp.raw.${name}"; } - ;; - "safesearch") - [ -s "${adb_tmpdir}/tmp.safesearch.${name}" ] && adb_cnt="$(wc -l 2>/dev/null <"${adb_tmpdir}/tmp.safesearch.${name}")" - ;; - "merge") - [ -s "${adb_tmpdir}/${adb_dnsfile}" ] && adb_cnt="$(wc -l 2>/dev/null <"${adb_tmpdir}/${adb_dnsfile}")" - ;; - "download" | "backup" | "restore") - [ -s "${src_tmpfile}" ] && adb_cnt="$(wc -l 2>/dev/null <"${src_tmpfile}")" - ;; - "final") - if [ -s "${adb_dnsdir}/${adb_dnsfile}" ]; then - adb_cnt="$(wc -l 2>/dev/null <"${adb_dnsdir}/${adb_dnsfile}")" - if [ -s "${adb_tmpdir}/tmp.add.whitelist" ]; then - adb_cnt="$((adb_cnt - $(wc -l 2>/dev/null <"${adb_tmpdir}/tmp.add.whitelist")))" - fi - for file in "${adb_tmpdir}/tmp.safesearch".*; do - if [ -r "${file}" ]; then - adb_cnt="$((adb_cnt - $(wc -l 2>/dev/null <"${file}")))" - fi - done - [ -n "${adb_dnsheader}" ] && adb_cnt="$(((adb_cnt - $(printf "%b" "${adb_dnsheader}" | grep -c "^")) / 2))" - fi - ;; - esac + if [ -s "${file}" ]; then + if [ -n "${var}" ] || [ "${mode}" != "final" ]; then + adb_cnt="$("${adb_awkcmd}" 'END{print NR}' "${file}")" + [ -n "${var}" ] && printf '%s' "${adb_cnt}" + else + # build argument list: main file first, then all subtraction files + # + files="${file}" + [ -s "${adb_tmpdir}/tmp.add.allowlist" ] && files="${files} ${adb_tmpdir}/tmp.add.allowlist" + for file in "${adb_tmpdir}/tmp.safesearch".*; do + [ -s "${file}" ] && files="${files} ${file}" + done + adb_cnt="$("${adb_awkcmd}" -v hdr="${adb_dnsheader}" ' + NR == FNR { main++; next } + { sub_cnt++ } + END { + cnt = main - sub_cnt + if (hdr != "") { + hlines = gsub(/\n/, "", hdr) + cnt = int((cnt - hlines) / 2) + } + if (cnt < 0) cnt = 0 + res = "" + pos = 0 + s = sprintf("%d", cnt) + for (i = length(s); i > 0; i--) { + res = substr(s, i, 1) res + if (++pos == 3 && i > 1) { res = " " res; pos = 0 } + } + print res + } + ' ${files})" + fi + fi } # set external config options # f_extconf() { - local config config_dir config_file section zone port fwcfg + local config case "${adb_dns}" in - "dnsmasq") - config="dhcp" - config_dir="$(uci_get dhcp "@dnsmasq[${adb_dnsinstance}]" confdir | grep -Fo "${adb_dnsdir}")" - if [ "${adb_enabled}" = "1" ] && [ -z "${config_dir}" ]; then - uci_set dhcp "@dnsmasq[${adb_dnsinstance}]" confdir "${adb_dnsdir}" 2>/dev/null - fi - ;; - "kresd") - config="resolver" - config_file="$(uci_get resolver kresd rpz_file | grep -Fo "${adb_dnsdir}/${adb_dnsfile}")" - if [ "${adb_enabled}" = "1" ] && [ -z "${config_file}" ]; then - uci -q add_list resolver.kresd.rpz_file="${adb_dnsdir}/${adb_dnsfile}" - elif [ "${adb_enabled}" = "0" ] && [ -n "${config_file}" ]; then - uci -q del_list resolver.kresd.rpz_file="${adb_dnsdir}/${adb_dnsfile}" - fi - ;; - esac - f_uci "${config}" - - config="firewall" - fwcfg="$(uci -qNX show "${config}" | "${adb_awk}" 'BEGIN{FS="[.=]"};/adblock_/{if(zone==$2){next}else{ORS=" ";zone=$2;print zone}}')" - if [ "${adb_enabled}" = "1" ] && [ "${adb_forcedns}" = "1" ] && - /etc/init.d/firewall enabled; then - for zone in ${adb_zonelist}; do - for port in ${adb_portlist}; do - if ! printf "%s" "${fwcfg}" | grep -q "adblock_${zone}${port}[ |\$]"; then - uci -q batch <<-EOC - set firewall."adblock_${zone}${port}"="redirect" - set firewall."adblock_${zone}${port}".name="Adblock DNS (${zone}, ${port})" - set firewall."adblock_${zone}${port}".src="${zone}" - set firewall."adblock_${zone}${port}".proto="tcp udp" - set firewall."adblock_${zone}${port}".src_dport="${port}" - set firewall."adblock_${zone}${port}".dest_port="${port}" - set firewall."adblock_${zone}${port}".target="DNAT" - set firewall."adblock_${zone}${port}".family="any" - set firewall."adblock_${zone}${port}".ipset="!tsdns_bypass" - add_list firewall."adblock_${zone}${port}".ns_tag="automated" - EOC - fi - fwcfg="${fwcfg/adblock_${zone}${port}[ |\$]/}" - done - done - fwcfg="${fwcfg#"${fwcfg%%[![:space:]]*}"}" - fwcfg="${fwcfg%"${fwcfg##*[![:space:]]}"}" - fi - if [ "${adb_enabled}" = "0" ] || [ "${adb_forcedns}" = "0" ] || [ -n "${fwcfg}" ]; then - for section in ${fwcfg}; do - uci_remove firewall "${section}" - done - fi - - # add adb_bypass - if [ "${adb_enabled}" = "1" ] && [ "${adb_forcedns}" = "1" ] && /etc/init.d/firewall enabled; then - if ! uci -q get firewall.tsdns_bypass >/dev/null; then - uci -q batch <<-EOC - set firewall.tsdns_bypass="ipset" - set firewall.tsdns_bypass.name="tsdns_bypass" - set firewall.tsdns_bypass.match="src_net" - set firewall.tsdns_bypass.enabled="1" - EOC + "dnsmasq") + config="dhcp" + if [ "${adb_dnsshift}" = "1" ] && + ! uci_get ${config} @dnsmasq[${adb_dnsinstance}] addnmount | "${adb_grepcmd}" -q "${adb_backupdir}"; then + uci -q add_list ${config}.@dnsmasq[${adb_dnsinstance}].addnmount="${adb_backupdir}" + elif [ "${adb_dnsshift}" = "0" ] && + uci_get ${config} @dnsmasq[${adb_dnsinstance}] addnmount | "${adb_grepcmd}" -q "${adb_backupdir}"; then + uci -q del_list ${config}.@dnsmasq[${adb_dnsinstance}].addnmount="${adb_backupdir}" fi - # note: adb_bypass var contains an extra space at the beginning - if [ " $(uci -q get firewall.tsdns_bypass.entry)" != "${adb_bypass}" ]; then - # make sure bypass list is always empty - uci -q delete firewall.tsdns_bypass.entry - for src in ${adb_bypass} - do - uci -q add_list firewall.tsdns_bypass.entry="${src}" - done + ;; + "kresd") + config="resolver" + if [ "${adb_enabled}" = "1" ] && + ! uci_get ${config} kresd rpz_file | "${adb_grepcmd}" -q "${adb_dnsdir}/${adb_dnsfile}"; then + uci -q add_list ${config}.kresd.rpz_file="${adb_dnsdir}/${adb_dnsfile}" + elif [ "${adb_enabled}" = "0" ] && + uci_get ${config} kresd rpz_file | "${adb_grepcmd}" -q "${adb_dnsdir}/${adb_dnsfile}"; then + uci -q del_list ${config}.kresd.rpz_file="${adb_dnsdir}/${adb_dnsfile}" fi - fi - - # remove adb_bypass - if [ "${adb_enabled}" = "0" ] || [ "${adb_forcedns}" = "0" ]; then - uci -q delete firewall.tsdns_bypass - fi - + ;; + "smartdns") + config="smartdns" + if [ "${adb_enabled}" = "1" ] && + ! uci_get ${config} @${config}[${adb_dnsinstance}] conf_files | "${adb_grepcmd}" -q "${adb_dnsdir}/${adb_dnsfile}"; then + uci -q add_list ${config}.@${config}[${adb_dnsinstance}].conf_files="${adb_dnsdir}/${adb_dnsfile}" + elif [ "${adb_enabled}" = "0" ] && + uci_get ${config} @${config}[${adb_dnsinstance}] conf_files | "${adb_grepcmd}" -q "${adb_dnsdir}/${adb_dnsfile}"; then + uci -q del_list ${config}.@${config}[${adb_dnsinstance}].conf_files="${adb_dnsdir}/${adb_dnsfile}" + fi + ;; + esac f_uci "${config}" } # restart dns backend # f_dnsup() { - local rset dns_service dns_up dns_pid restart_rc cnt="0" out_rc="4" in_rc="${1:-0}" + local restart_rc nft_rc cnt="0" out_rc="4" - if [ "${adb_dns}" = "raw" ]; then + if [ "${adb_dns}" = "raw" ] || [ -z "${adb_dns}" ]; then out_rc="0" else - if [ "${in_rc}" = "0" ] && [ "${adb_dnsflush}" = "0" ]; then + + # load external dns bridge + # + if { [ -n "${adb_bridgednsv4}" ] || [ -n "${adb_bridgednsv6}" ]; } && [ "${adb_nftbridge}" = "1" ]; then + if "${adb_nftcmd}" list chain inet adblock dns-bridge >/dev/null 2>&1; then + if [ -n "${adb_bridgednsv4}" ]; then + "${adb_nftcmd}" add rule inet adblock dns-bridge meta nfproto ipv4 meta l4proto { udp, tcp } th dport 53 counter dnat to ${adb_bridgednsv4}:53 2>>"${adb_errorlog}" + nft_rc="${?}" + fi + if [ -n "${adb_bridgednsv6}" ]; then + "${adb_nftcmd}" add rule inet adblock dns-bridge meta nfproto ipv6 meta l4proto { udp, tcp } th dport 53 counter dnat to [${adb_bridgednsv6}]:53 2>>"${adb_errorlog}" + nft_rc="$((nft_rc + $?))" + fi + if [ "${nft_rc}" = "0" ]; then + f_log "debug" "external DNS bridge loaded: ${adb_bridgednsv4:-"-"} / ${adb_bridgednsv6:-"-"}" + else + f_log "err" "failed to load external DNS bridge: ${adb_bridgednsv4:-"-"} / ${adb_bridgednsv6:-"-"}" + fi + fi + fi + + # restart dns backend + # + if [ "${adb_dnsflush}" = "0" ]; then case "${adb_dns}" in - "unbound") - if [ -x "${adb_dnscachecmd}" ] && [ -d "${adb_tmpdir}" ] && [ -f "${adb_dnsdir}/unbound.conf" ]; then - "${adb_dnscachecmd}" -c "${adb_dnsdir}/unbound.conf" dump_cache >"${adb_tmpdir}/adb_cache.dump" 2>/dev/null - fi - "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 + "unbound") + if [ -x "${adb_dnscachecmd}" ] && [ -d "${adb_tmpdir}" ] && [ -f "${adb_dnsdir}/unbound.conf" ]; then + "${adb_dnscachecmd}" -c "${adb_dnsdir}/unbound.conf" dump_cache >"${adb_tmpdir}/adb_cache.dump" 2>>"${adb_errorlog}" + fi + "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 + restart_rc="${?}" + ;; + "named") + if [ -x "${adb_dnscachecmd}" ] && [ -f "/etc/bind/rndc.conf" ]; then + "${adb_dnscachecmd}" -c "/etc/bind/rndc.conf" reload >/dev/null 2>&1 restart_rc="${?}" - ;; - "named") - if [ -x "${adb_dnscachecmd}" ] && [ -f "/etc/bind/rndc.conf" ]; then - "${adb_dnscachecmd}" -c "/etc/bind/rndc.conf" reload >/dev/null 2>&1 - restart_rc="${?}" - fi - if [ -z "${restart_rc}" ] || { [ -n "${restart_rc}" ] && [ "${restart_rc}" != "0" ]; }; then - "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 - restart_rc="${?}" - fi - ;; - *) + fi + if [ -z "${restart_rc}" ] || { [ -n "${restart_rc}" ] && [ "${restart_rc}" != "0" ]; }; then "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 restart_rc="${?}" - ;; + fi + ;; + *) + "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 + restart_rc="${?}" + ;; esac fi if [ -z "${restart_rc}" ]; then @@ -652,24 +783,14 @@ f_dnsup() { restart_rc="${?}" fi fi + + # check if dns backend is responsive, restore dns cache for unbound and get dns backend pid + # if [ "${restart_rc}" = "0" ]; then - rset="/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}" while [ "${cnt}" -le "${adb_dnstimeout}" ]; do - dns_service="$(ubus -S call service list "{\"name\":\"${adb_dns}\"}")" - dns_up="$(printf "%s" "${dns_service}" | jsonfilter -l1 -e "@[\"${adb_dns}\"].instances.*.running")" - dns_pid="$(printf "%s" "${dns_service}" | jsonfilter -l1 -e "@[\"${adb_dns}\"].instances.*.pid")" - if [ "${dns_up}" = "true" ] && [ -n "${dns_pid}" ] && ! ls "/proc/${dns_pid}/fd/${adb_dnsdir}/${adb_dnsfile}" >/dev/null 2>&1; then - if [ -x "${adb_lookupcmd}" ] && [ -n "$(printf "%s" "${adb_lookupdomain}" | "${adb_awk}" "${rset}")" ]; then - if "${adb_lookupcmd}" "${adb_lookupdomain}" >/dev/null 2>&1; then - out_rc="0" - break - fi - else - sleep ${adb_dnstimeout} - cnt=${adb_dnstimeout} - out_rc="0" - break - fi + if "${adb_lookupcmd}" "${adb_lookupdomain}." >/dev/null 2>&1; then + out_rc="0" + break fi cnt="$((cnt + 1))" sleep 1 @@ -681,231 +802,519 @@ f_dnsup() { fi fi fi - f_log "debug" "f_dnsup ::: dns: ${adb_dns}, cache_cmd: ${adb_dnscachecmd:-"-"}, lookup_cmd: ${adb_lookupcmd:-"-"}, lookup_domain: ${adb_lookupdomain:-"-"}, restart_rc: ${restart_rc:-"-"}, dns_flush: ${adb_dnsflush}, dns_timeout: ${adb_dnstimeout}, dns_cnt: ${cnt}, in_rc: ${in_rc}, out_rc: ${out_rc}" + + # remove external dns bridge + # + if [ "${adb_nftbridge}" = "1" ] && "${adb_nftcmd}" list chain inet adblock dns-bridge >/dev/null 2>&1; then + "${adb_nftcmd}" flush chain inet adblock dns-bridge 2>>"${adb_errorlog}" + nft_rc="${?}" + if [ "${nft_rc}" = "0" ]; then + f_log "debug" "external DNS bridge removed" + else + f_log "err" "failed to remove external DNS bridge" + fi + fi + + f_log "debug" "f_dnsup ::: dns: ${adb_dns}, cache_cmd: ${adb_dnscachecmd:-"-"}, lookup_domain: ${adb_lookupdomain:-"-"}, restart_rc: ${restart_rc:-"-"}, dns_flush: ${adb_dnsflush}, dns_timeout: ${adb_dnstimeout}, dns_cnt: ${cnt}, nft_rc: ${nft_rc:-"-"}, rc: ${out_rc}" return "${out_rc}" } +# handle etag http header +# +f_etag() { + local http_head http_code etag_id etag_cnt out_rc="4" feed="${1}" feed_url="${2}" feed_suffix="${3}" feed_cnt="${4:-"1"}" + + if [ -n "${adb_etagparm}" ]; then + + # ensure etag file exists + # + [ ! -f "${adb_backupdir}/adblock.etag" ] && : >"${adb_backupdir}/adblock.etag" + + # fetch http headers and extract http code and etag/last-modified header + # + http_head="$("${adb_fetchcmd}" ${adb_etagparm} "${feed_url}${feed_suffix}" 2>&1)" + http_code="$(printf '%s' "${http_head}" | "${adb_awkcmd}" 'tolower($0)~/^[[:space:]]*http\/[0-9.]+ /{printf "%s",$2}')" + etag_id="$(printf '%s' "${http_head}" | "${adb_awkcmd}" 'tolower($0)~/^[[:space:]]*etag: /{gsub("\"","");printf "%s",$2}')" + + # if etag header is not present, try to use last-modified header as fallback for change detection + # + if [ -z "${etag_id}" ]; then + etag_id="$(printf '%s' "${http_head}" | "${adb_awkcmd}" 'tolower($0)~/^[[:space:]]*last-modified: /{gsub(/[Ll]ast-[Mm]odified:|[[:space:]]|,|:/,"");printf "%s\n",$1}')" + fi + + # acquire exclusive lock on etag file to serialize concurrent read-modify-write from parallel feeds + # + exec 9>"${adb_etaglock}" + "${adb_flockcmd}" -x 9 + + # compare http code and etag id with stored values, update etag file and return code accordingly + # + etag_cnt="$("${adb_awkcmd}" -v f="${feed}" '$1 == f { n++ } END { print n+0 }' "${adb_backupdir}/adblock.etag")" + if [ "${http_code}" = "200" ] && [ "${etag_cnt}" = "${feed_cnt}" ] && [ -n "${etag_id}" ] && + "${adb_awkcmd}" -v f="${feed}" -v s="${feed_suffix}" -v e="${etag_id}" ' + BEGIN { rc = 1; p = f " " s } + index($0, p) == 1 { + rest = substr($0, length(p) + 1) + sub(/^[[:space:]]+/, "", rest) + if (rest == e) { rc = 0; exit } + } + END { exit rc }' "${adb_backupdir}/adblock.etag"; then + out_rc="0" + elif [ -n "${etag_id}" ]; then + + # if feed count is less than etag count, it means the feed source has been removed or disabled, so remove all entries for this feed, + # otherwise only remove the entry with the matching feed suffix (feed url) to allow multiple sources for the same feed + # + if [ "${feed_cnt}" -lt "${etag_cnt}" ]; then + "${adb_awkcmd}" -v f="${feed}" '$1 != f' \ + "${adb_backupdir}/adblock.etag" >"${adb_backupdir}/adblock.etag.new" + else + "${adb_awkcmd}" -v f="${feed}" -v s="${feed_suffix}" ' + BEGIN { p = f " " s } + index($0, p) != 1' \ + "${adb_backupdir}/adblock.etag" >"${adb_backupdir}/adblock.etag.new" + fi + "${adb_mvcmd}" -f "${adb_backupdir}/adblock.etag.new" "${adb_backupdir}/adblock.etag" + printf '%s\t%s\n' "${feed} ${feed_suffix}" "${etag_id}" >>"${adb_backupdir}/adblock.etag" + out_rc="2" + fi + + # release lock + # + exec 9>&- + fi + + f_log "debug" "f_etag ::: feed: ${feed}, suffix: ${feed_suffix:-"-"}, http_code: ${http_code:-"-"}, feed/etag: ${feed_cnt}/${etag_cnt:-"0"}, rc: ${out_rc}" + return "${out_rc}" +} + +# add adblock-related nft rules +# +f_nftadd() { + local devices device port file="${adb_tmpdir}/adb_nft.add" + + # only proceed if at least one feature is enabled + # + if [ "${adb_nftallow}" = "0" ] && [ "${adb_nftblock}" = "0" ] && + [ "${adb_nftremote}" = "0" ] && [ "${adb_nftforce}" = "0" ] && + [ "${adb_nftbridge}" = "0" ]; then + return + fi + + # remove existing adblock-related nft rules on restart + # + [ "${adb_action}" = "restart" ] && f_nftremove + + # do not proceed if adblock-related nft rules already exists + # + if "${adb_nftcmd}" -t list table inet adblock >/dev/null 2>&1; then + return + fi + + # do not proceed if action is stop/suspend/resume + # + if [ "${adb_action}" = "stop" ] || [ "${adb_action}" = "suspend" ] || [ "${adb_action}" = "resume" ]; then + return + fi + + # prepare nftables rules for adblock features + # + { + # nft header (tables, sets, base and regular chains) + # + printf '%s\n\n' "#!${adb_nftcmd} -f" + printf '%s\n' "add table inet adblock" + + # allow Set + # + if [ "${adb_nftallow}" = "1" ] && [ -n "${adb_nftmacallow}" ]; then + printf '%s\n' "add set inet adblock mac_allow { type ether_addr; flags interval; auto-merge; elements = { ${adb_nftmacallow// /, } }; }" + fi + + # remote allow Set with timeout, for MACs that should be temporary allowed to bypass dns blocking + # + if [ "${adb_nftremote}" = "1" ] && [ -n "${adb_nftmacremote}" ]; then + printf '%s\n' "add set inet adblock mac_remote { type ether_addr; flags timeout; timeout ${adb_nftremotetimeout}m; }" + fi + + # adblock pre-routing chain for allow/block rules + # + if [ "${adb_nftblock}" = "1" ] && [ -n "${adb_nftmacblock}" ]; then + printf '%s\n' "add set inet adblock mac_block { type ether_addr; flags interval; auto-merge; elements = { ${adb_nftmacblock// /, } }; }" + fi + printf '%s\n' "add chain inet adblock pre-routing { type nat hook prerouting priority -150; policy accept; }" + printf '%s\n' "add chain inet adblock _reject" + + # dns-bridge base chain + # + printf '%s\n' "add chain inet adblock dns-bridge { type nat hook prerouting priority -160; policy accept; }" + + # reject chain rules + # + printf '%s\n' "add rule inet adblock _reject meta l4proto tcp counter reject with tcp reset" + printf '%s\n' "add rule inet adblock _reject counter reject with icmpx host-unreachable" + + # external allow rules + # + if [ "${adb_nftallow}" = "1" ]; then + if [ -n "${adb_nftmacallow}" ]; then + [ -n "${adb_allowdnsv4}" ] && printf '%s\n' "add rule inet adblock pre-routing meta nfproto ipv4 ether saddr @mac_allow meta l4proto { udp, tcp } th dport 53 counter dnat to ${adb_allowdnsv4}:53" + [ -n "${adb_allowdnsv6}" ] && printf '%s\n' "add rule inet adblock pre-routing meta nfproto ipv6 ether saddr @mac_allow meta l4proto { udp, tcp } th dport 53 counter dnat to [${adb_allowdnsv6}]:53" + fi + for device in ${adb_nftdevallow}; do + [ -n "${adb_allowdnsv4}" ] && printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" meta nfproto ipv4 meta l4proto { udp, tcp } th dport 53 counter dnat to ${adb_allowdnsv4}:53" + [ -n "${adb_allowdnsv6}" ] && printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" meta nfproto ipv6 meta l4proto { udp, tcp } th dport 53 counter dnat to [${adb_allowdnsv6}]:53" + done + f_log "debug" "adblock-related nft allow rules prepared for external DNS ${adb_allowdnsv4:-"-"} / ${adb_allowdnsv6:-"-"}" + fi + + # external remote allow rules + # + if [ "${adb_nftremote}" = "1" ] && [ -n "${adb_nftmacremote}" ]; then + [ -n "${adb_remotednsv4}" ] && printf '%s\n' "add rule inet adblock pre-routing meta nfproto ipv4 ether saddr @mac_remote meta l4proto { udp, tcp } th dport 53 counter dnat to ${adb_remotednsv4}:53" + [ -n "${adb_remotednsv6}" ] && printf '%s\n' "add rule inet adblock pre-routing meta nfproto ipv6 ether saddr @mac_remote meta l4proto { udp, tcp } th dport 53 counter dnat to [${adb_remotednsv6}]:53" + f_log "debug" "adblock-related nft remote allow rules prepared for external DNS ${adb_remotednsv4:-"-"} / ${adb_remotednsv6:-"-"} with timeout of ${adb_nftremotetimeout} minutes" + fi + + # external block rules + # + if [ "${adb_nftblock}" = "1" ]; then + if [ -n "${adb_nftmacblock}" ]; then + [ -n "${adb_blockdnsv4}" ] && printf '%s\n' "add rule inet adblock pre-routing meta nfproto ipv4 ether saddr @mac_block meta l4proto { udp, tcp } th dport 53 counter dnat to ${adb_blockdnsv4}:53" + [ -n "${adb_blockdnsv6}" ] && printf '%s\n' "add rule inet adblock pre-routing meta nfproto ipv6 ether saddr @mac_block meta l4proto { udp, tcp } th dport 53 counter dnat to [${adb_blockdnsv6}]:53" + fi + for device in ${adb_nftdevblock}; do + [ -n "${adb_blockdnsv4}" ] && printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" meta nfproto ipv4 meta l4proto { udp, tcp } th dport 53 counter dnat to ${adb_blockdnsv4}:53" + [ -n "${adb_blockdnsv6}" ] && printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" meta nfproto ipv6 meta l4proto { udp, tcp } th dport 53 counter dnat to [${adb_blockdnsv6}]:53" + done + f_log "debug" "adblock-related nft block rules prepared for external DNS ${adb_blockdnsv4:-"-"} / ${adb_blockdnsv6:-"-"}" + fi + + # local dns enforcement + # + if [ "${adb_nftforce}" = "1" ]; then + + # device/vlan exceptions + # + for device in ${adb_nftdevallow} ${adb_nftdevblock}; do + case " ${devices} " in + *" ${device} "*) ;; + + *) + [ -n "${devices}" ] && devices="${devices} ${device}" || devices="${device}" + printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" return" + ;; + esac + done + + # mac exceptions + # + for device in ${adb_nftdevforce}; do + if [ "${adb_nftallow}" = "1" ] && [ -n "${adb_nftmacallow}" ]; then + printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" ether saddr @mac_allow return" + fi + if [ "${adb_nftblock}" = "1" ] && [ -n "${adb_nftmacblock}" ]; then + printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" ether saddr @mac_block return" + fi + + # NethSecurity: IP-based bypass exceptions + # + for bypass_ip in ${ns_tsdns_bypass}; do + printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" ip saddr ${bypass_ip} return" + done + + # dns enforce rules + # + for port in ${adb_nftportforce}; do + if [ "${port}" = "53" ]; then + printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" meta nfproto { ipv4, ipv6 } meta l4proto { udp, tcp } th dport ${port} counter redirect to :${port}" + else + printf '%s\n' "add rule inet adblock pre-routing iifname \"${device}\" meta nfproto { ipv4, ipv6 } meta l4proto { udp, tcp } th dport ${port} counter goto _reject" + fi + done + done + f_log "debug" "adblock-related nft local DNS enforcement rules prepared for devices: ${adb_nftdevforce// /, } and ports: ${adb_nftportforce// /, }" + fi + } >"${file}" + if "${adb_nftcmd}" -f "${file}" 2>>"${adb_errorlog}"; then + f_log "info" "adblock-related nft rules loaded" + else + f_log "err" "failed to load adblock-related nft rules" + fi +} + +# remove adblock-related nft rules +# +f_nftremove() { + local file="${adb_tmpdir}/adb_nft.remove" + + if "${adb_nftcmd}" -t list table inet adblock >/dev/null 2>&1; then + { + printf '%s\n' "#!${adb_nftcmd} -f" + printf '%s\n' "delete table inet adblock" + } >"${file}" + + if "${adb_nftcmd}" -f "${file}" 2>>"${adb_errorlog}"; then + f_log "info" "adblock-related nft rules removed" + else + f_log "err" "failed to remove adblock-related nft rules" + fi + fi +} + # backup/restore/remove blocklists # f_list() { - local hold file rset item array safe_url safe_ips safe_cname safe_domains ip out_rc mode="${1}" src_name="${2:-"${src_name}"}" in_rc="${src_rc:-0}" cnt ffiles="-maxdepth 1 -name ${adb_dnsprefix}.*.gz" + local file files item array safe_url safe_ips safe_cname safe_domains ip out_rc file_name name + local mode="${1}" src_name="${2:-"${src_name}"}" in_rc="${src_rc:-0}" use_cname="0" case "${mode}" in - "iplist") - src_name="${mode}" - if [ "${adb_dns}" = "named" ]; then - rset="BEGIN{FS=\"[.:]\";pfx=\"32\"}{if(match(\$0,/:/))pfx=\"128\"}{printf \"%s.\",pfx;for(seg=NF;seg>=1;seg--)if(seg==1)printf \"%s\n\",\$seg;else if(\$seg>=0)printf \"%s.\",\$seg; else printf \"%s.\",\"zz\"}" - if [ -n "${adb_allowip}" ]; then - : >"${adb_tmpdir}/tmp.raw.${src_name}" - for ip in ${adb_allowip}; do - printf "%s" "${ip}" | "${adb_awk}" "${rset}" >>"${adb_tmpdir}/tmp.raw.${src_name}" - done - eval "${adb_dnsallowip}" "${adb_tmpdir}/tmp.raw.${src_name}" >"${adb_tmpdir}/tmp.add.${src_name}" + "blocklist" | "allowlist") + src_name="${mode}" + case "${src_name}" in + "blocklist") + if [ -f "${adb_blocklist}" ]; then + file_name="${adb_tmpfile}.${src_name}" + f_chkdom local 1 <"${adb_blocklist}" >"${adb_tmpdir}/tmp.raw.${src_name}" + if [ "${adb_tld}" = "1" ]; then + if [ -s "${adb_tmpdir}/tmp.rem.allowlist" ]; then + "${adb_awkcmd}" ' + NR==FNR { member[$1]; next } + !($1 in member) { + n = split($1, seg, ".") + for (f = n; f > 1; f--) printf "%s.", seg[f] + print seg[1] + } + ' "${adb_tmpdir}/tmp.rem.allowlist" "${adb_tmpdir}/tmp.raw.${src_name}" + else + "${adb_awkcmd}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' "${adb_tmpdir}/tmp.raw.${src_name}" + fi | "${adb_sortcmd}" ${adb_srtopts} -u >"${file_name}" out_rc="${?}" - fi - if [ -n "${adb_denyip}" ] && { [ -z "${out_rc}" ] || [ "${out_rc}" = "0" ]; }; then - : >"${adb_tmpdir}/tmp.raw.${src_name}" - for ip in ${adb_denyip}; do - printf "%s" "${ip}" | "${adb_awk}" "${rset}" >>"${adb_tmpdir}/tmp.raw.${src_name}" - done - eval "${adb_dnsdenyip}" "${adb_tmpdir}/tmp.raw.${src_name}" >>"${adb_tmpdir}/tmp.add.${src_name}" + else + if [ -s "${adb_tmpdir}/tmp.rem.allowlist" ]; then + "${adb_awkcmd}" 'NR==FNR{member[$1];next}!($1 in member)' "${adb_tmpdir}/tmp.rem.allowlist" "${adb_tmpdir}/tmp.raw.${src_name}" | + "${adb_sortcmd}" ${adb_srtopts} -u >"${file_name}" 2>>"${adb_errorlog}" + else + "${adb_sortcmd}" ${adb_srtopts} -u "${adb_tmpdir}/tmp.raw.${src_name}" 2>>"${adb_errorlog}" >"${file_name}" + fi out_rc="${?}" fi - rm -f "${adb_tmpdir}/tmp.raw.${src_name}" fi ;; - "blacklist" | "whitelist") - src_name="${mode}" - if [ "${src_name}" = "blacklist" ] && [ -f "${adb_blacklist}" ]; then - rset="/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}" - "${adb_awk}" "${rset}" "${adb_blacklist}" >"${adb_tmpdir}/tmp.raw.${src_name}" - if [ -s "${adb_whitelist}" ]; then - "${adb_awk}" 'NR==FNR{member[$1];next}!($1 in member)' "${adb_whitelist}" "${adb_tmpdir}/tmp.raw.${src_name}" >"${adb_tmpdir}/tmp.deduplicate.${src_name}" - else - cat "${adb_tmpdir}/tmp.raw.${src_name}" >"${adb_tmpdir}/tmp.deduplicate.${src_name}" - fi - "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' "${adb_tmpdir}/tmp.deduplicate.${src_name}" >"${adb_tmpdir}/tmp.raw.${src_name}" - "${adb_sort}" ${adb_srtopts} -u "${adb_tmpdir}/tmp.raw.${src_name}" 2>/dev/null >"${adb_tmpfile}.${src_name}" - out_rc="${?}" - rm -f "${adb_tmpdir}/tmp.raw.${src_name}" - elif [ "${src_name}" = "whitelist" ] && [ -f "${adb_whitelist}" ]; then - rset="/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}" - printf "%s\n" "${adb_lookupdomain}" | "${adb_awk}" "${rset}" >"${adb_tmpdir}/tmp.raw.${src_name}" - "${adb_awk}" "${rset}" "${adb_whitelist}" >>"${adb_tmpdir}/tmp.raw.${src_name}" + "allowlist") + if [ -f "${adb_allowlist}" ] && [ "${adb_dnsallow}" = "1" ]; then + file_name="${adb_tmpdir}/tmp.raw.${src_name}" + [ "${adb_lookupdomain}" != "localhost" ] && { printf '%s\n' "${adb_lookupdomain}" | f_chkdom local 1; } >"${file_name}" + f_chkdom local 1 <"${adb_allowlist}" >>"${file_name}" + "${adb_catcmd}" "${file_name}" >"${adb_tmpdir}/tmp.rem.${src_name}" + f_dnsallow "${file_name}" >"${adb_tmpdir}/tmp.add.${src_name}" out_rc="${?}" - if [ "${out_rc}" = "0" ]; then - rset="/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{gsub(\"\\\\.\",\"\\\\.\",\$1);print tolower(\"^(|.*\\\\.)\"\$1\"$\")}" - "${adb_awk}" "${rset}" "${adb_tmpdir}/tmp.raw.${src_name}" >"${adb_tmpdir}/tmp.rem.${src_name}" - out_rc="${?}" - if [ "${out_rc}" = "0" ] && [ "${adb_dnsallow}" != "1" ]; then - eval "${adb_dnsallow}" "${adb_tmpdir}/tmp.raw.${src_name}" >"${adb_tmpdir}/tmp.add.${src_name}" - out_rc="${?}" - if [ "${out_rc}" = "0" ] && [ "${adb_jail}" = "1" ] && [ "${adb_dnsstop}" != "0" ]; then - : >"${adb_jaildir}/${adb_dnsjail}" - [ -n "${adb_dnsheader}" ] && printf "%b" "${adb_dnsheader}" >>"${adb_jaildir}/${adb_dnsjail}" - cat "${adb_tmpdir}/tmp.add.${src_name}" >>"${adb_jaildir}/${adb_dnsjail}" - printf "%s\n" "${adb_dnsstop}" >>"${adb_jaildir}/${adb_dnsjail}" - fi - fi + if [ "${adb_jail}" = "1" ] && [ -n "${adb_dnsstop}" ]; then + printf '%b' "${adb_dnsheader}" >"${adb_tmpdir}/${adb_dnsfile}" + "${adb_catcmd}" "${adb_tmpdir}/tmp.add.${src_name}" >>"${adb_tmpdir}/${adb_dnsfile}" + printf '%b\n' "${adb_dnsstop}" >>"${adb_tmpdir}/${adb_dnsfile}" fi fi ;; - "safesearch") - case "${src_name}" in - "google") - rset="/^\\.([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{printf \"%s\n%s\n\",tolower(\"www\"\$1),tolower(substr(\$1,2,length(\$1)))}" - safe_url="https://www.google.com/supported_domains" - safe_cname="forcesafesearch.google.com" - safe_domains="${adb_tmpdir}/tmp.load.safesearch.${src_name}" - if [ "${adb_backup}" = "1" ] && [ -s "${adb_backupdir}/safesearch.${src_name}.gz" ]; then - zcat "${adb_backupdir}/safesearch.${src_name}.gz" >"${safe_domains}" - out_rc="${?}" - else - "${adb_fetchutil}" ${adb_fetchparm} "${safe_domains}" "${safe_url}" 2>/dev/null - out_rc="${?}" - if [ "${adb_backup}" = "1" ] && [ "${out_rc}" = "0" ]; then - gzip -cf "${safe_domains}" >"${adb_backupdir}/safesearch.${src_name}.gz" - out_rc="${?}" - fi - fi - if [ "${out_rc}" = "0" ]; then - if [ -x "${adb_lookupcmd}" ]; then - safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>/dev/null | "${adb_awk}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" - [ -n "${safe_ips}" ] && "${adb_awk}" "${rset}" "${safe_domains}" >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" - fi - out_rc="${?}" - fi - ;; - "bing") - safe_cname="strict.bing.com" - safe_domains="www.bing.com" - if [ -x "${adb_lookupcmd}" ]; then - safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>/dev/null | "${adb_awk}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" - [ -n "${safe_ips}" ] && printf "%s\n" ${safe_domains} >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" - fi - out_rc="${?}" - ;; - "duckduckgo") - safe_cname="safe.duckduckgo.com" - safe_domains="duckduckgo.com" - if [ -x "${adb_lookupcmd}" ]; then - safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>/dev/null | "${adb_awk}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" - [ -n "${safe_ips}" ] && printf "%s\n" ${safe_domains} >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" - fi - out_rc="${?}" - ;; - "pixabay") - safe_cname="safesearch.pixabay.com" - safe_domains="pixabay.com" - if [ -x "${adb_lookupcmd}" ]; then - safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>/dev/null | "${adb_awk}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" - [ -n "${safe_ips}" ] && printf "%s\n" ${safe_domains} >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" - fi - out_rc="${?}" - ;; - "yandex") - safe_cname="familysearch.yandex.ru" - safe_domains="ya.ru yandex.ru yandex.com yandex.com.tr yandex.ua yandex.by yandex.ee yandex.lt yandex.lv yandex.md yandex.uz yandex.tm yandex.tj yandex.az yandex.kz" - if [ -x "${adb_lookupcmd}" ]; then - safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>/dev/null | "${adb_awk}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" - [ -n "${safe_ips}" ] && printf "%s\n" ${safe_domains} >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" - fi - out_rc="${?}" - ;; - "youtube") - if [ "${adb_safesearchmod}" = "0" ]; then - safe_cname="restrict.youtube.com" - else - safe_cname="restrictmoderate.youtube.com" - fi - safe_domains="www.youtube.com m.youtube.com youtubei.googleapis.com youtube.googleapis.com www.youtube-nocookie.com" - if [ -x "${adb_lookupcmd}" ]; then - safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>/dev/null | "${adb_awk}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" - [ -n "${safe_ips}" ] && printf "%s\n" ${safe_domains} >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" - fi - out_rc="${?}" - ;; - esac - if [ "${out_rc}" = "0" ] && [ -s "${adb_tmpdir}/tmp.raw.safesearch.${src_name}" ]; then - : >"${adb_tmpdir}/tmp.safesearch.${src_name}" - [ "${adb_dns}" = "named" ] && array="${safe_cname}" || array="${safe_ips}" - for item in ${array}; do - if ! eval "${adb_dnssafesearch}" "${adb_tmpdir}/tmp.raw.safesearch.${src_name}" >>"${adb_tmpdir}/tmp.safesearch.${src_name}"; then - rm -f "${adb_tmpdir}/tmp.safesearch.${src_name}" - break - fi - done - out_rc="${?}" - rm -f "${adb_tmpdir}/tmp.raw.safesearch.${src_name}" + esac + ;; + "safesearch") + file_name="${adb_tmpdir}/tmp.safesearch.${src_name}" + case "${adb_dns}" in + "named" | "kresd" | "smartdns") + use_cname="1" + ;; + esac + case "${src_name}" in + "google") + safe_url="https://www.google.com/supported_domains" + safe_cname="forcesafesearch.google.com" + + # refresh if cache is missing or older than 30 days + # + if [ -s "${adb_backupdir}/safesearch.${src_name}.gz" ] && + [ -z "$("${adb_findcmd}" "${adb_backupdir}/safesearch.${src_name}.gz" -mtime +30 2>>"${adb_errorlog}")" ]; then + "${adb_zcatcmd}" "${adb_backupdir}/safesearch.${src_name}.gz" >"${adb_tmpdir}/tmp.load.safesearch.${src_name}" + else + "${adb_fetchcmd}" ${adb_fetchparm} "${adb_tmpdir}/tmp.load.safesearch.${src_name}" "${safe_url}" 2>>"${adb_errorlog}" + if [ -s "${adb_tmpdir}/tmp.load.safesearch.${src_name}" ]; then + "${adb_gzipcmd}" -cf "${adb_tmpdir}/tmp.load.safesearch.${src_name}" >"${adb_backupdir}/safesearch.${src_name}.gz" + elif [ -s "${adb_backupdir}/safesearch.${src_name}.gz" ]; then + "${adb_zcatcmd}" "${adb_backupdir}/safesearch.${src_name}.gz" >"${adb_tmpdir}/tmp.load.safesearch.${src_name}" + fi fi ;; - "backup") - ( - gzip -cf "${src_tmpfile}" >"${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" - out_rc="${?}" - ) & + "bing") + safe_cname="strict.bing.com" + safe_domains="www.bing.com" ;; - "restore") - if [ -n "${src_name}" ] && [ -s "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" ]; then - zcat "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" >"${src_tmpfile}" - out_rc="${?}" - elif [ -z "${src_name}" ]; then - cnt="1" - for file in "${adb_backupdir}/${adb_dnsprefix}".*.gz; do - if [ -r "${file}" ]; then - name="${file##*/}" - name="${name%.*}" - zcat "${file}" >"${adb_tmpfile}.${name}" & - hold="$((cnt % adb_cores))" - if [ "${hold}" = "0" ]; then - wait - fi - cnt="$((cnt + 1))" - fi - done - wait - out_rc="${?}" + "brave") + safe_cname="forcesafe.search.brave.com" + safe_domains="search.brave.com" + ;; + "duckduckgo") + safe_cname="safe.duckduckgo.com" + safe_domains="duckduckgo.com" + ;; + "pixabay") + safe_cname="safesearch.pixabay.com" + safe_domains="pixabay.com" + ;; + "yandex") + safe_cname="familysearch.yandex.ru" + safe_domains="ya.ru yandex.ru yandex.com yandex.com.tr yandex.ua yandex.by yandex.ee yandex.lt yandex.lv yandex.md yandex.uz yandex.tm yandex.tj yandex.az yandex.kz" + ;; + "youtube") + safe_cname="restrict.youtube.com" + safe_domains="www.youtube.com m.youtube.com youtubei.googleapis.com youtube.googleapis.com www.youtube-nocookie.com" + ;; + esac + if [ -n "${safe_domains}" ] && [ -n "${safe_cname}" ]; then + if [ "${use_cname}" = "0" ]; then + safe_ips="$("${adb_lookupcmd}" "${safe_cname}" 2>>"${adb_errorlog}" | "${adb_awkcmd}" '/^Address[ 0-9]*: /{ORS=" ";print $NF}')" + fi + if [ -n "${safe_ips}" ] || [ "${use_cname}" = "1" ]; then + printf '%s\n' ${safe_domains} >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" + [ "${use_cname}" = "1" ] && array="${safe_cname}" || array="${safe_ips}" + fi + fi + if [ -s "${adb_tmpdir}/tmp.raw.safesearch.${src_name}" ]; then + : >"${file_name}" + for item in ${array}; do + if ! f_dnssafesearch "${item}" "${adb_tmpdir}/tmp.raw.safesearch.${src_name}" >>"${file_name}"; then + : >"${file_name}" + break + fi + done + : >"${adb_tmpdir}/tmp.raw.safesearch.${src_name}" + out_rc="0" + fi + ;; + "prepare") + file_name="${src_tmpfile}" + if [ -s "${src_tmpload}" ]; then + if [ "${adb_tld}" = "1" ]; then + f_chkdom ${src_rset} <"${src_tmpload}" | + "${adb_awkcmd}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' | + "${adb_sortcmd}" ${adb_srtopts} -u >"${src_tmpfile}" 2>>"${adb_errorlog}" else - out_rc=4 + f_chkdom ${src_rset} <"${src_tmpload}" | + "${adb_sortcmd}" ${adb_srtopts} -u >"${src_tmpfile}" 2>>"${adb_errorlog}" fi - if [ "${adb_action}" != "start" ] && [ "${adb_action}" != "resume" ] && [ -n "${src_name}" ] && [ "${out_rc}" != "0" ]; then - adb_sources="${adb_sources/${src_name}}" + out_rc="${?}" + if [ "${out_rc}" = "0" ] && [ -s "${src_tmpfile}" ]; then + f_list backup + elif [ "${adb_action}" != "boot" ] && [ "${adb_action}" != "start" ]; then + f_log "info" "preparation of '${src_name}' failed, rc: ${src_rc}" + f_list restore + out_rc="${?}" + : >"${src_tmpfile}" fi - ;; - "remove") - [ "${adb_backup}" = "1" ] && rm "${adb_backupdir}/${adb_dnsprefix}.${src_name}.gz" 2>/dev/null + else + f_log "info" "download of '${src_name}' failed, url: ${src_url}, rule: ${src_rset:-"-"}, categories: ${src_cat:-"-"}, rc: ${src_rc}" + if [ "${adb_action}" != "boot" ] && [ "${adb_action}" != "start" ]; then + f_list restore + out_rc="${?}" + fi + fi + ;; + "backup") + file_name="${src_tmpfile}" + "${adb_gzipcmd}" -cf "${src_tmpfile}" >"${adb_backupdir}/adb_list.${src_name}.gz" + out_rc="${?}" + ;; + "restore") + file_name="${src_tmpfile}" + if [ -n "${src_name}" ] && [ -s "${adb_backupdir}/adb_list.${src_name}.gz" ]; then + "${adb_zcatcmd}" "${adb_backupdir}/adb_list.${src_name}.gz" >"${src_tmpfile}" out_rc="${?}" - adb_sources="${adb_sources/${src_name}}" - ;; - "merge") - if [ "${adb_backup}" = "1" ]; then - for src_name in ${adb_sources}; do - ffiles="${ffiles} -a ! -name ${adb_dnsprefix}.${src_name}.gz" - done - if [ "${adb_safesearch}" = "1" ] && [ "${adb_dnssafesearch}" != "0" ]; then - ffiles="${ffiles} -a ! -name safesearch.google.gz" + elif [ -z "${src_name}" ]; then + for file in "${adb_backupdir}/adb_list."*.gz; do + if [ -r "${file}" ]; then + name="${file##*/}" + name="${name%.*}" + "${adb_zcatcmd}" "${file}" >"${adb_tmpfile}.${name}" + out_rc="${?}" + [ "${out_rc}" != "0" ] && break fi - find "${adb_backupdir}" ${ffiles} -print0 2>/dev/null | xargs -0 rm 2>/dev/null + done + else + out_rc="4" + fi + case "${adb_action}" in + "boot" | "start" | "restart" | "resume") ;; + + *) + if [ -n "${src_name}" ] && [ "${out_rc}" != "0" ]; then + adb_feed=" ${adb_feed} " + adb_feed="${adb_feed// ${src_name} / }" + adb_feed="${adb_feed# }" + adb_feed="${adb_feed% }" fi - unset src_name - "${adb_sort}" ${adb_srtopts} -mu "${adb_tmpfile}".* 2>/dev/null >"${adb_tmpdir}/${adb_dnsfile}" - out_rc="${?}" - rm -f "${adb_tmpfile}".* ;; - "final") - unset src_name - { [ -n "${adb_dnsheader}" ] && printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}"; } || : >"${adb_dnsdir}/${adb_dnsfile}" - [ -s "${adb_tmpdir}/tmp.add.iplist" ] && cat "${adb_tmpdir}/tmp.add.iplist" >>"${adb_dnsdir}/${adb_dnsfile}" - [ -s "${adb_tmpdir}/tmp.add.whitelist" ] && cat "${adb_tmpdir}/tmp.add.whitelist" >>"${adb_dnsdir}/${adb_dnsfile}" - for file in "${adb_tmpdir}/tmp.safesearch".*; do - [ -r "${file}" ] && cat "${file}" >>"${adb_dnsdir}/${adb_dnsfile}" - done - { [ "${adb_dnsdeny}" != "0" ] && eval "${adb_dnsdeny}" "${adb_tmpdir}/${adb_dnsfile}" >>"${adb_dnsdir}/${adb_dnsfile}"; } || mv "${adb_tmpdir}/${adb_dnsfile}" "${adb_dnsdir}/${adb_dnsfile}" + esac + ;; + "remove") + "${adb_rmcmd}" "${adb_backupdir}/adb_list.${src_name}.gz" 2>>"${adb_errorlog}" + out_rc="${?}" + adb_feed=" ${adb_feed} " + adb_feed="${adb_feed// ${src_name} / }" + adb_feed="${adb_feed# }" + adb_feed="${adb_feed% }" + ;; + "merge") + src_name="" + file_name="${adb_tmpdir}/${adb_dnsfile}" + + # remove stale backup files + # + files="" + for file in ${adb_feed}; do + files="${files} ! -name adb_list.${file}.gz" + done + if [ "${adb_safesearch}" = "1" ] && [ "${adb_dnssafesearch}" = "1" ]; then + files="${files} ! -name safesearch.google.gz" + fi + "${adb_findcmd}" "${adb_backupdir}" -maxdepth 1 -type f -name '*.gz' ${files} -print0 2>>"${adb_errorlog}" | "${adb_xargscmd}" -0r "${adb_rmcmd}" -f + + # merge files + # + for file in "${adb_tmpfile}".*; do + [ -e "${file}" ] || continue + "${adb_sortcmd}" ${adb_srtopts} -mu "${adb_tmpfile}".* 2>>"${adb_errorlog}" >"${file_name}" out_rc="${?}" - ;; + break + done + [ -z "${out_rc}" ] && { + : >"${file_name}" + out_rc="4" + } + "${adb_rmcmd}" -f "${adb_tmpfile}".* + ;; + "final") + src_name="" + file_name="${adb_finaldir}/${adb_dnsfile}" + { + [ -n "${adb_dnsheader}" ] && printf '%b' "${adb_dnsheader}" + [ -s "${adb_tmpdir}/tmp.add.allowlist" ] && "${adb_sortcmd}" ${adb_srtopts} -u "${adb_tmpdir}/tmp.add.allowlist" + [ "${adb_safesearch}" = "1" ] && "${adb_catcmd}" "${adb_tmpdir}/tmp.safesearch."* 2>>"${adb_errorlog}" + if [ "${adb_dnsdeny}" = "1" ]; then + f_dnsdeny "${adb_tmpdir}/${adb_dnsfile}" + else + "${adb_catcmd}" "${adb_tmpdir}/${adb_dnsfile}" + fi + } >"${file_name}" + if [ "${adb_dnsshift}" = "1" ] && [ ! -L "${adb_dnsdir}/${adb_dnsfile}" ]; then + "${adb_lncmd}" -fs "${file_name}" "${adb_dnsdir}/${adb_dnsfile}" + elif [ "${adb_dnsshift}" = "0" ] && [ -s "${adb_backupdir}/${adb_dnsfile}" ]; then + "${adb_rmcmd}" -f "${adb_backupdir}/${adb_dnsfile}" + fi + out_rc="0" + ;; esac - f_count "${mode}" "${src_name}" + f_count "${mode}" "${file_name}" out_rc="${out_rc:-"${in_rc}"}" + f_log "debug" "f_list ::: name: ${src_name:-"-"}, mode: ${mode}, cnt: ${adb_cnt}, in_rc: ${in_rc}, out_rc: ${out_rc}" return "${out_rc}" } @@ -913,188 +1322,404 @@ f_list() { # top level domain compression # f_tld() { - local cnt cnt_tld source="${1}" temp_tld="${1}.tld" + local cnt_tld cnt_rem source="${1}" temp_tld="${1}.tld" - if "${adb_awk}" '{if(NR==1){tld=$NF};while(getline){if(index($NF,tld".")==0){print tld;tld=$NF}}print tld}' "${source}" | - "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' >"${temp_tld}"; then - mv -f "${temp_tld}" "${source}" - cnt_tld="$(wc -l 2>/dev/null <"${source}")" - else - rm -f "${temp_tld}" + # reverse domain, get unique tlds and unreverse them back to original form + # + if "${adb_awkcmd}" ' + function unreverse(dom, n, seg, out, i) { + n = split(dom, seg, ".") + out = seg[n] + for (i = n-1; i >= 1; i--) out = out "." seg[i] + return out + } + { + if (NR == 1) { parent = $NF } + else if (index($NF, parent ".") == 0) { print unreverse(parent); parent = $NF } + } + END { if (parent) print unreverse(parent) } + ' "${source}" >"${temp_tld}"; then + [ "${adb_debug}" = "1" ] && cnt_tld="$(f_count tld "${temp_tld}" "var")" + + # remove allowlisted (sub-) domains from tld list if allowlist is enabled and not empty + # + if [ -s "${adb_tmpdir}/tmp.rem.allowlist" ]; then + "${adb_awkcmd}" ' + NR==FNR { + del[$0] + next + } + { + dominated = 0 + n = split($0, seg, ".") + for (i = 1; i <= n; i++) { + parent = seg[i] + for (j = i + 1; j <= n; j++) parent = parent "." seg[j] + if (parent in del) { dominated = 1; break } + } + if (!dominated) print + } + ' "${adb_tmpdir}/tmp.rem.allowlist" "${temp_tld}" >"${source}" + "${adb_rmcmd}" -f "${temp_tld}" + [ "${adb_debug}" = "1" ] && cnt_rem="$(f_count tld "${source}" "var")" + else + "${adb_mvcmd}" -f "${temp_tld}" "${source}" + fi fi - f_log "debug" "f_tld ::: source: ${source}, cnt: ${adb_cnt:-"-"}, cnt_tld: ${cnt_tld:-"-"}" + + f_log "debug" "f_tld ::: name: -, cnt: ${adb_cnt:-"-"}, cnt_tld: ${cnt_tld:-"-"}, cnt_rem: ${cnt_rem:-"-"}" } -# suspend/resume adblock processing -# -f_switch() { - local status entry done="false" mode="${1}" +# suspend/resume adblock processing +# +f_switch() { + local status switch rc="0" mode="${1}" + + json_init + json_load_file "${adb_rtfile}" >/dev/null 2>&1 + json_get_var status "adblock_status" + f_env + + # suspend adblock processing + # + if [ "${status}" = "enabled" ] && [ "${mode}" = "suspend" ]; then + + # suspend via external DNS bridge + # + if { [ -n "${adb_bridgednsv4}" ] || [ -n "${adb_bridgednsv6}" ]; } && [ "${adb_nftbridge}" = "1" ]; then + if "${adb_nftcmd}" list chain inet adblock dns-bridge >/dev/null 2>&1; then + if [ -n "${adb_bridgednsv4}" ]; then + "${adb_nftcmd}" add rule inet adblock dns-bridge meta nfproto ipv4 meta l4proto { udp, tcp } th dport 53 counter dnat to ${adb_bridgednsv4}:53 2>>"${adb_errorlog}" + rc="${?}" + fi + if [ -n "${adb_bridgednsv6}" ]; then + "${adb_nftcmd}" add rule inet adblock dns-bridge meta nfproto ipv6 meta l4proto { udp, tcp } th dport 53 counter dnat to [${adb_bridgednsv6}]:53 2>>"${adb_errorlog}" + rc="$((rc + $?))" + fi + [ "${rc}" = "0" ] && switch="nft" + fi + + # suspend via local DNS + # + else + if [ "${adb_dnsshift}" = "0" ] && [ -f "${adb_finaldir}/${adb_dnsfile}" ]; then + "${adb_mvcmd}" -f "${adb_finaldir}/${adb_dnsfile}" "${adb_backupdir}/${adb_dnsfile}" + printf '%b' "${adb_dnsheader}" >"${adb_finaldir}/${adb_dnsfile}" + switch="dns" + elif [ "${adb_dnsshift}" = "1" ] && [ -L "${adb_dnsdir}/${adb_dnsfile}" ]; then + "${adb_rmcmd}" -f "${adb_dnsdir}/${adb_dnsfile}" + printf '%b' "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" + switch="dns" + fi + fi + + # resume adblock processing + # + elif [ "${status}" = "paused" ] && [ "${mode}" = "resume" ]; then - json_init - json_load_file "${adb_rtfile}" >/dev/null 2>&1 - json_select "data" >/dev/null 2>&1 - json_get_var status "adblock_status" - if [ "${mode}" = "suspend" ] && [ "${status}" = "enabled" ]; then - f_env - printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" - if [ "${adb_jail}" = "1" ] && [ "${adb_jaildir}" = "${adb_dnsdir}" ]; then - printf "%b" "${adb_dnsheader}" >"${adb_jaildir}/${adb_dnsjail}" - elif [ -f "${adb_dnsdir}/${adb_dnsjail}" ]; then - rm -f "${adb_dnsdir}/${adb_dnsjail}" + # resume via external DNS bridge + # + if [ "${adb_nftbridge}" = "1" ] && "${adb_nftcmd}" list chain inet adblock dns-bridge >/dev/null 2>&1; then + if "${adb_nftcmd}" flush chain inet adblock dns-bridge 2>>"${adb_errorlog}"; then + switch="nft" + fi + f_count "final" "${adb_finaldir}/${adb_dnsfile}" + + # resume via local DNS + # + else + if [ "${adb_dnsshift}" = "0" ] && [ -f "${adb_backupdir}/${adb_dnsfile}" ]; then + "${adb_mvcmd}" -f "${adb_backupdir}/${adb_dnsfile}" "${adb_finaldir}/${adb_dnsfile}" + f_count "final" "${adb_finaldir}/${adb_dnsfile}" + switch="dns" + elif [ "${adb_dnsshift}" = "1" ] && [ ! -L "${adb_dnsdir}/${adb_dnsfile}" ]; then + "${adb_lncmd}" -fs "${adb_finaldir}/${adb_dnsfile}" "${adb_dnsdir}/${adb_dnsfile}" + f_count "final" "${adb_finaldir}/${adb_dnsfile}" + switch="dns" + fi fi - f_count - done="true" - elif [ "${mode}" = "resume" ] && [ "${status}" = "paused" ]; then - f_env - f_main - done="true" fi - if [ "${done}" = "true" ]; then - [ "${mode}" = "suspend" ] && f_dnsup + + # update runtime information and log action + # + if [ "${switch}" = "nft" ]; then + f_jsnup "${mode}" + f_log "info" "${mode} adblock service via external DNS bridge" + elif [ "${switch}" = "dns" ]; then + f_dnsup f_jsnup "${mode}" - f_log "info" "${mode} adblock processing" + f_log "info" "${mode} adblock service via local DNS" + else + f_count "final" "${adb_finaldir}/${adb_dnsfile}" + f_jsnup "${status}" fi f_rmtemp } -# query blocklist for certain (sub-)domains +# search blocklist for certain (sub-)domains # -f_query() { - local search result prefix suffix field query_start query_end query_timeout=30 domain="${1}" tld="${1#*.}" +f_search() { + local rc search res result tmp_result prefix suffix field search_start search_end search_timeout=30 domain="${1}" tld="${1#*.}" - if [ -z "${domain}" ] || [ "${domain}" = "${tld}" ]; then - printf "%s\n" "::: invalid input, please submit a single (sub-)domain :::" - else - case "${adb_dns}" in - "dnsmasq") - prefix=".*[\\/\\.]" - suffix="(\\/)" - field="2" - ;; - "unbound") - prefix=".*[\"\\.]" - suffix="(always_nxdomain)" - field="3" - ;; - "named") - prefix="[^\\*].*[\\.]" - suffix="( \\.)" - field="1" - ;; - "kresd") - prefix="[^\\*].*[\\.]" - suffix="( \\.)" - field="1" - ;; - "raw") - prefix=".*[\\.]" - suffix="" - field="1" - ;; - esac - query_start="$(date "+%s")" - while [ "${domain}" != "${tld}" ]; do - search="${domain//[+*~%\$&\"\']/}" - search="${search//./\\.}" - result="$("${adb_awk}" -F '/|\"|\t| ' "/^(${search}|${prefix}+${search}.*${suffix})$/{i++;if(i<=9){printf \" + %s\n\",\$${field}}else if(i==10){printf \" + %s\n\",\"[...]\";exit}}" "${adb_dnsdir}/${adb_dnsfile}")" - printf "%s\n%s\n%s\n" ":::" "::: domain '${domain}' in active blocklist" ":::" - printf "%s\n\n" "${result:-" - no match"}" - domain="${tld}" - tld="${domain#*.}" - done - if [ "${adb_backup}" = "1" ] && [ -d "${adb_backupdir}" ]; then - search="${1//[+*~%\$&\"\']/}" - search="${search//./\\.}" - printf "%s\n%s\n%s\n" ":::" "::: domain '${1}' in backups and black-/whitelist" ":::" - for file in "${adb_backupdir}/${adb_dnsprefix}".*.gz "${adb_blacklist}" "${adb_whitelist}"; do - suffix="${file##*.}" - if [ "${suffix}" = "gz" ]; then - zcat "${file}" 2>/dev/null | - "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' | "${adb_awk}" -v f="${file##*/}" "BEGIN{rc=1};/^($search|.*\\.${search})$/{i++;if(i<=3){printf \" + %-30s%s\n\",f,\$1;rc=0}else if(i==4){printf \" + %-30s%s\n\",f,\"[...]\"}};END{exit rc}" + # prepare result file + # + tmp_result="${adb_rundir}/adblock.search.tmp" + result="${adb_rundir}/adblock.search" + + # input validation + # + case "${domain}" in + "" | *[!a-zA-Z0-9.-]* | -* | *- | *..* | *.) + printf '%s\n' "::: invalid input, please submit a single (sub-)domain :::" + printf '%s\n' "::: invalid input, please submit a single (sub-)domain :::" >"${result}" + return + ;; + esac + + # length validation for domain part, max. 253 characters according to RFC 1035 + # + case "${#domain}" in + [0-9] | [1-9][0-9] | 1[0-9][0-9] | 2[0-4][0-9] | 25[0-3]) ;; + + *) + printf '%s\n' "::: invalid input, domain exceeds 253 characters :::" + printf '%s\n' "::: invalid input, domain exceeds 253 characters :::" >"${result}" + return + ;; + esac + + # search blocklist + # + case "${adb_dns}" in + "dnsmasq") + prefix='local=.*[\/\.]' + suffix='\/' + field="2" + ;; + "unbound") + prefix='local-zone: .*["\.]' + suffix='" always_nxdomain' + field="3" + ;; + "named") + prefix="" + suffix=' CNAME \.' + field="1" + ;; + "kresd") + prefix="" + suffix=' CNAME \.' + field="1" + ;; + "smartdns") + prefix='address .*.*[\/\.]' + suffix='\/#' + field="3" + ;; + "raw") + prefix="" + suffix="" + field="1" + ;; + esac + + # initialize tmp_result and start search + # + : >"${tmp_result}" + read -r search_start _ <"/proc/uptime" + search_start="${search_start%.*}" + + # search recursively for domain and its parent domains until tld is reached + # + while :; do + search="${domain//./\\.}" + res="$("${adb_awkcmd}" -F '/|\"|\t| ' "/^(${prefix}${search}${suffix})$/{i++;if(i<=9){printf \" + %s\n\",\$${field}}else if(i==10){printf \" + %s\n\",\"[...]\";exit}}" "${adb_finaldir}/${adb_dnsfile}")" + printf '%s\n%s\n%s\n' ":::" "::: domain '${domain}' in active blocklist" ":::" >>"${tmp_result}" + printf '%s\n\n' "${res:-" - no match"}" >>"${tmp_result}" + [ "${domain}" = "${tld}" ] && break + domain="${tld}" + tld="${domain#*.}" + done + + # search exactly for domain in backup files and local block-/allowlist + # + if [ -d "${adb_backupdir}" ]; then + search="${1//./\\.}" + printf '%s\n%s\n%s\n' ":::" "::: domain '${1}' in backups and in local block-/allowlist" ":::" >>"${tmp_result}" + for file in "${adb_backupdir}/adb_list".*.gz "${adb_blocklist}" "${adb_allowlist}"; do + suffix="${file##*.}" + if [ "${suffix}" = "gz" ]; then + if [ "${adb_tld}" = "1" ]; then + "${adb_zcatcmd}" "${file}" 2>>"${adb_errorlog}" | + "${adb_awkcmd}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' | + "${adb_awkcmd}" -v f="${file##*/}" "BEGIN{rc=1};/^($search|.*\\.${search})$/{i++;if(i<=3){printf \" + %-30s%s\n\",f,\$1;rc=0}else if(i==4){printf \" + %-30s%s\n\",f,\"[...]\"}};END{exit rc}" >>"${tmp_result}" else - "${adb_awk}" -v f="${file##*/}" "BEGIN{rc=1};/^($search|.*\\.${search})$/{i++;if(i<=3){printf \" + %-30s%s\n\",f,\$1;rc=0}else if(i==4){printf \" + %-30s%s\n\",f,\"[...]\"}};END{exit rc}" "${file}" + "${adb_zcatcmd}" "${file}" 2>>"${adb_errorlog}" | + "${adb_awkcmd}" -v f="${file##*/}" "BEGIN{rc=1};/^($search|.*\\.${search})$/{i++;if(i<=3){printf \" + %-30s%s\n\",f,\$1;rc=0}else if(i==4){printf \" + %-30s%s\n\",f,\"[...]\"}};END{exit rc}" >>"${tmp_result}" fi - if [ "${?}" = "0" ]; then - result="true" - query_end="$(date "+%s")" - if [ "$((query_end - query_start))" -gt "${query_timeout}" ]; then - printf "%s\n\n" " - [...]" - break - fi + rc="${?}" + else + "${adb_awkcmd}" -v f="${file##*/}" "BEGIN{rc=1};/^($search|.*\\.${search})$/{i++;if(i<=3){printf \" + %-30s%s\n\",f,\$1;rc=0}else if(i==4){printf \" + %-30s%s\n\",f,\"[...]\"}};END{exit rc}" "${file}" >>"${tmp_result}" + rc="${?}" + fi + if [ "${rc}" = "0" ]; then + res="true" + read -r search_end _ <"/proc/uptime" + search_end="${search_end%.*}" + if [ "$((search_end - search_start))" -gt "${search_timeout}" ]; then + printf '%s\n\n' " - [...]" >>"${tmp_result}" + break fi - done - [ "${result}" != "true" ] && printf "%s\n\n" " - no match" - fi + fi + done + [ "${res}" != "true" ] && printf '%s\n\n' " - no match" >>"${tmp_result}" fi + "${adb_mvcmd}" -f "${tmp_result}" "${result}" + "${adb_catcmd}" "${result}" 2>>"${adb_errorlog}" } # update runtime information # f_jsnup() { - local entry sources runtime utils bg_pid status="${1:-"enabled"}" + local s_shift s_custom s_unfiltered s_filtered s_remote s_bridge s_force s_flush s_tld s_search s_report s_mail + local s_jail s_debug pid pids object feeds end_time runtime dns dns_ver free_mem custom_feed="0" status="${1:-"enabled"}" + local vm_mem dns_mem="0" duration jail="0" nft_unfiltered="0" nft_filtered="0" nft_remote="0" nft_bridge="0" nft_force="0" - adb_memory="$("${adb_awk}" '/^MemTotal|^MemFree|^MemAvailable/{ORS="/"; print int($2/1000)}' "/proc/meminfo" 2>/dev/null | - "${adb_awk}" '{print substr($0,1,length($0)-1)}')" - - case "${status}" in - "enabled" | "error") - adb_endtime="$(date "+%s")" - if [ "$(((adb_endtime - adb_starttime) / 60))" -lt 60 ]; then - runtime="${adb_action}, $(((adb_endtime - adb_starttime) / 60))m $(((adb_endtime - adb_starttime) % 60))s, ${adb_memory:-0}, $(date -Iseconds)" - else - runtime="${adb_action}, n/a, ${adb_memory:-0}, $(date -Iseconds)" - fi - [ "${status}" = "error" ] && adb_cnt="0" + # get DNS memory usage and version + # + if adb_dnspid="$("${adb_ubuscmd}" -S call service list 2>>"${adb_errorlog}" | + "${adb_jsoncmd}" -l1 -e "@[\"${adb_dns}\"].instances.*.pid")" && [ -n "${adb_dnspid}" ]; then + pids="$("${adb_pgrepcmd}" -P "${adb_dnspid}" 2>>"${adb_errorlog}")" + for pid in ${adb_dnspid} ${pids}; do + vm_mem="$("${adb_awkcmd}" '/^VmRSS/{printf "%d", $2}' "/proc/${pid}/status" 2>>"${adb_errorlog}")" + dns_mem="$((dns_mem + ${vm_mem:-0}))" + done + case "${adb_dns}" in + "kresd") + dns="knot-resolver" ;; - "suspend") - status="paused" + "named") + dns="bind-server" ;; - "resume") - status="" + "unbound") + dns="unbound-daemon" + ;; + "dnsmasq") + dns='dnsmasq", "dnsmasq-full", "dnsmasq-dhcpv6' ;; + esac + dns_ver="$(printf '%s' "${adb_packages}" | "${adb_jsoncmd}" -ql1 -e "@.packages[\"${dns:-"${adb_dns}"}\"]")" + dns_mem="$("${adb_awkcmd}" -v mem="${dns_mem}" 'BEGIN{printf "%.2f", mem/1024}' 2>>"${adb_errorlog}")" + fi + free_mem="$("${adb_awkcmd}" '/^MemAvailable/{printf "%.2f", $2/1024}' "/proc/meminfo" 2>>"${adb_errorlog}")" + + # check for custom feed and nft rules + # + [ -s "${adb_customfeedfile}" ] && custom_feed="1" + if [ "${adb_nftforce}" = "1" ] && [ -n "${adb_nftdevforce}" ] && [ -n "${adb_nftportforce}" ]; then + nft_force="1" + fi + if [ "${adb_nftallow}" = "1" ] && + { [ -n "${adb_nftmacallow}" ] || [ -n "${adb_nftdevallow}" ]; } && + { [ -n "${adb_allowdnsv4}" ] || [ -n "${adb_allowdnsv6}" ]; }; then + nft_unfiltered="1" + fi + if [ "${adb_nftblock}" = "1" ] && + { [ -n "${adb_nftmacblock}" ] || [ -n "${adb_nftdevblock}" ]; } && + { [ -n "${adb_blockdnsv4}" ] || [ -n "${adb_blockdnsv6}" ]; }; then + nft_filtered="1" + fi + if [ "${adb_nftremote}" = "1" ] && [ -n "${adb_nftmacremote}" ] && + { [ -n "${adb_remotednsv4}" ] || [ -n "${adb_remotednsv6}" ]; }; then + nft_remote="1" + fi + if [ "${adb_nftbridge}" = "1" ] && + { [ -n "${adb_bridgednsv4}" ] || [ -n "${adb_bridgednsv6}" ]; }; then + nft_bridge="1" + fi + + # update runtime information based on status + # + case "${status}" in + "enabled") + if [ -n "${adb_starttime}" ]; then + read -r end_time _ <"/proc/uptime" + end_time="${end_time%.*}" + duration="$(((end_time - adb_starttime) / 60))m $(((end_time - adb_starttime) % 60))s" + fi + runtime="mode: ${adb_action}, date / time: $(date "+%d/%m/%Y %H:%M:%S"), duration: ${duration:-"-"}, memory: ${free_mem:-0} MB available" + ;; + "resume") + status="enabled" + ;; + "suspend") + adb_cnt="0" + status="paused" + ;; + *) + adb_cnt="0" + ;; esac + + # update runtime file and send mail if enabled + # json_init if json_load_file "${adb_rtfile}" >/dev/null 2>&1; then - utils="download: $(readlink -fn "${adb_fetchutil}"), sort: $(readlink -fn "${adb_sort}"), awk: $(readlink -fn "${adb_awk}")" - [ -z "${adb_cnt}" ] && { json_get_var adb_cnt "blocked_domains"; adb_cnt="${adb_cnt%% *}"; } + [ -z "${adb_cnt}" ] && json_get_var adb_cnt "blocked_domains" [ -z "${runtime}" ] && json_get_var runtime "last_run" - fi - if [ "${adb_jail}" = "1" ] && [ "${adb_jaildir}" = "${adb_dnsdir}" ]; then - adb_cnt="0" - sources="restrictive_jail" - else - sources="$(printf "%s\n" ${adb_sources} | "${adb_sort}" | "${adb_awk}" '{ORS=" ";print $0}')" + if [ "${status}" = "enabled" ]; then + if [ "${adb_jail}" = "1" ] && [ -n "${adb_dnsstop}" ]; then + jail="1" + adb_cnt="0" + feeds="restrictive jail (allowlist-only)" + else + feeds="$(printf '%s\n' ${adb_feed// /, } | "${adb_sortcmd}" | "${adb_xargscmd}")" + fi + fi fi - : >"${adb_rtfile}" + # map flag values to status characters + # + case "${adb_dnsshift}" in "1") s_shift="✔" ;; *) s_shift="✘" ;; esac + case "${custom_feed}" in "1") s_custom="✔" ;; *) s_custom="✘" ;; esac + case "${nft_unfiltered}" in "1") s_unfiltered="✔" ;; *) s_unfiltered="✘" ;; esac + case "${nft_filtered}" in "1") s_filtered="✔" ;; *) s_filtered="✘" ;; esac + case "${nft_remote}" in "1") s_remote="✔" ;; *) s_remote="✘" ;; esac + case "${nft_bridge}" in "1") s_bridge="✔" ;; *) s_bridge="✘" ;; esac + case "${nft_force}" in "1") s_force="✔" ;; *) s_force="✘" ;; esac + case "${adb_dnsflush}" in "1") s_flush="✔" ;; *) s_flush="✘" ;; esac + case "${adb_tld}" in "1") s_tld="✔" ;; *) s_tld="✘" ;; esac + case "${adb_safesearch}" in "1") s_search="✔" ;; *) s_search="✘" ;; esac + case "${adb_report}" in "1") s_report="✔" ;; *) s_report="✘" ;; esac + case "${adb_mail}" in "1") s_mail="✔" ;; *) s_mail="✘" ;; esac + case "${jail}" in "1") s_jail="✔" ;; *) s_jail="✘" ;; esac + case "${adb_debug}" in "1") s_debug="✔" ;; *) s_debug="✘" ;; esac + + # update runtime file + # + printf '%s\n' "{}" >"${adb_rtfile}" json_init json_load_file "${adb_rtfile}" >/dev/null 2>&1 - json_init - json_add_string "adblock_status" "${status:-"enabled"}" - json_add_string "adblock_version" "${adb_ver}" - json_add_string "blocked_domains" "${adb_cnt:-0}" - json_add_array "active_sources" - for entry in ${sources}; do - json_add_object - json_add_string "source" "${entry}" - json_close_object + json_add_string "adblock_status" "${status}" + json_add_string "frontend_ver" "${adb_fver}" + json_add_string "backend_ver" "${adb_bver}" + json_add_string "blocked_domains" "${adb_cnt:-"0"}" + json_add_array "active_feeds" + for object in ${feeds:-"-"}; do + json_add_string "${object}" "${object}" done json_close_array - json_add_string "dns_backend" "${adb_dns:-"-"} (${adb_dnscachecmd##*/}), ${adb_dnsdir:-"-"}" - json_add_string "run_utils" "${utils:-"-"}" + json_add_string "dns_backend" "${adb_dns:-"-"} (${dns_ver:-"-"}), ${adb_finaldir:-"-"}, ${dns_mem:-"0"} MB" json_add_string "run_ifaces" "trigger: ${adb_trigger:-"-"}, report: ${adb_repiface:-"-"}" - json_add_string "run_directories" "base: ${adb_tmpbase}, backup: ${adb_backupdir}, report: ${adb_reportdir}, jail: ${adb_jaildir}" - json_add_string "run_flags" "backup: $(f_char ${adb_backup}), flush: $(f_char ${adb_dnsflush}), force: $(f_char ${adb_forcedns}), search: $(f_char ${adb_safesearch}), report: $(f_char ${adb_report}), mail: $(f_char ${adb_mail}), jail: $(f_char ${adb_jail})" + json_add_string "run_information" "base: ${adb_basedir}, dns: ${adb_dnsdir}, backup: ${adb_backupdir}, report: ${adb_reportdir}, error: ${adb_errorlog}" + json_add_string "run_flags" "shift: ${s_shift}, custom feed: ${s_custom}, ext. DNS (std/prot/remote/bridge): ${s_unfiltered}/${s_filtered}/${s_remote}/${s_bridge}, force: ${s_force}, flush: ${s_flush}, tld: ${s_tld}, search: ${s_search}, report: ${s_report}, mail: ${s_mail}, jail: ${s_jail}, debug: ${s_debug}" json_add_string "last_run" "${runtime:-"-"}" - json_add_string "system" "${adb_sysver}" + json_add_string "system_info" "cores: ${adb_cores}, fetch: ${adb_fetchcmd##*/}, ${adb_sysver}" json_dump >"${adb_rtfile}" - if [ "${adb_mail}" = "1" ] && [ -x "${adb_mailservice}" ] && - { [ "${status}" = "error" ] || { [ "${status}" = "enabled" ] && [ "${adb_cnt}" -le "${adb_mailcnt}" ]; }; }; then - ("${adb_mailservice}" "${adb_ver}" >/dev/null 2>&1) & - bg_pid="${!}" + if [ "${adb_mail}" = "1" ] && [ -x "${adb_mailservice}" ] && [ "${status}" = "enabled" ] && [ "${adb_action}" != "resume" ]; then + "${adb_mailservice}" >/dev/null 2>&1 fi - f_log "debug" "f_jsnup ::: status: ${status:-"-"}, cnt: ${adb_cnt}, mail: ${adb_mail}, mail_service: ${adb_mailservice}, mail_cnt: ${adb_mailcnt}, mail_pid: ${bg_pid:-"-"}" } # write to syslog @@ -1103,10 +1728,13 @@ f_log() { local class="${1}" log_msg="${2}" if [ -n "${log_msg}" ] && { [ "${class}" != "debug" ] || [ "${adb_debug}" = "1" ]; }; then - [ -x "${adb_loggercmd}" ] && "${adb_loggercmd}" -p "${class}" -t "adblock-${adb_ver}[${$}]" "${log_msg}" || \ - printf "%s %s %s\n" "${class}" "adblock-${adb_ver}[${$}]" "${log_msg}" - if [ "${class}" = "err" ]; then - f_rmdns + if [ -x "${adb_loggercmd}" ]; then + "${adb_loggercmd}" -p "${class}" -t "adblock-${adb_bver}[${$}]" "${log_msg::512}" + else + printf '%s %s %s\n' "${class}" "adblock-${adb_bver}[${$}]" "${log_msg::512}" + fi + if [ "${class}" = "err" ] || [ "${class}" = "emerg" ]; then + [ "${adb_action}" != "mail" ] && f_rmdns f_jsnup "error" exit 1 fi @@ -1116,19 +1744,33 @@ f_log() { # main function for blocklist processing # f_main() { - local src_tmpload src_tmpfile src_name src_rset src_url src_log src_arc src_cat src_item src_list src_entries src_suffix src_rc entry cnt + local src_name src_domain src_rset src_url src_cat src_item src_list src_entries src_suffix src_rc entry cnt + local src_tmpcat src_tmparchive src_tmpload src_tmpfile seen_domains feed_restore map_domain - f_log "debug" "f_main ::: memory: ${adb_memory:-0}, cores: ${adb_cores}, safe_search: ${adb_safesearch}, force_dns: ${adb_forcedns}, awk: ${adb_awk}" - - # white- and blacklist preparation + # allow- and blocklist preparation # for entry in ${adb_locallist}; do - (f_list "${entry}" "${entry}") & + f_list "${entry}" "${entry}" done - if [ "${adb_dns}" != "raw" ] && [ "${adb_jail}" = "1" ] && [ "${adb_jaildir}" = "${adb_dnsdir}" ]; then - printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" - chown "${adb_dnsuser}" "${adb_jaildir}/${adb_dnsjail}" 2>/dev/null + # jail mode preparation + # + if [ "${adb_jail}" = "1" ] && [ -n "${adb_dnsstop}" ]; then + if [ -s "${adb_tmpdir}/${adb_dnsfile}" ]; then + "${adb_mvcmd}" -f "${adb_tmpdir}/${adb_dnsfile}" "${adb_finaldir}/${adb_dnsfile}" + else + f_log "info" "jail mode active without allowlist, blocking all queries" + { + printf '%b' "${adb_dnsheader}" + printf '%b\n' "${adb_dnsstop}" + } >"${adb_finaldir}/${adb_dnsfile}" + fi + chown "${adb_dnsuser}" "${adb_finaldir}/${adb_dnsfile}" 2>>"${adb_errorlog}" + if [ "${adb_dnsshift}" = "1" ] && [ ! -L "${adb_dnsdir}/${adb_dnsfile}" ]; then + "${adb_lncmd}" -fs "${adb_finaldir}/${adb_dnsfile}" "${adb_dnsdir}/${adb_dnsfile}" + elif [ "${adb_dnsshift}" = "0" ] && [ -s "${adb_backupdir}/${adb_dnsfile}" ]; then + "${adb_rmcmd}" -f "${adb_backupdir}/${adb_dnsfile}" + fi if f_dnsup; then if [ "${adb_action}" != "resume" ]; then f_jsnup "enabled" @@ -1139,160 +1781,233 @@ f_main() { fi f_rmtemp return - elif [ -f "${adb_dnsdir}/${adb_dnsjail}" ]; then - rm -f "${adb_dnsdir}/${adb_dnsjail}" - f_dnsup fi # safe search preparation # - if [ "${adb_safesearch}" = "1" ] && [ "${adb_dnssafesearch}" != "0" ]; then - [ -z "${adb_safesearchlist}" ] && adb_safesearchlist="google bing duckduckgo pixabay yandex youtube" + if [ "${adb_safesearch}" = "1" ] && [ "${adb_dnssafesearch}" = "1" ]; then + [ -z "${adb_safesearchlist}" ] && adb_safesearchlist="google bing brave duckduckgo pixabay yandex youtube" cnt="1" for entry in ${adb_safesearchlist}; do - (f_list safesearch "${entry}") & - hold="$((cnt % adb_cores))" - [ "${hold}" = "0" ] && wait + ( + f_list safesearch "${entry}" + ) & + [ "${cnt}" -gt "${adb_cores}" ] && wait -n cnt="$((cnt + 1))" done + wait + fi + + # add map service domain to allowlist if map is enabled + # + if [ "${adb_map}" = "1" ] && [ "${adb_dnsallow}" = "1" ] && [ -n "${adb_geourl}" ]; then + map_domain="${adb_geourl#*://}" + map_domain="${map_domain%%/*}" + if [ -n "${map_domain}" ]; then + printf '%s\n' "${map_domain}" | f_dnsallow >>"${adb_tmpdir}/tmp.add.allowlist" + printf '%s\n' "${map_domain}" >>"${adb_tmpdir}/tmp.rem.allowlist" + fi fi - wait # main loop # cnt="1" - for src_name in ${adb_sources}; do + seen_domains="${map_domain}" + + # determine if feed restore should be attempted based on action + # + case "${adb_action}" in + "boot" | "start" | "restart" | "resume") + feed_restore="1" + ;; + *) + feed_restore="0" + ;; + esac + + # loop through feeds defined in configuration and process them + # + for src_name in ${adb_feed}; do + + # check if feed is defined in configuration, if not remove it from feed list and continue with next one + # if ! json_select "${src_name}" >/dev/null 2>&1; then - adb_sources="${adb_sources/${src_name}/}" + adb_feed=" ${adb_feed} " + adb_feed="${adb_feed// ${src_name} / }" + adb_feed="${adb_feed# }" + adb_feed="${adb_feed% }" continue fi + + # get feed information + # json_get_var src_url "url" >/dev/null 2>&1 json_get_var src_rset "rule" >/dev/null 2>&1 json_select .. src_tmpcat="${adb_tmpload}.${src_name}.cat" src_tmpload="${adb_tmpload}.${src_name}.load" - src_tmpsort="${adb_tmpload}.${src_name}.sort" + src_tmparchive="${adb_tmpload}.${src_name}.archive" src_tmpfile="${adb_tmpfile}.${src_name}" src_rc=4 # basic pre-checks # - if [ -z "${src_url}" ] || [ -z "${src_rset}" ]; then + if [ -z "${src_url}" ] || [ -z "${src_rset}" ] || + [ "${src_rset%% *}" != "feed" ]; then f_list remove continue fi - # backup mode + # add domains of active feed URLs to the allowlist # - if [ "${adb_backup}" = "1" ] && { [ "${adb_action}" = "start" ] || [ "${adb_action}" = "resume" ]; }; then - if f_list restore && [ -s "${src_tmpfile}" ]; then - continue - fi + src_domain="${src_url#*://}" + src_domain="${src_domain%%/*}" + if [ -n "${src_domain}" ] && [ "${adb_dnsallow}" = "1" ]; then + case " ${seen_domains} " in + *" ${src_domain} "*) ;; + + *) + seen_domains="${seen_domains} ${src_domain}" + printf '%s\n' "${src_domain}" | f_dnsallow >>"${adb_tmpdir}/tmp.add.allowlist" + printf '%s\n' "${src_domain}" >>"${adb_tmpdir}/tmp.rem.allowlist" + ;; + esac fi # download queue processing # - unset src_cat src_entries - if [ "${src_name}" = "utcapitole" ] && [ -n "${adb_utc_sources}" ]; then - src_cat="${adb_utc_sources}" - if [ -n "${src_cat}" ]; then - ( - src_arc="${adb_tmpdir}/${src_url##*/}" - src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${src_arc}" "${src_url}" 2>&1)" - src_rc="${?}" - if [ "${src_rc}" = "0" ] && [ -s "${src_arc}" ]; then - src_suffix="$(eval printf "%s" \"\$\{adb_src_suffix_${src_name}:-\"domains\"\}\")" - src_list="$(tar -tzf "${src_arc}" 2>/dev/null)" - for src_item in ${src_cat}; do - src_entries="${src_entries} $(printf "%s" "${src_list}" | grep -E "${src_item}/${src_suffix}$")" - done - if [ -n "${src_entries}" ]; then - tar -xOzf "${src_arc}" ${src_entries} 2>/dev/null >"${src_tmpload}" - src_rc="${?}" - fi - : >"${src_arc}" - else - src_log="$(printf "%s" "${src_log}" | "${adb_awk}" '{ORS=" ";print $0}')" - f_log "info" "download of '${src_name}' failed, url: ${src_url}, rule: ${src_rset:-"-"}, categories: ${src_cat:-"-"}, rc: ${src_rc}, log: ${src_log:-"-"}" - fi - if [ "${src_rc}" = "0" ] && [ -s "${src_tmpload}" ]; then - if [ -s "${adb_tmpdir}/tmp.rem.whitelist" ]; then - "${adb_awk}" "${src_rset}" "${src_tmpload}" | sed "s/\r//g" | - grep -Evf "${adb_tmpdir}/tmp.rem.whitelist" | "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' >"${src_tmpsort}" - else - "${adb_awk}" "${src_rset}" "${src_tmpload}" | sed "s/\r//g" | - "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' >"${src_tmpsort}" - fi - : >"${src_tmpload}" - "${adb_sort}" ${adb_srtopts} -u "${src_tmpsort}" 2>/dev/null >"${src_tmpfile}" - src_rc="${?}" - : >"${src_tmpsort}" - if [ "${src_rc}" = "0" ] && [ -s "${src_tmpfile}" ]; then - f_list download - [ "${adb_backup}" = "1" ] && f_list backup - elif [ "${adb_backup}" = "1" ] && [ "${adb_action}" != "start" ]; then - f_log "info" "archive preparation of '${src_name}' failed, categories: ${src_cat:-"-"}, entries: ${src_entries}, rc: ${src_rc}" - f_list restore - rm -f "${src_tmpfile}" - fi - elif [ "${adb_backup}" = "1" ] && [ "${adb_action}" != "start" ]; then - f_log "info" "archive extraction of '${src_name}' failed, categories: ${src_cat:-"-"}, entries: ${src_entries}, rc: ${src_rc}" - f_list restore - fi - ) & + src_cat="" + src_entries="" + + # category handling for feeds with fixed categories + # + case "${src_name}" in + "1hosts") + src_cat="${adb_hst_feed}" + if [ -z "${src_cat}" ]; then + f_log "info" "feed '${src_name}' requires category configuration, skipping" + continue fi - else - if [ "${src_name}" = "energized" ] && [ -n "${adb_eng_sources}" ]; then - src_cat="${adb_eng_sources}" - elif [ "${src_name}" = "stevenblack" ] && [ -n "${adb_stb_sources}" ]; then - src_cat="${adb_stb_sources}" - elif { [ "${src_name}" = "energized" ] && [ -z "${adb_eng_sources}" ]; } || - { [ "${src_name}" = "stevenblack" ] && [ -z "${adb_stb_sources}" ]; }; then + ;; + "hagezi") + src_cat="${adb_hag_feed}" + if [ -z "${src_cat}" ]; then + f_log "info" "feed '${src_name}' requires category configuration, skipping" + continue + fi + ;; + "ipfire_dbl") + src_cat="${adb_ipf_feed}" + if [ -z "${src_cat}" ]; then + f_log "info" "feed '${src_name}' requires category configuration, skipping" + continue + fi + ;; + "stevenblack") + src_cat="${adb_stb_feed}" + if [ -z "${src_cat}" ]; then + f_log "info" "feed '${src_name}' requires category configuration, skipping" continue fi + ;; + esac + + # category handling for feeds with multiple categories + # + if [ -n "${src_cat}" ]; then ( - for suffix in ${src_cat:-${src_url}}; do - if [ "${src_url}" != "${suffix}" ]; then - src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${src_tmpcat}" "${src_url}${suffix}" 2>&1)" - src_rc="${?}" - if [ "${src_rc}" = "0" ] && [ -s "${src_tmpcat}" ]; then - cat "${src_tmpcat}" >>"${src_tmpload}" - : >"${src_tmpcat}" + # restore handling on boot, resume or (re-)start + # + if [ "${feed_restore}" = "1" ]; then + if f_list restore && [ -s "${src_tmpfile}" ]; then + exit 0 + fi + fi + + # etag handling on reload + # + if [ -n "${adb_etagparm}" ] && [ "${adb_action}" = "reload" ]; then + etag_rc="0" + src_cnt="0" + for _ in ${src_cat}; do + src_cnt="$((src_cnt + 1))" + done + for suffix in ${src_cat}; do + if ! f_etag "${src_name}" "${src_url}" "${suffix}" "${src_cnt}"; then + etag_rc="$((etag_rc + 1))" fi - else - src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${src_tmpload}" "${src_url}" 2>&1)" - src_rc="${?}" + done + if [ "${etag_rc}" = "0" ]; then + if f_list restore && [ -s "${src_tmpfile}" ]; then + exit 0 + fi + fi + fi + + # category download + # + for suffix in ${src_cat}; do + "${adb_fetchcmd}" ${adb_fetchparm} "${src_tmpcat}" "${src_url}${suffix}" 2>>"${adb_errorlog}" + src_rc="${?}" + if [ "${src_rc}" = "0" ] && [ -s "${src_tmpcat}" ]; then + "${adb_catcmd}" "${src_tmpcat}" >>"${src_tmpload}" + : >"${src_tmpcat}" fi done - if [ "${src_rc}" = "0" ] && [ -s "${src_tmpload}" ]; then - if [ -s "${adb_tmpdir}/tmp.rem.whitelist" ]; then - "${adb_awk}" "${src_rset}" "${src_tmpload}" | sed "s/\r//g" | - grep -Evf "${adb_tmpdir}/tmp.rem.whitelist" | "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' >"${src_tmpsort}" - else - "${adb_awk}" "${src_rset}" "${src_tmpload}" | sed "s/\r//g" | - "${adb_awk}" 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' >"${src_tmpsort}" + f_list prepare + ) & + + # normal handling for feeds without categories + # + else + ( + [ "${src_name}" = "utcapitole" ] && src_cat="${adb_utc_feed}" + + # restore handling on boot, resume or (re-)start + # + if [ "${feed_restore}" = "1" ]; then + if f_list restore && [ -s "${src_tmpfile}" ]; then + exit 0 fi - : >"${src_tmpload}" - "${adb_sort}" ${adb_srtopts} -u "${src_tmpsort}" 2>/dev/null >"${src_tmpfile}" - src_rc="${?}" - : >"${src_tmpsort}" - if [ "${src_rc}" = "0" ] && [ -s "${src_tmpfile}" ]; then - f_list download - [ "${adb_backup}" = "1" ] && f_list backup - elif [ "${adb_backup}" = "1" ] && [ "${adb_action}" != "start" ]; then - f_log "info" "preparation of '${src_name}' failed, rc: ${src_rc}" - f_list restore - rm -f "${src_tmpfile}" + fi + + # etag handling on reload + # + if [ -n "${adb_etagparm}" ] && [ "${adb_action}" = "reload" ]; then + if f_etag "${src_name}" "${src_url}"; then + if f_list restore && [ -s "${src_tmpfile}" ]; then + exit 0 + fi + fi + fi + + # download feed and extract categories if necessary + # + if [ "${src_name}" = "utcapitole" ]; then + if [ -n "${src_cat}" ]; then + "${adb_fetchcmd}" ${adb_fetchparm} "${src_tmparchive}" "${src_url}" 2>>"${adb_errorlog}" + src_rc="${?}" + if [ "${src_rc}" = "0" ] && [ -s "${src_tmparchive}" ]; then + src_suffix="${adb_src_suffix_utcapitole:-"domains"}" + src_entries="$(tar -tzf "${src_tmparchive}" 2>>"${adb_errorlog}" | "${adb_awkcmd}" \ + -v cats="${src_cat}" -v sfx="${src_suffix}" ' + BEGIN { n = split(cats, c, " ") } + { for (i = 1; i <= n; i++) if ($0 ~ "(^|/)" c[i] "/" sfx "$") print }')" + if [ -n "${src_entries}" ]; then + tar -xOzf "${src_tmparchive}" ${src_entries} 2>>"${adb_errorlog}" >"${src_tmpload}" + src_rc="${?}" + fi + : >"${src_tmparchive}" + fi fi else - src_log="$(printf "%s" "${src_log}" | "${adb_awk}" '{ORS=" ";print $0}')" - f_log "info" "download of '${src_name}' failed, url: ${src_url}, rule: ${src_rset:-"-"}, categories: ${src_cat:-"-"}, rc: ${src_rc}, log: ${src_log:-"-"}" - [ "${adb_backup}" = "1" ] && [ "${adb_action}" != "start" ] && f_list restore + "${adb_fetchcmd}" ${adb_fetchparm} "${src_tmpload}" "${src_url}" 2>>"${adb_errorlog}" + src_rc="${?}" fi + f_list prepare ) & fi - hold="$((cnt % adb_cores))" - [ "${hold}" = "0" ] && wait + [ "${cnt}" -gt "${adb_cores}" ] && wait -n cnt="$((cnt + 1))" done wait @@ -1300,15 +2015,16 @@ f_main() { # tld compression and dns restart # if f_list merge && [ -s "${adb_tmpdir}/${adb_dnsfile}" ]; then - f_tld "${adb_tmpdir}/${adb_dnsfile}" + [ "${adb_tld}" = "1" ] && f_tld "${adb_tmpdir}/${adb_dnsfile}" f_list final else - printf "%b" "${adb_dnsheader}" >"${adb_dnsdir}/${adb_dnsfile}" + printf '%b' "${adb_dnsheader}" >"${adb_finaldir}/${adb_dnsfile}" + f_log "info" "no merge input, only header written to ${adb_finaldir}/${adb_dnsfile}" fi - chown "${adb_dnsuser}" "${adb_dnsdir}/${adb_dnsfile}" 2>/dev/null + chown "${adb_dnsuser}" "${adb_finaldir}/${adb_dnsfile}" 2>>"${adb_errorlog}" if f_dnsup; then - [ "${adb_action}" != "resume" ] && f_jsnup "enabled" f_log "info" "blocklist with overall ${adb_cnt} blocked domains loaded successfully (${adb_sysver})" + [ "${adb_action}" != "resume" ] && f_jsnup "enabled" else f_log "err" "dns backend restart with adblock blocklist failed" fi @@ -1318,149 +2034,412 @@ f_main() { # trace dns queries via tcpdump and prepare a report # f_report() { - local report_raw report_txt content status total start end start_date start_time end_date end_time blocked percent top_list top array item index hold ports value key key_list cnt="0" resolve="-nn" action="${1}" top_count="${2:-"10"}" res_count="${3:-"50"}" search="${4:-"+"}" + local report_raw report_txt content status total start end start_date start_time end_date end_time blocked percent top_list top array item index value key key_list + local domain rc map_seen ip request requests iface_v4 iface_v6 ip_v4 ip_v6 map_jsn cnt report_srt report_jsn top_tmpclients top_tmpdomains top_tmpblocked + local file jsn resolve="-nn" action="${1}" top_count="${2:-"10"}" res_count="${3:-"50"}" search="${4:-"+"}" report_raw="${adb_reportdir}/adb_report.raw" report_srt="${adb_reportdir}/adb_report.srt" - report_jsn="${adb_reportdir}/adb_report.json" + report_jsn="${adb_reportdir}/adb_report.jsn" report_txt="${adb_reportdir}/adb_mailreport.txt" + top_tmpclients="${adb_reportdir}/top_clients.tmp" + top_tmpdomains="${adb_reportdir}/top_domains.tmp" + top_tmpblocked="${adb_reportdir}/top_blocked.tmp" + map_jsn="${adb_reportdir}/adb_map.jsn" - # build json file + # build report # if [ "${action}" != "json" ]; then - : >"${report_raw}" - : >"${report_srt}" - : >"${report_txt}" - : >"${report_jsn}" + : >"${report_srt}" >"${report_txt}" >"${report_jsn}" >"${map_jsn}" + : >"${top_tmpclients}" >"${top_tmpdomains}" >"${top_tmpblocked}" [ "${adb_represolve}" = "1" ] && resolve="" + cnt="1" for file in "${adb_reportdir}/adb_report.pcap"*; do + [ -s "${file}" ] || continue ( - "${adb_dumpcmd}" "${resolve}" -tttt -r "${file}" 2>/dev/null | - "${adb_awk}" -v cnt="${cnt}" '!/\.lan\. |PTR\? | SOA\? /&&/ A[\? ]+|NXDomain|0\.0\.0\.0/{a=$1;b=substr($2,0,8);c=$4;sub(/\.[0-9]+$/,"",c);gsub(/[^[:alnum:]\.:-]/,"",c);d=cnt $7;sub(/\*$/,"",d); - e=$(NF-1);sub(/[0-9]\/[0-9]\/[0-9]|0\.0\.0\.0/,"NX",e);sub(/\.$/,"",e);sub(/([0-9]{1,3}\.){3}[0-9]{1,3}/,"OK",e);gsub(/[^[:alnum:]\.-]/,"",e);if(e==""){e="err"};printf "%s\t%s\t%s\t%s\t%s\n",d,e,a,b,c}' >>"${report_raw}" + "${adb_dumpcmd}" ${resolve} --immediate-mode -tttt -T domain -r "${file}" 2>/dev/null | + "${adb_awkcmd}" -v repiface="${adb_repiface}" ' + BEGIN { + pending = 0 + } + # ignore Reverse DNS + /\.in-addr\.arpa/ || /\.ip6\.arpa/ { next } + # domain request parser (with optional EDNS marker support) + /\+[[:space:]]+(\[[0-9a-z]*\][[:space:]]+)?(A\?|AAAA\?)/ { + # drop unresolved previous query + if (pending) + pending = 0 + date = $1 + split($2, t, ":") + time = t[1] ":" t[2] ":" substr(t[3],1,2) + interface = repiface + client = $4 + if (repiface == "any") { + interface = $3 + client = $6 + } + sub(/\.[0-9]+$/, "", client) + domain = $(NF-1) + sub(/[,\.]+$/, "", domain) + if (domain ~ /\.lan$/) next + if (domain !~ /\./) next + if (domain ~ /[\/:]/) next + qtype = $(NF-2) + sub(/\?$/, "", qtype) + last_date = date + last_time = time + last_client = client + last_interface = interface + last_domain = domain + last_qtype = qtype + pending = 1 + next + } + # ok answer + / (A|AAAA|CNAME) / && !/NXDomain/ && !/ServFail/ { + if (pending) { + printf "%s\t%s\t%s\t%s\t%s\t%s\tOK\n", + last_date, last_time, last_client, last_interface, last_qtype, last_domain + pending = 0 + } + next + } + # nxdomain answer + / NXDomain/ { + if (pending) { + printf "%s\t%s\t%s\t%s\t%s\t%s\tNX\n", + last_date, last_time, last_client, last_interface, last_qtype, last_domain + pending = 0 + } + next + } + # servfail answer + / ServFail/ { + if (pending) { + printf "%s\t%s\t%s\t%s\t%s\t%s\tSF\n", + last_date, last_time, last_client, last_interface, last_qtype, last_domain + pending = 0 + } + next + } + ' >"${report_raw}.${cnt}" ) & - hold="$((cnt % adb_cores))" - [ "${hold}" = "0" ] && wait + [ "${cnt}" -gt "${adb_cores}" ] && wait -n cnt="$((cnt + 1))" done wait - if [ -s "${report_raw}" ]; then - "${adb_sort}" ${adb_srtopts} -k1 -k3 -k4 -k5 -k1 -ur "${report_raw}" | - "${adb_awk}" '{currA=($1+0);currB=$1;currC=substr($1,length($1),1);if(reqA==currB){reqA=0;printf "%s\t%s\n",d,$2}else if(currC=="+"){reqA=currA;d=$3"\t"$4"\t"$5"\t"$2}}' | - "${adb_sort}" ${adb_srtopts} -k1 -k2 -k3 -k4 -ur >"${report_srt}" - rm -f "${report_raw}" - fi + for file in "${report_raw}".*; do + [ -s "${file}" ] || continue + "${adb_sortcmd}" ${adb_srtopts} -ru "${report_raw}".* >"${report_srt}" + "${adb_rmcmd}" -f "${report_raw}".* + break + done + # build json + # if [ -s "${report_srt}" ]; then - start="$("${adb_awk}" 'END{printf "%s_%s",$1,$2}' "${report_srt}")" - end="$("${adb_awk}" 'NR==1{printf "%s_%s",$1,$2}' "${report_srt}")" - total="$(wc -l <"${report_srt}")" - blocked="$("${adb_awk}" '{if($5=="NX")cnt++}END{printf "%s",cnt}' "${report_srt}")" - percent="$("${adb_awk}" -v t="${total}" -v b="${blocked}" 'BEGIN{printf "%.2f%s",b/t*100,"%"}')" - : >"${report_jsn}" + start="$("${adb_awkcmd}" 'END{printf "%s_%s",$1,$2}' "${report_srt}")" + end="$("${adb_awkcmd}" 'NR==1{printf "%s_%s",$1,$2}' "${report_srt}")" + total="$(f_count tld "${report_srt}" "var")" + blocked="$("${adb_awkcmd}" '{if($7=="NX")cnt++}END{printf "%s",cnt}' "${report_srt}")" + percent="$("${adb_awkcmd}" -v t="${total}" -v b="${blocked}" 'BEGIN{ if(t>0) printf "%.2f%s",b/t*100,"%"; else printf "0.00%%"}')" { - printf "%s\n" "{ " - printf "\t%s\n" "\"start_date\": \"${start%_*}\", " - printf "\t%s\n" "\"start_time\": \"${start#*_}\", " - printf "\t%s\n" "\"end_date\": \"${end%_*}\", " - printf "\t%s\n" "\"end_time\": \"${end#*_}\", " - printf "\t%s\n" "\"total\": \"${total}\", " - printf "\t%s\n" "\"blocked\": \"${blocked}\", " - printf "\t%s\n" "\"percent\": \"${percent}\", " - } >>"${report_jsn}" + printf '%s\n' "{ " + printf '\t%s\n' "\"start_date\": \"${start%_*}\", " + printf '\t%s\n' "\"start_time\": \"${start#*_}\", " + printf '\t%s\n' "\"end_date\": \"${end%_*}\", " + printf '\t%s\n' "\"end_time\": \"${end#*_}\", " + printf '\t%s\n' "\"total\": \"${total}\", " + printf '\t%s\n' "\"blocked\": \"${blocked}\", " + printf '\t%s\n' "\"percent\": \"${percent}\", " + } >"${report_jsn}" + + # build top list counters + # + "${adb_awkcmd}" ' + { + if (NF < 7) { + next + } + client = $3 + domain = $6 + rc = $7 + + if (domain == "" || domain == "-") { + next + } + + sub(/[\.]+$/, "", domain) + domain = tolower(domain) + + clients[client]++ + if (rc == "OK") { + ok_domain[domain]++ + } + else if (rc == "NX") { + nx_domain[domain]++ + } + all_domain[domain]++ + } + END { + for (c in clients) { + printf "%d %s\n", clients[c], c > "'"${top_tmpclients}"'" + } + for (d in all_domain) { + if (d in ok_domain) { + printf "%d %s\n", ok_domain[d], d > "'"${top_tmpdomains}"'" + } + if (d in nx_domain) { + printf "%d %s\n", nx_domain[d], d > "'"${top_tmpblocked}"'" + } + } + } + ' "${report_srt}" + + # build json top lists + # top_list="top_clients top_domains top_blocked" for top in ${top_list}; do - printf "\t%s" "\"${top}\": [ " >>"${report_jsn}" + printf '\t"%s": [ ' "${top}" >>"${report_jsn}" case "${top}" in - "top_clients") - "${adb_awk}" '{print $3}' "${report_srt}" | "${adb_sort}" ${adb_srtopts} | uniq -c | - "${adb_sort}" ${adb_srtopts} -nr | - "${adb_awk}" "{ORS=\" \";if(NR==1)printf \"\n\t\t{\n\t\t\t\\\"count\\\": \\\"%s\\\",\n\t\t\t\\\"address\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2; else if(NR<=${top_count})printf \",\n\t\t{\n\t\t\t\\\"count\\\": \\\"%s\\\",\n\t\t\t\\\"address\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2}" >>"${report_jsn}" - ;; - "top_domains") - "${adb_awk}" '{if($5!="NX")print $4}' "${report_srt}" | "${adb_sort}" ${adb_srtopts} | uniq -c | - "${adb_sort}" ${adb_srtopts} -nr | - "${adb_awk}" "{ORS=\" \";if(NR==1)printf \"\n\t\t{\n\t\t\t\\\"count\\\": \\\"%s\\\",\n\t\t\t\\\"address\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2; else if(NR<=${top_count})printf \",\n\t\t{\n\t\t\t\\\"count\\\": \\\"%s\\\",\n\t\t\t\\\"address\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2}" >>"${report_jsn}" - ;; - "top_blocked") - "${adb_awk}" '{if($5=="NX")print $4}' "${report_srt}" | - "${adb_sort}" ${adb_srtopts} | uniq -c | "${adb_sort}" ${adb_srtopts} -nr | - "${adb_awk}" "{ORS=\" \";if(NR==1)printf \"\n\t\t{\n\t\t\t\\\"count\\\": \\\"%s\\\",\n\t\t\t\\\"address\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2; else if(NR<=${top_count})printf \",\n\t\t{\n\t\t\t\\\"count\\\": \\\"%s\\\",\n\t\t\t\\\"address\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2}" >>"${report_jsn}" - ;; + top_clients) + "${adb_sortcmd}" ${adb_srtopts} -nr "${top_tmpclients}" | + "${adb_awkcmd}" -v top_count="${top_count}" ' + BEGIN { ORS=""; OFS="" } + NR==1 { + printf "\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}", $1, $2 + } + NR>1 && NR<=top_count { + printf ",\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}", $1, $2 + } + ' >>"${report_jsn}" + ;; + top_domains) + "${adb_sortcmd}" ${adb_srtopts} -nr "${top_tmpdomains}" | + "${adb_awkcmd}" -v top_count="${top_count}" ' + BEGIN { ORS=""; OFS="" } + NR==1 { + printf "\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}", $1, $2 + } + NR>1 && NR<=top_count { + printf ",\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}", $1, $2 + } + ' >>"${report_jsn}" + ;; + top_blocked) + "${adb_sortcmd}" ${adb_srtopts} -nr "${top_tmpblocked}" | + "${adb_awkcmd}" -v top_count="${top_count}" ' + BEGIN { ORS=""; OFS="" } + NR==1 { + printf "\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}", $1, $2 + } + NR>1 && NR<=top_count { + printf ",\n\t\t{\n\t\t\t\"count\": \"%s\",\n\t\t\t\"address\": \"%s\"\n\t\t}", $1, $2 + } + ' >>"${report_jsn}" + ;; esac - printf "\n\t%s\n" "]," >>"${report_jsn}" + printf '\n\t],\n' >>"${report_jsn}" + done + "${adb_rmcmd}" -f "${top_tmpclients}" "${top_tmpdomains}" "${top_tmpblocked}" + + # build json request list + # + search="${search//[!a-zA-Z0-9._-]/}" + case "${res_count}" in + '' | *[!0-9]*) + res_count="50" + ;; + esac + "${adb_awkcmd}" -v search="${search}" -v res_count="${res_count}" ' + BEGIN { + i = 0 + printf "\t\"requests\": [\n" + } + + # only match if search is empty or non-empty and NF == 7 + ((search == "" || index($0, search)) && NF == 7) { + i++ + if (res_count > 0 && i > res_count) { + next + } + if (i > 1) { + printf ",\n" + } + + printf "\n\t\t{\n" + printf "\t\t\t\"date\": \"%s\",\n", $1 + printf "\t\t\t\"time\": \"%s\",\n", $2 + printf "\t\t\t\"client\": \"%s\",\n", $3 + printf "\t\t\t\"iface\": \"%s\",\n", $4 + printf "\t\t\t\"type\": \"%s\",\n", $5 + printf "\t\t\t\"domain\": \"%s\",\n", $6 + printf "\t\t\t\"rc\": \"%s\"\n", $7 + printf "\t\t}" + } + END { + printf "\n\t]\n}\n" + }' "${report_srt}" >>"${report_jsn}" + "${adb_rmcmd}" -f "${report_srt}" + fi + + # retrieve/prepare map data + # + if [ "${adb_map}" = "1" ] && [ -s "${report_jsn}" ]; then + cnt="1" + network_find_wan iface_v4 && network_get_ipaddr ip_v4 "${iface_v4}" + network_find_wan6 iface_v6 && network_get_ipaddr6 ip_v6 "${iface_v6}" + if [ -n "${ip_v4}" ] || [ -n "${ip_v6}" ]; then + f_fetch + printf '%s' ",[{}" >"${map_jsn}" + fi + for ip in ${ip_v4} ${ip_v6}; do + ( + "${adb_fetchcmd}" ${adb_geoparm} "${adb_geourl}/${ip}" 2>>"${adb_errorlog}" | + "${adb_awkcmd}" -v feed="homeIP" '{printf ",{\"%s\": %s}\n",feed,$0}' >"${map_jsn}.${cnt}" + ) & + [ "${cnt}" -gt "${adb_cores}" ] && wait -n + cnt="$((cnt + 1))" + done + wait + if [ -s "${map_jsn}" ] && [ "${cnt}" -lt "45" ]; then + map_seen="" + json_init + if json_load_file "${report_jsn}" >/dev/null 2>&1; then + json_select "requests" >/dev/null 2>&1 + json_get_keys requests >/dev/null 2>&1 + for request in ${requests}; do + json_select "${request}" >/dev/null 2>&1 + json_get_var rc "rc" >/dev/null 2>&1 + json_get_var domain "domain" >/dev/null 2>&1 + if [ "${rc}" = "NX" ]; then + case " ${map_seen} " in + *" ${domain} "*) ;; + + *) + map_seen="${map_seen} ${domain} " + ( + "${adb_fetchcmd}" ${adb_geoparm} "${adb_geourl}/${domain}" 2>>"${adb_errorlog}" | + "${adb_awkcmd}" -v feed="${domain}" '{printf ",{\"%s\": %s}\n",feed,$0}' >"${map_jsn}.${cnt}" + ) & + [ "${cnt}" -gt "${adb_cores}" ] && wait -n + cnt="$((cnt + 1))" + [ "${cnt}" -ge "45" ] && break + ;; + esac + fi + json_select ".." + done + wait + fi + fi + for file in "${map_jsn}".*; do + [ -s "${file}" ] || continue + "${adb_catcmd}" "${map_jsn}".* >>"${map_jsn}" 2>/dev/null + "${adb_rmcmd}" -f "${map_jsn}".* + break done - search="${search//./\\.}" - search="${search//[+*~%\$&\"\' ]/}" - "${adb_awk}" "BEGIN{i=0;printf \"\t\\\"requests\\\": [\n\"}/(${search})/{i++;if(i==1)printf \"\n\t\t{\n\t\t\t\\\"date\\\": \\\"%s\\\",\n\t\t\t\\\"time\\\": \\\"%s\\\",\n\t\t\t\\\"client\\\": \\\"%s\\\",\n\t\t\t\\\"domain\\\": \\\"%s\\\",\n\t\t\t\\\"rc\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2,\$3,\$4,\$5;else if(i<=${res_count})printf \",\n\t\t{\n\t\t\t\\\"date\\\": \\\"%s\\\",\n\t\t\t\\\"time\\\": \\\"%s\\\",\n\t\t\t\\\"client\\\": \\\"%s\\\",\n\t\t\t\\\"domain\\\": \\\"%s\\\",\n\t\t\t\\\"rc\\\": \\\"%s\\\"\n\t\t}\",\$1,\$2,\$3,\$4,\$5}END{printf \"\n\t]\n}\n\"}" "${adb_reportdir}/adb_report.srt" >>"${report_jsn}" - rm -f "${report_srt}" fi fi # output preparation # if [ -s "${report_jsn}" ] && { [ "${action}" = "cli" ] || [ "${action}" = "mail" ]; }; then - printf "%s\n%s\n%s\n" ":::" "::: Adblock DNS-Query Report" ":::" >>"${report_txt}" + printf '%s\n%s\n%s\n' ":::" "::: Adblock DNS Report" ":::" >>"${report_txt}" json_init json_load_file "${report_jsn}" json_get_keys key_list for key in ${key_list}; do json_get_var value "${key}" - eval "${key}=\"${value}\"" + case "${key}" in + "start_date") + start_date="${value}" + ;; + "start_time") + start_time="${value}" + ;; + "end_date") + end_date="${value}" + ;; + "end_time") + end_time="${value}" + ;; + "total") + total="${value}" + ;; + "blocked") + blocked="${value}" + ;; + "percent") + percent="${value}" + ;; + esac done - printf " + %s\n + %s\n" "Start ::: ${start_date}, ${start_time}" "End ::: ${end_date}, ${end_time}" >>"${report_txt}" - printf " + %s\n + %s %s\n" "Total ::: ${total}" "Blocked ::: ${blocked}" "(${percent})" >>"${report_txt}" + printf ' + %s\n + %s\n' "Start ::: ${start_date}, ${start_time}" "End ::: ${end_date}, ${end_time}" >>"${report_txt}" + printf ' + %s\n + %s %s\n' "Total ::: ${total}" "Blocked ::: ${blocked}" "(${percent})" >>"${report_txt}" top_list="top_clients top_domains top_blocked requests" for top in ${top_list}; do case "${top}" in - "top_clients") - item="::: Top Clients" - ;; - "top_domains") - item="::: Top Domains" - ;; - "top_blocked") - item="::: Top Blocked Domains" - ;; + "top_clients") + item="::: Top Clients" + ;; + "top_domains") + item="::: Top Domains" + ;; + "top_blocked") + item="::: Top Blocked Domains" + ;; esac if json_get_type status "${top}" && [ "${top}" != "requests" ] && [ "${status}" = "array" ]; then - printf "%s\n%s\n%s\n" ":::" "${item}" ":::" >>"${report_txt}" + printf '%s\n%s\n%s\n' ":::" "${item}" ":::" >>"${report_txt}" json_select "${top}" index="1" item="" while json_get_type status "${index}" && [ "${status}" = "object" ]; do json_get_values item "${index}" - printf " + %-9s::: %s\n" ${item} >>"${report_txt}" + printf ' + %-9s::: %s\n' ${item} >>"${report_txt}" index="$((index + 1))" done elif json_get_type status "${top}" && [ "${top}" = "requests" ] && [ "${status}" = "array" ]; then - printf "%s\n%s\n%s\n" ":::" "::: Latest DNS Queries" ":::" >>"${report_txt}" - printf "%-15s%-15s%-45s%-80s%s\n" "Date" "Time" "Client" "Domain" "Answer" >>"${report_txt}" + printf '%s\n%s\n%s\n' ":::" "::: Latest DNS Queries" ":::" >>"${report_txt}" + printf '%-11s%-9s%-40s%-15s%-5s%-70s%s\n' "Date" "Time" "Client" "Interface" "Type" "Domain" "Answer" >>"${report_txt}" json_select "${top}" index="1" while json_get_type status "${index}" && [ "${status}" = "object" ]; do json_get_values item "${index}" - printf "%-15s%-15s%-45s%-80s%s\n" ${item} >>"${report_txt}" + printf '%-11s%-9s%-40s%-15s%-5s%-70s%s\n' ${item} >>"${report_txt}" index="$((index + 1))" done fi json_select ".." done - content="$(cat "${report_txt}" 2>/dev/null)" - rm -f "${report_txt}" + content="$("${adb_catcmd}" "${report_txt}" 2>>"${adb_errorlog}")" + "${adb_rmcmd}" -f "${report_txt}" fi # report output # - if [ "${action}" = "cli" ]; then - printf "%s\n" "${content}" - elif [ "${action}" = "json" ]; then - cat "${report_jsn}" - elif [ "${action}" = "mail" ] && [ "${adb_mail}" = "1" ] && [ -x "${adb_mailservice}" ]; then - ("${adb_mailservice}" "${adb_ver}" "${content}" >/dev/null 2>&1) & - bg_pid="${!}" - fi - f_log "debug" "f_report ::: action: ${action}, top_count: ${top_count}, res_count: ${res_count}, search: ${search}, dump_util: ${adb_dumpcmd}, rep_dir: ${adb_reportdir}, rep_iface: ${adb_repiface:-"-"}, rep_listen: ${adb_replisten}, rep_chunksize: ${adb_repchunksize}, rep_chunkcnt: ${adb_repchunkcnt}, rep_resolve: ${adb_represolve}" + case "${action}" in + "cli") + printf '%s\n' "${content}" + ;; + "json") + if [ "${adb_map}" = "1" ] && [ -s "${map_jsn}" ]; then + jsn="$("${adb_catcmd}" ${report_jsn} ${map_jsn} 2>>"${adb_errorlog}")" + [ -n "${jsn}" ] && printf '[%s]]\n' "${jsn}" + else + jsn="$("${adb_catcmd}" "${report_jsn}" 2>>"${adb_errorlog}")" + [ -n "${jsn}" ] && printf '[%s]\n' "${jsn}" + fi + ;; + "mail") + [ "${adb_mail}" = "1" ] && [ -x "${adb_mailservice}" ] && "${adb_mailservice}" "${content}" >/dev/null 2>&1 + "${adb_rmcmd}" -f "${report_txt}" + ;; + "gen") + printf '%s\n' "1" >"${adb_rundir}/adblock.report" + ;; + esac } # source required system libraries @@ -1473,47 +2452,72 @@ else f_log "err" "system libraries not found" fi -# awk check +# create runtime directory if it doesn't exist # -adb_awk="$(command -v gawk)" -if [ ! -x "${adb_awk}" ]; then - adb_awk="$(command -v awk)" - [ ! -x "${adb_awk}" ] && f_log "err" "awk not found or not executable" -fi +[ ! -d "${adb_rundir}" ] && mkdir -p "${adb_rundir}" -# sort check +# reference required system utilities # -adb_sort="$(command -v sort)" -if [ ! -x "${adb_sort}" ] || ! "${adb_sort}" --version 2>/dev/null | grep -q "coreutils"; then - f_log "err" "coreutils sort not found or not executable" -fi +adb_mvcmd="$(f_cmd mv)" +adb_lncmd="$(f_cmd ln)" +adb_rmcmd="$(f_cmd rm)" +adb_catcmd="$(f_cmd cat)" +adb_zcatcmd="$(f_cmd zcat)" +adb_awkcmd="$(f_cmd gawk awk)" +adb_sortcmd="$(f_cmd sort)" +adb_grepcmd="$(f_cmd grep)" +adb_gzipcmd="$(f_cmd gzip)" +adb_pgrepcmd="$(f_cmd pgrep)" +adb_sedcmd="$(f_cmd sed)" +adb_findcmd="$(f_cmd find)" +adb_jsoncmd="$(f_cmd jsonfilter)" +adb_ubuscmd="$(f_cmd ubus)" +adb_loggercmd="$(f_cmd logger)" +adb_lookupcmd="$(f_cmd nslookup)" +adb_xargscmd="$(f_cmd xargs)" +adb_flockcmd="$(f_cmd flock)" +adb_dumpcmd="$(f_cmd tcpdump optional)" +adb_mailcmd="$(f_cmd msmtp optional)" +adb_logreadcmd="$(f_cmd logread optional)" +adb_nftcmd="$(f_cmd nft)" # handle different adblock actions # f_load case "${adb_action}" in - "stop") - f_rmdns - ;; - "restart") - f_rmdns - f_env - f_main - ;; - "suspend") - [ "${adb_dns}" != "raw" ] && f_switch suspend - ;; - "resume") - [ "${adb_dns}" != "raw" ] && f_switch resume - ;; - "report") - f_report "${2}" "${3}" "${4}" "${5}" - ;; - "query") - f_query "${2}" - ;; - "start" | "reload") - f_env - f_main - ;; +"stop") + f_temp + f_nftremove + f_rmdns + f_jsnup "stopped" + ;; +"suspend") + [ "${adb_dns}" != "raw" ] && f_switch suspend + ;; +"resume") + [ "${adb_dns}" != "raw" ] && f_switch resume + ;; +"report") + f_report "${2}" "${3}" "${4}" "${5}" + ;; +"search") + f_search "${2}" + ;; +"boot" | "start" | "reload") + f_env + f_main + ;; +# Start NethSecurity patch +"nft-reload") + f_nftremove + f_nftadd + ;; +# End NethSecurity patch +"restart") + f_temp + f_jsnup "processing" + f_rmdns + f_env + f_main + ;; esac diff --git a/packages/adblock/files/adblock.sources b/packages/adblock/files/adblock.sources deleted file mode 100644 index 85af8602b..000000000 --- a/packages/adblock/files/adblock.sources +++ /dev/null @@ -1,352 +0,0 @@ -{ - "adaway": { - "url": "https://raw.githubusercontent.com/AdAway/adaway.github.io/master/hosts.txt", - "rule": "/^127\\.0\\.0\\.1[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "S", - "focus": "mobile", - "descurl": "https://github.com/AdAway/adaway.github.io" - }, - "adguard": { - "url": "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt", - "rule": "BEGIN{FS=\"[\/|^|\\r]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+[\\/\\^\\r]+$/{print tolower($3)}", - "size": "L", - "focus": "general", - "descurl": "https://adguard.com" - }, - "adguard_tracking": { - "url": "https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/combined_disguised_trackers_justdomains.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "tracking", - "descurl": "https://github.com/AdguardTeam/cname-trackers" - }, - "android_tracking": { - "url": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/android-tracking.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "tracking", - "descurl": "https://github.com/Perflyst/PiHoleBlocklist" - }, - "andryou": { - "url": "https://gitlab.com/andryou/block/raw/master/kouhai-compressed-domains", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "L", - "focus": "compilation", - "descurl": "https://gitlab.com/andryou/block/-/blob/master/readme.md" - }, - "anti_ad": { - "url": "https://raw.githubusercontent.com/privacy-protection-tools/anti-AD/master/anti-ad-domains.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "L", - "focus": "compilation", - "descurl": "https://github.com/privacy-protection-tools/anti-AD/blob/master/README.md" - }, - "antipopads": { - "url": "https://raw.githubusercontent.com/AdroitAdorKhan/antipopads-re/master/formats/domains.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "L", - "focus": "compilation", - "descurl": "https://github.com/AdroitAdorKhan/antipopads-re" - }, - "anudeep": { - "url": "https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "M", - "focus": "compilation", - "descurl": "https://github.com/anudeepND/blacklist" - }, - "bitcoin": { - "url": "https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "S", - "focus": "mining", - "descurl": "https://github.com/hoshsadiq/adblock-nocoin-list" - }, - "cpbl": { - "url": "https://raw.githubusercontent.com/bongochong/CombinedPrivacyBlockLists/master/NoFormatting/cpbl-ctld.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "XL", - "focus": "compilation", - "descurl": "https://github.com/bongochong/CombinedPrivacyBlockLists" - }, - "disconnect": { - "url": "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "general", - "descurl": "https://disconnect.me" - }, - "doh_blocklist": { - "url": "https://raw.githubusercontent.com/dibdot/DoH-IP-blocklists/master/doh-domains_overall.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "doh_server", - "descurl": "https://github.com/dibdot/DoH-IP-blocklists" - }, - "easylist": { - "url": "https://easylist-downloads.adblockplus.org/easylist.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "M", - "focus": "compilation", - "descurl": "https://easylist.to" - }, - "easyprivacy": { - "url": "https://easylist-downloads.adblockplus.org/easyprivacy.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "M", - "focus": "tracking", - "descurl": "https://easylist.to" - }, - "firetv_tracking": { - "url": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/AmazonFireTV.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "tracking", - "descurl": "https://github.com/Perflyst/PiHoleBlocklist" - }, - "games_tracking": { - "url": "https://raw.githubusercontent.com/KodoPengin/GameIndustry-hosts-Template/master/Main-Template/hosts", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "S", - "focus": "tracking", - "descurl": "https://www.gameindustry.eu" - }, - "hblock": { - "url": "https://hblock.molinero.dev/hosts_domains.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "XL", - "focus": "compilation", - "descurl": "https://hblock.molinero.dev" - }, - "lightswitch05": { - "url": "https://www.github.developerdan.com/hosts/lists/ads-and-tracking-extended.txt", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "XL", - "focus": "compilation", - "descurl": "https://github.com/lightswitch05/hosts" - }, - "notracking": { - "url": "https://raw.githubusercontent.com/notracking/hosts-blocklists/master/dnscrypt-proxy/dnscrypt-proxy.blacklist.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "XL", - "focus": "tracking", - "descurl": "https://github.com/notracking/hosts-blocklists" - }, - "oisd_big": { - "url": "https://big.oisd.nl/domainswild", - "rule": "BEGIN{FS=\"\\\\*.\"}/^\\*\\.([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "XXL", - "focus": "general", - "descurl": "https://oisd.nl" - }, - "oisd_nsfw": { - "url": "https://nsfw.oisd.nl/domainswild", - "rule": "BEGIN{FS=\"\\\\*.\"}/^\\*\\.([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "XXL", - "focus": "porn", - "descurl": "https://oisd.nl" - }, - "oisd_small": { - "url": "https://small.oisd.nl/domainswild", - "rule": "BEGIN{FS=\"\\\\*.\"}/^\\*\\.([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "L", - "focus": "general", - "descurl": "https://oisd.nl" - }, - "openphish": { - "url": "https://openphish.com/feed.txt", - "rule": "BEGIN{FS=\"\/\"}/^http[s]?:\\/\\/([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+(\\/|$)/{print tolower($3)}", - "size": "S", - "focus": "phishing", - "descurl": "https://openphish.com" - }, - "phishing_army": { - "url": "https://phishing.army/download/phishing_army_blocklist_extended.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "phishing", - "descurl": "https://phishing.army" - }, - "reg_cn": { - "url": "https://easylist-downloads.adblockplus.org/easylistchina.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_china", - "descurl": "https://easylist.to" - }, - "reg_cz": { - "url": "https://easylist-downloads.adblockplus.org/easylistczechslovak.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_czech+slovak", - "descurl": "https://easylist.to" - }, - "reg_de": { - "url": "https://easylist-downloads.adblockplus.org/easylistgermany.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_germany", - "descurl": "https://easylist.to" - }, - "reg_es": { - "url": "https://easylist-downloads.adblockplus.org/easylistspanish.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_spain", - "descurl": "https://easylist.to" - }, - "reg_fi": { - "url": "https://raw.githubusercontent.com/finnish-easylist-addition/finnish-easylist-addition/master/Finland_adb.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_finland", - "descurl": "https://github.com/finnish-easylist-addition" - }, - "reg_fr": { - "url": "https://easylist-downloads.adblockplus.org/liste_fr.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "M", - "focus": "reg_france", - "descurl": "https://forums.lanik.us/viewforum.php?f=91" - }, - "reg_id": { - "url": "https://easylist-downloads.adblockplus.org/abpindo.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_indonesia", - "descurl": "https://easylist.to" - }, - "reg_it": { - "url": "https://easylist-downloads.adblockplus.org/easylistitaly.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_italy", - "descurl": "https://easylist.to" - }, - "reg_jp": { - "url": "https://raw.githubusercontent.com/k2jp/abp-japanese-filters/master/abpjf.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_japan", - "descurl": "https://github.com/k2jp/abp-japanese-filters" - }, - "reg_kr": { - "url": "https://raw.githubusercontent.com/List-KR/List-KR/master/filters-share/adservice.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_korea", - "descurl": "https://github.com/List-KR/List-KR" - }, - "reg_nl": { - "url": "https://easylist-downloads.adblockplus.org/easylistdutch.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_netherlands", - "descurl": "https://easylist.to" - }, - "reg_pl": { - "url": "https://raw.githubusercontent.com/PolishFiltersTeam/KADhosts/master/KADhosts.txt", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "M", - "focus": "reg_poland", - "descurl": "https://kadantiscam.netlify.app" - }, - "reg_ro": { - "url": "https://easylist-downloads.adblockplus.org/rolist.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_romania", - "descurl": "https://easylist.to" - }, - "reg_ru": { - "url": "https://easylist-downloads.adblockplus.org/ruadlist.txt", - "rule": "BEGIN{FS=\"[|^]\"}/^\\|\\|([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+\\^(\\$third-party)?$/{print tolower($3)}", - "size": "S", - "focus": "reg_russia", - "descurl": "https://easylist.to" - }, - "reg_se": { - "url": "https://raw.githubusercontent.com/lassekongo83/Frellwits-filter-lists/master/Frellwits-Swedish-Hosts-File.txt", - "rule": "/^127\\.0\\.0\\.1[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "S", - "focus": "reg_sweden", - "descurl": "https://github.com/lassekongo83/Frellwits-filter-lists" - }, - "reg_vn": { - "url": "https://raw.githubusercontent.com/bigdargon/hostsVN/master/hosts", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "S", - "focus": "reg_vietnam", - "descurl": "https://bigdargon.github.io/hostsVN" - }, - "smarttv_tracking": { - "url": "https://raw.githubusercontent.com/Perflyst/PiHoleBlocklist/master/SmartTV.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "tracking", - "descurl": "https://github.com/Perflyst/PiHoleBlocklist" - }, - "spam404": { - "url": "https://raw.githubusercontent.com/Dawsey21/Lists/master/main-blacklist.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "general", - "descurl": "https://github.com/Dawsey21" - }, - "stevenblack": { - "url": "https://raw.githubusercontent.com/StevenBlack/hosts/master/", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "VAR", - "focus": "compilation", - "descurl": "https://github.com/StevenBlack/hosts" - }, - "stopforumspam": { - "url": "https://www.stopforumspam.com/downloads/toxic_domains_whole.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "spam", - "descurl": "https://www.stopforumspam.com" - }, - "utcapitole": { - "url": "https://dsi.ut-capitole.fr/blacklists/download/blacklists.tar.gz", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "VAR", - "focus": "general", - "descurl": "https://dsi.ut-capitole.fr/blacklists/index_en.php" - }, - "wally3k": { - "url": "https://v.firebog.net/hosts/static/w3kbl.txt", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "compilation", - "descurl": "https://firebog.net/about" - }, - "whocares": { - "url": "https://someonewhocares.org/hosts/hosts", - "rule": "/^127\\.0\\.0\\.1[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "M", - "focus": "general", - "descurl": "https://someonewhocares.org" - }, - "winhelp": { - "url": "https://winhelp2002.mvps.org/hosts.txt", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "S", - "focus": "general", - "descurl": "https://winhelp2002.mvps.org" - }, - "winspy": { - "url": "https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt", - "rule": "/^0\\.0\\.0\\.0[[:space:]]+([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($2)}", - "size": "S", - "focus": "win_telemetry", - "descurl": "https://github.com/crazy-max/WindowsSpyBlocker" - }, - "yoyo": { - "url": "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=nohtml&showintro=0&mimetype=plaintext", - "rule": "/^([[:alnum:]_-]{1,63}\\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower($1)}", - "size": "S", - "focus": "general", - "descurl": "https://pgl.yoyo.org/as" - } -} diff --git a/packages/checkmk-agent/Makefile b/packages/checkmk-agent/Makefile index 8f33df9f8..30053f9f0 100644 --- a/packages/checkmk-agent/Makefile +++ b/packages/checkmk-agent/Makefile @@ -6,7 +6,9 @@ include $(TOPDIR)/rules.mk PKG_NAME:=checkmk-agent -PKG_VERSION:=2.4.0p24 +# renovate: datasource=github-tags depName=Checkmk/checkmk +CHECKMK_UPSTREAM_VERSION:=2.5.0 +PKG_VERSION:=$(subst p,_p,$(CHECKMK_UPSTREAM_VERSION)) PKG_RELEASE:=1 PKG_BUILD_DIR:=$(BUILD_DIR)/checkmk-agent-$(PKG_VERSION) @@ -32,7 +34,7 @@ endef # Download the Check_MK agent binary define Download/checkmk-agent-binary - URL:=https://raw.githubusercontent.com/Checkmk/checkmk/v$(PKG_VERSION)/agents + URL:=https://raw.githubusercontent.com/Checkmk/checkmk/v$(CHECKMK_UPSTREAM_VERSION)/agents URL_FILE:=check_mk_agent.openwrt FILE:=check_mk_agent.openwrt HASH:=skip diff --git a/packages/netifyd/Makefile b/packages/netifyd/Makefile index 54a598df9..291baa833 100644 --- a/packages/netifyd/Makefile +++ b/packages/netifyd/Makefile @@ -8,8 +8,8 @@ PKG_MAINTAINER:=Darryl Sokoloski PKG_LICENSE:=Unlicensed # Base URL for downloads -NETIFYD_BASE_URL:=https://updates.nethsecurity.nethserver.org/netifyd-dist/netifyd-$(NETIFYD_VERSION)/$(ARCH) -DL_DIR:=$(DL_DIR)/netifyd-$(NETIFYD_VERSION)-$(ARCH) +NETIFYD_BASE_URL:=https://updates.nethsecurity.nethserver.org/netifyd-dist/netifyd-$(PKG_VERSION)/25.12.2/$(ARCH) +DL_DIR:=$(DL_DIR)/netifyd-$(PKG_VERSION)-$(ARCH) include $(INCLUDE_DIR)/package.mk @@ -66,7 +66,7 @@ define Download/libnetifyd URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetifyd.so.4.0.0 FILE:=libnetifyd.so.4.0.0 - HASH:=a7bef78717e200eef177da8ee94557a565828ce796d3fd1a1f078e3f55959dd5 + HASH:=ffbbe5078a2b6575db4c478cf1b4d257f9b7241797c84aa1bcbca17ee1b23f3b endef $(eval $(call Download,libnetifyd)) @@ -74,7 +74,7 @@ define Download/libnetify-plm URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-plm.so.1.0.0 FILE:=libnetify-plm.so.1.0.0 - HASH:=b5c8994dc5f497ef1ccf8aae3685c1541702031726bc9f2819489cacfcb67e8f + HASH:=4320235873539f561238c92094702e74b75b8939a301b694255a5512fa036a00 endef $(eval $(call Download,libnetify-plm)) @@ -82,7 +82,7 @@ define Download/libnetify-proc-aggregator URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-proc-aggregator.so.0.0.0 FILE:=libnetify-proc-aggregator.so.0.0.0 - HASH:=c9e5981fd4410776a1808902fbe284f40882823fc4c711f54e14421c80e94c6b + HASH:=736acb4c22d891da66694eee5507ea70679885f0747dec7a07eec8c926dd76fc endef $(eval $(call Download,libnetify-proc-aggregator)) @@ -90,7 +90,7 @@ define Download/libnetify-proc-core URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-proc-core.so.0.0.0 FILE:=libnetify-proc-core.so.0.0.0 - HASH:=725415498d05fc29695a6f1fe14aed4c6b5967ff0d7d4703d06b80489c852222 + HASH:=8c9a1f26a498a6d1c88d76e6aa0367749779ca5f44ea3fa614bc1814387258ed endef $(eval $(call Download,libnetify-proc-core)) @@ -98,7 +98,7 @@ define Download/libnetify-proc-dev-discovery URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-proc-dev-discovery.so.0.0.0 FILE:=libnetify-proc-dev-discovery.so.0.0.0 - HASH:=c25c4a497925cc0cf9d48f353de76fd2b5c9aa223390c87f2163feb1b0d61efb + HASH:=756c01e22c3724311eb6952ca6d580c728592771948207b93bab9b9795c38d1c endef $(eval $(call Download,libnetify-proc-dev-discovery)) @@ -106,7 +106,7 @@ define Download/libnetify-proc-flow-actions URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-proc-flow-actions.so.0.0.0 FILE:=libnetify-proc-flow-actions.so.0.0.0 - HASH:=06ed65a3ec5d840edff008aaebbe294e4b0f7545ab295b287f6ed286bf28b7d7 + HASH:=1d6f03ead8760e314f98f5fe083515b19bf9c60e720d36bc03f70fa60a9b0d2b endef $(eval $(call Download,libnetify-proc-flow-actions)) @@ -114,7 +114,7 @@ define Download/libnetify-sink-http URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-sink-http.so.0.0.0 FILE:=libnetify-sink-http.so.0.0.0 - HASH:=aaaadb9179f3df215f08d29f3a46db314d3990adafa5f3c4abeff305f4bb9583 + HASH:=95d0d6fa2b47b3764318a2f8680441d126c94bb39c84440b3e056767e330991d endef $(eval $(call Download,libnetify-sink-http)) @@ -122,7 +122,7 @@ define Download/libnetify-sink-log URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-sink-log.so.0.0.0 FILE:=libnetify-sink-log.so.0.0.0 - HASH:=26d136562f7416ce4fe7c70d8023ae3f578a2d2dff4239791c65ad4289d28765 + HASH:=f9cafc30e150109dc3ab5e57eb203d51525a4ad0241fe76724206edf73c575b1 endef $(eval $(call Download,libnetify-sink-log)) @@ -130,7 +130,7 @@ define Download/libnetify-sink-socket URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-sink-socket.so.0.0.0 FILE:=libnetify-sink-socket.so.0.0.0 - HASH:=8772335da50b702edd2a1412c98c82e32058dee2158db6e874ba3964ac66590a + HASH:=7ed9fe8d4d952ccbedca1c2be505fe288080f19c5c60d4db46932078c97d5364 endef $(eval $(call Download,libnetify-sink-socket)) @@ -138,7 +138,7 @@ define Download/libnetify-sink-sqlite URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-sink-sqlite.so.0.0.0 FILE:=libnetify-sink-sqlite.so.0.0.0 - HASH:=6c2879cb8c9da36d592c85b9f8b4042339f0322fe2ebf335a5d15d06562ec65b + HASH:=0de0ef72cbd092fedd5cc278a4b57977891650aa10712f5ae0e6381a5a02fa52 endef $(eval $(call Download,libnetify-sink-sqlite)) @@ -146,7 +146,7 @@ define Download/netifyd URL:=$(NETIFYD_BASE_URL)/usr/sbin URL_FILE:=netifyd FILE:=netifyd - HASH:=02a8647d779f7dc4506fe9dd10295a06f6f1cb7b701edbfe791e6f319c4d7fd3 + HASH:=a1d8f40f87ba58876c7652dc0f82e699d6f8139793b8ede9d33cea043a693c72 endef $(eval $(call Download,netifyd)) diff --git a/packages/netifyd/files/etc/netifyd/profiles.d/00-default.conf b/packages/netifyd/files/etc/netifyd/profiles.d/00-default.conf index 61266c775..687f9d690 100644 --- a/packages/netifyd/files/etc/netifyd/profiles.d/00-default.conf +++ b/packages/netifyd/files/etc/netifyd/profiles.d/00-default.conf @@ -158,5 +158,6 @@ all = include [netlink] # Set the Netlink buffer size buffer_size = 32768 +bridge_pvid_discovery = no # vim: set ft=dosini : diff --git a/packages/ns-api/Makefile b/packages/ns-api/Makefile index a3bd5ce0a..5f4e21a87 100644 --- a/packages/ns-api/Makefile +++ b/packages/ns-api/Makefile @@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=ns-api -PKG_VERSION:=3.6.1 +PKG_VERSION:=3.6.2 PKG_RELEASE:=1 PKG_BUILD_DIR:=$(BUILD_DIR)/ns-api-$(PKG_VERSION) @@ -22,6 +22,7 @@ define Package/ns-api TITLE:=NethSecurity REST API URL:=https://github.com/NethServer/nethsecurity-controller/ DEPENDS:= \ + +adblock \ +coreutils-date \ +coreutils-stty \ +python3-idna \ @@ -32,6 +33,7 @@ define Package/ns-api +python3-urllib \ +sshpass \ +wireguard-tools + EXTRA_DEPENDS:=adblock (>=4.5.3) PKGARCH:=all endef diff --git a/packages/ns-api/README.md b/packages/ns-api/README.md index 6c2e38672..476ad6431 100644 --- a/packages/ns-api/README.md +++ b/packages/ns-api/README.md @@ -6118,6 +6118,8 @@ Response example: {"message": "success"} ``` +The selected blocklists are applied immediately by restarting adblock. + ### dns-list-settings Show current dns adblock settings: @@ -6142,6 +6144,8 @@ Response example: {"message": "success"} ``` +The updated DNS enforcement settings are applied immediately by restarting adblock. + ### dns-list-zones List firewall zones that can be configured on Threat shield DNS (all zones but WAN): @@ -6171,12 +6175,15 @@ Response example: { "data": [ { - "address": "nethesis.it" + "address": "nethesis.it", + "description": "my allow1" } ] } ``` +The allow and block list methods work on UCI-staged data. Changes are visible immediately through the API and are written to `/etc/adblock/adblock.allowlist` and `/etc/adblock/adblock.blocklist` during the next adblock reload triggered by `ns.commit` or `reload_config`. + ### dns-add-allowed Add a domain which is always allowed: @@ -6224,7 +6231,7 @@ It can raise the following validation errors: ### dns-list-bypass -List hosts that can bypass the adblock DNS redirect: +List hosts or subnets that can bypass the adblock DNS redirect: ``` api-cli ns.threatshield dns-list-bypass ``` @@ -6236,7 +6243,7 @@ Response example: ### dns-add-bypass -Add a host that can bypass the adblock DNS redirect: +Add a host or subnet that can bypass the adblock DNS redirect: ``` api-cli ns.threatshield dns-add-bypass --data '{"address": "192.168.1.22"}' ``` @@ -6246,9 +6253,11 @@ Response example: {"message": "success"} ``` +The new bypass entry is applied immediately by restarting adblock. + ### dns-delete-bypass -Delete a host that can bypass the adblock DNS redirect: +Delete a host or subnet that can bypass the adblock DNS redirect: ``` api-cli ns.threatshield dns-delete-bypass --data '{"address": "192.168.1.22"}' ``` @@ -6258,6 +6267,8 @@ Response example: {"message": "success"} ``` +The updated bypass list is applied immediately by restarting adblock. + ### dns-list-blocked List blocked domains from the local DNS block list: diff --git a/packages/ns-api/files/ns.dashboard b/packages/ns-api/files/ns.dashboard index 449fec46e..1e3954eed 100644 --- a/packages/ns-api/files/ns.dashboard +++ b/packages/ns-api/files/ns.dashboard @@ -154,7 +154,7 @@ def check_adblock(): if not adb_enabled: return "disabled" pa = subprocess.run(["service", "adblock", "status"], check=False, capture_output=True, text=True) - if adb_enabled and re.search('adblock_status\s+:\s+enabled', pa.stdout): + if adb_enabled and re.search(r'adblock_status\s+:\s+enabled', pa.stdout): return "ok" else: return "error" @@ -165,7 +165,7 @@ def check_banip(): if not bip_enabled: return "disabled" pa = subprocess.run(["service", "banip", "status"], check=False, capture_output=True, text=True) - if bip_enabled and re.search('status\s+:\s+active', pa.stdout): + if bip_enabled and re.search(r'status\s+:\s+active', pa.stdout): return "ok" def check_ts_ip(): diff --git a/packages/ns-api/files/ns.nathelpers b/packages/ns-api/files/ns.nathelpers index c674078ac..15afe300e 100755 --- a/packages/ns-api/files/ns.nathelpers +++ b/packages/ns-api/files/ns.nathelpers @@ -58,12 +58,12 @@ DEFAULT_PARAMS = { def get_nat_helper_names(): nat_helpers = [] - proc = subprocess.run("/bin/opkg files kmod-nf-nathelper | grep -e '\.ko$' | cut -d'/' -f 5 | cut -d'.' -f1", shell=True, check=True, + proc = subprocess.run("/bin/opkg files kmod-nf-nathelper | grep -e '\\.ko$' | cut -d'/' -f 5 | cut -d'.' -f1", shell=True, check=True, capture_output=True, text=True) nat_helpers = proc.stdout.splitlines() nat_helpers_extra = [] - proc = subprocess.run("/bin/opkg files kmod-nf-nathelper-extra | grep -e '\.ko$' | cut -d'/' -f 5 | cut -d'.' -f1", shell=True, check=True, + proc = subprocess.run("/bin/opkg files kmod-nf-nathelper-extra | grep -e '\\.ko$' | cut -d'/' -f 5 | cut -d'.' -f1", shell=True, check=True, capture_output=True, text=True) nat_helpers_extra = proc.stdout.splitlines() return nat_helpers + nat_helpers_extra diff --git a/packages/ns-api/files/ns.ovpnrw b/packages/ns-api/files/ns.ovpnrw index 147b7e86e..a43e24e10 100755 --- a/packages/ns-api/files/ns.ovpnrw +++ b/packages/ns-api/files/ns.ovpnrw @@ -63,8 +63,7 @@ def add_tap_to_bridge(u, bridge, interface): u.save("network") except: pass - finally: - return + return def remove_tap_from_bridge(u, bridge, interface): if bridge is None or interface is None: @@ -85,8 +84,7 @@ def remove_tap_from_bridge(u, bridge, interface): u.save("network") except: pass - finally: - return + return def get_ip_and_mask(bridge): u = EUci() diff --git a/packages/ns-api/files/ns.redirects b/packages/ns-api/files/ns.redirects index 8356c065c..4af2f2837 100755 --- a/packages/ns-api/files/ns.redirects +++ b/packages/ns-api/files/ns.redirects @@ -21,7 +21,7 @@ def get_services(): line = line.strip() if not line: continue - tmp = re.split("\s+", line) + tmp = re.split(r"\s+", line) port = tmp[1][0:tmp[1].index("/")] services[port] = tmp[0] return services diff --git a/packages/ns-api/files/ns.threatshield b/packages/ns-api/files/ns.threatshield index be8ed3de1..8a7e1fc04 100644 --- a/packages/ns-api/files/ns.threatshield +++ b/packages/ns-api/files/ns.threatshield @@ -101,14 +101,38 @@ def write_allow_list(allow_list, file='/etc/banip/banip.allowlist'): f.write('\n') subprocess.run(["/etc/init.d/banip", "reload"], capture_output=True) +def dns_get_local_list(file): + option = 'allowlist' if 'allowlist' in file else 'blocklist' + values = EUci().get('adblock', 'ns_lists', option, list=True, default=[]) + ret = [] + for value in values: + parts = value.split('#', 1) + ret.append({'address': parts[0].strip(), 'description': parts[1].strip() if len(parts) > 1 else ''}) + return ret + def dns_write_local_list(local_list, file): - with open(file, 'w') as f: - for x in local_list: - f.write(x['address']) - if x['description']: - f.write(' #' + x['description']) - f.write('\n') - subprocess.run(["/etc/init.d/adblock", "restart"], capture_output=True) + option = 'allowlist' if 'allowlist' in file else 'blocklist' + e_uci = EUci() + e_uci.set('adblock', 'ns_lists', 'ns_lists') + + values = tuple( + f"{entry['address']} #{entry['description']}" if entry.get('description') else entry['address'] + for entry in local_list + ) + if values: + e_uci.set('adblock', 'ns_lists', option, values) + elif e_uci.get('adblock', 'ns_lists', option, list=True, default=[]): + e_uci.delete('adblock', 'ns_lists', option) + + e_uci.save('adblock') + return None + +def restart_adblock(): + try: + subprocess.run(["/etc/init.d/adblock", "restart"], capture_output=True, check=True) + except subprocess.CalledProcessError: + return generic_error("restart_failed") + return None def write_block_list(block_list): write_allow_list(block_list, '/etc/banip/banip.blocklist') @@ -120,17 +144,53 @@ def read_gz(file): else: return {} -def list_dns_feeds(enterprise=False): - # Decompress and read the JSON file /etc/adblock/combined.sources.gz - ret = {} - sources = '/etc/adblock/combined.sources.gz' - if not os.path.exists(sources): - ret = read_gz('/usr/share/threat_shield/community-dns.sources.gz') - if enterprise: - ret.update(read_gz('/usr/share/threat_shield/nethesis-dns.sources.gz')) +def read_json(file): + if os.path.exists(file) and os.path.getsize(file) > 0: + with open(file, 'r') as f: + try: + return json.load(f) + except Exception: + return {} + return {} + +def normalize_uci_list(value): + if value is None: + return [] + + if isinstance(value, str): + items = value.split() + elif isinstance(value, (list, tuple)): + items = [] + buffer = "" + for item in value: + item = str(item) + if len(item) == 1: + buffer += item + continue + if buffer: + items.extend(buffer.split()) + buffer = "" + items.append(item) + if buffer: + items.extend(buffer.split()) else: - ret = read_gz(sources) + items = [str(value)] + + normalized = [] + for item in items: + if item and item not in normalized: + normalized.append(item) + return normalized +def list_dns_feeds(enterprise=False): + # Read feeds from adblock.feeds (builtin) and adblock.custom.feeds (NethSecurity/user overrides) + ret = read_json('/etc/adblock/adblock.feeds') + custom = read_json('/etc/adblock/adblock.custom.feeds') + if custom: + ret.update(custom) + elif enterprise: + # fallback: merge nethesis sources gz if no custom feeds file yet + ret.update(read_gz('/usr/share/threat_shield/nethesis-dns.sources.gz')) return ret def get_confidence(f, enterprise=False): @@ -391,7 +451,7 @@ def dns_list_blocklist(e_uci): has_bl = has_bl_entitlement(e_uci) feeds = list_dns_feeds(has_bl) try: - enabled_feeds = list(e_uci.get_all('adblock', 'global', 'adb_sources')) + enabled_feeds = list(e_uci.get_all('adblock', 'global', 'adb_feed')) except: enabled_feeds = [] for f in feeds: @@ -413,7 +473,7 @@ def dns_list_blocklist(e_uci): def dns_edit_blocklist(e_uci, payload): try: - enabled = list(e_uci.get_all('adblock', 'global', 'adb_sources')) + enabled = list(e_uci.get_all('adblock', 'global', 'adb_feed')) except: enabled = [] if payload['enabled'] and payload['blocklist'] not in enabled: @@ -424,8 +484,11 @@ def dns_edit_blocklist(e_uci, payload): has_bl = has_bl_entitlement(e_uci) feeds = list_dns_feeds(has_bl) enabled = [feed for feed in enabled if feed in feeds] - e_uci.set('adblock', 'global', 'adb_sources', enabled) + e_uci.set('adblock', 'global', 'adb_feed', enabled) e_uci.save('adblock') + error = restart_adblock() + if error is not None: + return error return {'message': 'success'} def dns_list_zones(e_uci): @@ -438,11 +501,11 @@ def dns_list_zones(e_uci): def dns_list_settings(e_uci): ts_enabled = e_uci.get('adblock', 'global', 'ts_enabled', default='0') try: - zones = list(e_uci.get_all('adblock', 'global', 'adb_zonelist')) + zones = list(e_uci.get_all('adblock', 'global', 'adb_nftdevforce')) except: zones = ['lan'] try: - ports = list(e_uci.get_all('adblock', 'global', 'adb_portlist')) + ports = list(e_uci.get_all('adblock', 'global', 'adb_nftportforce')) except: ports = ['53', '853'] return { 'data': {'enabled': ts_enabled == '1', "zones": zones, "ports": ports} } @@ -453,65 +516,75 @@ def dns_edit_settings(e_uci, payload): raise ValidationError('zones', 'wan_zone_not_allowed', payload['zones']) e_uci.set('adblock', 'global', 'ts_enabled', '1') e_uci.set('adblock', 'global', 'adb_enabled', '1') - e_uci.set('adblock', 'global', 'adb_backup', '1') - e_uci.set('adblock', 'global', 'adb_forcedns', '1') - e_uci.set('adblock', 'global', 'adb_zonelist', payload.get('zones', ['lan'])) - e_uci.set('adblock', 'global', 'adb_portlist', payload.get('ports', ['53', '853'])) + e_uci.set('adblock', 'global', 'adb_nftforce', '1') + e_uci.set('adblock', 'global', 'adb_nftdevforce', payload.get('zones', ['lan'])) + e_uci.set('adblock', 'global', 'adb_nftportforce', payload.get('ports', ['53', '853'])) else: e_uci.set('adblock', 'global', 'ts_enabled', '0') e_uci.set('adblock', 'global', 'adb_enabled', '0') - e_uci.set('adblock', 'global', 'adb_forcedns', '0') + e_uci.set('adblock', 'global', 'adb_nftforce', '0') e_uci.save('adblock') + error = restart_adblock() + if error is not None: + return error return {'message': 'success'} def dns_list_allowed(): - return { "data": get_allow_list('/etc/adblock/adblock.whitelist') } + return { "data": dns_get_local_list('/etc/adblock/adblock.allowlist') } def dns_list_blocked(): - return { "data": get_allow_list('/etc/adblock/adblock.blacklist') } + return { "data": dns_get_local_list('/etc/adblock/adblock.blocklist') } def dns_add_allowed(payload): - cur = get_allow_list('/etc/adblock/adblock.whitelist') + cur = dns_get_local_list('/etc/adblock/adblock.allowlist') # extract address from cur list if payload['address'] in [x['address'] for x in cur]: raise ValidationError('address', 'address_already_present', payload['address']) cur.append({ "address": payload['address'], "description": payload['description'] }) - dns_write_local_list(cur, '/etc/adblock/adblock.whitelist') + error = dns_write_local_list(cur, '/etc/adblock/adblock.allowlist') + if error is not None: + return error return {'message': 'success'} def dns_add_blocked(payload): - cur = get_allow_list('/etc/adblock/adblock.blacklist') + cur = dns_get_local_list('/etc/adblock/adblock.blocklist') # extract address from cur list if payload['address'] in [x['address'] for x in cur]: raise ValidationError('address', 'address_already_present', payload['address']) cur.append({ "address": payload['address'], "description": payload.get('description') }) - dns_write_local_list(cur, '/etc/adblock/adblock.blacklist') + error = dns_write_local_list(cur, '/etc/adblock/adblock.blocklist') + if error is not None: + return error return {'message': 'success'} def dns_edit_allowed(payload): - cur = get_allow_list('/etc/adblock/adblock.whitelist') + cur = dns_get_local_list('/etc/adblock/adblock.allowlist') if payload['address'] not in [x['address'] for x in cur]: raise ValidationError('address', 'address_not_found', payload['address']) for i in range(len(cur)): if cur[i]['address'] == payload['address']: cur[i]['description'] = payload['description'] break - dns_write_local_list(cur, '/etc/adblock/adblock.whitelist') + error = dns_write_local_list(cur, '/etc/adblock/adblock.allowlist') + if error is not None: + return error return {'message': 'success'} def dns_edit_blocked(payload): - cur = get_allow_list('/etc/adblock/adblock.blacklist') + cur = dns_get_local_list('/etc/adblock/adblock.blocklist') if payload['address'] not in [x['address'] for x in cur]: raise ValidationError('address', 'address_not_found', payload['address']) for i in range(len(cur)): if cur[i]['address'] == payload['address']: cur[i]['description'] = payload.get('description') break - dns_write_local_list(cur, '/etc/adblock/adblock.blacklist') + error = dns_write_local_list(cur, '/etc/adblock/adblock.blocklist') + if error is not None: + return error return {'message': 'success'} def dns_delete_allowed(payload): - cur = get_allow_list('/etc/adblock/adblock.whitelist') + cur = dns_get_local_list('/etc/adblock/adblock.allowlist') if payload['address'] not in [x['address'] for x in cur]: raise ValidationError('address', 'address_not_found', payload['address']) # remove address from cur list @@ -519,11 +592,13 @@ def dns_delete_allowed(payload): if cur[i]['address'] == payload['address']: del cur[i] break - dns_write_local_list(cur, '/etc/adblock/adblock.whitelist') + error = dns_write_local_list(cur, '/etc/adblock/adblock.allowlist') + if error is not None: + return error return {'message': 'success'} def dns_delete_blocked(payload): - cur = get_allow_list('/etc/adblock/adblock.blacklist') + cur = dns_get_local_list('/etc/adblock/adblock.blocklist') if payload['address'] not in [x['address'] for x in cur]: raise ValidationError('address', 'address_not_found', payload['address']) # remove address from cur list @@ -531,39 +606,47 @@ def dns_delete_blocked(payload): if cur[i]['address'] == payload['address']: del cur[i] break - dns_write_local_list(cur, '/etc/adblock/adblock.blacklist') + error = dns_write_local_list(cur, '/etc/adblock/adblock.blocklist') + if error is not None: + return error return {'message': 'success'} def dns_list_bypass(e_uci): - # adblock.global.adb_bypass + # adblock.global.ns_tsdns_bypass try: - bypass = e_uci.get_all('adblock', 'global', 'adb_bypass') + bypass = normalize_uci_list(e_uci.get_all('adblock', 'global', 'ns_tsdns_bypass')) except: bypass = [] return { "data": bypass } def dns_add_bypass(e_uci, payload): try: - bypass = list(e_uci.get_all('adblock', 'global', 'adb_bypass')) + bypass = normalize_uci_list(e_uci.get_all('adblock', 'global', 'ns_tsdns_bypass')) except: bypass = [] if payload['address'] in bypass: raise ValidationError('address', 'address_already_present', payload['address']) bypass.append(payload['address']) - e_uci.set('adblock', 'global', 'adb_bypass', bypass) + e_uci.set('adblock', 'global', 'ns_tsdns_bypass', bypass) e_uci.save('adblock') + error = restart_adblock() + if error is not None: + return error return {'message': 'success'} def dns_delete_bypass(e_uci, payload): try: - bypass = list(e_uci.get_all('adblock', 'global', 'adb_bypass')) + bypass = normalize_uci_list(e_uci.get_all('adblock', 'global', 'ns_tsdns_bypass')) except: bypass = [] if payload['address'] not in bypass: raise ValidationError('address', 'address_not_found', payload['address']) bypass.remove(payload['address']) - e_uci.set('adblock', 'global', 'adb_bypass', bypass) + e_uci.set('adblock', 'global', 'ns_tsdns_bypass', bypass) e_uci.save('adblock') + error = restart_adblock() + if error is not None: + return error return {'message': 'success'} diff --git a/packages/ns-api/files/ns.update b/packages/ns-api/files/ns.update index b1fef000d..9121fdffd 100755 --- a/packages/ns-api/files/ns.update +++ b/packages/ns-api/files/ns.update @@ -86,7 +86,7 @@ def check_system_update(): response = requests.get(f"{url}/latest_release", headers={"Accept": "application/json"}, timeout=5) response.raise_for_status() version = response.text.strip() - if semver.compare(version, current_version) > 0: + if semver.Version.compare(version, current_version) > 0: data["lastVersion"] = f'NethSecurity {version}' except requests.exceptions.ConnectionError: return utils.generic_error("connection_error") diff --git a/packages/ns-api/openapi.yml b/packages/ns-api/openapi.yml index c5372c489..1110c7c31 100644 --- a/packages/ns-api/openapi.yml +++ b/packages/ns-api/openapi.yml @@ -79,6 +79,29 @@ components: items: $ref: "#/components/schemas/ValidationErrorDetail" + SuccessResponse: + type: object + required: [message] + properties: + message: + type: string + example: success + + ThreatShieldDnsListEntry: + type: object + required: + - address + - description + properties: + address: + type: string + description: Domain name present in the local Threat Shield DNS list + example: nethesis.it + description: + type: string + description: Optional free-form description associated with the domain + example: my allow1 + securitySchemes: BearerAuth: type: http @@ -188,3 +211,404 @@ paths: example: success - $ref: "#/components/schemas/ValidationError" - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-list-allowed: + post: + summary: List local Threat Shield DNS allowlist entries + description: Returns the allowlist entries currently staged in UCI. They are written to the adblock file on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-list-allowed + tags: + - threatshield + responses: + "200": + description: Staged allowlist entries + content: + application/json: + schema: + oneOf: + - type: object + required: + - data + properties: + data: + type: array + items: + $ref: "#/components/schemas/ThreatShieldDnsListEntry" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-add-allowed: + post: + summary: Add a local Threat Shield DNS allowlist entry + description: Stages the new allowlist entry in UCI. The physical adblock file is updated on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-add-allowed + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + - description + properties: + address: + type: string + description: Domain to add to the local allowlist + example: nethesis.it + description: + type: string + description: Free-form description for the domain + example: my allow1 + responses: + "200": + description: Allowlist entry staged or validation failed + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/SuccessResponse" + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-edit-allowed: + post: + summary: Edit a local Threat Shield DNS allowlist entry + description: Updates the staged allowlist description in UCI. The physical adblock file is updated on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-edit-allowed + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + - description + properties: + address: + type: string + description: Existing domain in the local allowlist + example: nethesis.it + description: + type: string + description: Updated description for the domain + example: my new desc + responses: + "200": + description: Allowlist entry updated or validation failed + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/SuccessResponse" + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-delete-allowed: + post: + summary: Delete a local Threat Shield DNS allowlist entry + description: Removes the staged allowlist entry from UCI. The physical adblock file is updated on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-delete-allowed + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + properties: + address: + type: string + description: Existing domain in the local allowlist + example: nethesis.it + responses: + "200": + description: Allowlist entry deleted or validation failed + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/SuccessResponse" + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-list-blocked: + post: + summary: List local Threat Shield DNS blocklist entries + description: Returns the blocklist entries currently staged in UCI. They are written to the adblock file on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-list-blocked + tags: + - threatshield + responses: + "200": + description: Staged blocklist entries + content: + application/json: + schema: + oneOf: + - type: object + required: + - data + properties: + data: + type: array + items: + $ref: "#/components/schemas/ThreatShieldDnsListEntry" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-add-blocked: + post: + summary: Add a local Threat Shield DNS blocklist entry + description: Stages the new blocklist entry in UCI. The physical adblock file is updated on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-add-blocked + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + properties: + address: + type: string + description: Domain to add to the local blocklist + example: nastydomain.net + description: + type: string + description: Optional free-form description for the domain + example: my block1 + responses: + "200": + description: Blocklist entry staged or validation failed + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/SuccessResponse" + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-edit-blocked: + post: + summary: Edit a local Threat Shield DNS blocklist entry + description: Updates the staged blocklist description in UCI. The physical adblock file is updated on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-edit-blocked + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + properties: + address: + type: string + description: Existing domain in the local blocklist + example: nastydomain.net + description: + type: string + description: Updated description for the domain + example: My new desc + responses: + "200": + description: Blocklist entry updated or validation failed + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/SuccessResponse" + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-delete-blocked: + post: + summary: Delete a local Threat Shield DNS blocklist entry + description: Removes the staged blocklist entry from UCI. The physical adblock file is updated on the next reload triggered by `ns.commit` or `reload_config`. + operationId: ns.threatshield.dns-delete-blocked + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + properties: + address: + type: string + description: Existing domain in the local blocklist + example: nastydomain.net + responses: + "200": + description: Blocklist entry deleted or validation failed + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/SuccessResponse" + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-edit-blocklist: + post: + summary: Enable or disable a Threat Shield DNS blocklist + operationId: ns.threatshield.dns-edit-blocklist + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - blocklist + - enabled + properties: + blocklist: + type: string + description: DNS blocklist name + example: adguard + enabled: + type: boolean + description: Whether the blocklist should be enabled + example: true + responses: + "200": + description: Blocklist updated or validation failed + content: + application/json: + schema: + oneOf: + - type: object + required: + - message + properties: + message: + type: string + example: success + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-edit-settings: + post: + summary: Update Threat Shield DNS enforcement settings + operationId: ns.threatshield.dns-edit-settings + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - enabled + properties: + enabled: + type: boolean + description: Enable or disable Threat Shield DNS + example: true + zones: + type: array + description: Firewall zones where DNS redirection is enforced + items: + type: string + example: [lan] + ports: + type: array + description: DNS ports enforced locally + items: + type: string + example: ["53", "853"] + responses: + "200": + description: Settings updated or validation failed + content: + application/json: + schema: + oneOf: + - type: object + required: + - message + properties: + message: + type: string + example: success + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-add-bypass: + post: + summary: Add a Threat Shield DNS bypass entry + operationId: ns.threatshield.dns-add-bypass + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + properties: + address: + type: string + description: Source IP address or subnet that should bypass DNS redirection + example: 192.168.1.22 + responses: + "200": + description: Bypass entry added or validation failed + content: + application/json: + schema: + oneOf: + - type: object + required: + - message + properties: + message: + type: string + example: success + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" + POST /ubus/ns.threatshield/dns-delete-bypass: + post: + summary: Delete a Threat Shield DNS bypass entry + operationId: ns.threatshield.dns-delete-bypass + tags: + - threatshield + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - address + properties: + address: + type: string + description: Source IP address or subnet to remove from the DNS bypass list + example: 192.168.1.22 + responses: + "200": + description: Bypass entry removed or validation failed + content: + application/json: + schema: + oneOf: + - type: object + required: + - message + properties: + message: + type: string + example: success + - $ref: "#/components/schemas/ValidationError" + - $ref: "#/components/schemas/Error" diff --git a/packages/ns-migration/files/scripts/openvpn b/packages/ns-migration/files/scripts/openvpn index 06908f81a..09cf6a1b2 100755 --- a/packages/ns-migration/files/scripts/openvpn +++ b/packages/ns-migration/files/scripts/openvpn @@ -62,8 +62,7 @@ def add_tap_to_bridge(u, bridge, interface): u.commit("network") except: pass - finally: - return + return iname="ns_roadwarrior1" diff --git a/packages/ns-threat_shield/Makefile b/packages/ns-threat_shield/Makefile index e7cfbd8b7..2b66694bf 100644 --- a/packages/ns-threat_shield/Makefile +++ b/packages/ns-threat_shield/Makefile @@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=ns-threat_shield -PKG_VERSION:=0.0.8 +PKG_VERSION:=0.0.9 PKG_RELEASE:=1 PKG_BUILD_DIR:=$(BUILD_DIR)/ns-threat_shield-$(PKG_VERSION) @@ -22,7 +22,8 @@ define Package/ns-threat_shield CATEGORY:=NethSecurity TITLE:=Threat shield block list URL:=https://github.com/NethServer/nethsecurity/ - DEPENDS:=+wget-ssl +adblock +jq + DEPENDS:=+wget-ssl +adblock +jq +ns-api + EXTRA_DEPENDS:=adblock (>=4.5.3), ns-api (>=3.6.2) PKGARCH:=all endef diff --git a/packages/ns-threat_shield/README.md b/packages/ns-threat_shield/README.md index cc8ace162..6d33c4052 100644 --- a/packages/ns-threat_shield/README.md +++ b/packages/ns-threat_shield/README.md @@ -82,16 +82,11 @@ uci commit adblock /etc/init.d/adblock restart ``` -### Custom categories - -To add custom categories, create a file `/etc/adblock/custom.sources.gz` with the list of categories to block. -If such file is present, the `/usr/share/threat_shield/community-dns.sources.gz` will be ignored. - ### DNS redirect bypass -Allow bypass of DNS redirect for a specific source IP: +Allow bypass of DNS redirect for a specific source IP or subnet: ``` -uci add_list adblock.global.adb_bypass=192.168.100.2 +uci add_list adblock.global.ns_tsdns_bypass=192.168.100.2 uci commit adblock /etc/init.d/adblock restart ``` diff --git a/packages/ns-threat_shield/files/ts-dns b/packages/ns-threat_shield/files/ts-dns index f67484be0..01f6399f6 100755 --- a/packages/ns-threat_shield/files/ts-dns +++ b/packages/ns-threat_shield/files/ts-dns @@ -8,8 +8,8 @@ DEST_DIR=/etc/adblock NETHESIS_SOURCES=/usr/share/threat_shield/nethesis-dns.sources.gz COMMUNITY_SOURCES=/usr/share/threat_shield/community-dns.sources.gz -CUSTOM_SOURCES=/etc/adblock/custom.sources.gz -TMP_FILE=/tmp/combined.sources +CUSTOM_FEEDS="$DEST_DIR/adblock.custom.feeds" +TMP_FILE=/tmp/ts-dns.sources SYSTEM_ID=$(uci -q get ns-plug.config.system_id) SYSTEM_SECRET=$(uci -q get ns-plug.config.secret) @@ -17,42 +17,28 @@ TYPE=$(uci -q get ns-plug.config.type) TS_ENABLED=$(uci -q get adblock.global.ts_enabled) if [ "$TS_ENABLED" = 1 ]; then - # Setup new blacklist source - uci set adblock.global.adb_srcarc="$DEST_DIR"/combined.sources.gz - # Setup dnsmasq as backend uci set adblock.global.adb_dns='dnsmasq' uci set adblock.global.adb_dnsinstance='0' - # Setup wget with compression support - uci set adblock.global.adb_fetchutil='wget' - uci set adblock.global.adb_fetchparm="--compression=gzip --no-cache --no-cookies --max-redirect=0 --timeout=20 -O" - - if [ -f $CUSTOM_SOURCES ]; then - # Add custom sources, if present - gunzip -c $CUSTOM_SOURCES > $TMP_FILE - else - # Use community sources - gunzip -c $COMMUNITY_SOURCES > $TMP_FILE - fi + # Build custom feeds from community sources + gunzip -c "$COMMUNITY_SOURCES" > "$TMP_FILE" - # Setup Nethesis sources if the machine has a subscription - if [ ! -z "$SYSTEM_SECRET" ] && [ ! -z "$SYSTEM_ID" ]; then - # Replaces credentials and compress - gunzip -c $NETHESIS_SOURCES | sed -e "s/__USER__/$SYSTEM_ID/" -e "s/__PASSWORD__/$SYSTEM_SECRET/" -e "s/__TYPE__/$TYPE/" >> $TMP_FILE + # Merge Nethesis sources if the machine has a subscription + if [ -n "$SYSTEM_SECRET" ] && [ -n "$SYSTEM_ID" ]; then + gunzip -c "$NETHESIS_SOURCES" | sed -e "s/__USER__/$SYSTEM_ID/" -e "s/__PASSWORD__/$SYSTEM_SECRET/" -e "s/__TYPE__/$TYPE/" >> "$TMP_FILE" fi - # Merge the source list and compress it to the final file - jq -s 'reduce .[] as $item ({}; . * $item)' $TMP_FILE | gzip -c > "$DEST_DIR"/combined.sources.gz + # Merge all sources into a single JSON object and write to adblock.custom.feeds + jq -s 'reduce .[] as $item ({}; . * $item)' "$TMP_FILE" > "$CUSTOM_FEEDS" # Cleanup - rm -f $TMP_FILE + rm -f "$TMP_FILE" else - # Reset sources to origin file if threat shield is not enabled - uci -q delete adblock.global.adb_srcarc - uci -q delete adblock.global.adb_fetchparam - uci set adblock.global.adb_fetchutil='curl' + # Clear custom feeds when threat shield is disabled + : > "$CUSTOM_FEEDS" fi # Save changes uci commit adblock + diff --git a/packages/python-jinja2/Makefile b/packages/python-jinja2/Makefile deleted file mode 100644 index f29df71ec..000000000 --- a/packages/python-jinja2/Makefile +++ /dev/null @@ -1,50 +0,0 @@ -# This is free software, licensed under the GNU General Public License v2. -# See /LICENSE for more information. -# - -include $(TOPDIR)/rules.mk - -PKG_NAME:=python-jinja2 -PKG_VERSION:=3.1.2 -PKG_RELEASE:=2 - -PYPI_NAME:=Jinja2 -PYTHON3_PKG_WHEEL_NAME:=jinja2 -PKG_HASH:=31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 - -PKG_MAINTAINER:=Michal Vasilek -PKG_LICENSE:=BSD-3-Clause -PKG_LICENSE_FILES:=LICENSE.rst -PKG_CPE_ID:=cpe:/a:pocoo:jinja2 -HOST_BUILD_DEPENDS:= python-markupsafe/host - -include $(TOPDIR)/feeds/packages/lang/python/pypi.mk -include $(INCLUDE_DIR)/package.mk -include $(INCLUDE_DIR)/host-build.mk -include $(TOPDIR)/feeds/packages/lang/python/python3-package.mk -include $(TOPDIR)/feeds/packages/lang/python/python3-host-build.mk - -define Package/python3-jinja2 - SECTION:=lang - CATEGORY:=Languages - SUBMENU:=Python - TITLE:=Very fast and expressive template engine - URL:=https://palletsprojects.com/p/jinja/ - DEPENDS:= \ - +python3-light \ - +python3-asyncio \ - +python3-logging \ - +python3-urllib \ - +python3-markupsafe -endef - -define Package/python3-jinja2/description -Jinja2 is a full featured template engine for Python. It has full -unicode support, an optional integrated sandboxed execution -environment, widely used and BSD licensed. -endef - -$(eval $(call Py3Package,python3-jinja2)) -$(eval $(call BuildPackage,python3-jinja2)) -$(eval $(call BuildPackage,python3-jinja2-src)) -$(eval $(call HostBuild)) \ No newline at end of file diff --git a/packages/python-semver/Makefile b/packages/python-semver/Makefile index 9132e194e..4affdbf63 100644 --- a/packages/python-semver/Makefile +++ b/packages/python-semver/Makefile @@ -5,11 +5,11 @@ include $(TOPDIR)/rules.mk PKG_NAME:=python-semver -PKG_VERSION:=2.13.0 +PKG_VERSION:=3.0.4 PKG_RELEASE:=1 PYPI_NAME:=semver -PKG_HASH:=fa0fe2722ee1c3f57eac478820c3a5ae2f624af8264cbdf9000c980ff7f75e3f +PKG_HASH:=afc7d8c584a5ed0a11033af086e8af226a9c0b206f313e0301f8dd7b6b589602 PKG_MAINTAINER:=Tommaso Bailetti PKG_LICENSE:=BSD-3-Clause diff --git a/packages/telegraf/Makefile b/packages/telegraf/Makefile new file mode 100644 index 000000000..e63822acb --- /dev/null +++ b/packages/telegraf/Makefile @@ -0,0 +1,90 @@ +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +include $(TOPDIR)/rules.mk + +PKG_NAME:=telegraf +# renovate: datasource=github-tags depName=influxdata/telegraf +PKG_VERSION:=1.38.3 +PKG_RELEASE:=1 + +PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz +PKG_SOURCE_URL:=https://codeload.github.com/influxdata/telegraf/tar.gz/v$(PKG_VERSION)? +PKG_SOURCE_SUBDIR:=telegraf-$(PKG_VERSION) +PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) + +PKG_HASH:=skip +PKG_MAINTAINER:=Tommaso Bailetti +PKG_LICENSE:=MIT + +PKG_BUILD_DEPENDS:=golang/host +PKG_BUILD_PARALLEL:=1 +PKG_BUILD_FLAGS:=no-mips16 + +GO_PKG:=github.com/influxdata/telegraf/cmd/$(PKG_NAME) +GO_BUILD_PKG:=github.com/influxdata/telegraf/cmd/$(PKG_NAME) +GO_PKG_LDFLAGS_X:=github.com/influxdata/telegraf/internal.Version=$(PKG_VERSION) +GO_PKG_TAGS:= \ + custom \ + inputs.bond \ + inputs.cpu \ + inputs.disk \ + inputs.ethtool \ + inputs.mem \ + inputs.net \ + inputs.netstat \ + inputs.nstat \ + inputs.processes \ + inputs.sensors \ + inputs.system \ + outputs.influxdb \ + outputs.prometheus_client + +include $(INCLUDE_DIR)/package.mk +include $(TOPDIR)/feeds/packages/lang/golang/golang-package.mk + +define Package/telegraf + SECTION:=base + CATEGORY:=NethServer + TITLE:=Telegraf + URL:=https://github.com/influxdata/telegraf + DEPENDS:= \ + $(GO_ARCH_DEPENDS) \ + +lm-sensors \ + +victoria-metrics \ + +python3-uci \ + +python3-jinja2 +endef + +define Package/telegraf/description + Telegraf is an agent for collecting, processing, aggregating, and writing metrics. +endef + +define Package/telegraf/conffiles +/etc/config/telegraf +endef + +define Package/telegraf/install + $(call GoPackage/Package/Install/Bin,$(1)) + $(INSTALL_DIR) $(1)/etc/init.d + $(INSTALL_BIN) ./files/telegraf.initd $(1)/etc/init.d/telegraf + $(INSTALL_DIR) $(1)/etc/config + $(INSTALL_CONF) ./files/telegraf.uci $(1)/etc/config/telegraf + $(INSTALL_DIR) $(1)/etc/telegraf + $(INSTALL_DATA) ./files/telegraf.conf $(1)/etc/telegraf.conf + $(INSTALL_DIR) $(1)/etc/telegraf.conf.d + $(INSTALL_DATA) ./files/telegraf.conf.d/os.conf $(1)/etc/telegraf.conf.d/os.conf + $(INSTALL_DIR) $(1)/usr/sbin + $(INSTALL_BIN) ./files/telegraf-config $(1)/usr/sbin/telegraf-config +endef + +define Package/telegraf/postinst +#!/bin/sh +[ -z "$${IPKG_INSTROOT}" ] && /etc/init.d/telegraf restart +exit 0 +endef + +$(eval $(call GoBinPackage,telegraf)) +$(eval $(call BuildPackage,telegraf)) diff --git a/packages/telegraf/files/telegraf-config b/packages/telegraf/files/telegraf-config new file mode 100644 index 000000000..235476d86 --- /dev/null +++ b/packages/telegraf/files/telegraf-config @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +import os +import subprocess +import glob +from jinja2 import Environment, BaseLoader +from euci import EUci + +PROMETHEUS_TEMPLATE = """# This file is automatically generated by /usr/sbin/telegraf-config. +# Do not edit manually — changes will be overwritten. + +# Prometheus client output +{% if enabled == '1' -%} +[[outputs.prometheus_client]] + ## Address to listen on. + ## ex: + ## listen = ":9273" + ## listen = "vsock://:9273" + listen = "{{ listen_addr }}" + + ## Maximum duration before timing out read of the request + # read_timeout = "10s" + ## Maximum duration before timing out write of the response + # write_timeout = "10s" + + ## Metric version controls the mapping from Prometheus metrics into Telegraf metrics. + ## See "Metric Format Configuration" in plugins/inputs/prometheus/README.md for details. + ## Valid options: 1, 2 + # metric_version = 1 + + ## Use HTTP Basic Authentication. +{%- if basic_auth_username and basic_auth_password %} + basic_username = "{{ basic_auth_username }}" + basic_password = "{{ basic_auth_password }}" +{%- else %} + # basic_username = "Foo" + # basic_password = "Bar" +{%- endif %} + + ## If set, the IP Ranges which are allowed to access metrics. + ## ex: ip_range = ["192.168.0.0/24", "192.168.1.0/30"] + # ip_range = [] + + ## Path to publish the metrics on. + # path = "/metrics" + + ## Expiration interval for each metric. 0 == no expiration + # expiration_interval = "60s" + + ## Collectors to enable, valid entries are "gocollector" and "process". + ## If unset, both are enabled. + # collectors_exclude = ["gocollector", "process"] + + ## Send string metrics as Prometheus labels. + ## Unless set to false all string metrics will be sent as labels. + # string_as_label = true + + ## If set, enable TLS with the given certificate. + # tls_cert = "/etc/ssl/telegraf.crt" + # tls_key = "/etc/ssl/telegraf.key" + + ## Set one or more allowed client CA certificate file names to + ## enable mutually authenticated TLS connections + # tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"] + + ## Export metric collection time. + # export_timestamp = false + + ## Specify the metric type explicitly. + ## This overrides the metric-type of the Telegraf metric. Globbing is allowed. + # [outputs.prometheus_client.metric_types] + # counter = [] + # gauge = [] +{%- endif %} +""" + +SENSORS_TEMPLATE = """# This file is automatically generated by /usr/sbin/telegraf-config. +# Do not edit manually — changes will be overwritten. + +# Monitor sensors, requires lm-sensors package +# This plugin ONLY supports Linux +{% if sensors_available -%} +[[inputs.sensors]] + ## Remove numbers from field names. + ## If true, a field name like 'temp1_input' will be changed to 'temp_input'. + # remove_numbers = true + + ## Timeout is the maximum amount of time that the sensors command can run. + # timeout = "5s" + [inputs.sensors.tags] + influxdb_db = "os-metrics" +{%- endif %} +""" + + +def generate_prometheus_config(): + """Read UCI config and render Prometheus client output section.""" + e_uci = EUci() + + try: + enabled = e_uci.get('telegraf', 'output_prometheus', 'enabled', dtype=str, default='0') + listen_addr = e_uci.get('telegraf', 'output_prometheus', 'listen_addr', dtype=str, default=':9273') + basic_auth_username = e_uci.get('telegraf', 'output_prometheus', 'basic_auth_username', dtype=str, default='') + basic_auth_password = e_uci.get('telegraf', 'output_prometheus', 'basic_auth_password', dtype=str, default='') + except Exception as e: + print(f"Error reading UCI config: {e}") + return False + + # Render template + template = Environment(loader=BaseLoader()).from_string(PROMETHEUS_TEMPLATE) + rendered = template.render( + enabled=enabled, + listen_addr=listen_addr, + basic_auth_username=basic_auth_username, + basic_auth_password=basic_auth_password + ) + + # Write to drop-in directory + config_file = '/etc/telegraf.conf.d/prometheus.conf' + os.makedirs(os.path.dirname(config_file), exist_ok=True) + + try: + with open(config_file, 'w') as f: + f.write(rendered) + return True + except Exception as e: + print(f"Error writing config file: {e}") + return False + + +def sensors_available(): + """Check if sensors command works by running it with -A -u flags.""" + try: + result = subprocess.run(['sensors', '-A', '-u'], capture_output=True, timeout=5) + return result.returncode == 0 + except (subprocess.TimeoutExpired, FileNotFoundError): + return False + + +def generate_sensors_config(): + """Render sensors input section based on system availability.""" + has_sensors = sensors_available() + + # Render template + template = Environment(loader=BaseLoader()).from_string(SENSORS_TEMPLATE) + rendered = template.render(sensors_available=has_sensors) + + # Write to drop-in directory + config_file = '/etc/telegraf.conf.d/sensors.conf' + os.makedirs(os.path.dirname(config_file), exist_ok=True) + + try: + with open(config_file, 'w') as f: + f.write(rendered) + return True + except Exception as e: + print(f"Error writing config file: {e}") + return False + + +if __name__ == '__main__': + generate_prometheus_config() + generate_sensors_config() + exit(0) diff --git a/packages/telegraf/files/telegraf.conf b/packages/telegraf/files/telegraf.conf new file mode 100644 index 000000000..90f4b4061 --- /dev/null +++ b/packages/telegraf/files/telegraf.conf @@ -0,0 +1,24 @@ +# Telegraf Configuration +# Managed by NethSecurity — do not edit manually. + +[global_tags] + +[agent] + interval = "10s" + round_interval = true + metric_batch_size = 1000 + metric_buffer_limit = 10000 + collection_jitter = "0s" + flush_interval = "10s" + flush_jitter = "0s" + precision = "0s" + omit_hostname = true + skip_processors_after_aggregators = true + +# Victoria metrics output plugin configuration +[[outputs.influxdb]] + urls = ["http://127.0.0.1:8428"] + database = "nethsecurity" + database_tag = "influxdb_db" + exclude_database_tag = true + content_encoding = "gzip" diff --git a/packages/telegraf/files/telegraf.conf.d/os.conf b/packages/telegraf/files/telegraf.conf.d/os.conf new file mode 100644 index 000000000..04f84000e --- /dev/null +++ b/packages/telegraf/files/telegraf.conf.d/os.conf @@ -0,0 +1,60 @@ +# OS and system metrics collection +# Includes CPU, memory, disk, network, and kernel statistics +# All metrics from this section are tagged with influxdb_db=os-metrics + +[[inputs.cpu]] + percpu = true + totalcpu = true + collect_cpu_time = false + report_active = false + core_tags = false + [inputs.cpu.tags] + influxdb_db = "os-metrics" + +[[inputs.disk]] + ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"] + [inputs.disk.tags] + influxdb_db = "os-metrics" + +[[inputs.mem]] + [inputs.mem.tags] + influxdb_db = "os-metrics" + +[[inputs.processes]] + [inputs.processes.tags] + influxdb_db = "os-metrics" + +[[inputs.system]] + [inputs.system.tags] + influxdb_db = "os-metrics" + +[[inputs.bond]] + [inputs.bond.tags] + influxdb_db = "os-metrics" + +[[inputs.ethtool]] + ## List of interfaces to include (default: all up interfaces) + # interface_include = ["eth0"] + ## List of interfaces to ignore (default: none) + interface_exclude = ["wg*", "ipsec*", "tun*", "br*"] + + [inputs.ethtool.tags] + influxdb_db = "os-metrics" + +[[inputs.net]] + ## Skip protocol stats, use inputs.nstat instead (fixes deprecation warning) + ignore_protocol_stats = true + [inputs.net.tags] + influxdb_db = "os-metrics" + +[[inputs.netstat]] + [inputs.netstat.tags] + influxdb_db = "os-metrics" + +[[inputs.nstat]] + proc_net_netstat = "/proc/net/netstat" + proc_net_snmp = "/proc/net/snmp" + proc_net_snmp6 = "/proc/net/snmp6" + dump_zeros = true + [inputs.nstat.tags] + influxdb_db = "os-metrics" diff --git a/packages/telegraf/files/telegraf.initd b/packages/telegraf/files/telegraf.initd new file mode 100644 index 000000000..49197475e --- /dev/null +++ b/packages/telegraf/files/telegraf.initd @@ -0,0 +1,36 @@ +#!/bin/sh /etc/rc.common + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# shellcheck disable=SC3043 + +START=99 +USE_PROCD=1 + +PROG="/usr/bin/telegraf" + +start_service() { + /usr/sbin/telegraf-config + procd_open_instance + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_set_param respawn 3600 5 0 + procd_set_param command $PROG + procd_append_param command --watch-config notify + procd_append_param command --config /etc/telegraf.conf + procd_append_param command --config-directory /etc/telegraf.conf.d + procd_close_instance +} + +reload_service() +{ + /usr/sbin/telegraf-config +} + +service_triggers() +{ + procd_add_reload_trigger telegraf +} diff --git a/packages/telegraf/files/telegraf.uci b/packages/telegraf/files/telegraf.uci new file mode 100644 index 000000000..d1225d452 --- /dev/null +++ b/packages/telegraf/files/telegraf.uci @@ -0,0 +1,5 @@ +config output_prometheus 'output_prometheus' + option enabled '0' + option listen_addr ':9273' + option basic_auth_username '' + option basic_auth_password '' diff --git a/packages/victoria-logs/Makefile b/packages/victoria-logs/Makefile new file mode 100644 index 000000000..813fe68a3 --- /dev/null +++ b/packages/victoria-logs/Makefile @@ -0,0 +1,76 @@ +include $(TOPDIR)/rules.mk + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +PKG_NAME:=victoria-logs +# renovate: datasource=github-tags depName=VictoriaMetrics/VictoriaLogs +PKG_VERSION:=1.49.0 +PKG_RELEASE:=1 + +PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz +PKG_SOURCE_URL:=https://codeload.github.com/VictoriaMetrics/VictoriaLogs/tar.gz/v$(PKG_VERSION)? +PKG_SOURCE_SUBDIR:=VictoriaLogs-$(PKG_VERSION) +PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) + +PKG_HASH:=skip +PKG_MAINTAINER:=Tommaso Bailetti +PKG_LICENSE:=Apache-2.0 + +PKG_BUILD_DEPENDS:=golang/host +PKG_BUILD_PARALLEL:=1 +PKG_BUILD_FLAGS:=no-mips16 + +GO_PKG:=github.com/VictoriaMetrics/VictoriaLogs +GO_PKG_BUILD_PKG:=github.com/VictoriaMetrics/VictoriaLogs/app/victoria-logs \ + github.com/VictoriaMetrics/VictoriaLogs/app/vlogscli +GO_PKG_LDFLAGS:= \ + -extldflags \ + -static +GO_PKG_LDFLAGS_X:=github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo.Version=$(PKG_NAME)-v$(PKG_VERSION) +GO_PKG_TAGS:= \ + netgo \ + osusergo \ + musl + +include $(INCLUDE_DIR)/package.mk +include $(TOPDIR)/feeds/packages/lang/golang/golang-package.mk + +define Package/victoria-logs + SECTION:=base + CATEGORY:=NethServer + TITLE:=Victoria Logs + URL:=https://github.com/VictoriaMetrics/VictoriaLogs + DEPENDS:=$(GO_ARCH_DEPENDS) +rsyslog +endef + +define Package/victoria-logs/description + VictoriaLogs — fast and easy-to-use database for logs. +endef + +define Package/victoria-logs/conffiles +/etc/config/victoria-logs +endef + +define Package/victoria-logs/install + $(call GoPackage/Package/Install/Bin,$(1)) + $(INSTALL_DIR) $(1)/etc/init.d + $(INSTALL_BIN) ./files/victoria-logs.initd $(1)/etc/init.d/victoria-logs + $(INSTALL_DIR) $(1)/etc/config + $(INSTALL_DATA) ./files/victoria-logs.conf $(1)/etc/config/victoria-logs + $(INSTALL_DIR) $(1)/etc/rsyslog.d + $(INSTALL_CONF) ./files/rsyslog-victoria-logs.conf $(1)/etc/rsyslog.d/victoria-logs.conf + $(INSTALL_DIR) $(1)/etc/uci-defaults + $(INSTALL_BIN) ./files/35_victoria-logs $(1)/etc/uci-defaults/35_victoria-logs +endef + +define Package/victoria-logs/postinst +#!/bin/sh +[ -z "$${IPKG_INSTROOT}" ] && /etc/init.d/victoria-logs restart +exit 0 +endef + +$(eval $(call GoPackage,victoria-logs)) +$(eval $(call BuildPackage,victoria-logs)) diff --git a/packages/victoria-logs/files/35_victoria-logs b/packages/victoria-logs/files/35_victoria-logs new file mode 100644 index 000000000..079e1a4eb --- /dev/null +++ b/packages/victoria-logs/files/35_victoria-logs @@ -0,0 +1,14 @@ +#!/bin/sh + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +RSYSLOG_CONF="/etc/rsyslog.d/victoria-logs.conf" + +# Register the victoria-logs rsyslog drop-in in rsyslog UCI includes +if ! uci -q get rsyslog.syslog.includes | grep -qF "${RSYSLOG_CONF}"; then + uci add_list rsyslog.syslog.includes="${RSYSLOG_CONF}" + uci commit rsyslog +fi diff --git a/packages/victoria-logs/files/rsyslog-victoria-logs.conf b/packages/victoria-logs/files/rsyslog-victoria-logs.conf new file mode 100644 index 000000000..c2a66de3b --- /dev/null +++ b/packages/victoria-logs/files/rsyslog-victoria-logs.conf @@ -0,0 +1,18 @@ +# Rsyslog configuration for VictoriaLogs + +ruleset(name="victoria-logs") { + *.* action( + type="omfwd" + target="127.0.0.1" + port="5514" + protocol="tcp" + TCP_Framing="octet-counted" + Template="RSYSLOG_SyslogProtocol23Format" + + action.resumeRetryCount="-1" + queue.type="linkedList" + queue.size="10000" + ) +} + +*.* call victoria-logs diff --git a/packages/victoria-logs/files/victoria-logs.conf b/packages/victoria-logs/files/victoria-logs.conf new file mode 100644 index 000000000..84aba8dbe --- /dev/null +++ b/packages/victoria-logs/files/victoria-logs.conf @@ -0,0 +1,4 @@ +config victorialogs 'main' + option storage_path '/var/lib/victoria-logs' + option max_disk_usage '50MB' + option http_listen_addr '127.0.0.1:9428' diff --git a/packages/victoria-logs/files/victoria-logs.initd b/packages/victoria-logs/files/victoria-logs.initd new file mode 100644 index 000000000..820ae7877 --- /dev/null +++ b/packages/victoria-logs/files/victoria-logs.initd @@ -0,0 +1,43 @@ +#!/bin/sh /etc/rc.common + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# shellcheck disable=SC3043 + +START=99 +USE_PROCD=1 + +PROG="/usr/bin/victoria-logs" + +start_service() { + config_load victoria-logs + local storage_path max_disk_usage http_listen_addr + config_get storage_path main storage_path /var/lib/victoria-logs + config_get max_disk_usage main max_disk_usage 50MB + config_get http_listen_addr main http_listen_addr 127.0.0.1:9428 + + procd_open_instance + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_set_param respawn 3600 5 0 + procd_set_param command $PROG + procd_append_param command -storageDataPath="$storage_path" + procd_append_param command -retention.maxDiskSpaceUsageBytes="$max_disk_usage" + procd_append_param command -httpListenAddr="$http_listen_addr" + procd_append_param command -syslog.listenAddr.tcp=127.0.0.1:5514 + procd_close_instance +} + +service_triggers() +{ + procd_add_reload_trigger victoria-logs +} + +reload_service() +{ + stop + start +} diff --git a/packages/victoria-metrics/Makefile b/packages/victoria-metrics/Makefile new file mode 100644 index 000000000..5fe85f5a8 --- /dev/null +++ b/packages/victoria-metrics/Makefile @@ -0,0 +1,72 @@ +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +include $(TOPDIR)/rules.mk + +PKG_NAME:=victoria-metrics +# renovate: datasource=github-tags depName=VictoriaMetrics/VictoriaMetrics +PKG_VERSION:=1.139.0 +PKG_RELEASE:=1 + +PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz +PKG_SOURCE_URL:=https://codeload.github.com/VictoriaMetrics/VictoriaMetrics/tar.gz/v$(PKG_VERSION)? +PKG_SOURCE_SUBDIR:=VictoriaMetrics-$(PKG_VERSION) +PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) + +PKG_HASH:=skip +PKG_MAINTAINER:=Tommaso Bailetti +PKG_LICENSE:=Apache-2.0 + +PKG_BUILD_DEPENDS:=golang/host +PKG_BUILD_PARALLEL:=1 +PKG_BUILD_FLAGS:=no-mips16 + +GO_PKG:=github.com/VictoriaMetrics/VictoriaMetrics +GO_PKG_BUILD_PKG:=github.com/VictoriaMetrics/VictoriaMetrics/app/victoria-metrics \ + github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert +GO_PKG_LDFLAGS:= \ + -extldflags \ + -static +GO_PKG_LDFLAGS_X:=github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo.Version=$(PKG_NAME)-v$(PKG_VERSION) +GO_PKG_TAGS:= \ + netgo \ + osusergo \ + musl + +include $(INCLUDE_DIR)/package.mk +include $(TOPDIR)/feeds/packages/lang/golang/golang-package.mk + +define Package/victoria-metrics + SECTION:=base + CATEGORY:=NethServer + TITLE:=Victoria Metrics + URL:=https://github.com/VictoriaMetrics/VictoriaMetrics + DEPENDS:=$(GO_ARCH_DEPENDS) +endef + +define Package/victoria-metrics/description + VictoriaMetrics time series database / single-node server. +endef + +define Package/victoria-metrics/conffiles +/etc/config/victoria-metrics +endef + +define Package/victoria-metrics/install + $(call GoPackage/Package/Install/Bin,$(1)) + $(INSTALL_DIR) $(1)/etc/init.d + $(INSTALL_BIN) ./files/victoria-metrics.initd $(1)/etc/init.d/victoria-metrics + $(INSTALL_DIR) $(1)/etc/config + $(INSTALL_DATA) ./files/victoria-metrics.conf $(1)/etc/config/victoria-metrics +endef + +define Package/victoria-metrics/postinst +#!/bin/sh +[ -z "$${IPKG_INSTROOT}" ] && /etc/init.d/victoria-metrics restart +exit 0 +endef + +$(eval $(call GoBinPackage,victoria-metrics)) +$(eval $(call BuildPackage,victoria-metrics)) \ No newline at end of file diff --git a/packages/victoria-metrics/files/victoria-metrics.conf b/packages/victoria-metrics/files/victoria-metrics.conf new file mode 100644 index 000000000..5b6813a89 --- /dev/null +++ b/packages/victoria-metrics/files/victoria-metrics.conf @@ -0,0 +1,4 @@ +config victoriametrics 'main' + option storage_path '/var/lib/victoriametrics' + option retention_period '1d' + option http_listen_addr '127.0.0.1:8428' diff --git a/packages/victoria-metrics/files/victoria-metrics.initd b/packages/victoria-metrics/files/victoria-metrics.initd new file mode 100644 index 000000000..fd2cee851 --- /dev/null +++ b/packages/victoria-metrics/files/victoria-metrics.initd @@ -0,0 +1,42 @@ +#!/bin/sh /etc/rc.common + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# shellcheck disable=SC3043 + +START=99 +USE_PROCD=1 + +PROG="/usr/bin/victoria-metrics" + +start_service() { + config_load victoria-metrics + local storage_path retention_period http_listen_addr + config_get storage_path main storage_path /var/lib/victoriametrics + config_get retention_period main retention_period 1d + config_get http_listen_addr main http_listen_addr 127.0.0.1:8428 + + procd_open_instance + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_set_param respawn 3600 5 0 + procd_set_param command $PROG + procd_append_param command -storageDataPath="$storage_path" + procd_append_param command -retentionPeriod="$retention_period" + procd_append_param command -httpListenAddr="$http_listen_addr" + procd_close_instance +} + +service_triggers() +{ + procd_add_reload_trigger victoria-metrics +} + +reload_service() +{ + stop + start +} diff --git a/patches/package/100-pppd-edit-bcopy.patch b/patches/package/100-pppd-edit-bcopy.patch deleted file mode 100644 index 2874da9fa..000000000 --- a/patches/package/100-pppd-edit-bcopy.patch +++ /dev/null @@ -1,17 +0,0 @@ -diff --git a/network/services/ppp/patches/200-use-memmove-for-bcopy.patch b/network/services/ppp/patches/200-use-memmove-for-bcopy.patch -new file mode 100644 -index 0000000000..1632eb27b8 ---- /dev/null -+++ b/package/network/services/ppp/patches/200-use-memmove-for-bcopy.patch -@@ -0,0 +1,11 @@ -+--- a/pppd/pppd-private.h -++++ b/pppd/pppd-private.h -+@@ -523,7 +523,7 @@ int parse_dotted_ip(char *, u_int32_t *) -+ #define TIMEOUT(r, f, t) ppp_timeout((r), (f), (t), 0) -+ #define UNTIMEOUT(r, f) ppp_untimeout((r), (f)) -+ -+-#define BCOPY(s, d, l) memcpy(d, s, l) -++#define BCOPY(s, d, l) memmove(d, s, l) -+ #define BZERO(s, n) memset(s, 0, n) -+ #define BCMP(s1, s2, l) memcmp(s1, s2, l) -+ diff --git a/scripts/Readme.md b/scripts/Readme.md index f7b16b6cb..32fa85a37 100644 --- a/scripts/Readme.md +++ b/scripts/Readme.md @@ -99,7 +99,14 @@ This script retrieves open issues labeled "testing" from the NethServer/nethsecu ## netifyd-packages.sh -This scripts pulls the `.ipk` packages from the `netifyd-ipks` directory, unpacks them and puts the files inside the `netifyd` package. This script is useful when an update of `netifyd` requires changes in the integration meta package. +This script extracts Netify `.apk` packages from the `netifyd-apks` directory, unpacks them, and merges the files for analysis and integration. This script is useful when an update of `netifyd` requires changes in the integration meta package. + +### Prerequisites + +- **apk-tools**: The Alpine package extraction tool must be installed on your system. + - On Fedora/RHEL: `dnf install apk-tools` + - On Debian/Ubuntu: `apt-get install apk-tools` + - On Alpine Linux: Already included ### Usage @@ -107,4 +114,12 @@ This scripts pulls the `.ipk` packages from the `netifyd-ipks` directory, unpack ./netifyd-packages.sh ``` -All process will be handled by the script, there will be an output that lists the files that were being copied in the process, this output needs to be copied and pasted inside the `packages/netifyd/Makefile` file, inside the `define Package/netifyd/install` section. +The script will process all `.apk` files in the `netifyd-apks/{arch}/` directories and extract their contents into `netifyd-apks/tmp/{arch}/netifyd/`. The merged files can then be copied into the `packages/netifyd/` directory as needed. + +### How It Works + +1. Iterates over each architecture subdirectory in `netifyd-apks/` +2. For each `.apk` file found, extracts its contents to a temporary location +3. Merges all extracted files into a single `netifyd` output directory per architecture +4. Skips metadata files (`.PKGINFO`, install scripts, etc.) during extraction +5. Outputs the merged file structure in `netifyd-apks/tmp/{arch}/netifyd/` diff --git a/scripts/netifyd-packages.sh b/scripts/netifyd-packages.sh index 8425d380c..d3261af1c 100755 --- a/scripts/netifyd-packages.sh +++ b/scripts/netifyd-packages.sh @@ -1,28 +1,32 @@ #!/bin/bash -# This is a helper script to extract all Netify IPK packages -# into netifyd-ipks/tmp/{arch} directories for analysis and integration. +# This is a helper script to extract all Netify APK packages +# into netifyd-apks/tmp/{arch} directories for analysis and integration. # # Usage: ./netifyd-packages.sh # -# Input: netifyd-ipks/{arch}/*.ipk -# Output: netifyd-ipks/tmp/{arch}/netifyd/ — merged contents of all IPKs for that arch -# netifyd-ipks/tmp/{arch}/{pkg}/ — per-package extraction (intermediate) +# Input: netifyd-apks/{arch}/*.apk +# Output: netifyd-apks/tmp/{arch}/netifyd/ — merged contents of all APKs for that arch +# netifyd-apks/tmp/{arch}/{pkg}/ — per-package extraction (intermediate) set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -IPKS_DIR="${SCRIPT_DIR}/netifyd-ipks" -TMP_DIR="${IPKS_DIR}/tmp" +APKS_DIR="${SCRIPT_DIR}/netifyd-apks" +TMP_DIR="${APKS_DIR}/tmp" -if [ ! -d "${IPKS_DIR}" ]; then - echo "ERROR: IPKs directory not found: ${IPKS_DIR}" >&2 +if [ ! -d "${APKS_DIR}" ]; then + echo "ERROR: APKs directory not found: ${APKS_DIR}" >&2 exit 1 fi # Iterate over each arch subdirectory -for arch_dir in "${IPKS_DIR}"/*/; do +for arch_dir in "${APKS_DIR}"/*/; do [ -d "${arch_dir}" ] || continue + + # Skip the tmp directory + [ "$(basename "${arch_dir}")" = "tmp" ] && continue + arch="$(basename "${arch_dir}")" echo "==> Processing arch: ${arch}" @@ -31,13 +35,13 @@ for arch_dir in "${IPKS_DIR}"/*/; do rm -rf "${TMP_DIR:?}/${arch}" mkdir -p "${output_dir}" - # Extract each IPK into its own per-package subdirectory, then merge - for ipk_file in "${arch_dir}"*.ipk; do - [ -f "${ipk_file}" ] || continue + # Extract each APK into its own per-package subdirectory, then merge + for apk_file in "${arch_dir}"*.apk; do + [ -f "${apk_file}" ] || continue # Derive package name: strip version and arch suffix - # e.g. netify-plm_2026-01-01-v1.2.1-r8_x86_64.ipk -> netify-plm - filename="$(basename "${ipk_file}" .ipk)" + # e.g. netify-plm_2026-01-01-v1.2.1-r8_x86_64.apk -> netify-plm + filename="$(basename "${apk_file}" .apk)" pkg_name="${filename%%_*}" # Extract to a temporary directory first to avoid conflicts @@ -45,7 +49,7 @@ for arch_dir in "${IPKS_DIR}"/*/; do mkdir -p "${pkg_extract_dir}" echo " Extracting ${filename} -> ${pkg_name}/" - tar -xf "${ipk_file}" ./data.tar.gz -O | tar -xzf - -C "${pkg_extract_dir}" + apk extract --allow-untrusted --destination "${pkg_extract_dir}" "${apk_file}" # Merge extracted files into the single netifyd output directory cp -a "${pkg_extract_dir}/." "${output_dir}/" diff --git a/tools/cleanup/cleanup.py b/tools/cleanup/cleanup.py index 4c413e494..caa163bf4 100755 --- a/tools/cleanup/cleanup.py +++ b/tools/cleanup/cleanup.py @@ -2,13 +2,13 @@ # # Cleanup old development builds from DigitalOcean Spaces: -# - Keep all tagged releases -# - Keep at least 3 versions of each sub release +# - Keep the latest 5 versions of each channel # +# Version format: 8.7.2-dev.. or 8.7.2-branch.. import os import boto3 -from semver import VersionInfo +from semver import Version as VersionInfo region = "ams3" bucket_name = "nethsecurity" @@ -27,20 +27,35 @@ parsed_files = [] for file in files: file_name = file.lstrip(f'{prefix}/').rstrip('/') - version_parsed = VersionInfo.parse(file_name) - if version_parsed.build is None: + try: + version_parsed = VersionInfo.parse(file_name) + except ValueError: + print(f'Skipping {file_name} - not a valid semver version.') + continue + + # Check if it's a development build (has prerelease segment) + if version_parsed.prerelease is None: print(f'Skipping {file_name} as it is not a development build.') continue - build_split = version_parsed.build.split('.') + + # Extract timestamp from prerelease segment: "dev.." or "branch.." + prerelease_parts = version_parsed.prerelease.split('.') + if len(prerelease_parts) < 3: + print(f'Skipping {file_name} - prerelease segment does not contain timestamp.') + continue + + timestamp = prerelease_parts[1] # middle part is the timestamp parsed_files.append({ - 'timestamp': build_split[1], - 'file': file + 'timestamp': timestamp, + 'file': file, + 'version': file_name }) -# keep only the latest 5 dev builds +# Keep only the latest 5 dev builds, sorted by timestamp (descending) to_delete = sorted(parsed_files, key=lambda k: k['timestamp'], reverse=True)[min_versions:] for d in to_delete: - print(f"Deleting {d['file']} ...") + print(f"Deleting {d['version']} ...") objects_to_delete = s3_client.list_objects(Bucket=bucket_name, Prefix=d['file']) delete_keys = {'Objects': [{'Key': k['Key']} for k in objects_to_delete.get('Contents', [])]} s3_client.delete_objects(Bucket=bucket_name, Delete=delete_keys) +