feat: add hub-worker binary and refactor config to cleanenv (#50)

* chore: split into api and worker

* chore: lint fix

* chore: renamed property

* chore: river fix

* chore: expose additional river config

* chore: improvements

* chore: align run-worker with run (allow env vars without .env)

Made-with: Cursor

* chore: rename

* chore: fix stale comment in backfill-embeddings referencing workers in API

* docs: fix AGENTS.md run-worker to reflect .env is optional

* chore: fixes

---------

Co-authored-by: Bhagya Amarasinghe <b.sithumini@yahoo.com>

Author: xernobyl

.env.example

@@ -1,5 +1,6 @@
 # Formbricks Hub Configuration
-# Copy this file to .env and update the values as needed
+# Copy this file to .env and update the values as needed.
+# Config is loaded via cleanenv: .env is optional; environment variables override .env values.
 
 # API Key for authentication (required)
 # This key is used to authenticate requests to protected endpoints
@@ -40,6 +41,16 @@ MESSAGE_PUBLISHER_PER_EVENT_TIMEOUT_SECONDS=10
 # Graceful shutdown timeout in seconds (optional). Default: 30
 SHUTDOWN_TIMEOUT_SECONDS=30
 
+# River worker (hub-worker only). API does not run workers; these affect job execution and cleanup.
+# RIVER_JOB_TIMEOUT_SECONDS: max time a job may run before context is cancelled. 0 = River default (1m).
+# RIVER_RESCUE_STUCK_JOBS_AFTER_SECONDS: time after which a running job is considered stuck and retried/discarded. 0 = River default (1h).
+# RIVER_COMPLETED_JOB_RETENTION_SECONDS: how long to keep completed jobs before cleanup; -1 = disable. Default: 86400 (24h).
+# RIVER_CLIENT_ID: optional identifier for this worker (logs, attempted_by); empty = auto-generated.
+# RIVER_JOB_TIMEOUT_SECONDS=0
+# RIVER_RESCUE_STUCK_JOBS_AFTER_SECONDS=0
+# RIVER_COMPLETED_JOB_RETENTION_SECONDS=86400
+# RIVER_CLIENT_ID=
+
 # Webhook max fan-out per event (optional)
 # Max number of webhook jobs enqueued per event; excess is capped and logged. Default: 500
 WEBHOOK_MAX_FAN_OUT_PER_EVENT=500

.github/workflows/api-contract-tests.yml

@@ -87,7 +87,7 @@ jobs:
 
       - name: Start API server
         run: |
-          ./bin/api &
+          ./bin/hub-api &
           API_PID=$!
           echo "API_PID=$API_PID" >> $GITHUB_ENV
 
@@ -147,4 +147,4 @@ jobs:
           if [ ! -z "$API_PID" ]; then
             kill $API_PID || true
           fi
-          pkill -f "./bin/api" || true
+          pkill -f "./bin/hub-api" || true

AGENTS.md

@@ -1,19 +1,21 @@
 # Repository Guidelines
 
 ## Project Structure & Module Organization
-- `cmd/api/` holds the API server (package `main`: `main.go`, `app.go`). Build/run the package, e.g. `go run ./cmd/api` or `make run`.
-- `internal/` contains core application layers: `api/handlers`, `api/middleware`, `service`, `repository`, `models`, and `config`.
+- `cmd/api/` holds the API server (hub-api): HTTP API, ingestion, record retrieval, tenant/auth; enqueues jobs to River (insert-only). Build/run: `go run ./cmd/api` or `make run`.
+- `cmd/worker/` holds the worker (hub-worker): runs River job workers (webhook delivery, embeddings). No HTTP. Build/run: `go run ./cmd/worker` or `make run-worker`.
+- `internal/` contains core application layers: `api/handlers`, `api/middleware`, `service`, `repository`, `models`, `config`, and `workers`.
 - `pkg/` provides shared utilities (currently `pkg/database`).
 - `migrations/` stores SQL migration files (goose); use `-- +goose up` / `-- +goose down` annotations.
 - `tests/` contains integration tests.
 
 ## Build, Test, and Development Commands
 - `make dev-setup`: start Postgres via Docker, install Go deps/tools, and initialize database schema.
-- `make run`: create a default `.env` if missing and run the API server.
-- `make build`: build the API binary to `bin/api`.
+- `make run`: run hub-api (config from `.env` if present and environment variables; copy `.env.example` to `.env` or set env vars).
+- `make run-worker`: run hub-worker (requires DATABASE_URL from `.env` or environment variables).
+- `make build`: build both `bin/hub-api` and `bin/hub-worker`. Use `make build-api` or `make build-worker` for a single binary.
 - `make tests`: run integration tests in `tests/`.
 - `make tests-coverage`: generate `coverage.html`.
-- `make check-coverage`: run all tests with coverage and fail if below COVERAGE_THRESHOLD (excludes cmd/api: app.go, main.go).
+- `make check-coverage`: run all tests with coverage and fail if below COVERAGE_THRESHOLD (excludes cmd/api and cmd/worker main packages).
 - `make init-db`: run goose migrations up using `DATABASE_URL`. `make migrate-status` and `make migrate-validate` for status and validation. New migrations go in `migrations/` with goose annotations (`-- +goose up` / `-- +goose down`). Name files with a sequential number and short description (e.g. `002_add_webhooks_table.sql`); goose orders by the numeric prefix. For webhook delivery, run `make river-migrate` after `init-db` to apply River job queue migrations.
 - `make fmt`: format code (runs `golangci-lint run --fix`; uses gofumpt/gci from config).
 - `make lint`: run `golangci-lint` (includes format checks; requires `make install-tools`).

Dockerfile

@@ -18,24 +18,26 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go install github.com/pr
 COPY go.mod go.sum ./
 RUN go mod download
 
-# Build the application
+# Build the application (hub-api and hub-worker)
 COPY . .
-RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o /build/bin/api ./cmd/api
+RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o /build/bin/hub-api ./cmd/api && \
+    CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o /build/bin/hub-worker ./cmd/worker
 
 # =============================================================================
-# Stage 2: Runtime
+# Stage 2: Runtime (default: hub-api)
 # =============================================================================
 FROM alpine:3.21
 
-RUN apk add --no-cache ca-certificates tzdata
+RUN apk add --no-cache ca-certificates tzdata wget
 
 # Create non-root user
 RUN addgroup -S app && adduser -S app -G app
 
 WORKDIR /app
 
-# Copy binary and migration tools from builder
-COPY --from=builder /build/bin/api /app/api
+# Copy binaries and migration tools from builder
+COPY --from=builder /build/bin/hub-api /app/hub-api
+COPY --from=builder /build/bin/hub-worker /app/hub-worker
 COPY --from=builder /go/bin/goose /usr/local/bin/goose
 COPY --from=builder /go/bin/river /usr/local/bin/river
 
@@ -47,4 +49,10 @@ USER app
 
 EXPOSE 8080
 
-ENTRYPOINT ["/app/api"]
+# Health check for hub-api. Disable or override when running hub-worker (e.g. docker run ... hub-worker)
+# since workers do not expose HTTP.
+HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
+	CMD wget --no-verbose --tries=1 --spider http://localhost:8080/health || exit 1
+
+# Default: run hub-api. Override with command to run hub-worker: docker run ... hub-worker
+ENTRYPOINT ["/app/hub-api"]

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all test help tests tests-coverage check-coverage build build-backfill-embeddings run run-backfill-embeddings init-db clean docker-up docker-down docker-clean deps install-tools fmt lint lint-new lint-openapi dev-setup test-all test-unit schemathesis install-hooks migrate-status migrate-validate river-migrate
+.PHONY: all test help tests tests-coverage check-coverage build build-api build-worker build-backfill-embeddings run run-worker run-backfill-embeddings init-db clean docker-up docker-down docker-clean deps install-tools fmt lint lint-new lint-openapi dev-setup test-all test-unit schemathesis install-hooks migrate-status migrate-validate river-migrate
 
 # Aliases for checkmake/lint expectations
 all: build
@@ -9,9 +9,12 @@ help:
 	@echo "Available targets:"
 	@echo "  make help             - Show this help message"
 	@echo "  make dev-setup        - Set up development environment (docker, deps, tools, schema, hooks)"
-	@echo "  make build                    - Build the API server"
+	@echo "  make build                    - Build hub-api and hub-worker"
+	@echo "  make build-api                - Build hub-api only (bin/hub-api)"
+	@echo "  make build-worker             - Build hub-worker only (bin/hub-worker)"
 	@echo "  make build-backfill-embeddings - Build the backfill-embeddings command"
-	@echo "  make run              - Run the API server"
+	@echo "  make run              - Run the API server (hub-api)"
+	@echo "  make run-worker       - Run the worker (hub-worker)"
 	@echo "  make run-backfill-embeddings - Run the backfill-embeddings command (enqueues embedding jobs; loads .env)"
 	@echo "  make test-unit        - Run unit tests (fast, no database)"
 	@echo "  make tests            - Run integration tests"
@@ -79,11 +82,21 @@ check-coverage:
 	fi && \
 	echo "✅ Coverage $$COV% meets threshold $(COVERAGE_THRESHOLD)%"
 
-# Build the API server
-build:
-	@echo "Building API server..."
-	go build -o bin/api ./cmd/api
-	@echo "Binary created: bin/api"
+# Build hub-api and hub-worker
+build: build-api build-worker
+	@echo "Binaries created: bin/hub-api, bin/hub-worker"
+
+# Build the API server (hub-api)
+build-api:
+	@echo "Building hub-api..."
+	go build -o bin/hub-api ./cmd/api
+	@echo "Binary created: bin/hub-api"
+
+# Build the worker (hub-worker)
+build-worker:
+	@echo "Building hub-worker..."
+	go build -o bin/hub-worker ./cmd/worker
+	@echo "Binary created: bin/hub-worker"
 
 # Build the backfill-embeddings command (enqueues embedding jobs; requires DATABASE_URL)
 build-backfill-embeddings:
@@ -96,27 +109,17 @@ run-backfill-embeddings:
 	@if [ ! -f .env ]; then echo "Error: .env file required. Copy .env.example to .env and configure."; exit 1; fi && \
 	(set -a && . ./.env && set +a && go run ./cmd/backfill-embeddings)
 
-# Run the API server
+# Run the API server (hub-api).
+# Config: .env if present, else environment variables; env vars override .env. Copy .env.example to .env or set env vars.
 run:
-	@echo "Checking for .env file..."
-	@if [ ! -f .env ]; then \
-		echo "Creating .env file with default values..."; \
-		echo "# Formbricks Hub Configuration" > .env; \
-		echo "# Auto-generated by 'make run' - modify as needed" >> .env; \
-		echo "" >> .env; \
-		echo "# API Key for authentication (required)" >> .env; \
-		echo "API_KEY=test-api-key-12345" >> .env; \
-		echo "" >> .env; \
-		echo "# Database connection URL (optional: set POSTGRES_PORT in .env if 5432 is in use; keep port in sync here)" >> .env; \
-		echo "DATABASE_URL=postgres://postgres:postgres@localhost:5432/test_db?sslmode=disable" >> .env; \
-		echo "" >> .env; \
-		echo "# Server port (default: 8080)" >> .env; \
-		echo "PORT=8080" >> .env; \
-		echo ".env file created with default values."; \
-	fi
-	@echo "Starting API server..."
+	@echo "Starting hub-api..."
 	go run ./cmd/api
 
+# Run the worker (hub-worker). Config: .env if present, else env vars (same as run). Requires DATABASE_URL; API_KEY not required.
+run-worker:
+	@echo "Starting hub-worker..."
+	go run ./cmd/worker
+
 # Initialize database schema (run goose migrations up)
 init-db:
 	@echo "Running migrations..."
@@ -200,7 +203,7 @@ deps:
 
 # Install development tools
 # Tool versions - update these periodically
-GOLANGCI_LINT_VERSION := v2.10.1
+GOLANGCI_LINT_VERSION := v2.11.3
 GOVULNCHECK_VERSION := v1.1.4
 GOOSE_VERSION := v3.27.0
 RIVER_VERSION := v0.31.0