chore: test reliability and speed skill (#133)

* Stabilize integration network and ingress test flakes

* Speed up fork-from-running integration tests

* Reduce fork test agent wait latency

* Add agent prompt

* Convert to skill

Author: sjmiller609

lib/ingress/binaries_linux.go

@@ -9,6 +9,7 @@ import (
 	"log/slog"
 	"os"
 	"path/filepath"
+	"syscall"
 
 	"github.com/kernel/hypeman/lib/paths"
 )
@@ -57,13 +58,29 @@ func ExtractCaddyBinary(p *paths.Paths) (string, error) {
 		return "", fmt.Errorf("create caddy binary dir: %w", err)
 	}
 
-	// Write binary to filesystem
-	if err := os.WriteFile(extractPath, data, 0755); err != nil {
+	lockFile, err := os.OpenFile(extractPath+".lock", os.O_CREATE|os.O_RDWR, 0644)
+	if err != nil {
+		return "", fmt.Errorf("open extraction lock: %w", err)
+	}
+	defer lockFile.Close()
+	if err := syscall.Flock(int(lockFile.Fd()), syscall.LOCK_EX); err != nil {
+		return "", fmt.Errorf("lock extraction: %w", err)
+	}
+	defer syscall.Flock(int(lockFile.Fd()), syscall.LOCK_UN)
+
+	// Another process may have extracted it while we waited for the lock.
+	if _, err := os.Stat(extractPath); err == nil {
+		if storedHash, err := os.ReadFile(hashPath); err == nil && string(storedHash) == embeddedHash {
+			return extractPath, nil
+		}
+	}
+
+	if err := atomicWriteExecutable(extractPath, data); err != nil {
 		return "", fmt.Errorf("write caddy binary: %w", err)
 	}
 
 	// Write hash file for future comparisons
-	if err := os.WriteFile(hashPath, []byte(embeddedHash), 0644); err != nil {
+	if err := atomicWriteFile(hashPath, []byte(embeddedHash), 0644); err != nil {
 		// Non-fatal - binary is extracted, just won't have hash for next time
 		// This could cause unnecessary re-extractions but won't break functionality
 		slog.Info("failed to write caddy binary hash file", "path", hashPath, "error", err)
@@ -76,3 +93,39 @@ func ExtractCaddyBinary(p *paths.Paths) (string, error) {
 func GetCaddyBinaryPath(p *paths.Paths) (string, error) {
 	return ExtractCaddyBinary(p)
 }
+
+func atomicWriteExecutable(path string, data []byte) error {
+	return atomicWriteFile(path, data, 0755)
+}
+
+func atomicWriteFile(path string, data []byte, mode os.FileMode) error {
+	dir := filepath.Dir(path)
+	tmpFile, err := os.CreateTemp(dir, "caddy-*")
+	if err != nil {
+		return fmt.Errorf("create temp file: %w", err)
+	}
+	tmpPath := tmpFile.Name()
+	cleanupTmp := true
+	defer func() {
+		if cleanupTmp {
+			_ = os.Remove(tmpPath)
+		}
+	}()
+
+	if _, err := tmpFile.Write(data); err != nil {
+		_ = tmpFile.Close()
+		return fmt.Errorf("write temp file: %w", err)
+	}
+	if err := tmpFile.Chmod(mode); err != nil {
+		_ = tmpFile.Close()
+		return fmt.Errorf("chmod temp file: %w", err)
+	}
+	if err := tmpFile.Close(); err != nil {
+		return fmt.Errorf("close temp file: %w", err)
+	}
+	if err := os.Rename(tmpPath, path); err != nil {
+		return fmt.Errorf("install file: %w", err)
+	}
+	cleanupTmp = false
+	return nil
+}

lib/instances/firecracker_test.go

@@ -4,6 +4,9 @@ package instances
 
 import (
 	"context"
+	"fmt"
+	"net"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
@@ -87,6 +90,31 @@ func createNginxImageAndWait(t *testing.T, ctx context.Context, imageManager ima
 	t.Fatalf("timed out waiting for image %q to become ready", nginxImage.Name)
 }
 
+func startGatewayProbeServer(t *testing.T, gatewayIP string) (string, func()) {
+	t.Helper()
+
+	listener, err := net.Listen("tcp", net.JoinHostPort(gatewayIP, "0"))
+	require.NoError(t, err)
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/probe", func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte("Connection successful"))
+	})
+
+	server := &http.Server{Handler: mux}
+	go func() {
+		_ = server.Serve(listener)
+	}()
+
+	cleanup := func() {
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+		defer cancel()
+		_ = server.Shutdown(shutdownCtx)
+	}
+
+	return fmt.Sprintf("http://%s/probe", listener.Addr().String()), cleanup
+}
+
 func TestFirecrackerStandbyAndRestore(t *testing.T) {
 	t.Parallel()
 	requireFirecrackerIntegrationPrereqs(t)
@@ -247,14 +275,17 @@ func TestFirecrackerNetworkLifecycle(t *testing.T) {
 	_, isBridge := master.(*netlink.Bridge)
 	assert.True(t, isBridge, "TAP should be attached to a bridge")
 
+	probeURL, stopProbeServer := startGatewayProbeServer(t, alloc.Gateway)
+	t.Cleanup(stopProbeServer)
+
 	require.NoError(t, waitForLogMessage(ctx, mgr, inst.Id, "start worker processes", 15*time.Second))
 	require.NoError(t, waitForLogMessage(ctx, mgr, inst.Id, "[guest-agent] listening", 10*time.Second))
 
-	// Retry to reduce flakiness while guest network stack settles.
+	// Retry while guest network stack settles.
 	var output string
 	var exitCode int
 	for i := 0; i < 10; i++ {
-		output, exitCode, err = execCommand(ctx, inst, "curl", "-s", "--connect-timeout", "10", "https://public-ping-bucket-kernel.s3.us-east-1.amazonaws.com/index.html")
+		output, exitCode, err = execCommand(ctx, inst, "curl", "-sS", "--connect-timeout", "10", probeURL)
 		if err == nil && exitCode == 0 {
 			break
 		}
@@ -294,7 +325,7 @@ func TestFirecrackerNetworkLifecycle(t *testing.T) {
 	assert.Equal(t, uint8(netlink.OperUp), uint8(tapRestored.Attrs().OperState))
 
 	for i := 0; i < 10; i++ {
-		output, exitCode, err = execCommand(ctx, inst, "curl", "-s", "https://public-ping-bucket-kernel.s3.us-east-1.amazonaws.com/index.html")
+		output, exitCode, err = execCommand(ctx, inst, "curl", "-sS", "--connect-timeout", "10", probeURL)
 		if err == nil && exitCode == 0 {
 			break
 		}
@@ -379,7 +410,6 @@ func TestFirecrackerForkFromRunningNetwork(t *testing.T) {
 
 	assertHostCanReachNginx(t, sourceAfterFork.IP, 80, 60*time.Second)
 	assertHostCanReachNginx(t, forked.IP, 80, 60*time.Second)
-	assertHostCanReachNginx(t, sourceAfterFork.IP, 80, 60*time.Second)
 	assert.NotEqual(t, sourceAfterFork.IP, forked.IP)
 	assert.NotEqual(t, sourceAfterFork.MAC, forked.MAC)
 }

lib/instances/fork_test.go

@@ -419,7 +419,6 @@ func TestForkCloudHypervisorFromRunningNetwork(t *testing.T) {
 	assert.NotEqual(t, sourceAfterFork.MAC, forked.MAC)
 	assertGuestHasOnlyExpectedIPv4(t, forked, forked.IP, 30*time.Second)
 	assertHostCanReachNginx(t, forked.IP, 80, 60*time.Second)
-	assertHostCanReachNginx(t, sourceAfterFork.IP, 80, 60*time.Second)
 }
 
 func assertHostCanReachNginx(t *testing.T, ip string, port int, timeout time.Duration) {
@@ -495,7 +494,7 @@ func execInInstance(ctx context.Context, inst *Instance, command ...string) (str
 		Command:      command,
 		Stdout:       &stdout,
 		Stderr:       &stderr,
-		WaitForAgent: 30 * time.Second,
+		WaitForAgent: 2 * time.Second,
 	})
 	if err != nil {
 		return "", -1, err

lib/instances/qemu_test.go

@@ -955,7 +955,6 @@ func TestQEMUForkFromRunningNetwork(t *testing.T) {
 	assert.NotEqual(t, sourceAfterFork.IP, forked.IP)
 	assert.NotEqual(t, sourceAfterFork.MAC, forked.MAC)
 	assertHostCanReachNginx(t, forked.IP, 80, 60*time.Second)
-	assertHostCanReachNginx(t, sourceAfterFork.IP, 80, 60*time.Second)
 }
 
 func TestQEMUSnapshotFeature(t *testing.T) {

lib/network/allocate.go

@@ -7,6 +7,7 @@ import (
 	mathrand "math/rand"
 	"net"
 	"strings"
+	"time"
 
 	"github.com/kernel/hypeman/lib/logger"
 )
@@ -22,17 +23,9 @@ func (m *manager) CreateAllocation(ctx context.Context, req AllocateRequest) (*N
 	log := logger.FromContext(ctx)
 
 	// 1. Get default network
-	network, err := m.getDefaultNetwork(ctx)
+	network, err := m.getDefaultNetworkWithSelfHeal(ctx)
 	if err != nil {
-		// Self-heal if bridge state was externally removed after initialization.
-		// This keeps allocations robust under highly concurrent test workloads.
-		if initErr := m.Initialize(ctx, nil); initErr != nil {
-			return nil, fmt.Errorf("get default network: %w", err)
-		}
-		network, err = m.getDefaultNetwork(ctx)
-		if err != nil {
-			return nil, fmt.Errorf("get default network: %w", err)
-		}
+		return nil, fmt.Errorf("get default network: %w", err)
 	}
 
 	// 2. Check name uniqueness (exclude current instance to allow restarts)
@@ -113,16 +106,9 @@ func (m *manager) RecreateAllocation(ctx context.Context, instanceID string, dow
 	}
 
 	// 2. Get default network details
-	network, err := m.getDefaultNetwork(ctx)
+	network, err := m.getDefaultNetworkWithSelfHeal(ctx)
 	if err != nil {
-		// Same self-healing behavior as CreateAllocation.
-		if initErr := m.Initialize(ctx, nil); initErr != nil {
-			return fmt.Errorf("get default network: %w", err)
-		}
-		network, err = m.getDefaultNetwork(ctx)
-		if err != nil {
-			return fmt.Errorf("get default network: %w", err)
-		}
+		return fmt.Errorf("get default network: %w", err)
 	}
 
 	// 3. Recreate TAP device with same name and rate limits from instance metadata
@@ -239,6 +225,31 @@ func (m *manager) allocateNextIP(ctx context.Context, subnet string) (string, er
 	return "", fmt.Errorf("no available IPs in subnet %s after %d random attempts and full scan", subnet, maxRetries)
 }
 
+func (m *manager) getDefaultNetworkWithSelfHeal(ctx context.Context) (*Network, error) {
+	network, err := m.getDefaultNetwork(ctx)
+	if err == nil {
+		return network, nil
+	}
+
+	// Self-heal if bridge state was externally removed after initialization.
+	// After re-initialization, kernel bridge/IP state may take a brief moment to become visible.
+	if initErr := m.Initialize(ctx, nil); initErr != nil {
+		return nil, err
+	}
+
+	deadline := time.Now().Add(2 * time.Second)
+	for {
+		network, err = m.getDefaultNetwork(ctx)
+		if err == nil {
+			return network, nil
+		}
+		if time.Now().After(deadline) {
+			return nil, err
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
 // incrementIP increments IP address by n
 func incrementIP(ip net.IP, n int) net.IP {
 	// Ensure we're working with IPv4 (4 bytes)

skills/test-agent/SKILL.md

@@ -0,0 +1,8 @@
+---
+name: test-agent
+description: Improve repository test quality by eliminating flakes first, then reducing slow-test redundancy, and finally optimizing longest tests while preserving coverage. Use when asked to repeatedly run CI-equivalent no-cache test cycles, diagnose flaky tests, and document findings in this skill's notes.
+---
+
+Follow the workflow and constraints documented in [agents/test-agent/PROMPT.md](agents/test-agent/PROMPT.md). Do not modify this file.
+
+Record investigation and timing notes in [agents/test-agent/NOTES.md](agents/test-agent/NOTES.md), use this file as memory of the investigation, reference or search it where needed for context.