feat(mcp): expand CRUD server to near-complete CLI parity (#1440)

* feat(mcp): expand CRUD MCP server to near-complete CLI parity

Closes the CLI-vs-MCP tool coverage gap identified by auditing every
`goclaw` CLI command against the existing goclaw_* MCP tool set. Adds
skill grant/revoke, agent skill pin/unpin, and full CRUD/inspection
surfaces for memory, knowledge graph (including dedup/merge/prune),
tenants, providers, LLM traces, channel contacts, pending messages,
audit activity, system config, tenant storage (list/size/delete/move),
scoped agent config export/import, secure-CLI binary registry, and a
DB-backed health check.

Deliberately out of scope, documented inline where relevant:
- `goclaw credentials`: confirmed CLI-local (~/.goclaw/config.yaml +
  keychain), no server resource to wrap. goclaw_secure_cli_binaries_*
  covers the closest real, previously-uncovered server resource instead.
- Full tar-archive agent export/import (KG + workspace files): the CLI's
  version streams a multi-section archive with progress events, a shape
  that doesn't map to a single MCP tool call. Config + context files
  (the portable "brain") is covered.
- `kg extract` (LLM-driven text extraction): goclaw_kg_ingest accepts
  the same Entity/Relation shapes the extractor produces, so a caller
  can run extraction itself and hand off the result.

Wires 8 new store dependencies (Memory, KnowledgeGraph, Tracing,
Contacts, PendingMessages, Activity, SystemConfigs, SecureCLI) through
gateway.Server setters -> cmd/gateway.go -> CRUDDeps, following the
existing Providers/Tenants pattern. Storage and secure-CLI-binary
handlers duplicate internal/http's path-escape/symlink-hiding
validation logic (documented inline) since internal/http already
imports internal/mcp and the reverse would cycle.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* fix(mcp): return sessionKey in goclaw_chat_send response for persistent conversations

The goclaw_chat_send tool creates a new session internally when sessionKey is
empty, but never returned the key to the caller. This prevented using
goclaw_chat_history to fetch previous messages in the session.

Add SessionKey field to ChatSendResult so callers can:
1. Start a new agent chat without providing sessionKey
2. Receive the sessionKey back in the response
3. Use that sessionKey for follow-up messages and history queries

Fixes the training loop pattern: start chat → get sessionKey → call
goclaw_chat_history with that key → iterate skill based on actual failures.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* feat(mcp): add timing diagnostics to goclaw_chat_send for timeout troubleshooting

Log request arrival, processing duration, and errors with millisecond precision.
Helps identify whether timeouts occur at MCP client→goclaw, goclaw→ollama, or
during agent execution. Critical for production debugging.

Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>

* fix(gateway): add http.Server timeouts for defensive timeout handling

Set explicit timeouts in http.Server:
- ReadTimeout: 1h (allow large uploads, long-running agent operations)
- WriteTimeout: 1h (allow streaming responses to slow clients)
- IdleTimeout: 30s (close idle keep-alive connections quickly)

Provides defense-in-depth when Nginx/Traefik timeouts are misconfigured.
Matches Nginx timeout (3600s) to prevent race conditions.

Timeout chain: traefik 3600s = nginx 3600s = goclaw 3600s

Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>

---------

Co-authored-by: Bruno Clermont <bruno.clermont@gmail.com>
Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>

Author: bclermont

cmd/gateway.go

@@ -538,6 +538,30 @@ func runGateway() {
 	if pgStores.Tenants != nil {
 		server.SetTenantStore(pgStores.Tenants)
 	}
+	if pgStores.Memory != nil {
+		server.SetMemoryStore(pgStores.Memory)
+	}
+	if pgStores.KnowledgeGraph != nil {
+		server.SetKnowledgeGraphStore(pgStores.KnowledgeGraph)
+	}
+	if pgStores.Tracing != nil {
+		server.SetTracingStore(pgStores.Tracing)
+	}
+	if pgStores.Contacts != nil {
+		server.SetContactStore(pgStores.Contacts)
+	}
+	if pgStores.PendingMessages != nil {
+		server.SetPendingMessageStore(pgStores.PendingMessages)
+	}
+	if pgStores.Activity != nil {
+		server.SetActivityStore(pgStores.Activity)
+	}
+	if pgStores.SystemConfigs != nil {
+		server.SetSystemConfigStore(pgStores.SystemConfigs)
+	}
+	if pgStores.SecureCLI != nil {
+		server.SetSecureCLIStore(pgStores.SecureCLI)
+	}
 	server.SetSQLDB(pgStores.DB)
 
 	// Build OAuth token refresher before wireExtras so the resolver can inject tokens.

internal/gateway/chat_runner.go

@@ -67,17 +67,18 @@ func (r *agentChatRunner) Send(ctx context.Context, agentID, sessionKey, message
 	})
 	if err != nil {
 		if ctx.Err() != nil {
-			return &mcpbridge.ChatSendResult{Cancelled: true}, nil
+			return &mcpbridge.ChatSendResult{Cancelled: true, SessionKey: sessionKey}, nil
 		}
 		return nil, fmt.Errorf("run agent %q: %w", agentID, err)
 	}
 
 	return &mcpbridge.ChatSendResult{
-		RunID:    result.RunID,
-		Content:  result.Content,
-		Usage:    result.Usage,
-		Thinking: result.Thinking,
-		Media:    result.Media,
+		RunID:      result.RunID,
+		SessionKey: sessionKey,
+		Content:    result.Content,
+		Usage:      result.Usage,
+		Thinking:   result.Thinking,
+		Media:      result.Media,
 	}, nil
 }
 

internal/gateway/server.go

@@ -77,6 +77,14 @@ type Server struct {
 	quotaChecker     *channels.QuotaChecker
 	sqlDB            *sql.DB           // for the CRUD MCP server's quota usage tool (today's trace summary)
 	tenantStore      store.TenantStore // for the CRUD MCP server's "X-GoClaw-Tenant-Id" header resolution
+	memoryStore      store.MemoryStore
+	kgStore          store.KnowledgeGraphStore
+	tracingStore     store.TracingStore
+	contactStore     store.ContactStore
+	pendingMsgStore  store.PendingMessageStore
+	activityStore    store.ActivityStore
+	systemCfgStore   store.SystemConfigStore
+	secureCLIStore   store.SecureCLIStore
 
 	// Phase 3 CRUD MCP server (/api/mcp/) dependencies: chat/LLM/logs/send/voices.
 	llmProviders      *providers.Registry
@@ -317,6 +325,14 @@ func (s *Server) BuildMux() *http.ServeMux {
 			VoiceCache:        s.voiceCache,
 			VoiceSecretsStore: s.voiceSecretsStore,
 			Tenants:           s.tenantStore,
+			Memory:            s.memoryStore,
+			KnowledgeGraph:    s.kgStore,
+			Tracing:           s.tracingStore,
+			Contacts:          s.contactStore,
+			PendingMessages:   s.pendingMsgStore,
+			Activity:          s.activityStore,
+			SystemConfigs:     s.systemCfgStore,
+			SecureCLI:         s.secureCLIStore,
 		}, s.version)
 		mux.Handle("/api/mcp/", mcpServerTokenAuthMiddleware(s.cfg.Gateway.MCPServerToken, crudHandler))
 	} else {
@@ -496,8 +512,11 @@ func (s *Server) Start(ctx context.Context) error {
 
 	addr := fmt.Sprintf("%s:%d", s.cfg.Gateway.Host, s.cfg.Gateway.Port)
 	s.httpServer = &http.Server{
-		Addr:    addr,
-		Handler: handler,
+		Addr:         addr,
+		Handler:      handler,
+		ReadTimeout:  3600 * time.Second,  // 1h: allow large uploads, long-running reads
+		WriteTimeout: 3600 * time.Second,  // 1h: allow streaming responses, slow clients
+		IdleTimeout:  30 * time.Second,    // 30s: close idle connections
 	}
 
 	slog.Info("gateway starting", "addr", addr)
@@ -867,6 +886,41 @@ func (s *Server) SetProviderStore(ps store.ProviderStore) { s.providerStore = ps
 // request header (UUID or slug) to a concrete tenant for every CRUD MCP call.
 func (s *Server) SetTenantStore(ts store.TenantStore) { s.tenantStore = ts }
 
+// SetMemoryStore sets the memory store, used by the CRUD MCP server (see
+// internal/mcp/crud_server.go) to expose goclaw_memory_* tools.
+func (s *Server) SetMemoryStore(ms store.MemoryStore) { s.memoryStore = ms }
+
+// SetKnowledgeGraphStore sets the knowledge graph store, used by the CRUD
+// MCP server (see internal/mcp/crud_server.go) to expose goclaw_kg_* tools.
+func (s *Server) SetKnowledgeGraphStore(kg store.KnowledgeGraphStore) { s.kgStore = kg }
+
+// SetTracingStore sets the LLM call tracing store, used by the CRUD MCP
+// server (see internal/mcp/crud_server.go) to expose goclaw_traces_* tools.
+func (s *Server) SetTracingStore(ts store.TracingStore) { s.tracingStore = ts }
+
+// SetContactStore sets the channel contact store, used by the CRUD MCP
+// server (see internal/mcp/crud_server.go) to expose goclaw_contacts_* tools.
+func (s *Server) SetContactStore(cs store.ContactStore) { s.contactStore = cs }
+
+// SetPendingMessageStore sets the pending-message store, used by the CRUD
+// MCP server (see internal/mcp/crud_server.go) to expose
+// goclaw_pending_messages_* tools.
+func (s *Server) SetPendingMessageStore(pm store.PendingMessageStore) { s.pendingMsgStore = pm }
+
+// SetActivityStore sets the audit-log store, used by the CRUD MCP server
+// (see internal/mcp/crud_server.go) to expose goclaw_activity_list.
+func (s *Server) SetActivityStore(as store.ActivityStore) { s.activityStore = as }
+
+// SetSystemConfigStore sets the system config store, used by the CRUD MCP
+// server (see internal/mcp/crud_server.go) to expose goclaw_system_config_*
+// tools.
+func (s *Server) SetSystemConfigStore(sc store.SystemConfigStore) { s.systemCfgStore = sc }
+
+// SetSecureCLIStore sets the secure-CLI binary registry store, used by the
+// CRUD MCP server (see internal/mcp/crud_server.go) to expose
+// goclaw_secure_cli_binaries_* tools.
+func (s *Server) SetSecureCLIStore(sc store.SecureCLIStore) { s.secureCLIStore = sc }
+
 // SetExecApprovalManager sets the exec approval manager, used by the CRUD MCP
 // server (see internal/mcp/crud_server.go) to expose exec approval tools.
 func (s *Server) SetExecApprovalManager(m *tools.ExecApprovalManager) { s.execApprovalMgr = m }

internal/mcp/crud_activity.go

@@ -0,0 +1,51 @@
+package mcp
+
+import (
+	"context"
+
+	mcpgo "github.com/mark3labs/mcp-go/mcp"
+	mcpserver "github.com/mark3labs/mcp-go/server"
+
+	"github.com/nextlevelbuilder/goclaw/internal/store"
+)
+
+// registerActivityCRUDTools registers goclaw_activity_list, backed by
+// store.ActivityStore — closes a CLI-vs-MCP coverage gap (`goclaw activity
+// list`, the audit log of admin/agent actions emitted via emitAudit
+// throughout internal/http).
+func registerActivityCRUDTools(srv *mcpserver.MCPServer, activity store.ActivityStore) {
+	srv.AddTool(mcpgo.NewTool("goclaw_activity_list",
+		mcpgo.WithDescription("List audit log entries (admin/agent actions), optionally filtered."),
+		mcpgo.WithString("actor_type", mcpgo.Description("Filter by actor type (e.g. \"user\", \"agent\", \"system\").")),
+		mcpgo.WithString("actor_id", mcpgo.Description("Filter by actor ID.")),
+		mcpgo.WithString("action", mcpgo.Description("Filter by action name (e.g. \"skill.file_updated\").")),
+		mcpgo.WithString("entity_type", mcpgo.Description("Filter by entity type (e.g. \"skill\", \"agent\").")),
+		mcpgo.WithString("entity_id", mcpgo.Description("Filter by entity ID.")),
+		mcpgo.WithNumber("limit", mcpgo.Description("Maximum entries to return; defaults to 50.")),
+		mcpgo.WithNumber("offset", mcpgo.Description("Pagination offset.")),
+		mcpgo.WithReadOnlyHintAnnotation(true),
+	), handleActivityList(activity))
+}
+
+func handleActivityList(activity store.ActivityStore) mcpserver.ToolHandlerFunc {
+	return func(ctx context.Context, req mcpgo.CallToolRequest) (*mcpgo.CallToolResult, error) {
+		opts := store.ActivityListOpts{
+			ActorType:  req.GetString("actor_type", ""),
+			ActorID:    req.GetString("actor_id", ""),
+			Action:     req.GetString("action", ""),
+			EntityType: req.GetString("entity_type", ""),
+			EntityID:   req.GetString("entity_id", ""),
+			Limit:      intArg(req, "limit", 50),
+			Offset:     intArg(req, "offset", 0),
+		}
+		list, err := activity.List(ctx, opts)
+		if err != nil {
+			return toolError("activity.list", err)
+		}
+		total, err := activity.Count(ctx, opts)
+		if err != nil {
+			return toolError("activity.list", err)
+		}
+		return jsonToolResult(map[string]any{"entries": list, "total": total})
+	}
+}

internal/mcp/crud_agents.go

@@ -2,6 +2,7 @@ package mcp
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"slices"
 	"strings"
@@ -446,3 +447,143 @@ func handleAgentsFilesSet(agents store.AgentStore) mcpserver.ToolHandlerFunc {
 		})
 	}
 }
+
+// maxPinnedSkillsPerAgent mirrors the invariant documented on
+// store.AgentData.ParsePinnedSkills (agent_store.go) — pinned skills are
+// always-loaded into every turn's system prompt, so the count is capped to
+// bound prompt size.
+const maxPinnedSkillsPerAgent = 10
+
+// registerAgentSkillPinCRUDTools registers goclaw_agents_pin_skill/
+// goclaw_agents_unpin_skill. Pinning is distinct from granting access
+// (registerSkillGrantCRUDTools in crud_skills.go): a pinned skill is
+// auto-loaded into the agent's system prompt every turn (see
+// internal/agent/resolver.go's ParsePinnedSkills / PinnedSkillsSummary),
+// while a grant only makes the skill available for on-demand use_skill
+// calls. There is no dedicated pin/unpin RPC on the gateway — the web UI
+// sets other_config.pinned_skills via the general agents.update WS method
+// (internal/gateway/methods/agents_update.go), which replaces the whole
+// other_config JSONB blob wholesale. These handlers replicate that by
+// reading the agent's current other_config, splicing pinned_skills, and
+// writing the full blob back — a naive partial write would silently drop
+// every other other_config field (self_evolution_metrics, tts_params, etc).
+func registerAgentSkillPinCRUDTools(srv *mcpserver.MCPServer, agents store.AgentStore) {
+	srv.AddTool(mcpgo.NewTool("goclaw_agents_pin_skill",
+		mcpgo.WithDescription("Pin a skill onto an agent so it's auto-loaded into that agent's system prompt every turn (distinct from granting access)."),
+		mcpgo.WithString("id", mcpgo.Required(), mcpgo.Description("Agent UUID.")),
+		mcpgo.WithString("skill", mcpgo.Required(), mcpgo.Description("Skill slug/name to pin.")),
+	), handleAgentsPinSkill(agents))
+
+	srv.AddTool(mcpgo.NewTool("goclaw_agents_unpin_skill",
+		mcpgo.WithDescription("Unpin a skill from an agent (does not revoke access, only removes it from the always-loaded set)."),
+		mcpgo.WithString("id", mcpgo.Required(), mcpgo.Description("Agent UUID.")),
+		mcpgo.WithString("skill", mcpgo.Required(), mcpgo.Description("Skill slug/name to unpin.")),
+	), handleAgentsUnpinSkill(agents))
+}
+
+// spliceOtherConfigPinnedSkills reads ag's other_config JSONB, applies edit
+// to its pinned_skills list, and returns the full re-marshaled blob ready
+// to pass as agents.Update's "other_config" value (a full-blob replace, not
+// a merge — see registerAgentSkillPinCRUDTools doc comment).
+func spliceOtherConfigPinnedSkills(ag store.AgentData, edit func(current []string) ([]string, error)) (json.RawMessage, error) {
+	bag := map[string]any{}
+	if len(ag.OtherConfig) > 0 {
+		if err := json.Unmarshal(ag.OtherConfig, &bag); err != nil {
+			return nil, fmt.Errorf("cannot parse existing other_config: %w", err)
+		}
+	}
+	next, err := edit(ag.ParsePinnedSkills())
+	if err != nil {
+		return nil, err
+	}
+	bag["pinned_skills"] = next
+	return json.Marshal(bag)
+}
+
+func handleAgentsPinSkill(agents store.AgentStore) mcpserver.ToolHandlerFunc {
+	return func(ctx context.Context, req mcpgo.CallToolRequest) (*mcpgo.CallToolResult, error) {
+		idStr, err := req.RequireString("id")
+		if err != nil {
+			return toolError("agents.pin_skill", err)
+		}
+		id, err := uuid.Parse(idStr)
+		if err != nil {
+			return toolError("agents.pin_skill", fmt.Errorf("invalid id: %w", err))
+		}
+		skill, err := req.RequireString("skill")
+		if err != nil {
+			return toolError("agents.pin_skill", err)
+		}
+
+		ag, err := agents.GetByID(ctx, id)
+		if err != nil {
+			return toolError("agents.pin_skill", err)
+		}
+
+		raw, err := spliceOtherConfigPinnedSkills(*ag, func(current []string) ([]string, error) {
+			if slices.Contains(current, skill) {
+				return current, nil
+			}
+			if len(current) >= maxPinnedSkillsPerAgent {
+				return nil, fmt.Errorf("agent already has %d pinned skills (max %d)", len(current), maxPinnedSkillsPerAgent)
+			}
+			return append(current, skill), nil
+		})
+		if err != nil {
+			return toolError("agents.pin_skill", err)
+		}
+
+		if err := agents.Update(ctx, id, map[string]any{"other_config": []byte(raw)}); err != nil {
+			return toolError("agents.pin_skill", err)
+		}
+		updated, err := agents.GetByID(ctx, id)
+		if err != nil {
+			return toolError("agents.pin_skill", err)
+		}
+		return jsonToolResult(map[string]any{"ok": "true", "pinned_skills": updated.ParsePinnedSkills()})
+	}
+}
+
+func handleAgentsUnpinSkill(agents store.AgentStore) mcpserver.ToolHandlerFunc {
+	return func(ctx context.Context, req mcpgo.CallToolRequest) (*mcpgo.CallToolResult, error) {
+		idStr, err := req.RequireString("id")
+		if err != nil {
+			return toolError("agents.unpin_skill", err)
+		}
+		id, err := uuid.Parse(idStr)
+		if err != nil {
+			return toolError("agents.unpin_skill", fmt.Errorf("invalid id: %w", err))
+		}
+		skill, err := req.RequireString("skill")
+		if err != nil {
+			return toolError("agents.unpin_skill", err)
+		}
+
+		ag, err := agents.GetByID(ctx, id)
+		if err != nil {
+			return toolError("agents.unpin_skill", err)
+		}
+
+		raw, err := spliceOtherConfigPinnedSkills(*ag, func(current []string) ([]string, error) {
+			out := make([]string, 0, len(current))
+			for _, s := range current {
+				if s != skill {
+					out = append(out, s)
+				}
+			}
+			return out, nil
+		})
+		if err != nil {
+			return toolError("agents.unpin_skill", err)
+		}
+
+		if err := agents.Update(ctx, id, map[string]any{"other_config": []byte(raw)}); err != nil {
+			return toolError("agents.unpin_skill", err)
+		}
+		updated, err := agents.GetByID(ctx, id)
+		if err != nil {
+			return toolError("agents.unpin_skill", err)
+		}
+		return jsonToolResult(map[string]any{"ok": "true", "pinned_skills": updated.ParsePinnedSkills()})
+	}
+}