fix(ai,hooks,mcp): resolve AI bugfixes, MethodHook instance isolation, and MCP multi-tenancy (spec-02)

This commit is contained in:
fluzzi32 committed 2026-11-09 15:59:46 -03:00
1 parent 2da9b45548
commit 494a7dffe6
4 files changed
+708 -75

No files matched your search

+16 -5
View File
@@ -1,3 +1,5 @@
#!/usr/bin/env python3
# ruff: noqa
import os import os
import secrets import secrets
@@ -197,6 +199,9 @@ class ai:
self.session_path = os.path.join(self.sessions_dir, f"{self.session_id}.json") if self.session_id else None self.session_path = os.path.join(self.sessions_dir, f"{self.session_id}.json") if self.session_id else None
# Agnostic base prompts # Agnostic base prompts
self._update_base_prompts()
def _update_base_prompts(self):
architect_instructions = "" architect_instructions = ""
if self.has_architect: if self.has_architect:
architect_instructions = """ architect_instructions = """
@@ -237,7 +242,7 @@ class ai:
- NO WANDERING: Do not speculate. If stuck, report attempts. - NO WANDERING: Do not speculate. If stuck, report attempts.
- SAFETY: When you use 'run_commands' with configuration commands, the system automatically prompts the user for confirmation. Just execute - don't ask permission first. - SAFETY: When you use 'run_commands' with configuration commands, the system automatically prompts the user for confirmation. Just execute - don't ask permission first.
{architect_instructions} {architect_instructions}
Network Context: {{self.long_term_memory if self.long_term_memory else "Empty."}} Network Context: {self.long_term_memory if self.long_term_memory else "Empty."}
""").strip() """).strip()
self._architect_base_prompt = dedent(f""" self._architect_base_prompt = dedent(f"""
@@ -277,6 +282,7 @@ class ai:
@property @property
def engineer_system_prompt(self): def engineer_system_prompt(self):
"""Build engineer system prompt with plugin extensions.""" """Build engineer system prompt with plugin extensions."""
self._update_base_prompts()
if self.engineer_prompt_extensions: if self.engineer_prompt_extensions:
extensions = "\n".join(self.engineer_prompt_extensions) extensions = "\n".join(self.engineer_prompt_extensions)
return self._engineer_base_prompt + f"\n\nPlugin Capabilities:\n{extensions}" return self._engineer_base_prompt + f"\n\nPlugin Capabilities:\n{extensions}"
@@ -285,6 +291,7 @@ class ai:
@property @property
def architect_system_prompt(self): def architect_system_prompt(self):
"""Build architect system prompt with plugin extensions.""" """Build architect system prompt with plugin extensions."""
self._update_base_prompts()
prompt = self._architect_base_prompt prompt = self._architect_base_prompt
if getattr(self, "one_shot", False): if getattr(self, "one_shot", False):
prompt += "\n\nCRITICAL 1-SHOT DIAGNOSTICS DIRECTIVE:\nYou are running in a 1-shot offline diagnostics mode. There is no active conversation loop, and you are NOT conversing with a Network Engineer. You MUST deliver your complete strategic analysis immediately and directly to the user. Do not suggest or attempt to delegate/return control to the engineer." prompt += "\n\nCRITICAL 1-SHOT DIAGNOSTICS DIRECTIVE:\nYou are running in a 1-shot offline diagnostics mode. There is no active conversation loop, and you are NOT conversing with a Network Engineer. You MUST deliver your complete strategic analysis immediately and directly to the user. Do not suggest or attempt to delegate/return control to the engineer."
@@ -624,6 +631,7 @@ class ai:
# Reload memory after update # Reload memory after update
with open(self.memory_path, "r") as f: with open(self.memory_path, "r") as f:
self.long_term_memory = f.read() self.long_term_memory = f.read()
self._update_base_prompts()
return "Memory updated successfully." return "Memory updated successfully."
except PermissionError as e: except PermissionError as e:
@@ -859,7 +867,7 @@ class ai:
# Silently fail for LLM tools # Silently fail for LLM tools
pass pass
if self.architect_key: if self.has_architect:
base_tools.extend([ base_tools.extend([
{"type": "function", "function": {"name": "consult_architect", "description": "Ask the Strategic Reasoning Engine for advice on complex design, architecture, or troubleshooting decisions. You remain in control and will present the response to the user. Use this for: configuration planning, design validation, complex troubleshooting.", "parameters": {"type": "object", "properties": {"question": {"type": "string", "description": "Strategic question or decision needed."}, "technical_summary": {"type": "string", "description": "Technical findings and context gathered so far."}}, "required": ["question", "technical_summary"]}}}, {"type": "function", "function": {"name": "consult_architect", "description": "Ask the Strategic Reasoning Engine for advice on complex design, architecture, or troubleshooting decisions. You remain in control and will present the response to the user. Use this for: configuration planning, design validation, complex troubleshooting.", "parameters": {"type": "object", "properties": {"question": {"type": "string", "description": "Strategic question or decision needed."}, "technical_summary": {"type": "string", "description": "Technical findings and context gathered so far."}}, "required": ["question", "technical_summary"]}}},
{"type": "function", "function": {"name": "escalate_to_architect", "description": "Transfer full control to the Strategic Reasoning Engine. Use ONLY when the user explicitly requests the Architect or when the problem requires strategic oversight beyond consultation. After escalation, the Architect takes over the conversation.", "parameters": {"type": "object", "properties": {"reason": {"type": "string", "description": "Why you're escalating (e.g. 'User requested Architect', 'Complex multi-site design needed')."}, "context": {"type": "string", "description": "Full context and findings to hand over."}}, "required": ["reason", "context"]}}} {"type": "function", "function": {"name": "escalate_to_architect", "description": "Transfer full control to the Strategic Reasoning Engine. Use ONLY when the user explicitly requests the Architect or when the problem requires strategic oversight beyond consultation. After escalation, the Architect takes over the conversation.", "parameters": {"type": "object", "properties": {"reason": {"type": "string", "description": "Why you're escalating (e.g. 'User requested Architect', 'Complex multi-site design needed')."}, "context": {"type": "string", "description": "Full context and findings to hand over."}}, "required": ["reason", "context"]}}}
@@ -1216,14 +1224,17 @@ class ai:
if status: update_status("[architect]Engineer consulting Architect...") if status: update_status("[architect]Engineer consulting Architect...")
try: try:
# Consultation only - Engineer stays in control # Consultation only - Engineer stays in control
auth_kwargs = dict(self.architect_auth or {})
if self.architect_key and "api_key" not in auth_kwargs:
auth_kwargs["api_key"] = self.architect_key
claude_resp = completion( claude_resp = completion(
model=self.architect_model, model=self.architect_model,
messages=[ messages=[
{"role": "system", "content": self.architect_system_prompt}, {"role": "system", "content": self.architect_system_prompt},
{"role": "user", "content": f"The Engineer needs your strategic advice.\n\nTECHNICAL SUMMARY: {args['technical_summary']}\n\nQUESTION: {args['question']}\n\nProvide strategic guidance. The Engineer will continue handling the user."} {"role": "user", "content": f"The Engineer needs your strategic advice.\n\nTECHNICAL SUMMARY: {args['technical_summary']}\n\nQUESTION: {args['question']}\n\nProvide strategic guidance. The Engineer will continue handling the user."}
], ],
api_key=self.architect_key, num_retries=3,
num_retries=3 **auth_kwargs
) )
obs = claude_resp.choices[0].message.content obs = claude_resp.choices[0].message.content
if debug: if debug:
@@ -1790,7 +1801,7 @@ class PlaybookBuilderAgent:
import yaml import yaml
try: try:
# 1. Parse YAML # 1. Parse YAML
data = yaml.load(playbook_yaml, Loader=yaml.FullLoader) data = yaml.safe_load(playbook_yaml)
except Exception as e: except Exception as e:
return {"valid": False, "error": f"YAML Syntax Error: {e}"} return {"valid": False, "error": f"YAML Syntax Error: {e}"}
+92 -28
View File
@@ -1,6 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
#Imports #Imports
from functools import wraps, partial, update_wrapper from functools import update_wrapper, wraps
from . import printer from . import printer
#functions and classes #functions and classes
@@ -10,54 +11,118 @@ class MethodHook:
def __init__(self, func): def __init__(self, func):
self.func = func self.func = func
self.pre_hooks = [] # List to store registered pre-hooks self.pre_hooks = [] # List to store registered class-level pre-hooks
self.post_hooks = [] # List to store registered post-hooks self.post_hooks = [] # List to store registered class-level post-hooks
wraps(func)(self) wraps(func)(self)
def register_pre_hook(self, hook):
"""Register a function to be called before the original function"""
if hook not in self.pre_hooks:
self.pre_hooks.append(hook)
def register_post_hook(self, hook):
"""Register a function to be called after the original function"""
if hook not in self.post_hooks:
self.post_hooks.append(hook)
def __call__(self, *args, **kwargs): def __call__(self, *args, **kwargs):
# Execute pre-hooks before the original function # Fallback when called directly on descriptor or standalone function
for hook in self.pre_hooks: inst_pre = []
inst_post = []
if args and hasattr(args[0], "__dict__"):
inst_dict = args[0].__dict__
inst_pre = inst_dict.get("_method_pre_hooks", {}).get(self.func.__name__, []) if isinstance(inst_dict.get("_method_pre_hooks"), dict) else []
inst_post = inst_dict.get("_method_post_hooks", {}).get(self.func.__name__, []) if isinstance(inst_dict.get("_method_post_hooks"), dict) else []
all_pre = list(self.pre_hooks) + [h for h in inst_pre if h not in self.pre_hooks]
for hook in all_pre:
try: try:
args, kwargs = hook(*args, **kwargs) args, kwargs = hook(*args, **kwargs)
except Exception as e: except Exception as e: # noqa: BLE001
hook_name = getattr(hook, "__name__", str(hook)) hook_name = getattr(hook, "__name__", str(hook))
printer.error(f"{self.func.__name__} Pre-hook {hook_name} raised an exception: {e}") printer.error(f"{self.func.__name__} Pre-hook {hook_name} raised an exception: {e}")
result = self.func(*args, **kwargs) result = self.func(*args, **kwargs)
# Execute post-hooks after the original function all_post = list(self.post_hooks) + [h for h in inst_post if h not in self.post_hooks]
if self.post_hooks: for hook in all_post:
#printer.info(f"Executing {len(self.post_hooks)} post-hooks for {self.func.__name__}...")
pass
for hook in self.post_hooks:
try: try:
result = hook(*args, **kwargs, result=result) # Pass result to hooks result = hook(*args, **kwargs, result=result)
except Exception as e: except Exception as e: # noqa: BLE001
hook_name = getattr(hook, "__name__", str(hook)) hook_name = getattr(hook, "__name__", str(hook))
printer.error(f"{self.func.__name__} Post-hook {hook_name} raised an exception: {e}") printer.error(f"{self.func.__name__} Post-hook {hook_name} raised an exception: {e}")
return result return result
def __get__(self, instance, owner): def __get__(self, instance, owner):
if not instance: if instance is None:
return self return self
else: return self.make_callable(instance)
return self.make_callable(instance)
def make_callable(self, instance): def make_callable(self, instance):
# Returns a callable that also allows access to hook registration methods # Returns a callable bound to instance with per-instance hook registration
callable_instance = partial(self.__call__, instance) func_name = self.func.__name__
callable_instance.register_pre_hook = self.register_pre_hook
callable_instance.register_post_hook = self.register_post_hook
return callable_instance
def register_pre_hook(self, hook): def bound_method(*args, **kwargs):
"""Register a function to be called before the original function""" # Pre-hooks: class-level followed by instance-level
self.pre_hooks.append(hook) if hasattr(instance, "__dict__"):
inst_pre = instance.__dict__.get("_method_pre_hooks", {}).get(func_name, [])
inst_post = instance.__dict__.get("_method_post_hooks", {}).get(func_name, [])
else:
inst_pre = instance._method_pre_hooks.get(func_name, []) if hasattr(instance, "_method_pre_hooks") else []
inst_post = instance._method_post_hooks.get(func_name, []) if hasattr(instance, "_method_post_hooks") else []
all_pre = list(self.pre_hooks) + [h for h in inst_pre if h not in self.pre_hooks]
call_args = (instance,) + args
for hook in all_pre:
try:
call_args, kwargs = hook(*call_args, **kwargs)
except Exception as e: # noqa: BLE001
hook_name = getattr(hook, "__name__", str(hook))
printer.error(f"{func_name} Pre-hook {hook_name} raised an exception: {e}")
result = self.func(*call_args, **kwargs)
all_post = list(self.post_hooks) + [h for h in inst_post if h not in self.post_hooks]
for hook in all_post:
try:
result = hook(*call_args, **kwargs, result=result)
except Exception as e: # noqa: BLE001
hook_name = getattr(hook, "__name__", str(hook))
printer.error(f"{func_name} Post-hook {hook_name} raised an exception: {e}")
return result
wraps(self.func)(bound_method)
def register_pre_hook(hook):
if hasattr(instance, "__dict__"):
hooks = instance.__dict__.setdefault("_method_pre_hooks", {}).setdefault(func_name, [])
else:
if not hasattr(instance, "_method_pre_hooks"):
instance._method_pre_hooks = {}
hooks = instance._method_pre_hooks.setdefault(func_name, [])
if hook not in hooks:
hooks.append(hook)
def register_post_hook(hook):
if hasattr(instance, "__dict__"):
hooks = instance.__dict__.setdefault("_method_post_hooks", {}).setdefault(func_name, [])
else:
if not hasattr(instance, "_method_post_hooks"):
instance._method_post_hooks = {}
hooks = instance._method_post_hooks.setdefault(func_name, [])
if hook not in hooks:
hooks.append(hook)
bound_method.register_pre_hook = register_pre_hook
bound_method.register_post_hook = register_post_hook
bound_method.__self__ = instance
bound_method.__func__ = self.func
return bound_method
def register_post_hook(self, hook):
"""Register a function to be called after the original function"""
self.post_hooks.append(hook)
class ClassHook: class ClassHook:
"""Decorator class to enable Class Modifying""" """Decorator class to enable Class Modifying"""
@@ -91,4 +156,3 @@ class ClassHook:
"""Modify this specific instance.""" """Modify this specific instance."""
modification_func(instance) modification_func(instance)
return modify_instance return modify_instance
+99 -42
View File
@@ -1,9 +1,9 @@
import asyncio import asyncio
import json import logging
import os import os
import threading import threading
from typing import Any, Dict, List, Optional from contextlib import suppress
import logging from typing import Any, ClassVar
try: try:
from mcp import ClientSession from mcp import ClientSession
@@ -17,32 +17,54 @@ logging.getLogger("mcp").setLevel(logging.CRITICAL)
logging.getLogger("httpx").setLevel(logging.CRITICAL) logging.getLogger("httpx").setLevel(logging.CRITICAL)
logging.getLogger("httpcore").setLevel(logging.CRITICAL) logging.getLogger("httpcore").setLevel(logging.CRITICAL)
class MCPClientManager: class MCPClientManager:
"""Manages MCP SSE client connections for connpy.""" """Manages MCP SSE client connections for connpy."""
_instance = None _instances: ClassVar[dict[Any, "MCPClientManager"]] = {}
_instance: ClassVar["MCPClientManager | None"] = None
_lock = threading.Lock() _lock = threading.Lock()
def __new__(cls, *args, **kwargs): @classmethod
def _resolve_scope_key(cls, config=None, *args, **kwargs) -> Any:
cfg = config
if cfg is None and "config" in kwargs:
cfg = kwargs["config"]
if cfg is not None:
defaultdir = getattr(cfg, "defaultdir", None)
if defaultdir:
return os.path.abspath(str(defaultdir))
return id(cfg)
return "default"
def __new__(cls, config=None, *args, **kwargs):
key = cls._resolve_scope_key(config, *args, **kwargs)
with cls._lock: with cls._lock:
if cls._instance is None: if cls._instance is None and cls._instances:
cls._instance = super(MCPClientManager, cls).__new__(cls) cls._instances.clear()
cls._instance._initialized = False if key not in cls._instances:
return cls._instance instance = super().__new__(cls)
instance._initialized = False
instance._scope_key = key
cls._instances[key] = instance
cls._instance = cls._instances[key]
return cls._instances[key]
def __init__(self, config=None): def __init__(self, config=None):
if self._initialized: if config is not None:
self.config = config
if getattr(self, "_initialized", False):
return return
self.config = config if not hasattr(self, "config"):
self.sessions: Dict[str, Dict[str, Any]] = {} # name -> {session, stack} self.config = config
self.tool_cache: Dict[str, List[Dict[str, Any]]] = {} self.sessions: dict[str, dict[str, Any]] = {} # name -> {session, stack}
self._connecting: Dict[str, asyncio.Future] = {} self.tool_cache: dict[str, list[dict[str, Any]]] = {}
self._server_configs: dict[str, dict[str, Any]] = {}
self._connecting: dict[str, asyncio.Future] = {}
self._initialized = True self._initialized = True
async def get_tools_for_llm(self, os_filter: Optional[str] = None) -> List[Dict[str, Any]]: async def get_tools_for_llm(self, os_filter: str | None = None) -> list[dict[str, Any]]:
""" """Fetches tools from enabled MCP servers that match the OS filter."""
Fetches tools from enabled MCP servers that match the OS filter.
"""
if not MCP_AVAILABLE: if not MCP_AVAILABLE:
return [] return []
@@ -52,29 +74,53 @@ class MCPClientManager:
mcp_config = self.config.get_effective_setting("ai", {}).get("mcp_servers", {}) mcp_config = self.config.get_effective_setting("ai", {}).get("mcp_servers", {})
else: else:
mcp_config = self.config.config.get("ai", {}).get("mcp_servers", {}) if hasattr(self.config, "config") else {} mcp_config = self.config.config.get("ai", {}).get("mcp_servers", {}) if hasattr(self.config, "config") else {}
except Exception: except Exception: # noqa: BLE001
return [] return []
if not isinstance(mcp_config, dict):
mcp_config = {}
# Invalidate cache and reconnect if server configuration changed or server was removed
current_names = set(mcp_config.keys())
for old_name in list(self._server_configs.keys()):
if old_name not in current_names:
self.tool_cache.pop(old_name, None)
old_data = self.sessions.pop(old_name, None)
if old_data and old_data.get("stack"):
with suppress(Exception):
await old_data["stack"].aclose()
self._server_configs.pop(old_name, None)
for name, cfg in mcp_config.items():
cached_cfg = self._server_configs.get(name)
if cached_cfg is not None and cached_cfg != cfg:
self.tool_cache.pop(name, None)
old_data = self.sessions.pop(name, None)
if old_data and old_data.get("stack"):
with suppress(Exception):
await old_data["stack"].aclose()
self._server_configs[name] = dict(cfg) if isinstance(cfg, dict) else cfg
async def _fetch(name, cfg): async def _fetch(name, cfg):
if not cfg.get("enabled", True): return [] if not isinstance(cfg, dict) or not cfg.get("enabled", True):
return []
# Filter by OS if specified in config (primarily used for copilot strict matching) # Filter by OS if specified in config (primarily used for copilot strict matching)
auto_os = cfg.get("auto_load_on_os") auto_os = cfg.get("auto_load_on_os")
if os_filter is not None and auto_os and os_filter.lower() != auto_os.lower(): if os_filter is not None and auto_os and os_filter.lower() != auto_os.lower():
return [] return []
try: with suppress(Exception):
session = await self._ensure_connected(name, cfg) session = await self._ensure_connected(name, cfg)
if session: if session:
if name in self.tool_cache: return self.tool_cache[name] if name in self.tool_cache:
return self.tool_cache[name]
llm_tools = await self._fetch_tools_as_openai(name, session) llm_tools = await self._fetch_tools_as_openai(name, session)
self.tool_cache[name] = llm_tools self.tool_cache[name] = llm_tools
return llm_tools return llm_tools
except Exception:
pass
return [] return []
tasks = [ _fetch(name, cfg) for name, cfg in mcp_config.items() ] tasks = [_fetch(name, cfg) for name, cfg in mcp_config.items()]
if tasks: if tasks:
results = await asyncio.gather(*tasks) results = await asyncio.gather(*tasks)
@@ -83,8 +129,9 @@ class MCPClientManager:
return all_llm_tools return all_llm_tools
async def _ensure_connected(self, name: str, cfg: Dict[str, Any]) -> Optional[Any]: async def _ensure_connected(self, name: str, cfg: dict[str, Any]) -> Any | None:
if not MCP_AVAILABLE: return None if not MCP_AVAILABLE:
return None
if name in self.sessions and self.sessions[name].get("session"): if name in self.sessions and self.sessions[name].get("session"):
return self.sessions[name]["session"] return self.sessions[name]["session"]
@@ -96,7 +143,7 @@ class MCPClientManager:
if name in self._connecting: if name in self._connecting:
try: try:
return await asyncio.wait_for(asyncio.shield(self._connecting[name]), timeout=10.0) return await asyncio.wait_for(asyncio.shield(self._connecting[name]), timeout=10.0)
except Exception: except Exception: # noqa: BLE001
return None return None
loop = asyncio.get_running_loop() loop = asyncio.get_running_loop()
@@ -117,33 +164,32 @@ class MCPClientManager:
self.sessions[name] = {"session": session, "stack": stack} self.sessions[name] = {"session": session, "stack": stack}
fut.set_result(session) fut.set_result(session)
return session return session
except Exception: except Exception: # noqa: BLE001
fut.set_result(None) fut.set_result(None)
return None return None
finally: finally:
if name in self._connecting: if name in self._connecting:
del self._connecting[name] del self._connecting[name]
async def _fetch_tools_as_openai(self, server_name: str, session: Any) -> List[Dict[str, Any]]: async def _fetch_tools_as_openai(self, server_name: str, session: Any) -> list[dict[str, Any]]:
try: try:
result = await asyncio.wait_for(session.list_tools(), timeout=5.0) result = await asyncio.wait_for(session.list_tools(), timeout=5.0)
openai_tools = [] openai_tools = []
for tool in result.tools: for tool in result.tools:
# Use mcp_ prefix to ensure valid function name for LiteLLM/Gemini
prefixed_name = f"mcp_{server_name}__{tool.name}" prefixed_name = f"mcp_{server_name}__{tool.name}"
openai_tools.append({ openai_tools.append({
"type": "function", "type": "function",
"function": { "function": {
"name": prefixed_name, "name": prefixed_name,
"description": f"[{server_name}] {tool.description}", "description": f"[{server_name}] {tool.description}",
"parameters": tool.inputSchema "parameters": tool.inputSchema,
} },
}) })
return openai_tools return openai_tools
except Exception: except Exception: # noqa: BLE001
return [] return []
async def call_tool(self, full_tool_name: str, arguments: Dict[str, Any]) -> Any: async def call_tool(self, full_tool_name: str, arguments: dict[str, Any]) -> Any:
"""Calls an MCP tool and returns text result.""" """Calls an MCP tool and returns text result."""
if not MCP_AVAILABLE: if not MCP_AVAILABLE:
return "Error: MCP SDK is not installed." return "Error: MCP SDK is not installed."
@@ -151,7 +197,7 @@ class MCPClientManager:
if "__" not in full_tool_name: if "__" not in full_tool_name:
return f"Error: Tool {full_tool_name} is not a valid MCP tool." return f"Error: Tool {full_tool_name} is not a valid MCP tool."
clean_name = full_tool_name[4:] if full_tool_name.startswith("mcp_") else full_tool_name clean_name = full_tool_name.removeprefix("mcp_")
server_name, tool_name = clean_name.split("__", 1) server_name, tool_name = clean_name.split("__", 1)
if server_name not in self.sessions: if server_name not in self.sessions:
@@ -162,13 +208,24 @@ class MCPClientManager:
result = await asyncio.wait_for(session.call_tool(tool_name, arguments), timeout=60.0) result = await asyncio.wait_for(session.call_tool(tool_name, arguments), timeout=60.0)
text_outputs = [content.text for content in result.content if hasattr(content, "text")] text_outputs = [content.text for content in result.content if hasattr(content, "text")]
return "\n".join(text_outputs) if text_outputs else str(result) return "\n".join(text_outputs) if text_outputs else str(result)
except Exception as e: except Exception as e: # noqa: BLE001
return f"Error calling tool {tool_name} on {server_name}: {str(e)}" return f"Error calling tool {tool_name} on {server_name}: {e!s}"
async def shutdown(self): async def shutdown(self):
"""Close all SSE connections.""" """Close all SSE connections and clean up resources."""
for name, data in self.sessions.items(): for data in list(self.sessions.values()):
stack = data.get("stack") stack = data.get("stack")
if stack: if stack:
await stack.aclose() with suppress(Exception):
self.sessions = {} await stack.aclose()
self.sessions.clear()
self.tool_cache.clear()
self._server_configs.clear()
for fut in list(self._connecting.values()):
if not fut.done():
fut.cancel()
self._connecting.clear()
with self._lock:
if hasattr(self, "_scope_key") and self._scope_key in self._instances:
del self._instances[self._scope_key]
self._initialized = False
+501
View File
@@ -0,0 +1,501 @@
"""Unit tests for AI bugfixes, MethodHook instance isolation, and MCP multi-tenancy.
Covering specification: spec-02-ai-and-hooks-bugfixes.
"""
import os
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from connpy.ai import PlaybookBuilderAgent, ai
from connpy.hooks import MethodHook
from connpy.mcp_client import MCPClientManager
# =========================================================================
# 1. AI Long-Term Memory Bugfix Tests
# =========================================================================
class TestAILongTermMemory:
"""Tests ensuring long_term_memory interpolation and updates work correctly."""
def test_long_term_memory_injected_without_template_literal(self, ai_config, mock_litellm):
"""Verify no double braces or unresolved '{self.long_term_memory' literals exist."""
myai = ai(ai_config)
assert "{self.long_term_memory" not in myai.engineer_system_prompt
assert "{{self.long_term_memory" not in myai.engineer_system_prompt
assert "{self.long_term_memory" not in myai.architect_system_prompt
assert "Network Context: Empty." in myai.engineer_system_prompt
assert "Network Context: Empty." in myai.architect_system_prompt
def test_manage_memory_tool_updates_prompts_immediately(self, ai_config, mock_litellm):
"""Verify calling manage_memory_tool immediately updates engineer and architect prompts."""
myai = ai(ai_config)
res = myai.manage_memory_tool("DC-East ASN is 65010", action="append")
assert "Memory updated successfully" in res
assert "DC-East ASN is 65010" in myai.long_term_memory
assert "DC-East ASN is 65010" in myai.engineer_system_prompt
assert "DC-East ASN is 65010" in myai.architect_system_prompt
assert "{self.long_term_memory" not in myai.engineer_system_prompt
def test_init_with_preexisting_memory_file(self, ai_config, mock_litellm):
"""Verify preexisting ai_memory.md content is loaded into prompts on startup."""
mem_file = os.path.join(ai_config.defaultdir, "ai_memory.md")
with open(mem_file, "w") as f:
f.write("Border routers running BGP EVPN")
myai = ai(ai_config)
assert "Border routers running BGP EVPN" in myai.long_term_memory
assert "Border routers running BGP EVPN" in myai.engineer_system_prompt
assert "Border routers running BGP EVPN" in myai.architect_system_prompt
# =========================================================================
# 2. AI has_architect and architect_auth Tests
# =========================================================================
class TestAIArchitectAuthAndTools:
"""Tests ensuring architect tools and authentication function with architect_auth."""
def test_engineer_tools_include_architect_when_auth_configured_without_key(self, ai_config, mock_litellm):
"""consult_architect & escalate_to_architect must be available with architect_auth even without key."""
ai_config.config["ai"]["architect_api_key"] = None
ai_config.config["ai"]["architect_auth"] = {"service_account": "sa@google.com"}
myai = ai(ai_config)
assert myai.architect_key is None
assert myai.has_architect is True
tools = myai._get_engineer_tools()
tool_names = [t["function"]["name"] for t in tools]
assert "consult_architect" in tool_names
assert "escalate_to_architect" in tool_names
def test_engineer_tools_include_architect_for_keyless_model(self, ai_config, mock_litellm):
"""Vertex AI / Ollama keyless models must have architect tools enabled."""
ai_config.config["ai"]["architect_api_key"] = None
ai_config.config["ai"]["architect_auth"] = {}
ai_config.config["ai"]["architect_model"] = "vertex_ai/gemini-1.5-pro"
myai = ai(ai_config)
assert myai.has_architect is True
tools = myai._get_engineer_tools()
tool_names = [t["function"]["name"] for t in tools]
assert "consult_architect" in tool_names
assert "escalate_to_architect" in tool_names
def test_consult_architect_passes_auth_kwargs_cleanly(self, ai_config):
"""consult_architect must pass dict(architect_auth) to litellm.completion."""
ai_config.config["ai"]["architect_api_key"] = None
ai_config.config["ai"]["architect_auth"] = {
"api_base": "https://custom-ai.internal/v1",
"custom_header": "test-val",
}
myai = ai(ai_config)
with patch("connpy.ai.completion") as mock_comp:
fake_resp = MagicMock()
fake_msg = MagicMock()
fake_msg.content = "Architect advice on spine-leaf"
fake_msg.tool_calls = None
fake_msg.role = "assistant"
fake_msg.model_dump.return_value = {"role": "assistant", "content": "Advice"}
fake_choice = MagicMock()
fake_choice.message = fake_msg
fake_resp.choices = [fake_choice]
fake_resp.usage = MagicMock()
fake_resp.usage.prompt_tokens = 10
fake_resp.usage.completion_tokens = 10
fake_resp.usage.total_tokens = 20
# Craft first completion call that instructs engineer to consult architect
tc = MagicMock()
tc.function.name = "consult_architect"
tc.function.arguments = '{"question": "How to design MTU?", "technical_summary": "9000 vs 1500"}'
eng_msg = MagicMock()
eng_msg.role = "assistant"
eng_msg.content = ""
eng_msg.tool_calls = [tc]
eng_msg.model_dump.return_value = {
"role": "assistant",
"content": None,
"tool_calls": [{
"id": "call_1",
"type": "function",
"function": {
"name": "consult_architect",
"arguments": '{"question": "How to design MTU?", "technical_summary": "9000 vs 1500"}',
},
}],
}
eng_choice = MagicMock()
eng_choice.message = eng_msg
eng_resp = MagicMock()
eng_resp.choices = [eng_choice]
eng_resp.usage = MagicMock()
eng_resp.usage.prompt_tokens = 5
eng_resp.usage.completion_tokens = 5
eng_resp.usage.total_tokens = 10
# Final response after consultation
final_msg = MagicMock()
final_msg.role = "assistant"
final_msg.content = "Here is the guidance."
final_msg.tool_calls = None
final_msg.model_dump.return_value = {"role": "assistant", "content": "Here is the guidance."}
final_choice = MagicMock()
final_choice.message = final_msg
final_resp = MagicMock()
final_resp.choices = [final_choice]
final_resp.usage = MagicMock()
final_resp.usage.prompt_tokens = 5
final_resp.usage.completion_tokens = 5
final_resp.usage.total_tokens = 10
mock_comp.side_effect = [eng_resp, fake_resp, final_resp]
res = myai.ask("Design MTU question", stream=False)
assert "guidance" in res["response"]
# Second call to completion was consult_architect
assert mock_comp.call_count >= 2
consult_call = mock_comp.call_args_list[1]
kwargs = consult_call.kwargs
assert kwargs.get("api_base") == "https://custom-ai.internal/v1"
assert kwargs.get("custom_header") == "test-val"
assert "api_key" not in kwargs
def test_consult_architect_injects_architect_key_if_missing_from_auth(self, ai_config):
"""If architect_key exists and 'api_key' is not in architect_auth, it must be added."""
ai_config.config["ai"]["architect_api_key"] = "arch-key-12345"
ai_config.config["ai"]["architect_auth"] = {"api_base": "https://custom-ai.internal/v1"}
myai = ai(ai_config)
with patch("connpy.ai.completion") as mock_comp:
fake_resp = MagicMock()
fake_msg = MagicMock()
fake_msg.content = "Advice"
fake_msg.tool_calls = None
fake_msg.role = "assistant"
fake_msg.model_dump.return_value = {"role": "assistant", "content": "Advice"}
fake_choice = MagicMock()
fake_choice.message = fake_msg
fake_resp.choices = [fake_choice]
fake_resp.usage = MagicMock()
fake_resp.usage.prompt_tokens = 10
fake_resp.usage.completion_tokens = 10
fake_resp.usage.total_tokens = 20
tc = MagicMock()
tc.function.name = "consult_architect"
tc.function.arguments = '{"question": "Q?", "technical_summary": "S"}'
eng_msg = MagicMock()
eng_msg.role = "assistant"
eng_msg.content = ""
eng_msg.tool_calls = [tc]
eng_msg.model_dump.return_value = {
"role": "assistant",
"content": None,
"tool_calls": [{
"id": "call_1",
"type": "function",
"function": {
"name": "consult_architect",
"arguments": '{"question": "Q?", "technical_summary": "S"}',
},
}],
}
eng_choice = MagicMock()
eng_choice.message = eng_msg
eng_resp = MagicMock()
eng_resp.choices = [eng_choice]
eng_resp.usage = MagicMock()
eng_resp.usage.prompt_tokens = 5
eng_resp.usage.completion_tokens = 5
eng_resp.usage.total_tokens = 10
final_msg = MagicMock()
final_msg.role = "assistant"
final_msg.content = "Done."
final_msg.tool_calls = None
final_msg.model_dump.return_value = {"role": "assistant", "content": "Done."}
final_choice = MagicMock()
final_choice.message = final_msg
final_resp = MagicMock()
final_resp.choices = [final_choice]
final_resp.usage = MagicMock()
final_resp.usage.prompt_tokens = 5
final_resp.usage.completion_tokens = 5
final_resp.usage.total_tokens = 10
mock_comp.side_effect = [eng_resp, fake_resp, final_resp]
myai.ask("Consult question", stream=False)
consult_call = mock_comp.call_args_list[1]
kwargs = consult_call.kwargs
assert kwargs.get("api_key") == "arch-key-12345"
assert kwargs.get("api_base") == "https://custom-ai.internal/v1"
# =========================================================================
# 3. PlaybookBuilderAgent Safe YAML Validation Tests
# =========================================================================
class TestPlaybookValidationSafeLoad:
"""Tests verifying PlaybookBuilderAgent.validate_playbook uses safe_load."""
def test_validate_playbook_rejects_malicious_yaml_tags(self, config):
"""Ensure arbitrary Python object deserialization payloads are safely rejected."""
agent = PlaybookBuilderAgent(config)
malicious_yaml = """
tasks:
- name: Exploit
command: !!python/object/apply:os.system ["touch /tmp/hacked_marker"]
"""
res = agent.validate_playbook(malicious_yaml)
assert res["valid"] is False
assert "YAML Syntax Error" in res["error"]
assert not os.path.exists("/tmp/hacked_marker")
def test_validate_playbook_accepts_valid_yaml(self, config):
"""Ensure safe YAML with tasks structure validates successfully."""
agent = PlaybookBuilderAgent(config)
valid_yaml = """
tasks:
- name: Check BGP
action: run
nodes: "router.*"
commands:
- show ip bgp summary
output: stdout
"""
res = agent.validate_playbook(valid_yaml)
assert res["valid"] is True
def test_validate_playbook_invalid_structures(self, config):
"""Ensure non-dict or missing 'tasks' YAML is rejected cleanly."""
agent = PlaybookBuilderAgent(config)
assert agent.validate_playbook("- not a dict\n- just a list")["valid"] is False
assert agent.validate_playbook("simple_scalar_string")["valid"] is False
assert agent.validate_playbook("other_root: 123")["valid"] is False
assert agent.validate_playbook("tasks: 'not a list'")["valid"] is False
# =========================================================================
# 4. MethodHook Instance Isolation and Deduplication Tests
# =========================================================================
class TestMethodHookIsolation:
"""Tests ensuring MethodHook instances are strictly isolated and deduplicated."""
def test_instance_hooks_do_not_contaminate_other_instances(self):
"""Registering a hook on inst1 must not run on inst2 or modify class hooks."""
class NetworkService:
@MethodHook
def run(self, cmd):
return f"executed:{cmd}"
inst1 = NetworkService()
inst2 = NetworkService()
def inst1_post_hook(*args, **kwargs):
return kwargs["result"] + " [audited-by-1]"
inst1.run.register_post_hook(inst1_post_hook)
assert inst1.run("show clock") == "executed:show clock [audited-by-1]"
assert inst2.run("show clock") == "executed:show clock"
assert NetworkService.run.post_hooks == []
def test_instance_hooks_deduplication(self):
"""Registering the same callable multiple times on an instance runs only once."""
class NetworkService:
@MethodHook
def run(self, cmd):
return f"res:{cmd}"
calls = []
def counting_hook(*args, **kwargs):
calls.append("called")
return kwargs["result"]
inst = NetworkService()
inst.run.register_post_hook(counting_hook)
inst.run.register_post_hook(counting_hook)
inst.run.register_post_hook(counting_hook)
res = inst.run("test")
assert res == "res:test"
assert calls == ["called"]
def test_class_level_and_instance_level_execution_order(self):
"""Class pre-hooks run before instance pre-hooks; class post-hooks run before instance post-hooks."""
class ExecutionDevice:
@MethodHook
def execute(self, val):
trace.append(f"base:{val}")
return val
trace = []
def class_pre(self, val):
trace.append("class_pre")
return (self, val), {}
def inst_pre(self, val):
trace.append("inst_pre")
return (self, val), {}
def class_post(*args, **kwargs):
trace.append("class_post")
return kwargs["result"]
def inst_post(*args, **kwargs):
trace.append("inst_post")
return kwargs["result"]
ExecutionDevice.execute.register_pre_hook(class_pre)
ExecutionDevice.execute.register_post_hook(class_post)
dev = ExecutionDevice()
dev.execute.register_pre_hook(inst_pre)
dev.execute.register_post_hook(inst_post)
dev.execute("ping")
assert trace == ["class_pre", "inst_pre", "base:ping", "class_post", "inst_post"]
def test_configfile_instance_isolation(self, config):
"""Verify two configfile instances do not leak hooks across instances."""
from connpy.configfile import configfile
conf2 = configfile(conf=config.file, key=config.key)
captured = []
def custom_hook(*args, **kwargs):
captured.append("c1_hook")
return kwargs["result"]
config._getallnodes.register_post_hook(custom_hook)
config._getallnodes(".*")
assert captured == ["c1_hook"]
# Call on second instance — hook must NOT execute
captured.clear()
conf2._getallnodes(".*")
assert captured == []
# =========================================================================
# 5. MCPClientManager Multi-Tenancy and Invalidation Tests
# =========================================================================
class DummyConfig:
"""Mock config object with config attribute and defaultdir."""
def __init__(self, defaultdir: str, mcp_servers: dict | None = None):
self.defaultdir = defaultdir
self.config = {
"ai": {
"mcp_servers": mcp_servers or {},
},
}
class TestMCPClientManagerMultiTenancy:
"""Tests ensuring MCPClientManager isolates tenants and invalidates cache properly."""
@pytest.fixture(autouse=True)
def reset_mcp_singletons(self):
"""Clean MCPClientManager instances before and after each test."""
MCPClientManager._instances.clear()
MCPClientManager._instance = None
yield
MCPClientManager._instances.clear()
MCPClientManager._instance = None
def test_mcp_manager_scoped_by_defaultdir(self):
"""Different defaultdir paths produce distinct isolated instances."""
cfg_tenant1 = DummyConfig("/var/connpy/tenant1")
cfg_tenant2 = DummyConfig("/var/connpy/tenant2")
mgr1 = MCPClientManager(cfg_tenant1)
mgr2 = MCPClientManager(cfg_tenant2)
assert mgr1 is not mgr2
assert mgr1.config is cfg_tenant1
assert mgr2.config is cfg_tenant2
def test_mcp_manager_same_scope_returns_same_instance_and_updates_config(self):
"""Same defaultdir reuses manager and updates self.config."""
cfg_a = DummyConfig("/var/connpy/shared")
cfg_b = DummyConfig("/var/connpy/shared")
mgr_a = MCPClientManager(cfg_a)
mgr_b = MCPClientManager(cfg_b)
assert mgr_a is mgr_b
assert mgr_a.config is cfg_b
def test_mcp_manager_scoped_by_id_when_no_defaultdir(self):
"""Configs without defaultdir are isolated by id(config)."""
cfg1 = object()
cfg2 = object()
mgr1 = MCPClientManager(cfg1)
mgr2 = MCPClientManager(cfg2)
assert mgr1 is not mgr2
@pytest.mark.asyncio
async def test_get_tools_invalidates_cache_on_config_change(self):
"""When server config changes in mcp_servers, tool_cache is invalidated."""
cfg = DummyConfig(
"/tmp/test_mcp_invalidation",
mcp_servers={"srv1": {"url": "http://127.0.0.1:8001/sse", "enabled": True}},
)
mgr = MCPClientManager(cfg)
mgr.tool_cache["srv1"] = [{"type": "function", "function": {"name": "old_tool"}}]
mock_stack = AsyncMock()
mgr.sessions["srv1"] = {"session": MagicMock(), "stack": mock_stack}
mgr._server_configs["srv1"] = {"url": "http://127.0.0.1:8001/sse", "enabled": True}
# Modify config to point to new URL
cfg.config["ai"]["mcp_servers"]["srv1"] = {"url": "http://127.0.0.1:8002/sse", "enabled": True}
new_session = MagicMock()
with patch.object(mgr, "_ensure_connected", new_callable=AsyncMock) as mock_conn, \
patch.object(mgr, "_fetch_tools_as_openai", new_callable=AsyncMock) as mock_fetch:
mock_conn.return_value = new_session
mock_fetch.return_value = [{"type": "function", "function": {"name": "new_tool"}}]
tools = await mgr.get_tools_for_llm()
# Ensure old stack was closed
mock_stack.aclose.assert_awaited_once()
# Ensure new tools returned
assert tools == [{"type": "function", "function": {"name": "new_tool"}}]
assert mgr.tool_cache["srv1"] == [{"type": "function", "function": {"name": "new_tool"}}]
await mgr.shutdown()
@pytest.mark.asyncio
async def test_shutdown_cleans_resources_and_resets(self):
"""shutdown() closes all stacks, clears caches and removes from _instances."""
cfg = DummyConfig("/tmp/test_mcp_shutdown")
mgr = MCPClientManager(cfg)
mock_stack = AsyncMock()
mgr.sessions["test"] = {"session": MagicMock(), "stack": mock_stack}
mgr.tool_cache["test"] = [{"dummy": 1}]
await mgr.shutdown()
mock_stack.aclose.assert_awaited_once()
assert mgr.sessions == {}
assert mgr.tool_cache == {}
assert os.path.abspath(cfg.defaultdir) not in MCPClientManager._instances