Files
connpy/connpy/tests/test_ai_and_hooks_bugfixes.py
T

502 lines
20 KiB
Python

"""Unit tests for AI bugfixes, MethodHook instance isolation, and MCP multi-tenancy.
Covering specification: spec-02-ai-and-hooks-bugfixes.
"""
import os
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from connpy.ai import PlaybookBuilderAgent, ai
from connpy.hooks import MethodHook
from connpy.mcp_client import MCPClientManager
# =========================================================================
# 1. AI Long-Term Memory Bugfix Tests
# =========================================================================
class TestAILongTermMemory:
"""Tests ensuring long_term_memory interpolation and updates work correctly."""
def test_long_term_memory_injected_without_template_literal(self, ai_config, mock_litellm):
"""Verify no double braces or unresolved '{self.long_term_memory' literals exist."""
myai = ai(ai_config)
assert "{self.long_term_memory" not in myai.engineer_system_prompt
assert "{{self.long_term_memory" not in myai.engineer_system_prompt
assert "{self.long_term_memory" not in myai.architect_system_prompt
assert "Network Context: Empty." in myai.engineer_system_prompt
assert "Network Context: Empty." in myai.architect_system_prompt
def test_manage_memory_tool_updates_prompts_immediately(self, ai_config, mock_litellm):
"""Verify calling manage_memory_tool immediately updates engineer and architect prompts."""
myai = ai(ai_config)
res = myai.manage_memory_tool("DC-East ASN is 65010", action="append")
assert "Memory updated successfully" in res
assert "DC-East ASN is 65010" in myai.long_term_memory
assert "DC-East ASN is 65010" in myai.engineer_system_prompt
assert "DC-East ASN is 65010" in myai.architect_system_prompt
assert "{self.long_term_memory" not in myai.engineer_system_prompt
def test_init_with_preexisting_memory_file(self, ai_config, mock_litellm):
"""Verify preexisting ai_memory.md content is loaded into prompts on startup."""
mem_file = os.path.join(ai_config.defaultdir, "ai_memory.md")
with open(mem_file, "w") as f:
f.write("Border routers running BGP EVPN")
myai = ai(ai_config)
assert "Border routers running BGP EVPN" in myai.long_term_memory
assert "Border routers running BGP EVPN" in myai.engineer_system_prompt
assert "Border routers running BGP EVPN" in myai.architect_system_prompt
# =========================================================================
# 2. AI has_architect and architect_auth Tests
# =========================================================================
class TestAIArchitectAuthAndTools:
"""Tests ensuring architect tools and authentication function with architect_auth."""
def test_engineer_tools_include_architect_when_auth_configured_without_key(self, ai_config, mock_litellm):
"""consult_architect & escalate_to_architect must be available with architect_auth even without key."""
ai_config.config["ai"]["architect_api_key"] = None
ai_config.config["ai"]["architect_auth"] = {"service_account": "sa@google.com"}
myai = ai(ai_config)
assert myai.architect_key is None
assert myai.has_architect is True
tools = myai._get_engineer_tools()
tool_names = [t["function"]["name"] for t in tools]
assert "consult_architect" in tool_names
assert "escalate_to_architect" in tool_names
def test_engineer_tools_include_architect_for_keyless_model(self, ai_config, mock_litellm):
"""Vertex AI / Ollama keyless models must have architect tools enabled."""
ai_config.config["ai"]["architect_api_key"] = None
ai_config.config["ai"]["architect_auth"] = {}
ai_config.config["ai"]["architect_model"] = "vertex_ai/gemini-1.5-pro"
myai = ai(ai_config)
assert myai.has_architect is True
tools = myai._get_engineer_tools()
tool_names = [t["function"]["name"] for t in tools]
assert "consult_architect" in tool_names
assert "escalate_to_architect" in tool_names
def test_consult_architect_passes_auth_kwargs_cleanly(self, ai_config):
"""consult_architect must pass dict(architect_auth) to litellm.completion."""
ai_config.config["ai"]["architect_api_key"] = None
ai_config.config["ai"]["architect_auth"] = {
"api_base": "https://custom-ai.internal/v1",
"custom_header": "test-val",
}
myai = ai(ai_config)
with patch("connpy.ai.completion") as mock_comp:
fake_resp = MagicMock()
fake_msg = MagicMock()
fake_msg.content = "Architect advice on spine-leaf"
fake_msg.tool_calls = None
fake_msg.role = "assistant"
fake_msg.model_dump.return_value = {"role": "assistant", "content": "Advice"}
fake_choice = MagicMock()
fake_choice.message = fake_msg
fake_resp.choices = [fake_choice]
fake_resp.usage = MagicMock()
fake_resp.usage.prompt_tokens = 10
fake_resp.usage.completion_tokens = 10
fake_resp.usage.total_tokens = 20
# Craft first completion call that instructs engineer to consult architect
tc = MagicMock()
tc.function.name = "consult_architect"
tc.function.arguments = '{"question": "How to design MTU?", "technical_summary": "9000 vs 1500"}'
eng_msg = MagicMock()
eng_msg.role = "assistant"
eng_msg.content = ""
eng_msg.tool_calls = [tc]
eng_msg.model_dump.return_value = {
"role": "assistant",
"content": None,
"tool_calls": [{
"id": "call_1",
"type": "function",
"function": {
"name": "consult_architect",
"arguments": '{"question": "How to design MTU?", "technical_summary": "9000 vs 1500"}',
},
}],
}
eng_choice = MagicMock()
eng_choice.message = eng_msg
eng_resp = MagicMock()
eng_resp.choices = [eng_choice]
eng_resp.usage = MagicMock()
eng_resp.usage.prompt_tokens = 5
eng_resp.usage.completion_tokens = 5
eng_resp.usage.total_tokens = 10
# Final response after consultation
final_msg = MagicMock()
final_msg.role = "assistant"
final_msg.content = "Here is the guidance."
final_msg.tool_calls = None
final_msg.model_dump.return_value = {"role": "assistant", "content": "Here is the guidance."}
final_choice = MagicMock()
final_choice.message = final_msg
final_resp = MagicMock()
final_resp.choices = [final_choice]
final_resp.usage = MagicMock()
final_resp.usage.prompt_tokens = 5
final_resp.usage.completion_tokens = 5
final_resp.usage.total_tokens = 10
mock_comp.side_effect = [eng_resp, fake_resp, final_resp]
res = myai.ask("Design MTU question", stream=False)
assert "guidance" in res["response"]
# Second call to completion was consult_architect
assert mock_comp.call_count >= 2
consult_call = mock_comp.call_args_list[1]
kwargs = consult_call.kwargs
assert kwargs.get("api_base") == "https://custom-ai.internal/v1"
assert kwargs.get("custom_header") == "test-val"
assert "api_key" not in kwargs
def test_consult_architect_injects_architect_key_if_missing_from_auth(self, ai_config):
"""If architect_key exists and 'api_key' is not in architect_auth, it must be added."""
ai_config.config["ai"]["architect_api_key"] = "arch-key-12345"
ai_config.config["ai"]["architect_auth"] = {"api_base": "https://custom-ai.internal/v1"}
myai = ai(ai_config)
with patch("connpy.ai.completion") as mock_comp:
fake_resp = MagicMock()
fake_msg = MagicMock()
fake_msg.content = "Advice"
fake_msg.tool_calls = None
fake_msg.role = "assistant"
fake_msg.model_dump.return_value = {"role": "assistant", "content": "Advice"}
fake_choice = MagicMock()
fake_choice.message = fake_msg
fake_resp.choices = [fake_choice]
fake_resp.usage = MagicMock()
fake_resp.usage.prompt_tokens = 10
fake_resp.usage.completion_tokens = 10
fake_resp.usage.total_tokens = 20
tc = MagicMock()
tc.function.name = "consult_architect"
tc.function.arguments = '{"question": "Q?", "technical_summary": "S"}'
eng_msg = MagicMock()
eng_msg.role = "assistant"
eng_msg.content = ""
eng_msg.tool_calls = [tc]
eng_msg.model_dump.return_value = {
"role": "assistant",
"content": None,
"tool_calls": [{
"id": "call_1",
"type": "function",
"function": {
"name": "consult_architect",
"arguments": '{"question": "Q?", "technical_summary": "S"}',
},
}],
}
eng_choice = MagicMock()
eng_choice.message = eng_msg
eng_resp = MagicMock()
eng_resp.choices = [eng_choice]
eng_resp.usage = MagicMock()
eng_resp.usage.prompt_tokens = 5
eng_resp.usage.completion_tokens = 5
eng_resp.usage.total_tokens = 10
final_msg = MagicMock()
final_msg.role = "assistant"
final_msg.content = "Done."
final_msg.tool_calls = None
final_msg.model_dump.return_value = {"role": "assistant", "content": "Done."}
final_choice = MagicMock()
final_choice.message = final_msg
final_resp = MagicMock()
final_resp.choices = [final_choice]
final_resp.usage = MagicMock()
final_resp.usage.prompt_tokens = 5
final_resp.usage.completion_tokens = 5
final_resp.usage.total_tokens = 10
mock_comp.side_effect = [eng_resp, fake_resp, final_resp]
myai.ask("Consult question", stream=False)
consult_call = mock_comp.call_args_list[1]
kwargs = consult_call.kwargs
assert kwargs.get("api_key") == "arch-key-12345"
assert kwargs.get("api_base") == "https://custom-ai.internal/v1"
# =========================================================================
# 3. PlaybookBuilderAgent Safe YAML Validation Tests
# =========================================================================
class TestPlaybookValidationSafeLoad:
"""Tests verifying PlaybookBuilderAgent.validate_playbook uses safe_load."""
def test_validate_playbook_rejects_malicious_yaml_tags(self, config):
"""Ensure arbitrary Python object deserialization payloads are safely rejected."""
agent = PlaybookBuilderAgent(config)
malicious_yaml = """
tasks:
- name: Exploit
command: !!python/object/apply:os.system ["touch /tmp/hacked_marker"]
"""
res = agent.validate_playbook(malicious_yaml)
assert res["valid"] is False
assert "YAML Syntax Error" in res["error"]
assert not os.path.exists("/tmp/hacked_marker")
def test_validate_playbook_accepts_valid_yaml(self, config):
"""Ensure safe YAML with tasks structure validates successfully."""
agent = PlaybookBuilderAgent(config)
valid_yaml = """
tasks:
- name: Check BGP
action: run
nodes: "router.*"
commands:
- show ip bgp summary
output: stdout
"""
res = agent.validate_playbook(valid_yaml)
assert res["valid"] is True
def test_validate_playbook_invalid_structures(self, config):
"""Ensure non-dict or missing 'tasks' YAML is rejected cleanly."""
agent = PlaybookBuilderAgent(config)
assert agent.validate_playbook("- not a dict\n- just a list")["valid"] is False
assert agent.validate_playbook("simple_scalar_string")["valid"] is False
assert agent.validate_playbook("other_root: 123")["valid"] is False
assert agent.validate_playbook("tasks: 'not a list'")["valid"] is False
# =========================================================================
# 4. MethodHook Instance Isolation and Deduplication Tests
# =========================================================================
class TestMethodHookIsolation:
"""Tests ensuring MethodHook instances are strictly isolated and deduplicated."""
def test_instance_hooks_do_not_contaminate_other_instances(self):
"""Registering a hook on inst1 must not run on inst2 or modify class hooks."""
class NetworkService:
@MethodHook
def run(self, cmd):
return f"executed:{cmd}"
inst1 = NetworkService()
inst2 = NetworkService()
def inst1_post_hook(*args, **kwargs):
return kwargs["result"] + " [audited-by-1]"
inst1.run.register_post_hook(inst1_post_hook)
assert inst1.run("show clock") == "executed:show clock [audited-by-1]"
assert inst2.run("show clock") == "executed:show clock"
assert NetworkService.run.post_hooks == []
def test_instance_hooks_deduplication(self):
"""Registering the same callable multiple times on an instance runs only once."""
class NetworkService:
@MethodHook
def run(self, cmd):
return f"res:{cmd}"
calls = []
def counting_hook(*args, **kwargs):
calls.append("called")
return kwargs["result"]
inst = NetworkService()
inst.run.register_post_hook(counting_hook)
inst.run.register_post_hook(counting_hook)
inst.run.register_post_hook(counting_hook)
res = inst.run("test")
assert res == "res:test"
assert calls == ["called"]
def test_class_level_and_instance_level_execution_order(self):
"""Class pre-hooks run before instance pre-hooks; class post-hooks run before instance post-hooks."""
class ExecutionDevice:
@MethodHook
def execute(self, val):
trace.append(f"base:{val}")
return val
trace = []
def class_pre(self, val):
trace.append("class_pre")
return (self, val), {}
def inst_pre(self, val):
trace.append("inst_pre")
return (self, val), {}
def class_post(*args, **kwargs):
trace.append("class_post")
return kwargs["result"]
def inst_post(*args, **kwargs):
trace.append("inst_post")
return kwargs["result"]
ExecutionDevice.execute.register_pre_hook(class_pre)
ExecutionDevice.execute.register_post_hook(class_post)
dev = ExecutionDevice()
dev.execute.register_pre_hook(inst_pre)
dev.execute.register_post_hook(inst_post)
dev.execute("ping")
assert trace == ["class_pre", "inst_pre", "base:ping", "class_post", "inst_post"]
def test_configfile_instance_isolation(self, config):
"""Verify two configfile instances do not leak hooks across instances."""
from connpy.configfile import configfile
conf2 = configfile(conf=config.file, key=config.key)
captured = []
def custom_hook(*args, **kwargs):
captured.append("c1_hook")
return kwargs["result"]
config._getallnodes.register_post_hook(custom_hook)
config._getallnodes(".*")
assert captured == ["c1_hook"]
# Call on second instance — hook must NOT execute
captured.clear()
conf2._getallnodes(".*")
assert captured == []
# =========================================================================
# 5. MCPClientManager Multi-Tenancy and Invalidation Tests
# =========================================================================
class DummyConfig:
"""Mock config object with config attribute and defaultdir."""
def __init__(self, defaultdir: str, mcp_servers: dict | None = None):
self.defaultdir = defaultdir
self.config = {
"ai": {
"mcp_servers": mcp_servers or {},
},
}
class TestMCPClientManagerMultiTenancy:
"""Tests ensuring MCPClientManager isolates tenants and invalidates cache properly."""
@pytest.fixture(autouse=True)
def reset_mcp_singletons(self):
"""Clean MCPClientManager instances before and after each test."""
MCPClientManager._instances.clear()
MCPClientManager._instance = None
yield
MCPClientManager._instances.clear()
MCPClientManager._instance = None
def test_mcp_manager_scoped_by_defaultdir(self):
"""Different defaultdir paths produce distinct isolated instances."""
cfg_tenant1 = DummyConfig("/var/connpy/tenant1")
cfg_tenant2 = DummyConfig("/var/connpy/tenant2")
mgr1 = MCPClientManager(cfg_tenant1)
mgr2 = MCPClientManager(cfg_tenant2)
assert mgr1 is not mgr2
assert mgr1.config is cfg_tenant1
assert mgr2.config is cfg_tenant2
def test_mcp_manager_same_scope_returns_same_instance_and_updates_config(self):
"""Same defaultdir reuses manager and updates self.config."""
cfg_a = DummyConfig("/var/connpy/shared")
cfg_b = DummyConfig("/var/connpy/shared")
mgr_a = MCPClientManager(cfg_a)
mgr_b = MCPClientManager(cfg_b)
assert mgr_a is mgr_b
assert mgr_a.config is cfg_b
def test_mcp_manager_scoped_by_id_when_no_defaultdir(self):
"""Configs without defaultdir are isolated by id(config)."""
cfg1 = object()
cfg2 = object()
mgr1 = MCPClientManager(cfg1)
mgr2 = MCPClientManager(cfg2)
assert mgr1 is not mgr2
@pytest.mark.asyncio
async def test_get_tools_invalidates_cache_on_config_change(self):
"""When server config changes in mcp_servers, tool_cache is invalidated."""
cfg = DummyConfig(
"/tmp/test_mcp_invalidation",
mcp_servers={"srv1": {"url": "http://127.0.0.1:8001/sse", "enabled": True}},
)
mgr = MCPClientManager(cfg)
mgr.tool_cache["srv1"] = [{"type": "function", "function": {"name": "old_tool"}}]
mock_stack = AsyncMock()
mgr.sessions["srv1"] = {"session": MagicMock(), "stack": mock_stack}
mgr._server_configs["srv1"] = {"url": "http://127.0.0.1:8001/sse", "enabled": True}
# Modify config to point to new URL
cfg.config["ai"]["mcp_servers"]["srv1"] = {"url": "http://127.0.0.1:8002/sse", "enabled": True}
new_session = MagicMock()
with patch.object(mgr, "_ensure_connected", new_callable=AsyncMock) as mock_conn, \
patch.object(mgr, "_fetch_tools_as_openai", new_callable=AsyncMock) as mock_fetch:
mock_conn.return_value = new_session
mock_fetch.return_value = [{"type": "function", "function": {"name": "new_tool"}}]
tools = await mgr.get_tools_for_llm()
# Ensure old stack was closed
mock_stack.aclose.assert_awaited_once()
# Ensure new tools returned
assert tools == [{"type": "function", "function": {"name": "new_tool"}}]
assert mgr.tool_cache["srv1"] == [{"type": "function", "function": {"name": "new_tool"}}]
await mgr.shutdown()
@pytest.mark.asyncio
async def test_shutdown_cleans_resources_and_resets(self):
"""shutdown() closes all stacks, clears caches and removes from _instances."""
cfg = DummyConfig("/tmp/test_mcp_shutdown")
mgr = MCPClientManager(cfg)
mock_stack = AsyncMock()
mgr.sessions["test"] = {"session": MagicMock(), "stack": mock_stack}
mgr.tool_cache["test"] = [{"dummy": 1}]
await mgr.shutdown()
mock_stack.aclose.assert_awaited_once()
assert mgr.sessions == {}
assert mgr.tool_cache == {}
assert os.path.abspath(cfg.defaultdir) not in MCPClientManager._instances