feat(spec-04-adaptive-timeouts-ssm): worker (eng_flash_high) implementation

This commit is contained in:
fluzzi32 committed 2026-11-16 21:06:07 -03:00
1 parent 951961a5d3
commit c3995ea6a9
3 files changed
+2321 -363

No files matched your search

+1216 -320
View File
File diff suppressed because it is too large. Load diff
+153 -43
View File
@@ -1,36 +1,115 @@
from typing import List, Dict, Any, Callable, Optional
import os
from .base import BaseService
from collections.abc import Callable
from typing import Any
from connpy.core import nodes as Nodes
from .base import BaseService
from .exceptions import ConnpyError
class ExecutionService(BaseService):
"""Business logic for executing commands on nodes and running automation scripts."""
def run_commands(
self,
nodes_filter: str,
commands: List[str],
variables: Optional[Dict[str, Any]] = None,
parallel: int = 10,
timeout: int = 20,
folder: Optional[str] = None,
prompt: Optional[str] = None,
on_node_complete: Optional[Callable] = None,
logger: Optional[Callable] = None,
name: Optional[str] = None
) -> Dict[str, str]:
def __init__(self, config=None, remote_stub=None):
super().__init__(config)
self.remote_stub = remote_stub
def _is_remote_backend(self) -> bool:
"""Return True when execution is configured to run against a remote gRPC backend."""
if getattr(self, "remote_stub", None) is not None:
return True
cfg = getattr(self.config, "config", None)
return isinstance(cfg, dict) and cfg.get("service_mode") == "remote"
def _ensure_backend_supports_stage_timeouts(
self,
connect_timeout: int | None = None,
auth_timeout: int | None = None,
) -> None:
"""Raise ConnpyError if per-stage timeout overrides are used on gRPC remote backend."""
if self._is_remote_backend() and (connect_timeout is not None or auth_timeout is not None):
raise ConnpyError(
"connect_timeout and auth_timeout overrides are not supported in remote gRPC mode"
)
def _get_remote_stub(self):
"""Return the configured or lazily constructed ExecutionStub for remote gRPC mode."""
if getattr(self, "remote_stub", None) is not None:
return self.remote_stub
cfg = getattr(self.config, "config", None)
remote_host = cfg.get("remote_host") if isinstance(cfg, dict) else None
if not remote_host:
raise ConnpyError("Remote host must be specified in remote mode")
import grpc
from connpy.grpc_layer.stubs import AuthClientInterceptor, ExecutionStub
def get_token():
env_token = os.environ.get("CONNPY_TOKEN")
if env_token:
return env_token
defaultdir = getattr(self.config, "defaultdir", None)
if defaultdir:
token_path = os.path.join(defaultdir, ".token")
if os.path.exists(token_path):
try:
with open(token_path) as f:
return f.read().strip()
except OSError:
pass
return None
channel = grpc.insecure_channel(remote_host)
channel = grpc.intercept_channel(channel, AuthClientInterceptor(get_token))
return ExecutionStub(channel, remote_host=remote_host)
def run_commands(
self,
nodes_filter: str,
commands: list[str],
variables: dict[str, Any] | None = None,
parallel: int = 10,
timeout: int | None = None,
folder: str | None = None,
prompt: str | None = None,
on_node_complete: Callable | None = None,
logger: Callable | None = None,
name: str | None = None,
connect_timeout: int | None = None,
auth_timeout: int | None = None,
) -> dict[str, Any]:
"""Execute commands on a set of nodes."""
try:
self._ensure_backend_supports_stage_timeouts(
connect_timeout=connect_timeout,
auth_timeout=auth_timeout,
)
if self._is_remote_backend():
stub = self._get_remote_stub()
return stub.run_commands(
nodes_filter=nodes_filter,
commands=commands,
variables=variables,
parallel=parallel,
timeout=timeout if timeout is not None else 20,
folder=folder,
prompt=prompt,
on_node_complete=on_node_complete,
logger=logger,
name=name or "",
)
matched_names = self.config._getallnodes(nodes_filter)
if not matched_names:
raise ConnpyError(f"No nodes found matching filter: {nodes_filter}")
node_data = self.config.getitems(matched_names, extract=True)
executor = Nodes(node_data, config=self.config)
self.last_executor = executor
results = executor.run(
commands=commands,
vars=variables,
@@ -39,7 +118,10 @@ class ExecutionService(BaseService):
folder=folder,
prompt=prompt,
on_complete=on_node_complete,
logger=logger
logger=logger,
connect_timeout=connect_timeout,
auth_timeout=auth_timeout,
_default_cmd_timeout=20,
)
# Combine output and status for the caller
@@ -47,39 +129,63 @@ class ExecutionService(BaseService):
for unique in results:
full_results[unique] = {
"output": results[unique],
"status": executor.status.get(unique, 1)
"status": executor.status.get(unique, 1),
}
return full_results
except ConnpyError:
raise
except Exception as e:
raise ConnpyError(f"Execution failed: {e}")
raise ConnpyError(f"Execution failed: {e}") from e
def test_commands(
self,
nodes_filter: str,
commands: List[str],
expected: List[str],
variables: Optional[Dict[str, Any]] = None,
commands: list[str],
expected: list[str],
variables: dict[str, Any] | None = None,
parallel: int = 10,
timeout: int = 20,
folder: Optional[str] = None,
prompt: Optional[str] = None,
on_node_complete: Optional[Callable] = None,
logger: Optional[Callable] = None,
name: Optional[str] = None
) -> Dict[str, Dict[str, bool]]:
timeout: int | None = None,
folder: str | None = None,
prompt: str | None = None,
on_node_complete: Callable | None = None,
logger: Callable | None = None,
name: str | None = None,
connect_timeout: int | None = None,
auth_timeout: int | None = None,
) -> dict[str, dict[str, bool]]:
"""Run commands and verify expected output on a set of nodes."""
try:
self._ensure_backend_supports_stage_timeouts(
connect_timeout=connect_timeout,
auth_timeout=auth_timeout,
)
if self._is_remote_backend():
stub = self._get_remote_stub()
return stub.test_commands(
nodes_filter=nodes_filter,
commands=commands,
expected=expected,
variables=variables,
parallel=parallel,
timeout=timeout if timeout is not None else 20,
folder=folder or "",
prompt=prompt,
on_node_complete=on_node_complete,
logger=logger,
name=name or "",
)
matched_names = self.config._getallnodes(nodes_filter)
if not matched_names:
raise ConnpyError(f"No nodes found matching filter: {nodes_filter}")
node_data = self.config.getitems(matched_names, extract=True)
executor = Nodes(node_data, config=self.config)
self.last_executor = executor
results = executor.test(
return executor.test(
commands=commands,
expected=expected,
vars=variables,
@@ -88,22 +194,26 @@ class ExecutionService(BaseService):
folder=folder,
prompt=prompt,
on_complete=on_node_complete,
logger=logger
logger=logger,
connect_timeout=connect_timeout,
auth_timeout=auth_timeout,
_default_cmd_timeout=20,
)
return results
except ConnpyError:
raise
except Exception as e:
raise ConnpyError(f"Testing failed: {e}")
raise ConnpyError(f"Testing failed: {e}") from e
def run_cli_script(self, nodes_filter: str, script_path: str, parallel: int = 10) -> Dict[str, str]:
def run_cli_script(self, nodes_filter: str, script_path: str, parallel: int = 10) -> dict[str, str]:
"""Run a plain-text script containing one command per line."""
if not os.path.exists(script_path):
raise ConnpyError(f"Script file not found: {script_path}")
try:
with open(script_path, "r") as f:
with open(script_path) as f:
commands = [line.strip() for line in f if line.strip()]
except Exception as e:
raise ConnpyError(f"Failed to read script {script_path}: {e}")
except OSError as e:
raise ConnpyError(f"Failed to read script {script_path}: {e}") from e
return self.run_commands(nodes_filter, commands, parallel=parallel)
+952
View File
@@ -0,0 +1,952 @@
"""Tests for adaptive multi-hop timeouts, progress extensions, deadline, and hardened expects."""
import itertools
import os
import re
import shlex
from unittest.mock import MagicMock, patch
import pexpect
import pytest
import yaml
from connpy.configfile import configfile
from connpy.core import (
DEFAULT_ADAPTIVE_BASE_TIMEOUT,
DEFAULT_AUTH_TIMEOUT,
DEFAULT_DIRECT_CONNECT_TIMEOUT,
DEFAULT_MAX_PROGRESS_EXTENSIONS,
DEFAULT_MAX_TOTAL_CONNECT_SECONDS,
DEFAULT_MOSH_CONNECT_TIMEOUT,
DEFAULT_OTHER_HOP_COST,
DEFAULT_SSM_DIRECT_COST,
DEFAULT_SSM_HOP_COST,
MAX_PRE_AUTH_BUFFER_BYTES,
MAX_STAGE_TIMEOUT_CAP,
compile_expects,
node,
nodes,
resolve_timeouts,
validate_strict_timeout_int,
)
from connpy.services.exceptions import ConnpyError
from connpy.services.execution_service import ExecutionService
class FakeClock:
"""Deterministic monotonic clock helper for unit tests."""
def __init__(self, initial: float = 0.0):
self.now = initial
def __call__(self) -> float:
return self.now
def advance(self, seconds: float) -> float:
self.now += seconds
return self.now
_CONF_COUNTER = itertools.count(1)
def _make_config(tmp_path, connections, profiles=None):
if profiles is None:
profiles = {
"default": {
"host": "",
"protocol": "ssh",
"port": "",
"user": "",
"password": "",
"options": "",
"logs": "",
"tags": "",
"jumphost": "",
}
}
conf_dir = tmp_path / f"conf_{next(_CONF_COUNTER)}"
conf_dir.mkdir(parents=True, exist_ok=True)
conf_file = conf_dir / "config.yaml"
data = {
"config": {"case": False, "idletime": 30, "fzf": False},
"connections": connections,
"profiles": profiles,
}
conf_file.write_text(yaml.dump(data, default_flow_style=False, sort_keys=False))
os.chmod(str(conf_file), 0o600)
return configfile(conf=str(conf_file), key=str(conf_dir / ".osk"))
def _make_ssm_jh_router_config(tmp_path, *, ssm_host="i-0123456789abcdef0", ssm_tags=None, ssh_options=""):
connections = {
"ssm_bastion": {
"host": ssm_host,
"protocol": "ssm",
"port": "",
"user": "ec2-user",
"password": "",
"options": "",
"logs": "",
"tags": ssm_tags if ssm_tags is not None else {"profile": "prod", "region": "us-east-1"},
"jumphost": "",
"type": "connection",
},
"jh_ssh": {
"host": "10.10.0.2",
"protocol": "ssh",
"port": "2222",
"user": "jumpuser",
"password": "",
"options": ssh_options,
"logs": "",
"tags": "",
"jumphost": "ssm_bastion",
"type": "connection",
},
"router1": {
"host": "10.20.0.5",
"protocol": "ssh",
"port": "",
"user": "netadmin",
"password": "",
"options": "",
"logs": "",
"tags": "",
"jumphost": "jh_ssh",
"type": "connection",
},
}
return _make_config(tmp_path, connections)
# =============================================================================
# 1. validate_strict_timeout_int and resolve_timeouts
# =============================================================================
class TestResolveTimeouts:
def test_constants_values(self):
assert DEFAULT_DIRECT_CONNECT_TIMEOUT == 10
assert DEFAULT_MOSH_CONNECT_TIMEOUT == 15
assert DEFAULT_ADAPTIVE_BASE_TIMEOUT == 15
assert DEFAULT_SSM_DIRECT_COST == 20
assert DEFAULT_SSM_HOP_COST == 25
assert DEFAULT_OTHER_HOP_COST == 10
assert DEFAULT_AUTH_TIMEOUT == 25
assert DEFAULT_MAX_PROGRESS_EXTENSIONS == 3
assert MAX_STAGE_TIMEOUT_CAP == 600
assert DEFAULT_MAX_TOTAL_CONNECT_SECONDS == 180
assert MAX_PRE_AUTH_BUFFER_BYTES == 65536
def test_direct_ssh_defaults_and_legacy_override(self):
assert resolve_timeouts("ssh", {}, []) == (10, 25, 10, 3, 180)
assert resolve_timeouts("ssh", {}, [], _default_cmd_timeout=20) == (10, 25, 20, 3, 180)
assert resolve_timeouts("ssh", {}, [], timeout=5) == (5, 5, 5, 3, 180)
def test_direct_mosh_and_ssm_budgets(self):
assert resolve_timeouts("mosh", {}, [], timeout=5) == (15, 5, 5, 3, 180)
assert resolve_timeouts("mosh", {}, []) == (15, 25, 10, 3, 180)
assert resolve_timeouts("mosh", {}, [], timeout=30) == (30, 30, 30, 3, 180)
assert resolve_timeouts("ssm", {}, []) == (20, 25, 10, 3, 180)
assert resolve_timeouts("ssm", {}, [], timeout=5) == (5, 5, 5, 3, 180)
assert resolve_timeouts("ssm", {}, [], timeout=45) == (45, 45, 45, 3, 180)
def test_multihop_ssm_jh_router_and_short_overrides(self):
hop_meta = [
{"name": "ssm_bastion", "protocol": "ssm"},
{"name": "jh_ssh", "protocol": "ssh"},
]
# 15 + 25 + 10 = 50 connect timeout; 25 auth; 10 or 20 cmd
assert resolve_timeouts("ssh", {}, hop_meta) == (50, 25, 10, 3, 180)
assert resolve_timeouts("ssh", {}, hop_meta, _default_cmd_timeout=20) == (50, 25, 20, 3, 180)
assert resolve_timeouts("ssh", {}, hop_meta, timeout=75) == (75, 75, 75, 3, 180)
assert resolve_timeouts(
"ssh",
{},
hop_meta,
timeout=75,
connect_timeout=12,
auth_timeout=8,
) == (12, 8, 75, 3, 180)
assert resolve_timeouts("ssh", {}, hop_meta, connect_timeout=12) == (12, 25, 10, 3, 180)
def test_max_total_ceiling_acts_as_upper_bound(self):
assert resolve_timeouts("ssh", {}, [], _default_max_total_seconds=45) == (10, 25, 10, 3, 45)
assert resolve_timeouts("ssh", {}, [], _default_max_total_seconds=500) == (10, 25, 10, 3, 180)
@pytest.mark.parametrize("bad_val", [True, False, 10.5, "10", 0, -5, 601])
def test_rejects_non_strict_int_values(self, bad_val):
with pytest.raises(ValueError, match="Invalid"):
validate_strict_timeout_int(bad_val, "timeout")
with pytest.raises(ValueError, match="Invalid"):
resolve_timeouts("ssh", {}, [], timeout=bad_val)
with pytest.raises(ValueError, match="Invalid"):
resolve_timeouts("ssh", {}, [], connect_timeout=bad_val)
with pytest.raises(ValueError, match="Invalid"):
resolve_timeouts("ssh", {}, [], auth_timeout=bad_val)
def test_connect_and_run_return_code_12_on_invalid_timeout(self):
n = node("r1", "10.0.0.1", user="admin", protocol="ssh")
res_conn = n._connect(timeout=True)
assert res_conn.startswith("Connection failed code: 12\nInvalid timeout configuration:")
res_run = n.run(["show version"], connect_timeout="bad")
assert res_run.startswith("Connection failed code: 12\nInvalid timeout configuration:")
assert n.status == 1
res_test = n.test(["show version"], "v1", auth_timeout=False)
assert res_test.startswith("Connection failed code: 12\nInvalid timeout configuration:")
assert n.status == 1
assert n.result is None
# =============================================================================
# 2. ExecutionService local vs remote gRPC
# =============================================================================
class TestExecutionServiceTimeouts:
def test_local_execution_service_defaults_cmd_timeout_20_and_auth_25(self, tmp_path):
cfg = _make_config(
tmp_path,
{
"r1": {
"host": "10.0.0.1",
"protocol": "ssh",
"port": "",
"user": "admin",
"password": "mysecretpassword",
"options": "",
"logs": "",
"tags": "",
"jumphost": "",
"type": "connection",
}
},
)
svc = ExecutionService(cfg)
mock_child = MagicMock()
# _connect: [8 (Password:), 9 (prompt)], run(): [9 (prompt), 9 (SIGWINCH consume), 9 (after cmd)]
mock_child.expect.side_effect = [8, 9, 0, 0, 0]
mock_child.before = b""
mock_child.after = b"r1#"
mock_child.readline.return_value = b""
mock_child.child_fd = 3
with (
patch("connpy.core.pexpect.spawn", return_value=mock_child),
patch("pexpect.fdpexpect.fdspawn", return_value=MagicMock()),
):
res = svc.run_commands("r1", ["show version"])
assert res["r1"]["status"] == 0
timeouts_seen = [call.kwargs["timeout"] for call in mock_child.expect.call_args_list]
# 1) connect stage -> 10, 2) auth stage -> 25, 3) run pre-cmd -> 20, 4) winsize -> 1, 5) run post-cmd -> 20
assert timeouts_seen == [10, 25, 20, 1, 20]
def test_local_execution_service_test_commands_propagates_stage_timeouts(self, tmp_path):
cfg = _make_config(
tmp_path,
{
"r1": {
"host": "10.0.0.1",
"protocol": "ssh",
"port": "",
"user": "admin",
"password": "",
"options": "",
"logs": "",
"tags": "",
"jumphost": "",
"type": "connection",
}
},
)
svc = ExecutionService(cfg)
with patch("connpy.services.execution_service.Nodes") as mock_nodes:
mock_exec = mock_nodes.return_value
mock_exec.test.return_value = {"r1": {"ok": True}}
svc.test_commands(
"r1",
["show version"],
["ok"],
connect_timeout=18,
auth_timeout=35,
)
_, kwargs = mock_exec.test.call_args
assert kwargs["timeout"] is None
assert kwargs["connect_timeout"] == 18
assert kwargs["auth_timeout"] == 35
assert kwargs["_default_cmd_timeout"] == 20
def test_remote_grpc_uses_default_20_and_rejects_stage_overrides(self, tmp_path):
cfg = _make_config(tmp_path, {})
cfg.config["service_mode"] = "remote"
cfg.config["remote_host"] = "127.0.0.1:8048"
fake_stub = MagicMock()
fake_stub.run_commands.return_value = {"r1": {"output": "ok", "status": 0}}
fake_stub.test_commands.return_value = {"r1": {"ok": True}}
svc = ExecutionService(cfg, remote_stub=fake_stub)
res_run = svc.run_commands("r1", ["show version"])
assert res_run == {"r1": {"output": "ok", "status": 0}}
assert fake_stub.run_commands.call_args.kwargs["timeout"] == 20
res_test = svc.test_commands("r1", ["show version"], ["ok"], timeout=45)
assert res_test == {"r1": {"ok": True}}
assert fake_stub.test_commands.call_args.kwargs["timeout"] == 45
with pytest.raises(ConnpyError, match="not supported in remote gRPC mode"):
svc.run_commands("r1", ["show version"], connect_timeout=15)
with pytest.raises(ConnpyError, match="not supported in remote gRPC mode"):
svc.test_commands("r1", ["show version"], ["ok"], auth_timeout=30)
with pytest.raises(ConnpyError, match="not supported in remote gRPC mode"):
svc._ensure_backend_supports_stage_timeouts(connect_timeout=12, auth_timeout=None)
# =============================================================================
# 3. _build_jumphost_chain & SSM/SSH security
# =============================================================================
class TestJumphostChainAndSecurity:
def test_default_2_tuple_and_include_meta_3_tuple_all_branches(self, tmp_path):
connections = {
"ssm_hop": {
"host": "i-0123456789abcdef0",
"protocol": "ssm",
"port": "",
"user": "ec2-user",
"password": "",
"options": "",
"logs": "",
"tags": {"profile": "prod", "region": "us-east-1"},
"jumphost": "",
"type": "connection",
},
"ssh_hop": {
"host": "10.0.0.2",
"protocol": "ssh",
"port": "2222",
"user": "DOMAIN\\user",
"password": "",
"options": "-o StrictHostKeyChecking=no",
"logs": "",
"tags": "",
"jumphost": "ssm_hop",
"type": "connection",
},
"mosh_hop": {
"host": "10.0.0.3",
"protocol": "mosh",
"port": "22",
"user": "muser",
"password": "",
"options": "",
"logs": "",
"tags": {"mosh_ssh_options": "-4"},
"jumphost": "",
"type": "connection",
},
"kube_hop": {
"host": "jump-pod",
"protocol": "kubectl",
"port": "",
"user": "",
"password": "",
"options": "-n ops",
"logs": "",
"tags": "",
"jumphost": "",
"type": "connection",
},
"docker_hop": {
"host": "jump-ctr",
"protocol": "docker",
"port": "",
"user": "",
"password": "",
"options": "",
"logs": "",
"tags": "",
"jumphost": "",
"type": "connection",
},
"unknown_hop": {
"host": "10.0.0.9",
"protocol": "serial",
"port": "",
"user": "",
"password": "",
"options": "",
"logs": "",
"tags": "",
"jumphost": "",
"type": "connection",
},
"router1": {
"host": "10.20.0.5",
"protocol": "ssh",
"port": "",
"user": "admin",
"password": "",
"options": "",
"logs": "",
"tags": "",
"jumphost": "ssh_hop",
"type": "connection",
},
}
cfg = _make_config(tmp_path, connections)
r = node("router1", "10.20.0.5", user="admin", jumphost="ssh_hop", config=cfg)
# Default returns 2-tuple
res_2 = r._build_jumphost_chain("ssh_hop", cfg)
assert isinstance(res_2, tuple)
assert len(res_2) == 2
# include_meta=True returns 3-tuple for all branches
raw_cmd, pws, meta = r._build_jumphost_chain("ssh_hop", cfg, include_meta=True)
assert pws == []
assert [m["protocol"] for m in meta] == ["ssm", "ssh"]
assert "--profile prod" in raw_cmd
assert "--region us-east-1" in raw_cmd
assert "DOMAIN\\user@10.0.0.2" in raw_cmd
for jh_name, expected_proto in [
("mosh_hop", "mosh"),
("kube_hop", "kubectl"),
("docker_hop", "docker"),
]:
cmd_b, _, meta_b = r._build_jumphost_chain(jh_name, cfg, include_meta=True)
assert cmd_b != ""
assert meta_b == [{"name": jh_name, "protocol": expected_proto}]
cmd_unk, _, meta_unk = r._build_jumphost_chain("unknown_hop", cfg, include_meta=True)
assert cmd_unk == ""
assert meta_unk == []
# Adaptive timeout derived from router1's _hop_chain_meta
assert resolve_timeouts(r.protocol, r.tags, r._hop_chain_meta) == (50, 25, 10, 3, 180)
def test_ssm_target_arn_and_managed_instance_accepted_and_bad_rejected(self, tmp_path):
for good_target in [
"i-12345",
"i-0123abcd",
"i-0123456789abcdef0",
"mi-0123456789abcdef0",
"arn:aws:ssm:us-east-1:123456789012:managed-instance/mi-0123456789abcdef0",
]:
cfg = _make_ssm_jh_router_config(tmp_path, ssm_host=good_target)
r = node("router1", "10.20.0.5", user="netadmin", jumphost="jh_ssh", config=cfg)
assert good_target in r.jumphost
for bad_target in [
"i-123",
"i-0123456789abcdef01",
"-oProxyCommand=evil",
"i-01234567..89",
"i-01234567;id",
"i-01234567\nid",
"arn:aws:ssm:us-east-1:١٢٣٤٥٦٧٨٩٠١٢:node/1",
]:
cfg = _make_ssm_jh_router_config(tmp_path, ssm_host=bad_target)
with pytest.raises(ValueError, match="SSM jumphost target|Control characters"):
node("router1", "10.20.0.5", user="netadmin", jumphost="jh_ssh", config=cfg)
def test_ssm_profile_region_and_control_char_validation(self, tmp_path):
for bad_tag in ["../prod", "-flag", "bad profile", "prod;id", "prod\x00"]:
cfg = _make_ssm_jh_router_config(tmp_path, ssm_tags={"profile": bad_tag})
with pytest.raises(ValueError, match="SSM jumphost profile|Control characters"):
node("router1", "10.20.0.5", user="netadmin", jumphost="jh_ssh", config=cfg)
cfg2 = _make_ssm_jh_router_config(tmp_path, ssm_tags={"region": bad_tag})
with pytest.raises(ValueError, match="SSM jumphost region|Control characters"):
node("router1", "10.20.0.5", user="netadmin", jumphost="jh_ssh", config=cfg2)
# Unconditional control char check even when allow_unsafe_options=True
cfg_ctrl = _make_config(
tmp_path,
{
"bad_hop": {
"host": "10.0.0.1",
"protocol": "ssh",
"port": "22\x1b",
"user": "admin",
"password": "",
"options": "",
"logs": "",
"tags": "",
"jumphost": "",
"type": "connection",
}
},
)
with pytest.raises(ValueError, match="Control characters"):
node(
"r1",
"10.0.0.2",
jumphost="bad_hop",
config=cfg_ctrl,
allow_unsafe_options=True,
)
def test_connect_timeout_and_server_alive_injection_respects_user_options(self, tmp_path):
# 1) Without user overrides: injects -o ConnectTimeout=50 and -o ServerAliveInterval=15
cfg = _make_ssm_jh_router_config(tmp_path)
r = node(
"router1",
"10.20.0.5",
user="netadmin",
options="-o StrictHostKeyChecking=no",
jumphost="jh_ssh",
config=cfg,
)
# Default _get_cmd() without connect_timeout does not inject
assert "ConnectTimeout" not in r._get_cmd()
cmd_with_ct = r._get_cmd(connect_timeout=50)
assert "-o ConnectTimeout=50" in cmd_with_ct
assert "-o ServerAliveInterval=15" in cmd_with_ct
# 2) With user ConnectTimeout=7 & ServerAliveInterval=20 on SSM and 9/30 on JH: does NOT overwrite
cfg_user = _make_ssm_jh_router_config(
tmp_path,
ssm_tags={
"profile": "prod",
"ssh_options": "-o ConnectTimeout=7 -o ServerAliveInterval=20",
},
ssh_options="-oConnectTimeout=9 -o ServerAliveInterval=30",
)
r2 = node(
"router1",
"10.20.0.5",
user="netadmin",
options="-o ConnectTimeout=4 -o ServerAliveInterval=60",
jumphost="jh_ssh",
config=cfg_user,
)
raw_jh, _, _ = r2._build_jumphost_chain("jh_ssh", cfg_user, include_meta=True, connect_timeout=50)
assert "-oConnectTimeout=9" in raw_jh
assert "ConnectTimeout=50" not in raw_jh
assert "-o ServerAliveInterval=30" in raw_jh
assert "ServerAliveInterval=15" not in raw_jh
assert "-o ConnectTimeout=7" in raw_jh
outer_cmd = r2._get_cmd(connect_timeout=50)
outer_tokens = shlex.split(outer_cmd)
assert "ConnectTimeout=4" in outer_tokens
assert "ServerAliveInterval=60" in outer_tokens
assert "ConnectTimeout=50" not in outer_tokens
assert "ServerAliveInterval=15" not in outer_tokens
def test_ssm_ssh_options_rejects_disallowed_directives(self, tmp_path):
for bad_ssh_opt in ["-o ProxyCommand=evil", "-o LocalCommand=id", "-o ProxyJump=bad"]:
cfg = _make_ssm_jh_router_config(
tmp_path,
ssm_tags={"profile": "prod", "ssh_options": bad_ssh_opt},
)
with pytest.raises(ValueError, match="cannot contain|Disallowed"):
node("router1", "10.20.0.5", user="netadmin", jumphost="jh_ssh", config=cfg)
# =============================================================================
# 4. _connect progress extension, global deadline, finally, concurrency, redaction
# =============================================================================
class TestConnectProgressDeadlineAndCleanup:
def test_progress_extension_with_fake_clock_and_single_warning(self):
clock = FakeClock(100.0)
n = node("r1", "10.0.0.1", user="admin", protocol="ssh")
logs = []
mock_child = MagicMock()
mock_child.logfile_read = None
mock_child.child_fd = 7
call_count = {"n": 0}
def expect_side_effect(patterns, timeout):
call_count["n"] += 1
idx = call_count["n"]
clock.advance(timeout)
if idx == 1:
mock_child.before = b"SSH-2.0-OpenSSH\r\n"
return 12 # TIMEOUT
if idx == 2:
mock_child.before = b"SSH-2.0-OpenSSH\r\nkex banner...\r\n"
return 12 # TIMEOUT
if idx == 3:
mock_child.before = b"SSH-2.0-OpenSSH\r\nkex banner...\r\nmotd..."
return 12 # TIMEOUT
mock_child.before = b"router1#"
return 9 # prompt
mock_child.expect.side_effect = expect_side_effect
with (
patch("connpy.core.pexpect.spawn", return_value=mock_child),
patch("pexpect.fdpexpect.fdspawn", return_value=MagicMock()),
):
res = n._connect(
max_attempts=1,
logger=lambda lvl, msg: logs.append((lvl, msg)),
connect_timeout=10,
clock=clock,
)
assert res is True
assert call_count["n"] == 4
warnings = [m for lvl, m in logs if lvl == "warning"]
assert len(warnings) == 1
assert "extending" in warnings[0].lower()
mock_child.terminate.assert_not_called()
def test_progress_extension_capped_at_max_ext_and_resets_handshake_per_attempt(self):
clock = FakeClock(0.0)
n = node("r1", "10.0.0.1", user="admin", protocol="ssh")
children = []
def spawn_side_effect(*args, **kwargs):
c = MagicMock()
c.logfile_read = None
c.before = b""
c.after = b""
c.readline.return_value = b""
step = {"k": 0}
def exp_se(pats, timeout):
step["k"] += 1
clock.advance(timeout)
# Keep growing bytes every timeout to prove cap at max_ext=3 (4 calls total per attempt)
c.before = b"x" * (step["k"] * 10)
return 12
c.expect.side_effect = exp_se
children.append(c)
return c
with patch("connpy.core.pexpect.spawn", side_effect=spawn_side_effect):
res = n._connect(
debug=True,
max_attempts=2,
connect_timeout=5,
_default_max_total_seconds=100,
clock=clock,
)
assert "Connection failed code: 12" in res
assert "Connection timeout" in res
# 2 attempts, each allowed 3 progress extensions -> 4 expect calls per child
assert len(children) == 2
for c in children:
assert c.expect.call_count == 4
assert c.logfile_read is None
c.terminate.assert_called_with(force=True)
assert n.mylog.closed is True
# Idempotent close verification
n.mylog.close()
assert n.mylog.closed is True
def test_global_deadline_bounds_multiple_attempts_and_clamps_expect_timeout(self):
clock = FakeClock(0.0)
n = node("r1", "10.0.0.1", user="admin", protocol="ssh")
seen_timeouts = []
children = []
def spawn_side_effect(*args, **kwargs):
c = MagicMock()
c.logfile_read = None
c.before = b""
c.after = b""
c.readline.return_value = b""
def exp_se(pats, timeout):
seen_timeouts.append(timeout)
clock.advance(timeout)
return 12 # TIMEOUT without progress -> triggers retry if remaining > 0
c.expect.side_effect = exp_se
children.append(c)
return c
with patch("connpy.core.pexpect.spawn", side_effect=spawn_side_effect):
res = n._connect(
max_attempts=3,
connect_timeout=15,
_default_max_total_seconds=35,
clock=clock,
)
assert "Connection failed code: 12" in res
assert "deadline" in res.lower()
assert seen_timeouts == [15, 15, 5]
assert clock() == 35.0
assert len(children) == 3
for c in children:
c.terminate.assert_called_with(force=True)
def test_pre_auth_buffer_cap_terminates_and_cleans_up(self):
n = node("r1", "10.0.0.1", user="admin", protocol="ssh")
mock_child = MagicMock()
mock_child.logfile_read = None
mock_child.expect.return_value = 9
mock_child.before = b"A" * (MAX_PRE_AUTH_BUFFER_BYTES + 1)
with patch("connpy.core.pexpect.spawn", return_value=mock_child):
res = n._connect(debug=True)
assert "Connection failed code: 12" in res
assert "Pre-auth buffer exceeded" in res
assert mock_child.logfile_read is None
mock_child.terminate.assert_called_with(force=True)
assert n.mylog.closed is True
def test_secret_redaction_in_returned_errors_and_debug_logs(self):
secret = "SuperSecretToken!42"
n = node("r1", "10.0.0.1", user="admin", password=secret, protocol="ssh")
logs = []
mock_child = MagicMock()
mock_child.logfile_read = None
mock_child.expect.return_value = 2
mock_child.before = f"Echoed: {secret}\r\n".encode()
mock_child.after = b"Permission denied (publickey,password).\r\n"
mock_child.readline.return_value = f"bad {secret}\r\n".encode()
with patch("connpy.core.pexpect.spawn", return_value=mock_child):
res = n._connect(debug=True, logger=lambda lvl, msg: logs.append((lvl, msg)))
assert isinstance(res, str)
assert secret not in res
assert "***" in res
assert all(secret not in msg for _, msg in logs)
mock_child.terminate.assert_called_with(force=True)
assert n.mylog.closed is True
def test_parallel_8_nodes_isolated_cleanup_and_results(self):
node_defs = {
f"r{i}": {
"host": f"10.0.0.{i}",
"user": "admin",
"password": f"secretPass{i}!",
"protocol": "ssh",
}
for i in range(1, 9)
}
group = nodes(node_defs)
def fake_spawn(prog, args, **kwargs):
target_arg = args[-1]
c = MagicMock()
c.logfile_read = None
c.child_fd = 100
c.before = b""
c.readline.return_value = b""
if target_arg.endswith((".1", ".5")):
c.expect.return_value = 1
c.after = b"ssh: connect to host 10.0.0.1 port 22: Connection refused\r\n"
else:
c.expect.return_value = 9
c.after = b"router#"
return c
with (
patch("connpy.core.pexpect.spawn", side_effect=fake_spawn),
patch("pexpect.fdpexpect.fdspawn", return_value=MagicMock()),
):
out = group.run(["show version"], parallel=8)
assert len(out) == 8
assert group.status["r1"] == 1
assert group.status["r5"] == 1
for i in [2, 3, 4, 6, 7, 8]:
assert group.status[f"r{i}"] == 0
# =============================================================================
# 5. compile_expects, bytes matching, adversarial MOTD, pexpect.EOF
# =============================================================================
class TestHardenedExpectsAndMotd:
def test_compile_expects_compiles_ascii_to_bytes_with_dotall(self):
raw_map = {
"ssh": [
r"(?m:^Connection refused(?:\r?\n))",
pexpect.EOF,
pexpect.TIMEOUT,
]
}
compiled = compile_expects(raw_map)
assert isinstance(compiled["ssh"][0], re.Pattern)
assert isinstance(compiled["ssh"][0].pattern, bytes)
assert compiled["ssh"][0].flags & re.DOTALL
assert compiled["ssh"][1] is pexpect.EOF
assert compiled["ssh"][2] is pexpect.TIMEOUT
def test_adversarial_motd_banners_do_not_false_positive_and_connect(self):
# 1) SSH adversarial MOTD: TACACS+ unavailable + Permission denied retry + chunked Connection refused + closed + not supported
n_ssh = node("r_ssh", "10.0.0.1", user="admin", password="pw1234", protocol="ssh")
banner_bytes = (
b"Welcome to Corporate Router\r\n"
b"TACACS+ server unavailable - using local fallback\r\n"
b"Feature X not supported in legacy mode\r\n"
b"Session timeout is 300s\r\n"
b"Ticket closed by admin\r\n"
b"Connection refused"
b" by policy\r\n"
b"Permission denied, please try again.\r\n"
b"Password: "
)
# Verify none of the hardened ssh error regexes match the adversarial banner
mock_child = MagicMock()
mock_child.logfile_read = None
mock_child.child_fd = 3
# Expect checks using real pexpect pattern matching on a BytesIO-like feed
captured_patterns = {}
def _real_match_side_effect(patterns, timeout):
# Check error patterns [1..7, 13..16] do NOT match banner_bytes
for idx in [1, 2, 3, 4, 5, 6, 7, 13, 14, 15, 16]:
pat = patterns[idx]
assert pat.search(banner_bytes) is None, f"False positive on ssh index {idx}"
# Verify password_prompt [8] DOES match the end of banner_bytes
assert patterns[8].search(banner_bytes) is not None
if not captured_patterns.get("calls"):
captured_patterns["calls"] = [1]
return 8 # password_prompt
# After password sent, prompt arrives
assert patterns[9].search(b"r_ssh#") is not None
return 9
mock_child.expect.side_effect = _real_match_side_effect
with (
patch("connpy.core.pexpect.spawn", return_value=mock_child),
patch("pexpect.fdpexpect.fdspawn", return_value=MagicMock()),
):
res = n_ssh._connect()
assert res is True
mock_child.sendline.assert_any_call("pw1234")
# 2) SSM adversarial MOTD: "error: motd"
n_ssm = node("r_ssm", "i-0123456789abcdef0", protocol="ssm")
ssm_banner = b"MOTD:\r\nerror: motd notice only\r\nsh-5.1$ "
ssm_child = MagicMock()
ssm_child.child_fd = 4
def _ssm_match_side_effect(patterns, timeout):
for idx in [1, 2, 3, 4, 5, 6]:
assert patterns[idx].search(ssm_banner) is None, f"False positive on ssm index {idx}"
assert patterns[9].search(ssm_banner) is not None
return 9
ssm_child.expect.side_effect = _ssm_match_side_effect
with (
patch("connpy.core.pexpect.spawn", return_value=ssm_child),
patch("pexpect.fdpexpect.fdspawn", return_value=MagicMock()),
):
res_ssm = n_ssm._connect()
assert res_ssm is True
# 3) Telnet adversarial MOTD
n_tel = node("r_tel", "10.0.0.1", user="admin", protocol="telnet")
tel_banner = b"TACACS+ server unavailable\r\nConnection refused by policy\r\nrouter# "
tel_child = MagicMock()
tel_child.child_fd = 5
def _tel_match_side_effect(patterns, timeout):
for idx in [1, 2, 3, 4, 5, 6, 7, 13, 14, 15, 16]:
assert patterns[idx].search(tel_banner) is None, f"False positive on telnet index {idx}"
assert patterns[9].search(tel_banner) is not None
return 9
tel_child.expect.side_effect = _tel_match_side_effect
with (
patch("connpy.core.pexpect.spawn", return_value=tel_child),
patch("pexpect.fdpexpect.fdspawn", return_value=MagicMock()),
):
res_tel = n_tel._connect()
assert res_tel is True
def test_eof_evaluates_errors_without_trailing_newline(self):
# SSH EOF without trailing newline preserves exact error code 1
n_ssh = node("r1", "10.0.0.1", user="admin", protocol="ssh")
mock_child = MagicMock()
mock_child.expect.return_value = 11 # pexpect.EOF
mock_child.before = b"ssh: connect to host 10.0.0.1 port 22: Connection refused"
mock_child.after = pexpect.EOF
mock_child.readline.return_value = b""
with patch("connpy.core.pexpect.spawn", return_value=mock_child):
res = n_ssh._connect()
assert isinstance(res, str)
assert "Connection failed code: 1" in res
assert "Connection refused" in res
mock_child.terminate.assert_called_with(force=True)
# SSM EOF without trailing newline preserves exact error code 3
n_ssm = node("s1", "i-0123456789abcdef0", protocol="ssm")
ssm_child = MagicMock()
ssm_child.expect.return_value = 10 # pexpect.EOF for ssm
ssm_child.before = b"TargetNotConnected: i-0123456789abcdef0"
ssm_child.after = pexpect.EOF
ssm_child.readline.return_value = b""
with patch("connpy.core.pexpect.spawn", return_value=ssm_child):
res_ssm = n_ssm._connect()
assert isinstance(res_ssm, str)
assert "Connection failed code: 3" in res_ssm
assert "TargetNotConnected" in res_ssm
ssm_child.terminate.assert_called_with(force=True)
def test_real_ssh_and_ssm_error_patterns_match_in_bytes(self):
n_ssh = node("r1", "10.0.0.1", user="admin", protocol="ssh")
mock_child = MagicMock()
mock_child.expect.return_value = 2
mock_child.before = b""
mock_child.after = b"Permission denied (publickey,password).\r\n"
mock_child.readline.return_value = b""
def _cap(pats, timeout):
assert pats[2].search(b"Permission denied (publickey,password).\r\n") is not None
assert pats[1].search(b"ssh: connect to host 10.0.0.1 port 22: Connection refused\r\n") is not None
assert pats[5].search(b"ssh: connect to host 10.0.0.1 port 22: Operation timed out\r\n") is not None
assert pats[6].search(b"ssh: Could not resolve hostname bad: Name or service not known\r\n") is not None
assert pats[7].search(b"Connection closed by remote host\r\n") is not None
return 2
mock_child.expect.side_effect = _cap
with patch("connpy.core.pexpect.spawn", return_value=mock_child):
res = n_ssh._connect()
assert "Connection failed code: 2" in res
# Ensure real error patterns also match for ssm [1,2,3,5,6]
n_ssm = node("s1", "i-0123456789abcdef0", protocol="ssm")
ssm_child = MagicMock()
def _cap_ssm(pats, timeout):
assert pats[1].search(b"Cannot perform start-session: timeout\r\n") is not None
assert pats[2].search(b"An error occurred (AccessDeniedException) when calling StartSession\r\n") is not None
assert pats[3].search(b"TargetNotConnected: i-0123456789abcdef0\r\n") is not None
assert pats[5].search(b"UnknownClientException: error\r\n") is not None
assert pats[6].search(b"Unable to locate credentials\r\n") is not None
return 2
ssm_child.expect.side_effect = _cap_ssm
ssm_child.before = b""
ssm_child.after = b"An error occurred (AccessDeniedException)\r\n"
ssm_child.readline.return_value = b""
with patch("connpy.core.pexpect.spawn", return_value=ssm_child):
res_ssm = n_ssm._connect()
assert "Connection failed code: 2" in res_ssm