Merge feature/spec-04-adaptive-timeouts-ssm: adaptive SSM/jumphost timeouts & hardened expects

This commit is contained in:
fluzzi32 committed 2026-11-16 21:25:41 -03:00
commit ef6d1da5e5
3 files changed
+2351 -298

No files matched your search

+1167 -264
View File
File diff suppressed because it is too large. Load diff
+144 -34
View File
@@ -1,28 +1,107 @@
from typing import List, Dict, Any, Callable, Optional
import os import os
from .base import BaseService from collections.abc import Callable
from typing import Any
from connpy.core import nodes as Nodes from connpy.core import nodes as Nodes
from .base import BaseService
from .exceptions import ConnpyError from .exceptions import ConnpyError
class ExecutionService(BaseService): class ExecutionService(BaseService):
"""Business logic for executing commands on nodes and running automation scripts.""" """Business logic for executing commands on nodes and running automation scripts."""
def __init__(self, config=None, remote_stub=None):
super().__init__(config)
self.remote_stub = remote_stub
def _is_remote_backend(self) -> bool:
"""Return True when execution is configured to run against a remote gRPC backend."""
if getattr(self, "remote_stub", None) is not None:
return True
cfg = getattr(self.config, "config", None)
return isinstance(cfg, dict) and cfg.get("service_mode") == "remote"
def _ensure_backend_supports_stage_timeouts(
self,
connect_timeout: int | None = None,
auth_timeout: int | None = None,
) -> None:
"""Raise ConnpyError if per-stage timeout overrides are used on gRPC remote backend."""
if self._is_remote_backend() and (connect_timeout is not None or auth_timeout is not None):
raise ConnpyError(
"connect_timeout and auth_timeout overrides are not supported in remote gRPC mode"
)
def _get_remote_stub(self):
"""Return the configured or lazily constructed ExecutionStub for remote gRPC mode."""
if getattr(self, "remote_stub", None) is not None:
return self.remote_stub
cfg = getattr(self.config, "config", None)
remote_host = cfg.get("remote_host") if isinstance(cfg, dict) else None
if not remote_host:
raise ConnpyError("Remote host must be specified in remote mode")
import grpc
from connpy.grpc_layer.stubs import AuthClientInterceptor, ExecutionStub
def get_token():
env_token = os.environ.get("CONNPY_TOKEN")
if env_token:
return env_token
defaultdir = getattr(self.config, "defaultdir", None)
if defaultdir:
token_path = os.path.join(defaultdir, ".token")
if os.path.exists(token_path):
try:
with open(token_path) as f:
return f.read().strip()
except OSError:
pass
return None
channel = grpc.insecure_channel(remote_host)
channel = grpc.intercept_channel(channel, AuthClientInterceptor(get_token))
return ExecutionStub(channel, remote_host=remote_host)
def run_commands( def run_commands(
self, self,
nodes_filter: str, nodes_filter: str,
commands: List[str], commands: list[str],
variables: Optional[Dict[str, Any]] = None, variables: dict[str, Any] | None = None,
parallel: int = 10, parallel: int = 10,
timeout: int = 20, timeout: int | None = None,
folder: Optional[str] = None, folder: str | None = None,
prompt: Optional[str] = None, prompt: str | None = None,
on_node_complete: Optional[Callable] = None, on_node_complete: Callable | None = None,
logger: Optional[Callable] = None, logger: Callable | None = None,
name: Optional[str] = None name: str | None = None,
) -> Dict[str, str]: connect_timeout: int | None = None,
auth_timeout: int | None = None,
) -> dict[str, Any]:
"""Execute commands on a set of nodes.""" """Execute commands on a set of nodes."""
try: try:
self._ensure_backend_supports_stage_timeouts(
connect_timeout=connect_timeout,
auth_timeout=auth_timeout,
)
if self._is_remote_backend():
stub = self._get_remote_stub()
return stub.run_commands(
nodes_filter=nodes_filter,
commands=commands,
variables=variables,
parallel=parallel,
timeout=timeout if timeout is not None else 20,
folder=folder,
prompt=prompt,
on_node_complete=on_node_complete,
logger=logger,
name=name or "",
)
matched_names = self.config._getallnodes(nodes_filter) matched_names = self.config._getallnodes(nodes_filter)
if not matched_names: if not matched_names:
raise ConnpyError(f"No nodes found matching filter: {nodes_filter}") raise ConnpyError(f"No nodes found matching filter: {nodes_filter}")
@@ -39,7 +118,10 @@ class ExecutionService(BaseService):
folder=folder, folder=folder,
prompt=prompt, prompt=prompt,
on_complete=on_node_complete, on_complete=on_node_complete,
logger=logger logger=logger,
connect_timeout=connect_timeout,
auth_timeout=auth_timeout,
_default_cmd_timeout=20,
) )
# Combine output and status for the caller # Combine output and status for the caller
@@ -47,30 +129,54 @@ class ExecutionService(BaseService):
for unique in results: for unique in results:
full_results[unique] = { full_results[unique] = {
"output": results[unique], "output": results[unique],
"status": executor.status.get(unique, 1) "status": executor.status.get(unique, 1),
} }
return full_results return full_results
except ConnpyError:
raise
except Exception as e: except Exception as e:
raise ConnpyError(f"Execution failed: {e}") raise ConnpyError(f"Execution failed: {e}") from e
def test_commands( def test_commands(
self, self,
nodes_filter: str, nodes_filter: str,
commands: List[str], commands: list[str],
expected: List[str], expected: list[str],
variables: Optional[Dict[str, Any]] = None, variables: dict[str, Any] | None = None,
parallel: int = 10, parallel: int = 10,
timeout: int = 20, timeout: int | None = None,
folder: Optional[str] = None, folder: str | None = None,
prompt: Optional[str] = None, prompt: str | None = None,
on_node_complete: Optional[Callable] = None, on_node_complete: Callable | None = None,
logger: Optional[Callable] = None, logger: Callable | None = None,
name: Optional[str] = None name: str | None = None,
) -> Dict[str, Dict[str, bool]]: connect_timeout: int | None = None,
auth_timeout: int | None = None,
) -> dict[str, dict[str, bool]]:
"""Run commands and verify expected output on a set of nodes.""" """Run commands and verify expected output on a set of nodes."""
try: try:
self._ensure_backend_supports_stage_timeouts(
connect_timeout=connect_timeout,
auth_timeout=auth_timeout,
)
if self._is_remote_backend():
stub = self._get_remote_stub()
return stub.test_commands(
nodes_filter=nodes_filter,
commands=commands,
expected=expected,
variables=variables,
parallel=parallel,
timeout=timeout if timeout is not None else 20,
folder=folder or "",
prompt=prompt,
on_node_complete=on_node_complete,
logger=logger,
name=name or "",
)
matched_names = self.config._getallnodes(nodes_filter) matched_names = self.config._getallnodes(nodes_filter)
if not matched_names: if not matched_names:
raise ConnpyError(f"No nodes found matching filter: {nodes_filter}") raise ConnpyError(f"No nodes found matching filter: {nodes_filter}")
@@ -79,7 +185,7 @@ class ExecutionService(BaseService):
executor = Nodes(node_data, config=self.config) executor = Nodes(node_data, config=self.config)
self.last_executor = executor self.last_executor = executor
results = executor.test( return executor.test(
commands=commands, commands=commands,
expected=expected, expected=expected,
vars=variables, vars=variables,
@@ -88,22 +194,26 @@ class ExecutionService(BaseService):
folder=folder, folder=folder,
prompt=prompt, prompt=prompt,
on_complete=on_node_complete, on_complete=on_node_complete,
logger=logger logger=logger,
connect_timeout=connect_timeout,
auth_timeout=auth_timeout,
_default_cmd_timeout=20,
) )
return results except ConnpyError:
raise
except Exception as e: except Exception as e:
raise ConnpyError(f"Testing failed: {e}") raise ConnpyError(f"Testing failed: {e}") from e
def run_cli_script(self, nodes_filter: str, script_path: str, parallel: int = 10) -> Dict[str, str]: def run_cli_script(self, nodes_filter: str, script_path: str, parallel: int = 10) -> dict[str, str]:
"""Run a plain-text script containing one command per line.""" """Run a plain-text script containing one command per line."""
if not os.path.exists(script_path): if not os.path.exists(script_path):
raise ConnpyError(f"Script file not found: {script_path}") raise ConnpyError(f"Script file not found: {script_path}")
try: try:
with open(script_path, "r") as f: with open(script_path) as f:
commands = [line.strip() for line in f if line.strip()] commands = [line.strip() for line in f if line.strip()]
except Exception as e: except OSError as e:
raise ConnpyError(f"Failed to read script {script_path}: {e}") raise ConnpyError(f"Failed to read script {script_path}: {e}") from e
return self.run_commands(nodes_filter, commands, parallel=parallel) return self.run_commands(nodes_filter, commands, parallel=parallel)
File diff suppressed because it is too large. Load diff