Merge feature/spec-04-adaptive-timeouts-ssm: adaptive SSM/jumphost timeouts & hardened expects

This commit is contained in:
fluzzi32 committed 2026-11-16 21:25:41 -03:00
commit ef6d1da5e5
3 files changed
+2402 -349

No files matched your search

+1218 -315
View File
File diff suppressed because it is too large. Load diff
+144 -34
View File
@@ -1,28 +1,107 @@
from typing import List, Dict, Any, Callable, Optional
import os
from .base import BaseService
from collections.abc import Callable
from typing import Any
from connpy.core import nodes as Nodes
from .base import BaseService
from .exceptions import ConnpyError
class ExecutionService(BaseService):
"""Business logic for executing commands on nodes and running automation scripts."""
def __init__(self, config=None, remote_stub=None):
super().__init__(config)
self.remote_stub = remote_stub
def _is_remote_backend(self) -> bool:
"""Return True when execution is configured to run against a remote gRPC backend."""
if getattr(self, "remote_stub", None) is not None:
return True
cfg = getattr(self.config, "config", None)
return isinstance(cfg, dict) and cfg.get("service_mode") == "remote"
def _ensure_backend_supports_stage_timeouts(
self,
connect_timeout: int | None = None,
auth_timeout: int | None = None,
) -> None:
"""Raise ConnpyError if per-stage timeout overrides are used on gRPC remote backend."""
if self._is_remote_backend() and (connect_timeout is not None or auth_timeout is not None):
raise ConnpyError(
"connect_timeout and auth_timeout overrides are not supported in remote gRPC mode"
)
def _get_remote_stub(self):
"""Return the configured or lazily constructed ExecutionStub for remote gRPC mode."""
if getattr(self, "remote_stub", None) is not None:
return self.remote_stub
cfg = getattr(self.config, "config", None)
remote_host = cfg.get("remote_host") if isinstance(cfg, dict) else None
if not remote_host:
raise ConnpyError("Remote host must be specified in remote mode")
import grpc
from connpy.grpc_layer.stubs import AuthClientInterceptor, ExecutionStub
def get_token():
env_token = os.environ.get("CONNPY_TOKEN")
if env_token:
return env_token
defaultdir = getattr(self.config, "defaultdir", None)
if defaultdir:
token_path = os.path.join(defaultdir, ".token")
if os.path.exists(token_path):
try:
with open(token_path) as f:
return f.read().strip()
except OSError:
pass
return None
channel = grpc.insecure_channel(remote_host)
channel = grpc.intercept_channel(channel, AuthClientInterceptor(get_token))
return ExecutionStub(channel, remote_host=remote_host)
def run_commands(
self,
nodes_filter: str,
commands: List[str],
variables: Optional[Dict[str, Any]] = None,
commands: list[str],
variables: dict[str, Any] | None = None,
parallel: int = 10,
timeout: int = 20,
folder: Optional[str] = None,
prompt: Optional[str] = None,
on_node_complete: Optional[Callable] = None,
logger: Optional[Callable] = None,
name: Optional[str] = None
) -> Dict[str, str]:
timeout: int | None = None,
folder: str | None = None,
prompt: str | None = None,
on_node_complete: Callable | None = None,
logger: Callable | None = None,
name: str | None = None,
connect_timeout: int | None = None,
auth_timeout: int | None = None,
) -> dict[str, Any]:
"""Execute commands on a set of nodes."""
try:
self._ensure_backend_supports_stage_timeouts(
connect_timeout=connect_timeout,
auth_timeout=auth_timeout,
)
if self._is_remote_backend():
stub = self._get_remote_stub()
return stub.run_commands(
nodes_filter=nodes_filter,
commands=commands,
variables=variables,
parallel=parallel,
timeout=timeout if timeout is not None else 20,
folder=folder,
prompt=prompt,
on_node_complete=on_node_complete,
logger=logger,
name=name or "",
)
matched_names = self.config._getallnodes(nodes_filter)
if not matched_names:
raise ConnpyError(f"No nodes found matching filter: {nodes_filter}")
@@ -39,7 +118,10 @@ class ExecutionService(BaseService):
folder=folder,
prompt=prompt,
on_complete=on_node_complete,
logger=logger
logger=logger,
connect_timeout=connect_timeout,
auth_timeout=auth_timeout,
_default_cmd_timeout=20,
)
# Combine output and status for the caller
@@ -47,30 +129,54 @@ class ExecutionService(BaseService):
for unique in results:
full_results[unique] = {
"output": results[unique],
"status": executor.status.get(unique, 1)
"status": executor.status.get(unique, 1),
}
return full_results
except ConnpyError:
raise
except Exception as e:
raise ConnpyError(f"Execution failed: {e}")
raise ConnpyError(f"Execution failed: {e}") from e
def test_commands(
self,
nodes_filter: str,
commands: List[str],
expected: List[str],
variables: Optional[Dict[str, Any]] = None,
commands: list[str],
expected: list[str],
variables: dict[str, Any] | None = None,
parallel: int = 10,
timeout: int = 20,
folder: Optional[str] = None,
prompt: Optional[str] = None,
on_node_complete: Optional[Callable] = None,
logger: Optional[Callable] = None,
name: Optional[str] = None
) -> Dict[str, Dict[str, bool]]:
timeout: int | None = None,
folder: str | None = None,
prompt: str | None = None,
on_node_complete: Callable | None = None,
logger: Callable | None = None,
name: str | None = None,
connect_timeout: int | None = None,
auth_timeout: int | None = None,
) -> dict[str, dict[str, bool]]:
"""Run commands and verify expected output on a set of nodes."""
try:
self._ensure_backend_supports_stage_timeouts(
connect_timeout=connect_timeout,
auth_timeout=auth_timeout,
)
if self._is_remote_backend():
stub = self._get_remote_stub()
return stub.test_commands(
nodes_filter=nodes_filter,
commands=commands,
expected=expected,
variables=variables,
parallel=parallel,
timeout=timeout if timeout is not None else 20,
folder=folder or "",
prompt=prompt,
on_node_complete=on_node_complete,
logger=logger,
name=name or "",
)
matched_names = self.config._getallnodes(nodes_filter)
if not matched_names:
raise ConnpyError(f"No nodes found matching filter: {nodes_filter}")
@@ -79,7 +185,7 @@ class ExecutionService(BaseService):
executor = Nodes(node_data, config=self.config)
self.last_executor = executor
results = executor.test(
return executor.test(
commands=commands,
expected=expected,
vars=variables,
@@ -88,22 +194,26 @@ class ExecutionService(BaseService):
folder=folder,
prompt=prompt,
on_complete=on_node_complete,
logger=logger
logger=logger,
connect_timeout=connect_timeout,
auth_timeout=auth_timeout,
_default_cmd_timeout=20,
)
return results
except ConnpyError:
raise
except Exception as e:
raise ConnpyError(f"Testing failed: {e}")
raise ConnpyError(f"Testing failed: {e}") from e
def run_cli_script(self, nodes_filter: str, script_path: str, parallel: int = 10) -> Dict[str, str]:
def run_cli_script(self, nodes_filter: str, script_path: str, parallel: int = 10) -> dict[str, str]:
"""Run a plain-text script containing one command per line."""
if not os.path.exists(script_path):
raise ConnpyError(f"Script file not found: {script_path}")
try:
with open(script_path, "r") as f:
with open(script_path) as f:
commands = [line.strip() for line in f if line.strip()]
except Exception as e:
raise ConnpyError(f"Failed to read script {script_path}: {e}")
except OSError as e:
raise ConnpyError(f"Failed to read script {script_path}: {e}") from e
return self.run_commands(nodes_filter, commands, parallel=parallel)
File diff suppressed because it is too large. Load diff