Merge feature/spec-04-adaptive-timeouts-ssm: adaptive SSM/jumphost timeouts & hardened expects
This commit is contained in:
commit
ef6d1da5e5
3 files changed
+2402
-349
No files matched your search
+1218
-315
File diff suppressed because it is too large.
Load diff
@@ -1,28 +1,107 @@
|
||||
from typing import List, Dict, Any, Callable, Optional
|
||||
import os
|
||||
from .base import BaseService
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from connpy.core import nodes as Nodes
|
||||
|
||||
from .base import BaseService
|
||||
from .exceptions import ConnpyError
|
||||
|
||||
|
||||
class ExecutionService(BaseService):
|
||||
"""Business logic for executing commands on nodes and running automation scripts."""
|
||||
|
||||
def __init__(self, config=None, remote_stub=None):
|
||||
super().__init__(config)
|
||||
self.remote_stub = remote_stub
|
||||
|
||||
def _is_remote_backend(self) -> bool:
|
||||
"""Return True when execution is configured to run against a remote gRPC backend."""
|
||||
if getattr(self, "remote_stub", None) is not None:
|
||||
return True
|
||||
cfg = getattr(self.config, "config", None)
|
||||
return isinstance(cfg, dict) and cfg.get("service_mode") == "remote"
|
||||
|
||||
def _ensure_backend_supports_stage_timeouts(
|
||||
self,
|
||||
connect_timeout: int | None = None,
|
||||
auth_timeout: int | None = None,
|
||||
) -> None:
|
||||
"""Raise ConnpyError if per-stage timeout overrides are used on gRPC remote backend."""
|
||||
if self._is_remote_backend() and (connect_timeout is not None or auth_timeout is not None):
|
||||
raise ConnpyError(
|
||||
"connect_timeout and auth_timeout overrides are not supported in remote gRPC mode"
|
||||
)
|
||||
|
||||
def _get_remote_stub(self):
|
||||
"""Return the configured or lazily constructed ExecutionStub for remote gRPC mode."""
|
||||
if getattr(self, "remote_stub", None) is not None:
|
||||
return self.remote_stub
|
||||
cfg = getattr(self.config, "config", None)
|
||||
remote_host = cfg.get("remote_host") if isinstance(cfg, dict) else None
|
||||
if not remote_host:
|
||||
raise ConnpyError("Remote host must be specified in remote mode")
|
||||
|
||||
import grpc
|
||||
|
||||
from connpy.grpc_layer.stubs import AuthClientInterceptor, ExecutionStub
|
||||
|
||||
def get_token():
|
||||
env_token = os.environ.get("CONNPY_TOKEN")
|
||||
if env_token:
|
||||
return env_token
|
||||
defaultdir = getattr(self.config, "defaultdir", None)
|
||||
if defaultdir:
|
||||
token_path = os.path.join(defaultdir, ".token")
|
||||
if os.path.exists(token_path):
|
||||
try:
|
||||
with open(token_path) as f:
|
||||
return f.read().strip()
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
|
||||
channel = grpc.insecure_channel(remote_host)
|
||||
channel = grpc.intercept_channel(channel, AuthClientInterceptor(get_token))
|
||||
return ExecutionStub(channel, remote_host=remote_host)
|
||||
|
||||
def run_commands(
|
||||
self,
|
||||
nodes_filter: str,
|
||||
commands: List[str],
|
||||
variables: Optional[Dict[str, Any]] = None,
|
||||
commands: list[str],
|
||||
variables: dict[str, Any] | None = None,
|
||||
parallel: int = 10,
|
||||
timeout: int = 20,
|
||||
folder: Optional[str] = None,
|
||||
prompt: Optional[str] = None,
|
||||
on_node_complete: Optional[Callable] = None,
|
||||
logger: Optional[Callable] = None,
|
||||
name: Optional[str] = None
|
||||
) -> Dict[str, str]:
|
||||
|
||||
timeout: int | None = None,
|
||||
folder: str | None = None,
|
||||
prompt: str | None = None,
|
||||
on_node_complete: Callable | None = None,
|
||||
logger: Callable | None = None,
|
||||
name: str | None = None,
|
||||
connect_timeout: int | None = None,
|
||||
auth_timeout: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Execute commands on a set of nodes."""
|
||||
try:
|
||||
self._ensure_backend_supports_stage_timeouts(
|
||||
connect_timeout=connect_timeout,
|
||||
auth_timeout=auth_timeout,
|
||||
)
|
||||
|
||||
if self._is_remote_backend():
|
||||
stub = self._get_remote_stub()
|
||||
return stub.run_commands(
|
||||
nodes_filter=nodes_filter,
|
||||
commands=commands,
|
||||
variables=variables,
|
||||
parallel=parallel,
|
||||
timeout=timeout if timeout is not None else 20,
|
||||
folder=folder,
|
||||
prompt=prompt,
|
||||
on_node_complete=on_node_complete,
|
||||
logger=logger,
|
||||
name=name or "",
|
||||
)
|
||||
|
||||
matched_names = self.config._getallnodes(nodes_filter)
|
||||
if not matched_names:
|
||||
raise ConnpyError(f"No nodes found matching filter: {nodes_filter}")
|
||||
@@ -39,7 +118,10 @@ class ExecutionService(BaseService):
|
||||
folder=folder,
|
||||
prompt=prompt,
|
||||
on_complete=on_node_complete,
|
||||
logger=logger
|
||||
logger=logger,
|
||||
connect_timeout=connect_timeout,
|
||||
auth_timeout=auth_timeout,
|
||||
_default_cmd_timeout=20,
|
||||
)
|
||||
|
||||
# Combine output and status for the caller
|
||||
@@ -47,30 +129,54 @@ class ExecutionService(BaseService):
|
||||
for unique in results:
|
||||
full_results[unique] = {
|
||||
"output": results[unique],
|
||||
"status": executor.status.get(unique, 1)
|
||||
"status": executor.status.get(unique, 1),
|
||||
}
|
||||
|
||||
return full_results
|
||||
except ConnpyError:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise ConnpyError(f"Execution failed: {e}")
|
||||
raise ConnpyError(f"Execution failed: {e}") from e
|
||||
|
||||
def test_commands(
|
||||
self,
|
||||
nodes_filter: str,
|
||||
commands: List[str],
|
||||
expected: List[str],
|
||||
variables: Optional[Dict[str, Any]] = None,
|
||||
commands: list[str],
|
||||
expected: list[str],
|
||||
variables: dict[str, Any] | None = None,
|
||||
parallel: int = 10,
|
||||
timeout: int = 20,
|
||||
folder: Optional[str] = None,
|
||||
prompt: Optional[str] = None,
|
||||
on_node_complete: Optional[Callable] = None,
|
||||
logger: Optional[Callable] = None,
|
||||
name: Optional[str] = None
|
||||
) -> Dict[str, Dict[str, bool]]:
|
||||
|
||||
timeout: int | None = None,
|
||||
folder: str | None = None,
|
||||
prompt: str | None = None,
|
||||
on_node_complete: Callable | None = None,
|
||||
logger: Callable | None = None,
|
||||
name: str | None = None,
|
||||
connect_timeout: int | None = None,
|
||||
auth_timeout: int | None = None,
|
||||
) -> dict[str, dict[str, bool]]:
|
||||
"""Run commands and verify expected output on a set of nodes."""
|
||||
try:
|
||||
self._ensure_backend_supports_stage_timeouts(
|
||||
connect_timeout=connect_timeout,
|
||||
auth_timeout=auth_timeout,
|
||||
)
|
||||
|
||||
if self._is_remote_backend():
|
||||
stub = self._get_remote_stub()
|
||||
return stub.test_commands(
|
||||
nodes_filter=nodes_filter,
|
||||
commands=commands,
|
||||
expected=expected,
|
||||
variables=variables,
|
||||
parallel=parallel,
|
||||
timeout=timeout if timeout is not None else 20,
|
||||
folder=folder or "",
|
||||
prompt=prompt,
|
||||
on_node_complete=on_node_complete,
|
||||
logger=logger,
|
||||
name=name or "",
|
||||
)
|
||||
|
||||
matched_names = self.config._getallnodes(nodes_filter)
|
||||
if not matched_names:
|
||||
raise ConnpyError(f"No nodes found matching filter: {nodes_filter}")
|
||||
@@ -79,7 +185,7 @@ class ExecutionService(BaseService):
|
||||
executor = Nodes(node_data, config=self.config)
|
||||
self.last_executor = executor
|
||||
|
||||
results = executor.test(
|
||||
return executor.test(
|
||||
commands=commands,
|
||||
expected=expected,
|
||||
vars=variables,
|
||||
@@ -88,22 +194,26 @@ class ExecutionService(BaseService):
|
||||
folder=folder,
|
||||
prompt=prompt,
|
||||
on_complete=on_node_complete,
|
||||
logger=logger
|
||||
logger=logger,
|
||||
connect_timeout=connect_timeout,
|
||||
auth_timeout=auth_timeout,
|
||||
_default_cmd_timeout=20,
|
||||
)
|
||||
return results
|
||||
except ConnpyError:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise ConnpyError(f"Testing failed: {e}")
|
||||
raise ConnpyError(f"Testing failed: {e}") from e
|
||||
|
||||
def run_cli_script(self, nodes_filter: str, script_path: str, parallel: int = 10) -> Dict[str, str]:
|
||||
def run_cli_script(self, nodes_filter: str, script_path: str, parallel: int = 10) -> dict[str, str]:
|
||||
"""Run a plain-text script containing one command per line."""
|
||||
if not os.path.exists(script_path):
|
||||
raise ConnpyError(f"Script file not found: {script_path}")
|
||||
|
||||
try:
|
||||
with open(script_path, "r") as f:
|
||||
with open(script_path) as f:
|
||||
commands = [line.strip() for line in f if line.strip()]
|
||||
except Exception as e:
|
||||
raise ConnpyError(f"Failed to read script {script_path}: {e}")
|
||||
except OSError as e:
|
||||
raise ConnpyError(f"Failed to read script {script_path}: {e}") from e
|
||||
|
||||
return self.run_commands(nodes_filter, commands, parallel=parallel)
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user