Merge feature/spec-04-adaptive-timeouts-ssm: adaptive SSM/jumphost timeouts & hardened expects
This commit is contained in:
commit
ef6d1da5e5
3 files changed
+2402
-349
No files matched your search
+1218
-315
File diff suppressed because it is too large.
Load diff
@@ -1,28 +1,107 @@
|
|||||||
from typing import List, Dict, Any, Callable, Optional
|
|
||||||
import os
|
import os
|
||||||
from .base import BaseService
|
from collections.abc import Callable
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from connpy.core import nodes as Nodes
|
from connpy.core import nodes as Nodes
|
||||||
|
|
||||||
|
from .base import BaseService
|
||||||
from .exceptions import ConnpyError
|
from .exceptions import ConnpyError
|
||||||
|
|
||||||
|
|
||||||
class ExecutionService(BaseService):
|
class ExecutionService(BaseService):
|
||||||
"""Business logic for executing commands on nodes and running automation scripts."""
|
"""Business logic for executing commands on nodes and running automation scripts."""
|
||||||
|
|
||||||
|
def __init__(self, config=None, remote_stub=None):
|
||||||
|
super().__init__(config)
|
||||||
|
self.remote_stub = remote_stub
|
||||||
|
|
||||||
|
def _is_remote_backend(self) -> bool:
|
||||||
|
"""Return True when execution is configured to run against a remote gRPC backend."""
|
||||||
|
if getattr(self, "remote_stub", None) is not None:
|
||||||
|
return True
|
||||||
|
cfg = getattr(self.config, "config", None)
|
||||||
|
return isinstance(cfg, dict) and cfg.get("service_mode") == "remote"
|
||||||
|
|
||||||
|
def _ensure_backend_supports_stage_timeouts(
|
||||||
|
self,
|
||||||
|
connect_timeout: int | None = None,
|
||||||
|
auth_timeout: int | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Raise ConnpyError if per-stage timeout overrides are used on gRPC remote backend."""
|
||||||
|
if self._is_remote_backend() and (connect_timeout is not None or auth_timeout is not None):
|
||||||
|
raise ConnpyError(
|
||||||
|
"connect_timeout and auth_timeout overrides are not supported in remote gRPC mode"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _get_remote_stub(self):
|
||||||
|
"""Return the configured or lazily constructed ExecutionStub for remote gRPC mode."""
|
||||||
|
if getattr(self, "remote_stub", None) is not None:
|
||||||
|
return self.remote_stub
|
||||||
|
cfg = getattr(self.config, "config", None)
|
||||||
|
remote_host = cfg.get("remote_host") if isinstance(cfg, dict) else None
|
||||||
|
if not remote_host:
|
||||||
|
raise ConnpyError("Remote host must be specified in remote mode")
|
||||||
|
|
||||||
|
import grpc
|
||||||
|
|
||||||
|
from connpy.grpc_layer.stubs import AuthClientInterceptor, ExecutionStub
|
||||||
|
|
||||||
|
def get_token():
|
||||||
|
env_token = os.environ.get("CONNPY_TOKEN")
|
||||||
|
if env_token:
|
||||||
|
return env_token
|
||||||
|
defaultdir = getattr(self.config, "defaultdir", None)
|
||||||
|
if defaultdir:
|
||||||
|
token_path = os.path.join(defaultdir, ".token")
|
||||||
|
if os.path.exists(token_path):
|
||||||
|
try:
|
||||||
|
with open(token_path) as f:
|
||||||
|
return f.read().strip()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
||||||
|
channel = grpc.insecure_channel(remote_host)
|
||||||
|
channel = grpc.intercept_channel(channel, AuthClientInterceptor(get_token))
|
||||||
|
return ExecutionStub(channel, remote_host=remote_host)
|
||||||
|
|
||||||
def run_commands(
|
def run_commands(
|
||||||
self,
|
self,
|
||||||
nodes_filter: str,
|
nodes_filter: str,
|
||||||
commands: List[str],
|
commands: list[str],
|
||||||
variables: Optional[Dict[str, Any]] = None,
|
variables: dict[str, Any] | None = None,
|
||||||
parallel: int = 10,
|
parallel: int = 10,
|
||||||
timeout: int = 20,
|
timeout: int | None = None,
|
||||||
folder: Optional[str] = None,
|
folder: str | None = None,
|
||||||
prompt: Optional[str] = None,
|
prompt: str | None = None,
|
||||||
on_node_complete: Optional[Callable] = None,
|
on_node_complete: Callable | None = None,
|
||||||
logger: Optional[Callable] = None,
|
logger: Callable | None = None,
|
||||||
name: Optional[str] = None
|
name: str | None = None,
|
||||||
) -> Dict[str, str]:
|
connect_timeout: int | None = None,
|
||||||
|
auth_timeout: int | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
"""Execute commands on a set of nodes."""
|
"""Execute commands on a set of nodes."""
|
||||||
try:
|
try:
|
||||||
|
self._ensure_backend_supports_stage_timeouts(
|
||||||
|
connect_timeout=connect_timeout,
|
||||||
|
auth_timeout=auth_timeout,
|
||||||
|
)
|
||||||
|
|
||||||
|
if self._is_remote_backend():
|
||||||
|
stub = self._get_remote_stub()
|
||||||
|
return stub.run_commands(
|
||||||
|
nodes_filter=nodes_filter,
|
||||||
|
commands=commands,
|
||||||
|
variables=variables,
|
||||||
|
parallel=parallel,
|
||||||
|
timeout=timeout if timeout is not None else 20,
|
||||||
|
folder=folder,
|
||||||
|
prompt=prompt,
|
||||||
|
on_node_complete=on_node_complete,
|
||||||
|
logger=logger,
|
||||||
|
name=name or "",
|
||||||
|
)
|
||||||
|
|
||||||
matched_names = self.config._getallnodes(nodes_filter)
|
matched_names = self.config._getallnodes(nodes_filter)
|
||||||
if not matched_names:
|
if not matched_names:
|
||||||
raise ConnpyError(f"No nodes found matching filter: {nodes_filter}")
|
raise ConnpyError(f"No nodes found matching filter: {nodes_filter}")
|
||||||
@@ -39,7 +118,10 @@ class ExecutionService(BaseService):
|
|||||||
folder=folder,
|
folder=folder,
|
||||||
prompt=prompt,
|
prompt=prompt,
|
||||||
on_complete=on_node_complete,
|
on_complete=on_node_complete,
|
||||||
logger=logger
|
logger=logger,
|
||||||
|
connect_timeout=connect_timeout,
|
||||||
|
auth_timeout=auth_timeout,
|
||||||
|
_default_cmd_timeout=20,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Combine output and status for the caller
|
# Combine output and status for the caller
|
||||||
@@ -47,30 +129,54 @@ class ExecutionService(BaseService):
|
|||||||
for unique in results:
|
for unique in results:
|
||||||
full_results[unique] = {
|
full_results[unique] = {
|
||||||
"output": results[unique],
|
"output": results[unique],
|
||||||
"status": executor.status.get(unique, 1)
|
"status": executor.status.get(unique, 1),
|
||||||
}
|
}
|
||||||
|
|
||||||
return full_results
|
return full_results
|
||||||
|
except ConnpyError:
|
||||||
|
raise
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise ConnpyError(f"Execution failed: {e}")
|
raise ConnpyError(f"Execution failed: {e}") from e
|
||||||
|
|
||||||
def test_commands(
|
def test_commands(
|
||||||
self,
|
self,
|
||||||
nodes_filter: str,
|
nodes_filter: str,
|
||||||
commands: List[str],
|
commands: list[str],
|
||||||
expected: List[str],
|
expected: list[str],
|
||||||
variables: Optional[Dict[str, Any]] = None,
|
variables: dict[str, Any] | None = None,
|
||||||
parallel: int = 10,
|
parallel: int = 10,
|
||||||
timeout: int = 20,
|
timeout: int | None = None,
|
||||||
folder: Optional[str] = None,
|
folder: str | None = None,
|
||||||
prompt: Optional[str] = None,
|
prompt: str | None = None,
|
||||||
on_node_complete: Optional[Callable] = None,
|
on_node_complete: Callable | None = None,
|
||||||
logger: Optional[Callable] = None,
|
logger: Callable | None = None,
|
||||||
name: Optional[str] = None
|
name: str | None = None,
|
||||||
) -> Dict[str, Dict[str, bool]]:
|
connect_timeout: int | None = None,
|
||||||
|
auth_timeout: int | None = None,
|
||||||
|
) -> dict[str, dict[str, bool]]:
|
||||||
"""Run commands and verify expected output on a set of nodes."""
|
"""Run commands and verify expected output on a set of nodes."""
|
||||||
try:
|
try:
|
||||||
|
self._ensure_backend_supports_stage_timeouts(
|
||||||
|
connect_timeout=connect_timeout,
|
||||||
|
auth_timeout=auth_timeout,
|
||||||
|
)
|
||||||
|
|
||||||
|
if self._is_remote_backend():
|
||||||
|
stub = self._get_remote_stub()
|
||||||
|
return stub.test_commands(
|
||||||
|
nodes_filter=nodes_filter,
|
||||||
|
commands=commands,
|
||||||
|
expected=expected,
|
||||||
|
variables=variables,
|
||||||
|
parallel=parallel,
|
||||||
|
timeout=timeout if timeout is not None else 20,
|
||||||
|
folder=folder or "",
|
||||||
|
prompt=prompt,
|
||||||
|
on_node_complete=on_node_complete,
|
||||||
|
logger=logger,
|
||||||
|
name=name or "",
|
||||||
|
)
|
||||||
|
|
||||||
matched_names = self.config._getallnodes(nodes_filter)
|
matched_names = self.config._getallnodes(nodes_filter)
|
||||||
if not matched_names:
|
if not matched_names:
|
||||||
raise ConnpyError(f"No nodes found matching filter: {nodes_filter}")
|
raise ConnpyError(f"No nodes found matching filter: {nodes_filter}")
|
||||||
@@ -79,7 +185,7 @@ class ExecutionService(BaseService):
|
|||||||
executor = Nodes(node_data, config=self.config)
|
executor = Nodes(node_data, config=self.config)
|
||||||
self.last_executor = executor
|
self.last_executor = executor
|
||||||
|
|
||||||
results = executor.test(
|
return executor.test(
|
||||||
commands=commands,
|
commands=commands,
|
||||||
expected=expected,
|
expected=expected,
|
||||||
vars=variables,
|
vars=variables,
|
||||||
@@ -88,22 +194,26 @@ class ExecutionService(BaseService):
|
|||||||
folder=folder,
|
folder=folder,
|
||||||
prompt=prompt,
|
prompt=prompt,
|
||||||
on_complete=on_node_complete,
|
on_complete=on_node_complete,
|
||||||
logger=logger
|
logger=logger,
|
||||||
|
connect_timeout=connect_timeout,
|
||||||
|
auth_timeout=auth_timeout,
|
||||||
|
_default_cmd_timeout=20,
|
||||||
)
|
)
|
||||||
return results
|
except ConnpyError:
|
||||||
|
raise
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise ConnpyError(f"Testing failed: {e}")
|
raise ConnpyError(f"Testing failed: {e}") from e
|
||||||
|
|
||||||
def run_cli_script(self, nodes_filter: str, script_path: str, parallel: int = 10) -> Dict[str, str]:
|
def run_cli_script(self, nodes_filter: str, script_path: str, parallel: int = 10) -> dict[str, str]:
|
||||||
"""Run a plain-text script containing one command per line."""
|
"""Run a plain-text script containing one command per line."""
|
||||||
if not os.path.exists(script_path):
|
if not os.path.exists(script_path):
|
||||||
raise ConnpyError(f"Script file not found: {script_path}")
|
raise ConnpyError(f"Script file not found: {script_path}")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with open(script_path, "r") as f:
|
with open(script_path) as f:
|
||||||
commands = [line.strip() for line in f if line.strip()]
|
commands = [line.strip() for line in f if line.strip()]
|
||||||
except Exception as e:
|
except OSError as e:
|
||||||
raise ConnpyError(f"Failed to read script {script_path}: {e}")
|
raise ConnpyError(f"Failed to read script {script_path}: {e}") from e
|
||||||
|
|
||||||
return self.run_commands(nodes_filter, commands, parallel=parallel)
|
return self.run_commands(nodes_filter, commands, parallel=parallel)
|
||||||
|
|
||||||
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user