4106 lines
176 KiB
Python
Executable File
4106 lines
176 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import ast
|
|
import asyncio
|
|
import datetime
|
|
import fcntl
|
|
import io
|
|
import ipaddress
|
|
import logging
|
|
import os
|
|
import re
|
|
import shlex
|
|
import shutil
|
|
import sys
|
|
import threading
|
|
import time as _time_mod
|
|
from contextlib import contextmanager, suppress
|
|
from copy import deepcopy
|
|
from pathlib import Path
|
|
from time import sleep, time
|
|
|
|
import pexpect
|
|
|
|
from . import printer
|
|
from .auth_utils import (
|
|
DEFAULT_MAX_AUTH_PROMPTS,
|
|
DEFAULT_MAX_HOP_HOST_ATTEMPTS,
|
|
DEFAULT_MAX_TOTAL_AUTH_ATTEMPTS,
|
|
MAX_AUTH_PROMPTS_CAP,
|
|
MAX_AUTH_STREAM_BYTES,
|
|
MAX_COMPOSE_VARIANTS,
|
|
MAX_EOF_FALLBACK_ADVANCES,
|
|
MAX_HOP_HOST_ATTEMPTS_CAP,
|
|
MAX_PROFILE_DEPTH,
|
|
MAX_TOTAL_AUTH_ATTEMPTS_CAP,
|
|
MIN_AUTH_PROMPTS,
|
|
MIN_HOP_HOST_ATTEMPTS,
|
|
MIN_TOTAL_AUTH_ATTEMPTS,
|
|
AuthAttemptTracker,
|
|
AuthCandidate,
|
|
RuntimeAuthCandidate,
|
|
SanitizedAuthError,
|
|
has_insecure_hostkey_checking,
|
|
host_exact_re,
|
|
is_compose_expression,
|
|
normalize_prompt_line,
|
|
parse_compose_expression,
|
|
resolve_fallback_entries,
|
|
sanitize_exception,
|
|
sanitize_log_message,
|
|
validate_strict_int_range,
|
|
)
|
|
from .hooks import ClassHook, MethodHook
|
|
from .tunnels import LocalStream
|
|
|
|
_LOGGER = logging.getLogger(__name__)
|
|
|
|
DEFAULT_DIRECT_CONNECT_TIMEOUT = 10
|
|
DEFAULT_MOSH_CONNECT_TIMEOUT = 15
|
|
DEFAULT_ADAPTIVE_BASE_TIMEOUT = 15
|
|
DEFAULT_SSM_DIRECT_COST = 20
|
|
DEFAULT_SSM_HOP_COST = 25
|
|
DEFAULT_OTHER_HOP_COST = 10
|
|
DEFAULT_AUTH_TIMEOUT = 25
|
|
DEFAULT_MAX_PROGRESS_EXTENSIONS = 3
|
|
MAX_STAGE_TIMEOUT_CAP = 600
|
|
DEFAULT_MAX_TOTAL_CONNECT_SECONDS = 180
|
|
MAX_PRE_AUTH_BUFFER_BYTES = 65536
|
|
|
|
_CONTROL_CHAR_RE = re.compile(r"[\x00-\x1f\x7f]")
|
|
_SSM_TARGET_RE = re.compile(
|
|
r"\A(?:(?:i|mi)-[0-9a-f]{4,17}|arn:aws[a-z-]*:ssm:[a-z0-9-]+:\d{12}:[A-Za-z0-9._/-]{1,256})\Z",
|
|
re.ASCII,
|
|
)
|
|
_SSM_TAG_RE = re.compile(r"\A[A-Za-z0-9._-]{1,128}\Z", re.ASCII)
|
|
_SSM_UNSAFE_OPTIONS_RE = re.compile(r'[;|&$`><"\'\\\x00-\x1f\x7f]')
|
|
|
|
|
|
def validate_strict_timeout_int(
|
|
value,
|
|
name: str,
|
|
min_val: int = 1,
|
|
max_val: int = MAX_STAGE_TIMEOUT_CAP,
|
|
) -> int:
|
|
"""Strictly validate that value is a plain int within [min_val, max_val]."""
|
|
if type(value) is not int or value < min_val or value > max_val:
|
|
raise ValueError(
|
|
f"Invalid {name}: expected integer in [{min_val}, {max_val}], got {value!r}"
|
|
)
|
|
return value
|
|
|
|
|
|
def resolve_timeouts(
|
|
protocol,
|
|
tags,
|
|
hop_chain_meta,
|
|
timeout=None,
|
|
connect_timeout=None,
|
|
auth_timeout=None,
|
|
*,
|
|
_default_cmd_timeout=10,
|
|
_default_max_total_seconds=None,
|
|
) -> tuple[int, int, int, int, int]:
|
|
"""Resolve effective connect, auth, command, extension, and total deadline budgets."""
|
|
default_cmd = validate_strict_timeout_int(_default_cmd_timeout, "_default_cmd_timeout")
|
|
tags_dict = tags if isinstance(tags, dict) else {}
|
|
|
|
if timeout is not None:
|
|
validate_strict_timeout_int(timeout, "timeout")
|
|
if connect_timeout is not None:
|
|
validate_strict_timeout_int(connect_timeout, "connect_timeout")
|
|
if auth_timeout is not None:
|
|
validate_strict_timeout_int(auth_timeout, "auth_timeout")
|
|
|
|
tag_ct = tags_dict.get("connect_timeout")
|
|
tag_at = tags_dict.get("auth_timeout")
|
|
tag_cmd = tags_dict.get("command_timeout")
|
|
if tag_ct is not None:
|
|
validate_strict_timeout_int(tag_ct, "tags.connect_timeout")
|
|
if tag_at is not None:
|
|
validate_strict_timeout_int(tag_at, "tags.auth_timeout")
|
|
if tag_cmd is not None:
|
|
validate_strict_timeout_int(tag_cmd, "tags.command_timeout")
|
|
|
|
hops = hop_chain_meta or []
|
|
if hops:
|
|
adaptive_ct = DEFAULT_ADAPTIVE_BASE_TIMEOUT
|
|
for hop in hops:
|
|
hop_proto = hop.get("protocol", "ssh") if isinstance(hop, dict) else str(hop)
|
|
if hop_proto == "ssm":
|
|
adaptive_ct += DEFAULT_SSM_HOP_COST
|
|
else:
|
|
adaptive_ct += DEFAULT_OTHER_HOP_COST
|
|
if protocol == "mosh":
|
|
adaptive_ct = max(adaptive_ct, DEFAULT_MOSH_CONNECT_TIMEOUT)
|
|
elif protocol == "ssm":
|
|
adaptive_ct = DEFAULT_SSM_DIRECT_COST
|
|
elif protocol == "mosh":
|
|
adaptive_ct = DEFAULT_MOSH_CONNECT_TIMEOUT
|
|
else:
|
|
adaptive_ct = DEFAULT_DIRECT_CONNECT_TIMEOUT
|
|
|
|
adaptive_ct = min(adaptive_ct, MAX_STAGE_TIMEOUT_CAP)
|
|
|
|
if timeout is not None:
|
|
eff_cmd = timeout
|
|
elif tag_cmd is not None:
|
|
eff_cmd = tag_cmd
|
|
else:
|
|
eff_cmd = default_cmd
|
|
|
|
if auth_timeout is not None:
|
|
eff_at = auth_timeout
|
|
elif timeout is not None:
|
|
eff_at = timeout
|
|
elif tag_at is not None:
|
|
eff_at = tag_at
|
|
else:
|
|
eff_at = DEFAULT_AUTH_TIMEOUT
|
|
|
|
if connect_timeout is not None:
|
|
eff_ct = connect_timeout
|
|
elif timeout is not None:
|
|
eff_ct = max(timeout, DEFAULT_MOSH_CONNECT_TIMEOUT) if protocol == "mosh" else timeout
|
|
elif tag_ct is not None:
|
|
eff_ct = max(tag_ct, DEFAULT_MOSH_CONNECT_TIMEOUT) if protocol == "mosh" else tag_ct
|
|
else:
|
|
eff_ct = adaptive_ct
|
|
|
|
tag_max_ext = tags_dict.get("max_progress_extensions")
|
|
if tag_max_ext is not None:
|
|
max_ext = validate_strict_timeout_int(
|
|
tag_max_ext,
|
|
"tags.max_progress_extensions",
|
|
min_val=0,
|
|
max_val=10,
|
|
)
|
|
else:
|
|
max_ext = DEFAULT_MAX_PROGRESS_EXTENSIONS
|
|
|
|
max_total = DEFAULT_MAX_TOTAL_CONNECT_SECONDS
|
|
if _default_max_total_seconds is not None:
|
|
ceiling = validate_strict_timeout_int(
|
|
_default_max_total_seconds,
|
|
"_default_max_total_seconds",
|
|
min_val=1,
|
|
max_val=MAX_STAGE_TIMEOUT_CAP,
|
|
)
|
|
max_total = min(max_total, ceiling)
|
|
|
|
return eff_ct, eff_at, eff_cmd, max_ext, max_total
|
|
|
|
|
|
def compile_expects(expects_map):
|
|
"""Compile ASCII expect regex strings into bytes patterns with re.DOTALL."""
|
|
def _compile_list(items):
|
|
compiled = []
|
|
for pat in items:
|
|
if isinstance(pat, str):
|
|
compiled.append(re.compile(pat.encode("ascii"), re.DOTALL))
|
|
elif isinstance(pat, bytes):
|
|
compiled.append(re.compile(pat, re.DOTALL))
|
|
elif isinstance(pat, re.Pattern):
|
|
if isinstance(pat.pattern, str):
|
|
compiled.append(
|
|
re.compile(
|
|
pat.pattern.encode("ascii"),
|
|
(pat.flags & ~re.UNICODE) | re.DOTALL,
|
|
)
|
|
)
|
|
else:
|
|
compiled.append(pat)
|
|
else:
|
|
compiled.append(pat)
|
|
return compiled
|
|
|
|
if isinstance(expects_map, dict):
|
|
return {proto: _compile_list(pats) for proto, pats in expects_map.items()}
|
|
return _compile_list(expects_map)
|
|
|
|
|
|
def _extract_ssh_o_keys(tokens: list[str]) -> set[str]:
|
|
"""Return lowercased -o directive keys present in tokenized SSH options."""
|
|
keys: set[str] = set()
|
|
idx = 0
|
|
while idx < len(tokens):
|
|
tok = tokens[idx]
|
|
if tok == "-o" and idx + 1 < len(tokens):
|
|
spec = tokens[idx + 1]
|
|
if "=" in spec:
|
|
keys.add(spec.split("=", 1)[0].strip().lower())
|
|
idx += 2
|
|
elif tok.startswith("-o") and len(tok) > 2:
|
|
spec = tok.removeprefix("-o")
|
|
if "=" in spec:
|
|
keys.add(spec.split("=", 1)[0].strip().lower())
|
|
idx += 1
|
|
else:
|
|
idx += 1
|
|
return keys
|
|
|
|
|
|
def _redact_known_secrets(text: str, secrets: list[str]) -> str:
|
|
"""Replace any non-trivial known secret strings with '***'."""
|
|
if not text or not secrets:
|
|
return text
|
|
out = text
|
|
unique_secrets = sorted(
|
|
{s for s in secrets if isinstance(s, str) and len(s) >= 4},
|
|
key=len,
|
|
reverse=True,
|
|
)
|
|
for sec in unique_secrets:
|
|
out = out.replace(sec, "***")
|
|
return out
|
|
|
|
@contextmanager
|
|
def copilot_terminal_mode():
|
|
import termios
|
|
import tty
|
|
|
|
fd = sys.stdin.fileno()
|
|
try:
|
|
old_settings = termios.tcgetattr(fd)
|
|
|
|
# First we switch to absolute raw mode to disable ISIG, ICANON, ECHO, etc.
|
|
tty.setraw(fd)
|
|
|
|
# Then we re-enable OPOST so rich.Live renders correctly
|
|
new_settings = termios.tcgetattr(fd)
|
|
new_settings[1] = new_settings[1] | termios.OPOST
|
|
termios.tcsetattr(fd, termios.TCSANOW, new_settings)
|
|
|
|
yield
|
|
finally:
|
|
try:
|
|
termios.tcsetattr(fd, termios.TCSANOW, old_settings)
|
|
except Exception as exc: # noqa: BLE001
|
|
_LOGGER.debug("Failed to restore terminal attributes: %s", exc)
|
|
|
|
_ALLOWED_SSH_NOARG_FLAGS = frozenset({
|
|
"-4", "-6", "-A", "-C", "-X", "-Y", "-a", "-q", "-t", "-v", "-vv", "-vvv", "-x",
|
|
})
|
|
_NON_TUNNEL_SSH_FLAGS = frozenset({
|
|
"-A", "-X", "-Y", "-t", "-v", "-vv", "-vvv", "-x",
|
|
})
|
|
_ALLOWED_SSH_ARG_FLAGS = frozenset({
|
|
"-c", "-i", "-m",
|
|
})
|
|
_ALLOWED_SSH_O_KEYS = frozenset({
|
|
"addkeystoagent",
|
|
"ciphers",
|
|
"compression",
|
|
"connectionattempts",
|
|
"connecttimeout",
|
|
"fingerprinthash",
|
|
"forwardagent",
|
|
"gssapiauthentication",
|
|
"hostkeyalgorithms",
|
|
"identitiesonly",
|
|
"identityfile",
|
|
"ipqos",
|
|
"kbdinteractiveauthentication",
|
|
"kexalgorithms",
|
|
"loglevel",
|
|
"macs",
|
|
"numberofpasswordprompts",
|
|
"preferredauthentications",
|
|
"pubkeyacceptedalgorithms",
|
|
"pubkeyacceptedkeytypes",
|
|
"pubkeyauthentication",
|
|
"serveralivecountmax",
|
|
"serveraliveinterval",
|
|
"stricthostkeychecking",
|
|
"tcpkeepalive",
|
|
"userknownhostsfile",
|
|
"verifyhostkeydns",
|
|
})
|
|
_SAFE_SSH_PATH_RE = re.compile(r"^[A-Za-z0-9._/@:,+~= -]+$", re.ASCII)
|
|
_SAFE_SSH_O_VALUE_RE = re.compile(r"^[A-Za-z0-9._/@:,+~=-]+$", re.ASCII)
|
|
_LEGACY_SSH_UNSAFE_RE = re.compile(r'[\s\x00-\x1f\x7f;|&$`><"\']')
|
|
|
|
#functions and classes
|
|
@ClassHook
|
|
class node:
|
|
''' This class generates a node object. Containts all the information and methods to connect and interact with a device using ssh or telnet.
|
|
|
|
### Attributes:
|
|
|
|
- output (str): Output of the commands you ran with run or test
|
|
method.
|
|
|
|
- result(bool): True if expected value is found after running
|
|
the commands using test method.
|
|
|
|
- status (int): 0 if the method run or test run successfully.
|
|
1 if connection failed.
|
|
2 if expect timeouts without prompt or EOF.
|
|
|
|
'''
|
|
|
|
def __init__(
|
|
self,
|
|
unique,
|
|
host,
|
|
options='',
|
|
logs='',
|
|
password='',
|
|
port='',
|
|
protocol='',
|
|
user='',
|
|
config='',
|
|
tags='',
|
|
jumphost='',
|
|
allow_unsafe_options=None,
|
|
fallback=None,
|
|
disable_fallback_on_insecure_hostkey=None,
|
|
):
|
|
'''
|
|
|
|
### Parameters:
|
|
|
|
- unique (str): Unique name to assign to the node.
|
|
|
|
- host (str): IP address or hostname of the node.
|
|
|
|
### Optional Parameters:
|
|
|
|
- options (str): Additional options to pass the ssh/telnet for
|
|
connection.
|
|
|
|
- logs (str): Path/file for storing the logs. You can use
|
|
${unique},${host}, ${port}, ${user}, ${protocol}
|
|
as variables.
|
|
|
|
- password (str): Encrypted or plaintext password.
|
|
|
|
- port (str): Port to connect to node, default 22 for ssh and 23
|
|
for telnet.
|
|
|
|
- protocol (str): Select ssh, mosh, telnet, kubectl, docker or ssm. Default is ssh.
|
|
|
|
- user (str): Username to of the node.
|
|
|
|
- config (obj): Pass the object created with class configfile with
|
|
key for decryption and extra configuration if you
|
|
are using connection manager.
|
|
|
|
- tags (dict) : Tags useful for automation and personal porpuse
|
|
like "os", "prompt" and "screenleght_command"
|
|
|
|
- jumphost (str): Reference another node to be used as a jumphost
|
|
|
|
- allow_unsafe_options (bool): Explicitly override safety restrictions.
|
|
'''
|
|
self.config = config
|
|
if config == '':
|
|
self.idletime = 0
|
|
self.key = None
|
|
else:
|
|
self.idletime = config.config["idletime"] if (hasattr(config, "config") and isinstance(config.config, dict) and "idletime" in config.config) else 0
|
|
self.key = getattr(config, "key", None)
|
|
if allow_unsafe_options is not None:
|
|
self.allow_unsafe_options = bool(allow_unsafe_options)
|
|
elif config != '' and config is not None and hasattr(config, "config") and isinstance(config.config, dict):
|
|
self.allow_unsafe_options = bool(config.config.get("allow_unsafe_options", False))
|
|
else:
|
|
self.allow_unsafe_options = False
|
|
self.unique = unique
|
|
profiles_map = getattr(config, "profiles", {}) if (config != '' and config is not None) else {}
|
|
attr = {"host": host, "logs": logs, "options":options, "port": port, "protocol": protocol, "user": user, "tags": tags, "jumphost": jumphost}
|
|
for key, attr_val in attr.items():
|
|
profile = re.search("^@(.*)", str(attr_val))
|
|
if profile and config != '':
|
|
try:
|
|
setattr(self, key, deepcopy(profiles_map[profile.group(1)][key]))
|
|
except KeyError:
|
|
setattr(self, key, "")
|
|
elif attr_val == '' and key == "protocol":
|
|
try:
|
|
setattr(self, key, profiles_map["default"][key])
|
|
except (KeyError, AttributeError):
|
|
setattr(self, key, "ssh")
|
|
else:
|
|
setattr(self, key, attr_val)
|
|
self._orig_protocol = self.protocol
|
|
if isinstance(password, list):
|
|
self.password = []
|
|
for item in password:
|
|
if (
|
|
isinstance(item, str)
|
|
and not is_compose_expression(item)
|
|
and item.startswith("@")
|
|
and config != ''
|
|
and item[1:] in profiles_map
|
|
):
|
|
self.password.append(deepcopy(profiles_map[item[1:]].get("password", "")))
|
|
else:
|
|
self.password.append(item)
|
|
else:
|
|
if (
|
|
isinstance(password, str)
|
|
and not is_compose_expression(password)
|
|
and password.startswith("@")
|
|
and config != ''
|
|
and password[1:] in profiles_map
|
|
):
|
|
resolved_pw = profiles_map[password[1:]].get("password", "")
|
|
self.password = deepcopy(resolved_pw) if isinstance(resolved_pw, list) else [resolved_pw]
|
|
else:
|
|
self.password = [password]
|
|
|
|
target_passwords_snapshot = list(self.password)
|
|
|
|
raw_fallback = fallback
|
|
if raw_fallback in (None, "", []) and isinstance(self.tags, dict):
|
|
if "_fallback" in self.tags:
|
|
raw_fallback = self.tags.pop("_fallback", None)
|
|
elif "fallback" in self.tags:
|
|
raw_fallback = self.tags.get("fallback")
|
|
|
|
if raw_fallback not in (None, "", []):
|
|
resolved_fb_entries = resolve_fallback_entries(raw_fallback, profiles_map)
|
|
else:
|
|
resolved_fb_entries = []
|
|
|
|
candidates: list[AuthCandidate] = []
|
|
primary_specs = tuple(
|
|
p for p in target_passwords_snapshot if isinstance(p, str) and p != ""
|
|
)
|
|
if primary_specs:
|
|
candidates.append(
|
|
AuthCandidate(
|
|
user=str(self.user or ""),
|
|
password_spec=primary_specs if len(primary_specs) > 1 else primary_specs[0],
|
|
is_fallback=False,
|
|
label="primary",
|
|
)
|
|
)
|
|
elif self.user:
|
|
candidates.append(
|
|
AuthCandidate(
|
|
user=str(self.user or ""),
|
|
password_spec="",
|
|
is_fallback=False,
|
|
label="primary",
|
|
)
|
|
)
|
|
|
|
for fb_idx, entry in enumerate(resolved_fb_entries, start=1):
|
|
fb_user = str(entry.get("user") or self.user or "")
|
|
fb_pw = entry.get("password", "")
|
|
if isinstance(fb_pw, list):
|
|
pw_spec: str | tuple[str, ...] = tuple(str(x) for x in fb_pw if x != "")
|
|
else:
|
|
pw_spec = str(fb_pw or "")
|
|
candidates.append(
|
|
AuthCandidate(
|
|
user=fb_user,
|
|
password_spec=pw_spec,
|
|
is_fallback=True,
|
|
label=f"fallback#{fb_idx}",
|
|
)
|
|
)
|
|
self._auth_candidates: tuple[AuthCandidate, ...] = tuple(candidates)
|
|
|
|
if disable_fallback_on_insecure_hostkey is not None:
|
|
self.disable_fallback_on_insecure_hostkey = bool(disable_fallback_on_insecure_hostkey)
|
|
elif isinstance(self.tags, dict) and "disable_fallback_on_insecure_hostkey" in self.tags:
|
|
self.disable_fallback_on_insecure_hostkey = bool(
|
|
self.tags["disable_fallback_on_insecure_hostkey"]
|
|
)
|
|
elif (
|
|
config != ''
|
|
and config is not None
|
|
and hasattr(config, "config")
|
|
and isinstance(config.config, dict)
|
|
):
|
|
self.disable_fallback_on_insecure_hostkey = bool(
|
|
config.config.get("disable_fallback_on_insecure_hostkey", False)
|
|
)
|
|
else:
|
|
self.disable_fallback_on_insecure_hostkey = False
|
|
|
|
self._jumphost_name = self.jumphost if (self.jumphost != "" and config != '') else ""
|
|
self._hop_chain_meta = []
|
|
self._jumphost_hops: tuple[dict, ...] = ()
|
|
if self.jumphost != "" and config != '':
|
|
chain_res = self._build_jumphost_chain(self.jumphost, config, include_meta=True)
|
|
if len(chain_res) == 3:
|
|
raw_cmd, jh_passwords, self._hop_chain_meta = chain_res
|
|
else:
|
|
raw_cmd, jh_passwords = chain_res
|
|
self._hop_chain_meta = []
|
|
self._jumphost_hops = tuple(self._collect_jumphost_hops(self._jumphost_name, config))
|
|
if jh_passwords:
|
|
self.password = jh_passwords + self.password
|
|
if raw_cmd:
|
|
escaped = raw_cmd.replace('\\', '\\\\').replace('"', '\\"')
|
|
self.jumphost = f'-o ProxyCommand="{escaped}"'
|
|
else:
|
|
self.jumphost = ""
|
|
|
|
self.last_fallback_used = False
|
|
self.output = ""
|
|
self.status = 1
|
|
self.result = {}
|
|
self.cmd_byte_positions = [(0, None)]
|
|
|
|
def _collect_jumphost_hops(self, jumphost_name, config, visited=None, depth=0):
|
|
"""Collect structured hop metadata in SSH ProxyCommand prompt order."""
|
|
if depth >= 5:
|
|
return []
|
|
if visited is None:
|
|
visited = []
|
|
if jumphost_name in visited:
|
|
return []
|
|
visited = [*visited, jumphost_name]
|
|
try:
|
|
jh = config.getitem(jumphost_name)
|
|
except Exception: # noqa: BLE001
|
|
return []
|
|
jh = self._resolve_jumphost_data(deepcopy(jh), config)
|
|
hops = []
|
|
if jh.get("jumphost", "") != "":
|
|
hops.extend(
|
|
self._collect_jumphost_hops(
|
|
jh["jumphost"],
|
|
config,
|
|
visited,
|
|
depth + 1,
|
|
)
|
|
)
|
|
jh_tags = jh.get("tags")
|
|
resolved_fqdn = ""
|
|
if isinstance(jh_tags, dict) and jh_tags.get("fqdn"):
|
|
resolved_fqdn = str(jh_tags["fqdn"])
|
|
hops.append(
|
|
{
|
|
"name": jumphost_name,
|
|
"host": str(jh.get("host", "") or ""),
|
|
"port": str(jh.get("port", "") or ""),
|
|
"protocol": str(jh.get("protocol", "ssh") or "ssh"),
|
|
"user": str(jh.get("user", "") or ""),
|
|
"options": str(jh.get("options", "") or ""),
|
|
"tags": deepcopy(jh_tags) if isinstance(jh_tags, dict) else jh_tags,
|
|
"resolved_fqdn": resolved_fqdn,
|
|
"passwords": tuple(p for p in (jh.get("password") or []) if p != ""),
|
|
}
|
|
)
|
|
return hops
|
|
|
|
@staticmethod
|
|
def _resolve_jumphost_data(jh_dict, config):
|
|
'''Resolve @profile references and normalize passwords in a jumphost dict.'''
|
|
profiles_map = getattr(config, "profiles", {}) if config else {}
|
|
for key, jh_val in list(jh_dict.items()):
|
|
if key == "password":
|
|
continue
|
|
profile = re.search("^@(.*)", str(jh_val))
|
|
if profile:
|
|
try:
|
|
jh_dict[key] = deepcopy(profiles_map[profile.group(1)][key])
|
|
except KeyError:
|
|
jh_dict[key] = ""
|
|
elif jh_val == '' and key == "protocol":
|
|
try:
|
|
jh_dict[key] = profiles_map["default"][key]
|
|
except KeyError:
|
|
jh_dict[key] = "ssh"
|
|
raw_pw = jh_dict.get("password", "")
|
|
if isinstance(raw_pw, list):
|
|
resolved = []
|
|
for p in raw_pw:
|
|
if (
|
|
isinstance(p, str)
|
|
and not is_compose_expression(p)
|
|
and p.startswith("@")
|
|
and p[1:] in profiles_map
|
|
):
|
|
resolved.append(deepcopy(profiles_map[p[1:]].get("password", "")))
|
|
else:
|
|
resolved.append(p)
|
|
jh_dict["password"] = resolved
|
|
else:
|
|
if (
|
|
isinstance(raw_pw, str)
|
|
and not is_compose_expression(raw_pw)
|
|
and raw_pw.startswith("@")
|
|
and raw_pw[1:] in profiles_map
|
|
):
|
|
res = profiles_map[raw_pw[1:]].get("password", "")
|
|
jh_dict["password"] = deepcopy(res) if isinstance(res, list) else [res]
|
|
else:
|
|
jh_dict["password"] = [raw_pw]
|
|
return jh_dict
|
|
|
|
def _is_unsafe_allowed(self) -> bool:
|
|
if getattr(self, "allow_unsafe_options", False):
|
|
return True
|
|
if hasattr(self, "config") and self.config and hasattr(self.config, "config") and isinstance(self.config.config, dict):
|
|
return bool(self.config.config.get("allow_unsafe_options", False))
|
|
return False
|
|
|
|
def _validate_base_fields(self, check_port=True):
|
|
"""Sanitize host, user, and port against argument injection and shell metacharacters."""
|
|
host_s = str(self.host) if self.host is not None else ""
|
|
if not host_s:
|
|
raise ValueError("Host cannot be empty")
|
|
if host_s.startswith("-"):
|
|
raise ValueError(f"Host cannot start with '-': '{self.host}'")
|
|
host_bare = (
|
|
host_s[1:-1]
|
|
if (host_s.startswith("[") and host_s.endswith("]"))
|
|
else host_s
|
|
)
|
|
if not host_bare or host_bare.startswith("-"):
|
|
raise ValueError(f"Invalid characters in host: '{self.host}'")
|
|
if _LEGACY_SSH_UNSAFE_RE.search(host_bare):
|
|
raise ValueError(f"Invalid characters in host: '{self.host}'")
|
|
|
|
user_s = str(self.user) if self.user is not None else ""
|
|
if user_s:
|
|
if user_s.startswith("-"):
|
|
raise ValueError(f"User cannot start with '-': '{self.user}'")
|
|
if _LEGACY_SSH_UNSAFE_RE.search(user_s):
|
|
raise ValueError(f"Invalid characters in user: '{self.user}'")
|
|
|
|
if check_port and self.port != "" and self.port is not None:
|
|
if isinstance(self.port, (bool, float)):
|
|
raise ValueError(f"Invalid TCP SSH port '{self.port}'")
|
|
port_s = str(self.port).strip()
|
|
if port_s and (
|
|
not port_s.isascii()
|
|
or not re.fullmatch(r"[0-9]+", port_s, re.ASCII)
|
|
or not (1 <= int(port_s) <= 65535)
|
|
):
|
|
raise ValueError(f"Invalid TCP SSH port '{self.port}'")
|
|
|
|
def _validate_options_and_safety(self):
|
|
"""Validate options string for unbalanced quotes and dangerous control characters."""
|
|
if not self.options:
|
|
return
|
|
opts_str = str(self.options).strip()
|
|
if not opts_str:
|
|
return
|
|
if not self._is_unsafe_allowed():
|
|
if any(c in opts_str for c in ("\n", "\r", "\x00")):
|
|
raise ValueError("Options cannot contain newlines or null bytes")
|
|
try:
|
|
shlex.split(opts_str)
|
|
except ValueError as e:
|
|
raise ValueError(f"Invalid options: unbalanced quotes ({e})") from e
|
|
|
|
def _validate_ssh_options(self):
|
|
"""Sanitize SSH options against local command execution directives."""
|
|
self._validate_options_and_safety()
|
|
if not self.options:
|
|
return
|
|
opts_str = str(self.options).strip()
|
|
if not opts_str:
|
|
return
|
|
if not self._is_unsafe_allowed():
|
|
dangerous_directives = ("ProxyCommand", "LocalCommand", "PermitLocalCommand", "KnownHostsCommand")
|
|
pattern = r'(?:' + '|'.join(dangerous_directives) + r')'
|
|
match = re.search(pattern, opts_str, re.IGNORECASE)
|
|
if match:
|
|
raise ValueError(f"Disallowed SSH directive in options: '{match.group(0)}'")
|
|
|
|
@staticmethod
|
|
def _validate_legacy_ssh_hop_fields(host, user, port):
|
|
"""Minimal injection guard for legacy SSH jumphost fields (preserves DOMAIN\\user, %scope, etc.)."""
|
|
host_s = str(host) if host is not None else ""
|
|
if host_s.startswith("-"):
|
|
raise ValueError("Host cannot start with '-'")
|
|
if not host_s or _LEGACY_SSH_UNSAFE_RE.search(host_s):
|
|
raise ValueError(f"Invalid characters in host: '{host}'")
|
|
host_bare = (
|
|
host_s[1:-1]
|
|
if (host_s.startswith("[") and host_s.endswith("]"))
|
|
else host_s
|
|
)
|
|
if not host_bare or host_bare.startswith("-"):
|
|
raise ValueError(f"Invalid characters in host: '{host}'")
|
|
|
|
user_s = str(user) if user is not None else ""
|
|
if user_s.startswith("-"):
|
|
raise ValueError("User cannot start with '-'")
|
|
if user_s and _LEGACY_SSH_UNSAFE_RE.search(user_s):
|
|
raise ValueError(f"Invalid characters in user: '{user}'")
|
|
|
|
port_out = ""
|
|
if port != "" and port is not None:
|
|
if isinstance(port, (bool, float)):
|
|
raise ValueError(f"Invalid TCP SSH port '{port}'")
|
|
port_s = str(port).strip()
|
|
if port_s:
|
|
if (
|
|
not port_s.isascii()
|
|
or not re.fullmatch(r"[0-9]+", port_s, re.ASCII)
|
|
or not (1 <= int(port_s) <= 65535)
|
|
):
|
|
raise ValueError(f"Invalid TCP SSH port '{port}'")
|
|
port_out = port_s
|
|
|
|
return host_bare, port_out
|
|
|
|
@staticmethod
|
|
def _validate_mosh_ssh_fields(
|
|
host,
|
|
user,
|
|
port,
|
|
tags,
|
|
*,
|
|
strip_non_tunnel_flags=False,
|
|
allow_reverse_tunnel_port=None,
|
|
):
|
|
"""Validate host, user, TCP SSH port, and mosh_ssh_options for Mosh SSH backend.
|
|
|
|
Accepts IPv4, hostnames, and bare or bracketed IPv6 literals (including
|
|
IPv4-mapped IPv6 like ::ffff:1.2.3.4, without scope-id '%').
|
|
Returns:
|
|
tuple[str, str, str]: (host_bare_str, validated_port_str, validated_ssh_opts_str)
|
|
"""
|
|
host_s = str(host) if host is not None else ""
|
|
if host_s.startswith("-"):
|
|
raise ValueError("Host cannot start with '-'")
|
|
if not host_s:
|
|
raise ValueError(f"Invalid characters in host: '{host}'")
|
|
|
|
if (host_s.startswith("[") and host_s.endswith("]")) or ":" in host_s:
|
|
inner = host_s[1:-1] if (host_s.startswith("[") and host_s.endswith("]")) else host_s
|
|
if "%" in inner or not re.fullmatch(r"[0-9A-Fa-f:.]+", inner, re.ASCII):
|
|
raise ValueError(f"Invalid characters in host: '{host}'")
|
|
try:
|
|
ipaddress.IPv6Address(inner)
|
|
except ValueError as exc:
|
|
raise ValueError(f"Invalid characters in host: '{host}'") from exc
|
|
host_bare = inner
|
|
else:
|
|
if not re.fullmatch(r"[A-Za-z0-9._-]+", host_s, re.ASCII):
|
|
raise ValueError(f"Invalid characters in host: '{host}'")
|
|
host_bare = host_s
|
|
|
|
user_s = str(user) if user is not None else ""
|
|
if user_s.startswith("-"):
|
|
raise ValueError("User cannot start with '-'")
|
|
if user_s and not re.fullmatch(
|
|
r"[A-Za-z0-9._-]+(?:@[A-Za-z0-9._-]+)?", user_s, re.ASCII
|
|
):
|
|
raise ValueError(f"Invalid characters in user: '{user}'")
|
|
|
|
port_out = ""
|
|
if port != "" and port is not None:
|
|
if isinstance(port, (bool, float)):
|
|
raise ValueError(f"Invalid TCP SSH port '{port}'")
|
|
port_s = str(port).strip()
|
|
if port_s:
|
|
if (
|
|
not port_s.isascii()
|
|
or not re.fullmatch(r"[0-9]+", port_s, re.ASCII)
|
|
or not (1 <= int(port_s) <= 65535)
|
|
):
|
|
raise ValueError(f"Invalid TCP SSH port '{port}'")
|
|
port_out = port_s
|
|
|
|
allowed_r_port = None
|
|
if allow_reverse_tunnel_port is not None:
|
|
r_port_s = str(allow_reverse_tunnel_port).strip()
|
|
if (
|
|
isinstance(allow_reverse_tunnel_port, (bool, float))
|
|
or not r_port_s.isascii()
|
|
or not re.fullmatch(r"[0-9]+", r_port_s, re.ASCII)
|
|
or not (1 <= int(r_port_s) <= 65535)
|
|
):
|
|
raise ValueError(
|
|
f"Invalid allow_reverse_tunnel_port '{allow_reverse_tunnel_port}'"
|
|
)
|
|
allowed_r_port = r_port_s
|
|
|
|
opts_out = ""
|
|
raw_opts = tags.get("mosh_ssh_options", "") if isinstance(tags, dict) else ""
|
|
if raw_opts:
|
|
ssh_opts = str(raw_opts).strip()
|
|
if "\x00" in ssh_opts or "\n" in ssh_opts or "\r" in ssh_opts:
|
|
raise ValueError("Disallowed control characters in mosh_ssh_options")
|
|
if ssh_opts:
|
|
try:
|
|
tokens = shlex.split(ssh_opts)
|
|
except ValueError as exc:
|
|
raise ValueError(
|
|
"Invalid mosh_ssh_options: unbalanced quotes"
|
|
) from exc
|
|
r_count = 0
|
|
exit_fwd_count = 0
|
|
out_tokens = []
|
|
idx = 0
|
|
# Normative branch evaluation order:
|
|
# 1. NOARG -> 2. ARG separate -> 3. ARG attached -> 4. -R -> 5. -o/-oXXX -> 6. other
|
|
while idx < len(tokens):
|
|
tok = tokens[idx]
|
|
if tok in _ALLOWED_SSH_NOARG_FLAGS:
|
|
if not (strip_non_tunnel_flags and tok in _NON_TUNNEL_SSH_FLAGS):
|
|
out_tokens.append(tok)
|
|
idx += 1
|
|
elif tok in _ALLOWED_SSH_ARG_FLAGS:
|
|
if idx + 1 >= len(tokens) or not tokens[idx + 1].strip():
|
|
raise ValueError(
|
|
f"Missing argument for ssh option '{tok}' in mosh_ssh_options"
|
|
)
|
|
arg_val = tokens[idx + 1]
|
|
val_re = _SAFE_SSH_PATH_RE if tok == "-i" else _SAFE_SSH_O_VALUE_RE
|
|
if arg_val.startswith("-") or not val_re.fullmatch(arg_val):
|
|
raise ValueError(
|
|
f"Invalid argument for ssh option '{tok}' in mosh_ssh_options"
|
|
)
|
|
out_tokens.extend([tok, arg_val])
|
|
idx += 2
|
|
elif tok[:2] in _ALLOWED_SSH_ARG_FLAGS and len(tok) > 2:
|
|
flag_prefix = tok[:2]
|
|
arg_val = tok[2:]
|
|
val_re = _SAFE_SSH_PATH_RE if flag_prefix == "-i" else _SAFE_SSH_O_VALUE_RE
|
|
if (
|
|
not arg_val.strip()
|
|
or arg_val.startswith("-")
|
|
or not val_re.fullmatch(arg_val)
|
|
):
|
|
raise ValueError(
|
|
f"Invalid argument for ssh option '{flag_prefix}' in mosh_ssh_options"
|
|
)
|
|
out_tokens.append(tok)
|
|
idx += 1
|
|
elif tok == "-R":
|
|
if (
|
|
allowed_r_port is None
|
|
or r_count >= 1
|
|
or idx + 1 >= len(tokens)
|
|
or tokens[idx + 1]
|
|
!= f"{allowed_r_port}:localhost:{allowed_r_port}"
|
|
):
|
|
raise ValueError(
|
|
f"Disallowed ssh option in mosh_ssh_options: '{tok}'"
|
|
)
|
|
r_count += 1
|
|
out_tokens.extend([tok, tokens[idx + 1]])
|
|
idx += 2
|
|
elif tok == "-o" or tok.startswith("-o"):
|
|
if tok == "-o":
|
|
if idx + 1 >= len(tokens) or not tokens[idx + 1].strip():
|
|
raise ValueError(
|
|
"Missing argument for ssh option '-o' in mosh_ssh_options"
|
|
)
|
|
opt_spec = tokens[idx + 1]
|
|
step = 2
|
|
else:
|
|
opt_spec = tok[2:]
|
|
step = 1
|
|
if re.search(r"(?:localcommand|proxycommand)", opt_spec, re.IGNORECASE):
|
|
raise ValueError(
|
|
"mosh_ssh_options cannot contain LocalCommand or ProxyCommand"
|
|
)
|
|
m = re.fullmatch(r"([A-Za-z0-9]+)=(.+)", opt_spec, re.ASCII)
|
|
if (
|
|
not m
|
|
or not m.group(2).strip()
|
|
or not _SAFE_SSH_O_VALUE_RE.fullmatch(m.group(2).strip())
|
|
):
|
|
raise ValueError(
|
|
f"Disallowed -o directive in mosh_ssh_options: '{opt_spec}'"
|
|
)
|
|
key_raw = m.group(1)
|
|
key_lower = key_raw.lower()
|
|
val_str = m.group(2).strip()
|
|
if key_lower == "exitonforwardfailure":
|
|
if (
|
|
allowed_r_port is None
|
|
or exit_fwd_count >= 1
|
|
or val_str.lower() != "yes"
|
|
):
|
|
raise ValueError(
|
|
f"Disallowed -o directive in mosh_ssh_options: '{opt_spec}'"
|
|
)
|
|
exit_fwd_count += 1
|
|
elif key_lower not in _ALLOWED_SSH_O_KEYS:
|
|
raise ValueError(
|
|
f"Disallowed -o directive in mosh_ssh_options: '{opt_spec}'"
|
|
)
|
|
if not (strip_non_tunnel_flags and key_lower == "forwardagent"):
|
|
if tok == "-o":
|
|
out_tokens.extend(["-o", f"{key_raw}={val_str}"])
|
|
else:
|
|
out_tokens.append(f"-o{key_raw}={val_str}")
|
|
idx += step
|
|
else:
|
|
if re.search(r"(?:localcommand|proxycommand)", tok, re.IGNORECASE):
|
|
raise ValueError(
|
|
"mosh_ssh_options cannot contain LocalCommand or ProxyCommand"
|
|
)
|
|
raise ValueError(
|
|
f"Disallowed ssh option in mosh_ssh_options: '{tok}'"
|
|
)
|
|
opts_out = shlex.join(out_tokens)
|
|
|
|
return host_bare, port_out, opts_out
|
|
|
|
@staticmethod
|
|
def _validate_and_prepare_ssh_options(
|
|
raw_opts,
|
|
*,
|
|
field_name: str = "options",
|
|
strip_non_tunnel_flags: bool = False,
|
|
allow_unsafe: bool = False,
|
|
connect_timeout: int | None = None,
|
|
) -> str:
|
|
"""Validate SSH options against _ALLOWED_SSH_O_KEYS and optionally inject ConnectTimeout/ServerAliveInterval."""
|
|
out_tokens: list[str] = []
|
|
if raw_opts:
|
|
ssh_opts = str(raw_opts).strip()
|
|
if _CONTROL_CHAR_RE.search(ssh_opts):
|
|
raise ValueError(f"Disallowed control characters in {field_name}")
|
|
if ssh_opts:
|
|
try:
|
|
tokens = shlex.split(ssh_opts)
|
|
except ValueError as exc:
|
|
raise ValueError(f"Invalid {field_name}: unbalanced quotes") from exc
|
|
if allow_unsafe:
|
|
out_tokens = list(tokens)
|
|
else:
|
|
idx = 0
|
|
while idx < len(tokens):
|
|
tok = tokens[idx]
|
|
if tok in _ALLOWED_SSH_NOARG_FLAGS:
|
|
if not (strip_non_tunnel_flags and tok in _NON_TUNNEL_SSH_FLAGS):
|
|
out_tokens.append(tok)
|
|
idx += 1
|
|
elif tok in _ALLOWED_SSH_ARG_FLAGS:
|
|
if idx + 1 >= len(tokens) or not tokens[idx + 1].strip():
|
|
raise ValueError(
|
|
f"Missing argument for ssh option '{tok}' in {field_name}"
|
|
)
|
|
arg_val = tokens[idx + 1]
|
|
val_re = _SAFE_SSH_PATH_RE if tok == "-i" else _SAFE_SSH_O_VALUE_RE
|
|
if arg_val.startswith("-") or not val_re.fullmatch(arg_val):
|
|
raise ValueError(
|
|
f"Invalid argument for ssh option '{tok}' in {field_name}"
|
|
)
|
|
out_tokens.extend([tok, arg_val])
|
|
idx += 2
|
|
elif tok[:2] in _ALLOWED_SSH_ARG_FLAGS and len(tok) > 2:
|
|
flag_prefix = tok[:2]
|
|
arg_val = tok[2:]
|
|
val_re = _SAFE_SSH_PATH_RE if flag_prefix == "-i" else _SAFE_SSH_O_VALUE_RE
|
|
if (
|
|
not arg_val.strip()
|
|
or arg_val.startswith("-")
|
|
or not val_re.fullmatch(arg_val)
|
|
):
|
|
raise ValueError(
|
|
f"Invalid argument for ssh option '{flag_prefix}' in {field_name}"
|
|
)
|
|
out_tokens.append(tok)
|
|
idx += 1
|
|
elif tok == "-o" or tok.startswith("-o"):
|
|
if tok == "-o":
|
|
if idx + 1 >= len(tokens) or not tokens[idx + 1].strip():
|
|
raise ValueError(
|
|
f"Missing argument for ssh option '-o' in {field_name}"
|
|
)
|
|
opt_spec = tokens[idx + 1]
|
|
step = 2
|
|
else:
|
|
opt_spec = tok[2:]
|
|
step = 1
|
|
if re.search(
|
|
r"(?:localcommand|proxycommand|permitlocalcommand|knownhostscommand)",
|
|
opt_spec,
|
|
re.IGNORECASE,
|
|
):
|
|
raise ValueError(
|
|
f"{field_name} cannot contain LocalCommand or ProxyCommand"
|
|
)
|
|
m = re.fullmatch(r"([A-Za-z0-9]+)=(.+)", opt_spec, re.ASCII)
|
|
if (
|
|
not m
|
|
or not m.group(2).strip()
|
|
or not _SAFE_SSH_O_VALUE_RE.fullmatch(m.group(2).strip())
|
|
):
|
|
raise ValueError(
|
|
f"Disallowed -o directive in {field_name}: '{opt_spec}'"
|
|
)
|
|
key_raw = m.group(1)
|
|
key_lower = key_raw.lower()
|
|
val_str = m.group(2).strip()
|
|
if key_lower not in _ALLOWED_SSH_O_KEYS:
|
|
raise ValueError(
|
|
f"Disallowed -o directive in {field_name}: '{opt_spec}'"
|
|
)
|
|
if not (strip_non_tunnel_flags and key_lower == "forwardagent"):
|
|
if tok == "-o":
|
|
out_tokens.extend(["-o", f"{key_raw}={val_str}"])
|
|
else:
|
|
out_tokens.append(f"-o{key_raw}={val_str}")
|
|
idx += step
|
|
else:
|
|
if re.search(
|
|
r"(?:localcommand|proxycommand|permitlocalcommand|knownhostscommand)",
|
|
tok,
|
|
re.IGNORECASE,
|
|
):
|
|
raise ValueError(
|
|
f"{field_name} cannot contain LocalCommand or ProxyCommand"
|
|
)
|
|
raise ValueError(
|
|
f"Disallowed ssh option in {field_name}: '{tok}'"
|
|
)
|
|
|
|
if connect_timeout is not None:
|
|
ct_val = validate_strict_timeout_int(connect_timeout, "connect_timeout")
|
|
existing_keys = _extract_ssh_o_keys(out_tokens)
|
|
if "connecttimeout" not in existing_keys and "connecttimeout" in _ALLOWED_SSH_O_KEYS:
|
|
out_tokens.extend(["-o", f"ConnectTimeout={ct_val}"])
|
|
if (
|
|
"serveraliveinterval" not in existing_keys
|
|
and "serveraliveinterval" in _ALLOWED_SSH_O_KEYS
|
|
):
|
|
out_tokens.extend(["-o", "ServerAliveInterval=15"])
|
|
|
|
return shlex.join(out_tokens) if out_tokens else ""
|
|
|
|
def _build_jumphost_chain(
|
|
self,
|
|
jumphost_name,
|
|
config,
|
|
visited=None,
|
|
depth=0,
|
|
target_host="%h",
|
|
target_port="%p",
|
|
*,
|
|
include_meta=False,
|
|
connect_timeout=None,
|
|
):
|
|
"""Recursively build ProxyCommand for chained jumphosts.
|
|
|
|
Returns:
|
|
tuple: (raw_proxy_command, passwords_list) when include_meta=False, or
|
|
(raw_proxy_command, passwords_list, hop_meta) when include_meta=True.
|
|
|
|
Raises:
|
|
ValueError: On circular references, invalid hop fields, or exceeding max depth (5).
|
|
"""
|
|
if depth >= 5:
|
|
raise ValueError("Jumphost chain exceeds maximum depth of 5 hops")
|
|
if visited is None:
|
|
visited = []
|
|
if jumphost_name in visited:
|
|
cycle = " -> ".join(visited + [jumphost_name])
|
|
raise ValueError(f"Circular jumphost reference detected: {cycle}")
|
|
visited = visited + [jumphost_name]
|
|
|
|
jh = config.getitem(jumphost_name)
|
|
jh = self._resolve_jumphost_data(jh, config)
|
|
|
|
# Unconditional control-character guard on all hop metadata fields
|
|
for fld_name in ("host", "user", "port"):
|
|
fld_val = jh.get(fld_name)
|
|
if fld_val not in (None, "") and _CONTROL_CHAR_RE.search(str(fld_val)):
|
|
raise ValueError(f"Control characters disallowed in jumphost {fld_name}")
|
|
jh_tags = jh.get("tags")
|
|
if isinstance(jh_tags, dict):
|
|
for tag_fld in ("profile", "region"):
|
|
tag_val = jh_tags.get(tag_fld)
|
|
if tag_val not in (None, "") and _CONTROL_CHAR_RE.search(str(tag_val)):
|
|
raise ValueError(f"Control characters disallowed in jumphost {tag_fld}")
|
|
|
|
if connect_timeout is not None:
|
|
connect_timeout = validate_strict_timeout_int(connect_timeout, "connect_timeout")
|
|
|
|
jh_proto = jh.get("protocol", "ssh")
|
|
if jh_proto == "mosh":
|
|
jh_host, jh_port, jh_opts = self._validate_mosh_ssh_fields(
|
|
jh.get("host", ""),
|
|
jh.get("user", ""),
|
|
jh.get("port", ""),
|
|
jh.get("tags", ""),
|
|
strip_non_tunnel_flags=True,
|
|
)
|
|
if connect_timeout is not None:
|
|
jh_opts = self._validate_and_prepare_ssh_options(
|
|
jh_opts,
|
|
field_name="mosh_ssh_options",
|
|
strip_non_tunnel_flags=True,
|
|
connect_timeout=connect_timeout,
|
|
)
|
|
elif jh_proto == "ssh":
|
|
jh_host, jh_port = self._validate_legacy_ssh_hop_fields(
|
|
jh.get("host", ""),
|
|
jh.get("user", ""),
|
|
jh.get("port", ""),
|
|
)
|
|
jh_opts = self._validate_and_prepare_ssh_options(
|
|
jh.get("options", ""),
|
|
field_name="jumphost options",
|
|
allow_unsafe=self._is_unsafe_allowed(),
|
|
connect_timeout=connect_timeout,
|
|
)
|
|
else:
|
|
jh_host = str(jh.get("host", "") or "")
|
|
jh_port = str(jh["port"]).strip() if jh.get("port") not in ("", None) else ""
|
|
jh_opts = jh.get("options", "")
|
|
|
|
passwords = []
|
|
inner_proxy_opt = ""
|
|
inner_meta: list[dict[str, str]] = []
|
|
|
|
# Recursively resolve inner jumphost
|
|
if jh.get("jumphost", "") != "":
|
|
if jh_proto not in ["ssh", "mosh"]:
|
|
raise ValueError(
|
|
f"Jumphost '{jumphost_name}' uses protocol '{jh_proto}' "
|
|
"which does not support chained jumphosts. "
|
|
"Only SSH or Mosh jumphosts can have their own jumphosts."
|
|
)
|
|
parent_port = jh_port if jh_port != "" else "22"
|
|
inner_res = self._build_jumphost_chain(
|
|
jh["jumphost"],
|
|
config,
|
|
visited,
|
|
depth + 1,
|
|
target_host=jh_host,
|
|
target_port=parent_port,
|
|
include_meta=True,
|
|
connect_timeout=connect_timeout,
|
|
)
|
|
if len(inner_res) == 3:
|
|
inner_raw_cmd, inner_passwords, inner_meta = inner_res
|
|
else:
|
|
inner_raw_cmd, inner_passwords = inner_res
|
|
inner_meta = []
|
|
passwords = inner_passwords
|
|
escaped = inner_raw_cmd.replace("\\", "\\\\").replace('"', '\\"')
|
|
inner_proxy_opt = f'-o ProxyCommand="{escaped}"'
|
|
|
|
# Collect this hop's passwords
|
|
if jh["password"] != [""]:
|
|
passwords = passwords + jh["password"]
|
|
|
|
hop_meta = [*inner_meta, {"name": jumphost_name, "protocol": jh_proto}]
|
|
|
|
t_port = target_port if target_port != "" else "22"
|
|
if jh_proto in ["ssh", "mosh", "ssm"]:
|
|
if target_host == "%h" and t_port == "%p":
|
|
w_target = "%h"
|
|
w_port = "%p"
|
|
else:
|
|
t_host_bare, t_port_norm = self._validate_legacy_ssh_hop_fields(
|
|
target_host, "", t_port
|
|
)
|
|
w_target = f"[{t_host_bare}]" if ":" in t_host_bare else t_host_bare
|
|
w_port = t_port_norm if t_port_norm != "" else "22"
|
|
else:
|
|
w_target = target_host
|
|
w_port = t_port
|
|
|
|
# Build raw command based on protocol
|
|
if jh_proto in ["ssh", "mosh"]:
|
|
cmd = f"ssh -W {w_target}:{w_port}"
|
|
if inner_proxy_opt:
|
|
cmd += f" {inner_proxy_opt}"
|
|
if jh_port != "":
|
|
cmd += f" -p {jh_port}"
|
|
if jh_opts != "":
|
|
cmd += f" {jh_opts}"
|
|
jh_user = str(jh.get("user", "") or "")
|
|
user_host = f"{jh_user}@{jh_host}" if jh_user else jh_host
|
|
cmd += f" {user_host}"
|
|
elif jh_proto == "ssm":
|
|
ssm_target = str(jh.get("host", "") or "")
|
|
if (
|
|
".." in ssm_target
|
|
or ssm_target.startswith("-")
|
|
or not _SSM_TARGET_RE.fullmatch(ssm_target)
|
|
):
|
|
raise ValueError(f"Invalid SSM jumphost target: '{ssm_target}'")
|
|
|
|
ssm_user = str(jh.get("user", "") or "")
|
|
if ssm_user and (ssm_user.startswith("-") or _LEGACY_SSH_UNSAFE_RE.search(ssm_user)):
|
|
raise ValueError(f"Invalid characters in SSM jumphost user: '{ssm_user}'")
|
|
|
|
ssm_parts = [
|
|
"aws",
|
|
"ssm",
|
|
"start-session",
|
|
"--target",
|
|
ssm_target,
|
|
"--document-name",
|
|
"AWS-StartSSHSession",
|
|
"--parameters",
|
|
"portNumber=22",
|
|
]
|
|
if isinstance(jh_tags, dict):
|
|
for tag_key in ("profile", "region"):
|
|
raw_val = jh_tags.get(tag_key)
|
|
if raw_val not in (None, ""):
|
|
tag_str = str(raw_val)
|
|
if (
|
|
".." in tag_str
|
|
or tag_str.startswith("-")
|
|
or not _SSM_TAG_RE.fullmatch(tag_str)
|
|
):
|
|
raise ValueError(f"Invalid SSM jumphost {tag_key}: '{tag_str}'")
|
|
ssm_parts.extend([f"--{tag_key}", tag_str])
|
|
|
|
ssm_opts_raw = str(jh.get("options", "") or "").strip()
|
|
if ssm_opts_raw:
|
|
if _CONTROL_CHAR_RE.search(ssm_opts_raw) or (
|
|
not self._is_unsafe_allowed() and _SSM_UNSAFE_OPTIONS_RE.search(ssm_opts_raw)
|
|
):
|
|
raise ValueError("Invalid characters in SSM jumphost options")
|
|
try:
|
|
ssm_opt_tokens = shlex.split(ssm_opts_raw)
|
|
except ValueError as exc:
|
|
raise ValueError("Invalid SSM jumphost options: unbalanced quotes") from exc
|
|
ssm_parts.extend(ssm_opt_tokens)
|
|
|
|
ssm_cmd = shlex.join(ssm_parts)
|
|
raw_ssh_opts = jh_tags.get("ssh_options", "") if isinstance(jh_tags, dict) else ""
|
|
ssh_opts = self._validate_and_prepare_ssh_options(
|
|
raw_ssh_opts,
|
|
field_name="ssm ssh_options",
|
|
strip_non_tunnel_flags=True,
|
|
allow_unsafe=self._is_unsafe_allowed(),
|
|
connect_timeout=connect_timeout,
|
|
)
|
|
bastion_user_part = f"{ssm_user}@{ssm_target}" if ssm_user else ssm_target
|
|
outer_parts = ["ssh"]
|
|
if ssh_opts:
|
|
outer_parts.extend(shlex.split(ssh_opts))
|
|
outer_parts.extend(
|
|
["-o", f"ProxyCommand={ssm_cmd}", "-W", f"{w_target}:{w_port}", bastion_user_part]
|
|
)
|
|
cmd = shlex.join(outer_parts)
|
|
elif jh_proto in ["kubectl", "docker"]:
|
|
if not jh_host or jh_host.startswith("-") or _LEGACY_SSH_UNSAFE_RE.search(jh_host):
|
|
raise ValueError(f"Invalid characters in {jh_proto} jumphost host: '{jh_host}'")
|
|
nc_cmd = "nc"
|
|
if isinstance(jh_tags, dict) and "nc_command" in jh_tags:
|
|
nc_cmd = str(jh_tags["nc_command"])
|
|
if _CONTROL_CHAR_RE.search(nc_cmd) or (
|
|
not self._is_unsafe_allowed() and _SSM_UNSAFE_OPTIONS_RE.search(nc_cmd)
|
|
):
|
|
raise ValueError(f"Invalid nc_command in {jh_proto} jumphost")
|
|
jh_opt_joined = ""
|
|
jh_opt_str = str(jh.get("options", "") or "").strip()
|
|
if jh_opt_str:
|
|
if _CONTROL_CHAR_RE.search(jh_opt_str):
|
|
raise ValueError(f"Control characters disallowed in {jh_proto} options")
|
|
if not self._is_unsafe_allowed() and _SSM_UNSAFE_OPTIONS_RE.search(jh_opt_str):
|
|
raise ValueError(f"Invalid characters in {jh_proto} options")
|
|
try:
|
|
jh_opt_tokens = shlex.split(jh_opt_str)
|
|
except ValueError as exc:
|
|
raise ValueError(f"Invalid {jh_proto} options: unbalanced quotes") from exc
|
|
jh_opt_joined = shlex.join(jh_opt_tokens)
|
|
if jh_proto == "kubectl":
|
|
cmd = "kubectl exec "
|
|
if jh_opt_joined:
|
|
cmd += f"{jh_opt_joined} "
|
|
cmd += f"{jh_host} -i -- {nc_cmd} {w_target} {w_port}"
|
|
else:
|
|
cmd = "docker "
|
|
if jh_opt_joined:
|
|
cmd += f"{jh_opt_joined} "
|
|
cmd += f"exec -i {jh_host} {nc_cmd} {w_target} {w_port}"
|
|
else:
|
|
if include_meta:
|
|
return "", passwords, inner_meta
|
|
return "", passwords
|
|
|
|
if include_meta:
|
|
return cmd, passwords, hop_meta
|
|
return cmd, passwords
|
|
|
|
@MethodHook
|
|
def _passtx(self, passwords, *, keyfile=None):
|
|
# decrypts passwords, used by other methdos.
|
|
dpass = []
|
|
if keyfile is None:
|
|
keyfile = self.key
|
|
if keyfile is not None:
|
|
from Crypto.Cipher import PKCS1_OAEP
|
|
from Crypto.PublicKey import RSA
|
|
with open(keyfile) as f:
|
|
key = RSA.import_key(f.read())
|
|
decryptor = PKCS1_OAEP.new(key)
|
|
for passwd in passwords:
|
|
if not re.match('^b[\"\'].+[\"\']$', passwd):
|
|
dpass.append(passwd)
|
|
else:
|
|
try:
|
|
decrypted = decryptor.decrypt(ast.literal_eval(passwd)).decode("utf-8")
|
|
dpass.append(decrypted)
|
|
except Exception: # noqa: BLE001
|
|
printer.error("Decryption failed: Missing or corrupted key.")
|
|
printer.info("Verify your RSA key and configuration settings.")
|
|
sys.exit(1)
|
|
return dpass
|
|
|
|
|
|
|
|
@MethodHook
|
|
def _logfile(self, logfile = None):
|
|
# translate logs variables and generate logs path.
|
|
if logfile is None:
|
|
logfile = self.logs
|
|
logfile = logfile.replace("${unique}", self.unique)
|
|
logfile = logfile.replace("${host}", self.host)
|
|
logfile = logfile.replace("${port}", self.port)
|
|
logfile = logfile.replace("${user}", self.user)
|
|
logfile = logfile.replace("${protocol}", self.protocol)
|
|
now = datetime.datetime.now()
|
|
dateconf = re.search(r'\$\{date \'(.*)\'}', logfile)
|
|
if dateconf:
|
|
logfile = re.sub(r'\$\{date (.*)}',now.strftime(dateconf.group(1)), logfile)
|
|
return logfile
|
|
|
|
@MethodHook
|
|
def _logclean(self, logfile, var = False):
|
|
"""Remove special ascii characters and process terminal cursor movements to clean logs."""
|
|
from .utils import log_cleaner
|
|
|
|
if not var:
|
|
try:
|
|
with open(logfile) as f:
|
|
t = f.read()
|
|
except Exception: # noqa: BLE001
|
|
return
|
|
else:
|
|
t = logfile
|
|
|
|
result = log_cleaner(t)
|
|
|
|
if not var:
|
|
try:
|
|
with open(logfile, "w") as f:
|
|
f.write(result)
|
|
except Exception as exc: # noqa: BLE001
|
|
_LOGGER.debug("Failed writing cleaned log %s: %s", logfile, exc)
|
|
return
|
|
else:
|
|
return result
|
|
|
|
@MethodHook
|
|
def _savelog(self):
|
|
'''Save the log buffer to the file at regular intervals if there are changes.'''
|
|
t = threading.current_thread()
|
|
prev_size = 0 # Store the previous size of the buffer
|
|
|
|
while getattr(t, "do_run", True): # Check if thread is signaled to stop
|
|
current_size = self.mylog.tell() # Current size of the buffer
|
|
|
|
# Only save if the buffer size has changed
|
|
if current_size != prev_size:
|
|
with open(self.logfile, "w") as f: # Use "w" to overwrite the file
|
|
f.write(self._logclean(self.mylog.getvalue().decode(), True))
|
|
prev_size = current_size # Update the previous size
|
|
sleep(5)
|
|
|
|
@MethodHook
|
|
def _filter(self, a):
|
|
#Set time for last input when using interact
|
|
self.lastinput = time()
|
|
return a
|
|
|
|
@MethodHook
|
|
def _keepalive(self):
|
|
#Send keepalive ctrl+e when idletime passed without new inputs on interact
|
|
self.lastinput = time()
|
|
while True:
|
|
if time() - self.lastinput >= self.idletime:
|
|
self.child.sendcontrol("e")
|
|
self.lastinput = time()
|
|
sleep(1)
|
|
|
|
|
|
def _setup_interact_environment(self, debug=False, logger=None, async_mode=False):
|
|
try:
|
|
size = re.search('columns=([0-9]+).*lines=([0-9]+)',str(os.get_terminal_size()))
|
|
self.child.setwinsize(int(size.group(2)),int(size.group(1)))
|
|
except OSError:
|
|
pass
|
|
if logger and self.protocol != "local":
|
|
port_str = f":{self.port}" if self.port and self.protocol not in ["ssm", "kubectl", "docker"] else ""
|
|
logger("success", f"Connected to {self.unique} at {self.host}{port_str} via: {self.protocol}")
|
|
|
|
# Always initialize self.mylog to capture terminal context for the AI Copilot
|
|
if not hasattr(self, 'mylog'):
|
|
self.mylog = io.BytesIO()
|
|
|
|
if not async_mode:
|
|
self.child.logfile_read = self.mylog
|
|
|
|
# Only start disk-logging tasks if logfile is configured
|
|
if hasattr(self, "logfile") and not async_mode:
|
|
# Start the _savelog thread (sync mode)
|
|
log_thread = threading.Thread(target=self._savelog)
|
|
log_thread.daemon = True
|
|
log_thread.start()
|
|
if getattr(self, "missingtext", False) and not async_mode:
|
|
print(self.child.after.decode(), end="")
|
|
if self.idletime > 0 and not async_mode:
|
|
x = threading.Thread(target=self._keepalive)
|
|
x.daemon = True
|
|
x.start()
|
|
if debug and hasattr(self, "mylog") and not async_mode:
|
|
print(self.mylog.getvalue().decode())
|
|
|
|
def _teardown_interact_environment(self):
|
|
if 'logfile' in dir(self) and hasattr(self, 'mylog'):
|
|
with open(self.logfile, "w") as f:
|
|
f.write(self._logclean(self.mylog.getvalue().decode(), True))
|
|
|
|
def _write_clean_logfile(self, raw_log: str) -> None:
|
|
"""Clean and synchronously write raw log output to self.logfile."""
|
|
cleaned_log = self._logclean(raw_log, True)
|
|
with open(self.logfile, "w") as f:
|
|
f.write(cleaned_log)
|
|
|
|
def _is_child_connpy_active(self, child_fd: int) -> bool:
|
|
if self.protocol != "local":
|
|
return False
|
|
try:
|
|
fg_pgid = os.tcgetpgrp(child_fd)
|
|
if fg_pgid <= 0:
|
|
return False
|
|
cmdline_path = f"/proc/{fg_pgid}/cmdline"
|
|
if os.path.exists(cmdline_path):
|
|
with open(cmdline_path, "rb") as f:
|
|
raw_args = f.read().split(b"\x00")
|
|
is_active = False
|
|
for raw_arg in raw_args:
|
|
arg_str = raw_arg.decode(errors="ignore")
|
|
if not arg_str:
|
|
continue
|
|
base_name = os.path.basename(arg_str)
|
|
if base_name in ["conn", "connpy", "connapp"] or "connpy" in arg_str or "connapp" in arg_str:
|
|
is_active = True
|
|
break
|
|
|
|
return is_active
|
|
except Exception as exc: # noqa: BLE001
|
|
_LOGGER.debug("Failed checking active connpy process: %s", exc)
|
|
return False
|
|
|
|
async def _async_interact_loop(self, local_stream, resize_callback, copilot_handler=None):
|
|
local_stream.setup(resize_callback=resize_callback)
|
|
self.current_local_stream = local_stream
|
|
try:
|
|
child_fd = self.child.child_fd
|
|
|
|
# 1. Flush ghost buffer (Clean UX)
|
|
ghost_buffer = b''
|
|
if getattr(self, 'missingtext', False):
|
|
# If we are missing the password, we MUST show the password prompt
|
|
ghost_buffer = (self.child.after or b'') + (self.child.buffer or b'')
|
|
else:
|
|
# We auto-logged in. Hide the messy password negotiation and just keep any pending live stream.
|
|
ghost_buffer = self.child.buffer or b''
|
|
|
|
# Fix user's pet peeve: Strip leading newlines to avoid the empty lines
|
|
# the router echoes after receiving the password or blank line.
|
|
if not getattr(self, 'missingtext', False):
|
|
ghost_buffer = ghost_buffer.lstrip(b'\r\n ')
|
|
|
|
if ghost_buffer:
|
|
# Add a single clean newline so it doesn't merge with the Connected message
|
|
await local_stream.write(b'\r\n' + ghost_buffer)
|
|
if hasattr(self, 'mylog'):
|
|
self.mylog.write(b'\n' + ghost_buffer)
|
|
|
|
self.child.buffer = b''
|
|
self.child.before = b''
|
|
|
|
# 2. Set child fd non-blocking
|
|
flags = fcntl.fcntl(child_fd, fcntl.F_GETFL)
|
|
fcntl.fcntl(child_fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
|
|
|
|
loop = asyncio.get_running_loop()
|
|
child_reader_queue = asyncio.Queue()
|
|
|
|
# Reset and track command byte positions for copilot context navigation
|
|
# Each entry is (byte_position, command_text_or_None)
|
|
self.cmd_byte_positions = [(self.mylog.tell() if hasattr(self, 'mylog') else 0, None)]
|
|
|
|
def _child_read_ready():
|
|
try:
|
|
# Increase buffer to 64KB for better high-speed handling
|
|
data = os.read(child_fd, 65536)
|
|
if data:
|
|
child_reader_queue.put_nowait(data)
|
|
else:
|
|
child_reader_queue.put_nowait(b'')
|
|
except BlockingIOError:
|
|
pass
|
|
except OSError:
|
|
child_reader_queue.put_nowait(b'')
|
|
|
|
loop.add_reader(child_fd, _child_read_ready)
|
|
self.lastinput = time()
|
|
|
|
async def ingress_task():
|
|
while True:
|
|
data = await local_stream.read()
|
|
if not data:
|
|
break
|
|
|
|
# Copilot interception
|
|
if copilot_handler and b'\x00' in data:
|
|
if self._is_child_connpy_active(child_fd):
|
|
try:
|
|
os.write(child_fd, data)
|
|
except OSError:
|
|
break
|
|
self.lastinput = time()
|
|
continue
|
|
|
|
# Build node info from available metadata and ensure values are strings (not bytes)
|
|
def to_str(val):
|
|
if isinstance(val, bytes):
|
|
return val.decode(errors='replace')
|
|
return str(val) if val is not None else "unknown"
|
|
|
|
node_info = {
|
|
"name": to_str(getattr(self, 'unique', 'unknown')),
|
|
"host": to_str(getattr(self, 'host', 'unknown'))
|
|
}
|
|
if isinstance(getattr(self, 'tags', None), dict):
|
|
node_info["os"] = to_str(self.tags.get("os", "unknown"))
|
|
node_info["prompt"] = to_str(self.tags.get("prompt", r'>$|#$|\$$|>.$|#.$|\$.$'))
|
|
|
|
# Invoke copilot (async callback handles UI)
|
|
await copilot_handler(self.mylog.getvalue(), node_info, local_stream, child_fd, self.cmd_byte_positions)
|
|
continue
|
|
|
|
# Remove any stray \x00 bytes and forward normally
|
|
clean_data = data.replace(b'\x00', b'')
|
|
if clean_data:
|
|
# Track command boundaries when user hits Enter or presses Ctrl+C
|
|
if hasattr(self, 'mylog') and (b'\r' in clean_data or b'\n' in clean_data or b'\x03' in clean_data):
|
|
pos = self.mylog.tell()
|
|
marker_cmd = "CANCELLED" if b'\x03' in clean_data else None
|
|
self.cmd_byte_positions.append((pos, marker_cmd))
|
|
if hasattr(self, 'current_local_stream') and self.current_local_stream is not None:
|
|
try:
|
|
await self.current_local_stream.write(f'\x1b]133;B;{pos}\x07'.encode())
|
|
except Exception as exc: # noqa: BLE001
|
|
_LOGGER.debug("Failed writing OSC 133 marker: %s", exc)
|
|
|
|
try:
|
|
os.write(child_fd, clean_data)
|
|
except OSError:
|
|
break
|
|
self.lastinput = time()
|
|
|
|
async def egress_task():
|
|
# Continue stripping newlines from the live stream until we hit real text
|
|
skip_newlines = not getattr(self, 'missingtext', False) and not ghost_buffer
|
|
while True:
|
|
data = await child_reader_queue.get()
|
|
if not data:
|
|
break
|
|
|
|
# Batching Optimization: Drain the queue to batch writes during high-volume bursts
|
|
# Helps the terminal parse ANSI faster and reduces syscalls.
|
|
chunks = [data]
|
|
while not child_reader_queue.empty():
|
|
try:
|
|
extra = child_reader_queue.get_nowait()
|
|
if not extra:
|
|
chunks.append(b'') # Re-put EOF later or handle it
|
|
break
|
|
chunks.append(extra)
|
|
except asyncio.QueueEmpty:
|
|
break
|
|
|
|
has_eof = chunks[-1] == b''
|
|
if has_eof:
|
|
chunks.pop()
|
|
|
|
if chunks:
|
|
combined_data = b''.join(chunks)
|
|
if skip_newlines:
|
|
stripped = combined_data.lstrip(b'\r\n')
|
|
if stripped:
|
|
skip_newlines = False
|
|
combined_data = stripped
|
|
else:
|
|
if has_eof:
|
|
break
|
|
continue
|
|
|
|
await local_stream.write(combined_data)
|
|
if hasattr(self, 'mylog'):
|
|
self.mylog.write(combined_data)
|
|
|
|
if has_eof:
|
|
break
|
|
|
|
async def keepalive_task():
|
|
while True:
|
|
await asyncio.sleep(1)
|
|
if time() - self.lastinput >= self.idletime:
|
|
try:
|
|
self.child.sendcontrol("e")
|
|
self.lastinput = time()
|
|
except Exception as exc: # noqa: BLE001
|
|
_LOGGER.debug("Keepalive sendcontrol failed: %s", exc)
|
|
|
|
async def savelog_task():
|
|
prev_size = 0
|
|
while True:
|
|
await asyncio.sleep(5)
|
|
current_size = self.mylog.tell()
|
|
if current_size != prev_size:
|
|
try:
|
|
# Move heavy log cleaning & file I/O to a thread to avoid freezing the interaction loop
|
|
raw_log = self.mylog.getvalue().decode(errors='replace')
|
|
await asyncio.to_thread(self._write_clean_logfile, raw_log)
|
|
prev_size = current_size
|
|
except Exception as exc: # noqa: BLE001
|
|
_LOGGER.debug("Periodic savelog sync failed: %s", exc)
|
|
|
|
async def pwd_tracker_task():
|
|
import socket
|
|
hostname = socket.gethostname()
|
|
last_cwd = None
|
|
while True:
|
|
await asyncio.sleep(0.3)
|
|
try:
|
|
child_pid = getattr(self.child, 'pid', None)
|
|
if child_pid:
|
|
new_cwd = os.readlink(f"/proc/{child_pid}/cwd")
|
|
if new_cwd != last_cwd:
|
|
last_cwd = new_cwd
|
|
try:
|
|
os.chdir(new_cwd)
|
|
except Exception as exc: # noqa: BLE001
|
|
_LOGGER.debug("Failed syncing cwd: %s", exc)
|
|
try:
|
|
await local_stream.write(f"\033]7;file://{hostname}{new_cwd}\007".encode())
|
|
except Exception as exc: # noqa: BLE001
|
|
_LOGGER.debug("Failed writing OSC 7 update: %s", exc)
|
|
if isinstance(self.tags, dict):
|
|
self.tags["cwd"] = new_cwd
|
|
except Exception as exc: # noqa: BLE001
|
|
_LOGGER.debug("pwd_tracker_task iteration failed: %s", exc)
|
|
|
|
try:
|
|
# We wait for either the user (ingress) or the child (egress) to finish
|
|
tasks = [
|
|
asyncio.create_task(ingress_task()),
|
|
asyncio.create_task(egress_task())
|
|
]
|
|
if self.protocol == "local":
|
|
tasks.append(asyncio.create_task(pwd_tracker_task()))
|
|
if self.idletime > 0:
|
|
tasks.append(asyncio.create_task(keepalive_task()))
|
|
if hasattr(self, 'logfile') and hasattr(self, 'mylog'):
|
|
tasks.append(asyncio.create_task(savelog_task()))
|
|
|
|
done, _pending = await asyncio.wait(
|
|
[tasks[0], tasks[1]],
|
|
return_when=asyncio.FIRST_COMPLETED
|
|
)
|
|
|
|
# If ingress finished first (user quit), give egress a small window to catch up
|
|
# on the remaining output in the queue.
|
|
if tasks[0] in done and tasks[1] not in done:
|
|
try:
|
|
await asyncio.wait_for(tasks[1], timeout=0.2)
|
|
except (asyncio.TimeoutError, TimeoutError, asyncio.CancelledError): # noqa: UP041
|
|
pass
|
|
|
|
for t in tasks:
|
|
if t not in done:
|
|
t.cancel()
|
|
|
|
# Final log sync on thread to avoid losing last lines
|
|
if hasattr(self, 'logfile') and hasattr(self, 'mylog'):
|
|
try:
|
|
raw_log = self.mylog.getvalue().decode(errors='replace')
|
|
await asyncio.to_thread(self._write_clean_logfile, raw_log)
|
|
except Exception as exc: # noqa: BLE001
|
|
_LOGGER.debug("Final savelog sync failed: %s", exc)
|
|
|
|
finally:
|
|
loop.remove_reader(child_fd)
|
|
try:
|
|
flags = fcntl.fcntl(child_fd, fcntl.F_GETFL)
|
|
fcntl.fcntl(child_fd, fcntl.F_SETFL, flags & ~os.O_NONBLOCK)
|
|
except Exception as exc: # noqa: BLE001
|
|
_LOGGER.debug("Failed restoring blocking mode on child_fd: %s", exc)
|
|
finally:
|
|
self.current_local_stream = None
|
|
local_stream.teardown()
|
|
|
|
@MethodHook
|
|
async def inject_commands(self, commands, child_fd, on_inject=None):
|
|
"""
|
|
Inject a list of commands into the node's PTY.
|
|
Handles screen_length_command, history tracking and delays.
|
|
"""
|
|
if not commands:
|
|
return
|
|
|
|
# 0. Clear line
|
|
os.write(child_fd, b'\x15')
|
|
await asyncio.sleep(0.1)
|
|
|
|
# 1. Prepare list (prepend screen_length if exists)
|
|
slc = self.tags.get("screen_length_command") if hasattr(self, 'tags') and isinstance(self.tags, dict) else None
|
|
|
|
to_send = list(commands)
|
|
if slc and slc not in to_send: # avoid duplicates if already there
|
|
to_send.insert(0, slc)
|
|
|
|
# 2. Inject one by one
|
|
for cmd in to_send:
|
|
# Register in node's official history (SKIP if it's the administrative screen length command)
|
|
if cmd != slc and hasattr(self, 'cmd_byte_positions') and self.cmd_byte_positions is not None:
|
|
log_pos = self.mylog.tell() if hasattr(self, 'mylog') else 0
|
|
self.cmd_byte_positions.append((log_pos, cmd))
|
|
if hasattr(self, 'current_local_stream') and self.current_local_stream is not None:
|
|
try:
|
|
await self.current_local_stream.write(f'\x1b]133;B;{log_pos}\x07'.encode())
|
|
except Exception as exc: # noqa: BLE001
|
|
_LOGGER.debug("Failed writing command boundary marker: %s", exc)
|
|
|
|
# Write physically to PTY
|
|
os.write(child_fd, (cmd + "\n").encode())
|
|
|
|
# Notify (e.g., for gRPC or logs) - SKIP for administrative SLC
|
|
if on_inject and cmd != slc:
|
|
if asyncio.iscoroutinefunction(on_inject):
|
|
await on_inject(cmd)
|
|
else:
|
|
on_inject(cmd)
|
|
|
|
# Delay to avoid overwhelming the router
|
|
await asyncio.sleep(0.8)
|
|
|
|
@MethodHook
|
|
def interact(self, debug=False, logger=None):
|
|
'''
|
|
Asynchronous interactive session using Smart Tunnel architecture.
|
|
Allows multiplexing I/O and handling SIGWINCH events locally without blocking.
|
|
'''
|
|
connect = self._connect(debug=debug, logger=logger)
|
|
if connect is True:
|
|
try:
|
|
self._setup_interact_environment(debug=debug, logger=logger, async_mode=True)
|
|
|
|
local_stream = LocalStream()
|
|
|
|
def resize_callback(rows, cols):
|
|
try:
|
|
self.child.setwinsize(rows, cols)
|
|
except Exception as exc: # noqa: BLE001
|
|
_LOGGER.debug("Failed resizing child window: %s", exc)
|
|
|
|
# Build local copilot handler
|
|
copilot_handler = self._build_local_copilot_handler()
|
|
|
|
asyncio.run(self._async_interact_loop(local_stream, resize_callback, copilot_handler=copilot_handler))
|
|
finally:
|
|
self._teardown_interact_environment()
|
|
else:
|
|
if logger:
|
|
logger("error", str(connect))
|
|
else:
|
|
printer.error(f"Connection failed: {connect!s}")
|
|
sys.exit(1)
|
|
|
|
def _build_local_copilot_handler(self):
|
|
"""Build copilot handler for local CLI sessions using rich for rendering."""
|
|
config = getattr(self, 'config', None) if hasattr(self, 'config') else None
|
|
return self._copilot_handler(config)
|
|
|
|
def _copilot_handler(self, config):
|
|
"""Unified copilot handler for local session."""
|
|
|
|
async def handler(buffer, node_info, stream, child_fd, cmd_byte_positions=None):
|
|
try:
|
|
from .cli.terminal_ui import CopilotInterface
|
|
from .services.ai_service import AIService
|
|
|
|
interface = CopilotInterface(
|
|
config,
|
|
history=getattr(stream, 'copilot_history', None),
|
|
session_state=getattr(stream, 'copilot_state', None)
|
|
)
|
|
# Save history back to stream for persistence in current session
|
|
stream.copilot_history = interface.history
|
|
stream.copilot_state = interface.session_state
|
|
interface.session_state['banner_shown'] = False
|
|
|
|
ai_service = AIService(config)
|
|
|
|
async def on_ai_call(active_buffer, question, chunk_callback, merged_node_info):
|
|
return await ai_service.aask_copilot(
|
|
active_buffer,
|
|
question,
|
|
node_info=merged_node_info,
|
|
chunk_callback=chunk_callback
|
|
)
|
|
# Get raw bytes from BytesIO
|
|
raw_bytes = self.mylog.getvalue()
|
|
|
|
# Stop terminal reading so prompt_toolkit (in run_session)
|
|
# has exclusive control of stdin without LocalStream interference.
|
|
if hasattr(stream, 'stop_reading'):
|
|
stream.stop_reading()
|
|
elif hasattr(stream, '_loop') and hasattr(stream, 'stdin_fd'):
|
|
# Fallback if the method is missing (in LocalStream)
|
|
stream._loop.remove_reader(stream.stdin_fd)
|
|
|
|
try:
|
|
with copilot_terminal_mode():
|
|
while True:
|
|
action, commands, custom_cmd = await interface.run_session(
|
|
raw_bytes=raw_bytes,
|
|
cmd_byte_positions=self.cmd_byte_positions,
|
|
node_info=node_info,
|
|
on_ai_call=on_ai_call
|
|
)
|
|
if action in ("send_all", "custom"):
|
|
cmds_to_send = commands if action == "send_all" else custom_cmd
|
|
await self.inject_commands(cmds_to_send, child_fd)
|
|
|
|
# Dynamic Wait for Device Prompt Settle
|
|
prompt_pattern = node_info.get("prompt", r'>$|#$|\$$|>.$|#.$|\$.$')
|
|
start_t = time()
|
|
last_len = 0
|
|
quiet_count = 0
|
|
while time() - start_t < 60:
|
|
await asyncio.sleep(0.15)
|
|
cur_bytes = self.mylog.getvalue()
|
|
if len(cur_bytes) == last_len:
|
|
quiet_count += 1
|
|
else:
|
|
quiet_count = 0
|
|
last_len = len(cur_bytes)
|
|
|
|
clean_txt = self._logclean(cur_bytes.decode(errors='replace'), True).strip()
|
|
lines = [line.strip() for line in clean_txt.split('\n') if line.strip()]
|
|
last_line = lines[-1] if lines else ""
|
|
if quiet_count >= 2 and re.search(prompt_pattern, last_line):
|
|
break
|
|
|
|
raw_bytes = self.mylog.getvalue()
|
|
continue
|
|
if action == "continue":
|
|
continue
|
|
break
|
|
finally:
|
|
print("\033[2m Returning to session...\033[0m", flush=True)
|
|
# Restart terminal reading to return to interactive SSH/Telnet mode
|
|
if hasattr(stream, 'start_reading'):
|
|
stream.start_reading()
|
|
elif hasattr(stream, '_loop') and hasattr(stream, 'stdin_fd'):
|
|
stream._loop.add_reader(stream.stdin_fd, stream._read_ready)
|
|
|
|
os.write(child_fd, b'\x15\r')
|
|
except Exception as e: # noqa: BLE001
|
|
import traceback
|
|
print(f"\n[ERROR in Copilot Handler] {e}", flush=True)
|
|
traceback.print_exc()
|
|
os.write(child_fd, b'\x15\r')
|
|
|
|
return handler
|
|
|
|
@MethodHook
|
|
def run(
|
|
self,
|
|
commands,
|
|
vars=None,
|
|
*,
|
|
folder='',
|
|
prompt=r'>$|#$|\$$|>.$|#.$|\$.$',
|
|
stdout=False,
|
|
timeout=None,
|
|
logger=None,
|
|
connect_timeout=None,
|
|
auth_timeout=None,
|
|
_default_cmd_timeout=10,
|
|
unattended=True,
|
|
attempt_tracker=None,
|
|
):
|
|
'''
|
|
Run a command or list of commands on the node and return the output.
|
|
|
|
|
|
### Parameters:
|
|
|
|
- commands (str/list): Commands to run on the node. Should be
|
|
str or a list of str. You can use variables
|
|
as {varname} and defining them in optional
|
|
parameter vars.
|
|
|
|
### Optional Parameters:
|
|
|
|
- vars (dict): Dictionary containing the definition of variables
|
|
used in commands parameter.
|
|
Keys: Variable names.
|
|
Values: strings.
|
|
|
|
### Optional Named Parameters:
|
|
|
|
- folder (str): Path where output log should be stored, leave
|
|
empty to disable logging.
|
|
|
|
- prompt (str): Prompt to be expected after a command is finished
|
|
running. Usually linux uses ">" or EOF while
|
|
routers use ">" or "#". The default value should
|
|
work for most nodes. Change it if your connection
|
|
need some special symbol.
|
|
|
|
- stdout (bool):Set True to send the command output to stdout.
|
|
default False.
|
|
|
|
- timeout (int):Time in seconds for expect to wait for prompt/EOF.
|
|
default 10.
|
|
|
|
### Returns:
|
|
|
|
str: Output of the commands you ran on the node.
|
|
|
|
'''
|
|
force_ssh = self.protocol == "mosh"
|
|
effective_protocol = "ssh" if force_ssh else self.protocol
|
|
try:
|
|
_, _, eff_cmd, _, _ = resolve_timeouts(
|
|
effective_protocol,
|
|
self.tags,
|
|
getattr(self, "_hop_chain_meta", None) or [],
|
|
timeout=timeout,
|
|
connect_timeout=connect_timeout,
|
|
auth_timeout=auth_timeout,
|
|
_default_cmd_timeout=_default_cmd_timeout,
|
|
)
|
|
except ValueError as e:
|
|
err = f"Connection failed code: 12\nInvalid timeout configuration: {e}"
|
|
self.output = err
|
|
self.status = 1
|
|
if logger:
|
|
logger("error", f"Connection failed: {err}")
|
|
return err
|
|
|
|
connect = self._connect(
|
|
timeout=timeout,
|
|
logger=logger,
|
|
force_ssh=force_ssh,
|
|
connect_timeout=connect_timeout,
|
|
auth_timeout=auth_timeout,
|
|
_default_cmd_timeout=_default_cmd_timeout,
|
|
unattended=unattended,
|
|
attempt_tracker=attempt_tracker,
|
|
)
|
|
now = datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S')
|
|
if connect is True:
|
|
if logger:
|
|
port_str = f":{self.port}" if self.port and effective_protocol not in ["ssm", "kubectl", "docker"] else ""
|
|
logger("success", f"Connected to {self.unique} at {self.host}{port_str} via: {effective_protocol}")
|
|
|
|
if isinstance(self.tags, dict) and "prompt" in self.tags:
|
|
prompt = self.tags["prompt"]
|
|
prompt_pat = re.compile(prompt.encode("utf-8"), re.DOTALL)
|
|
expects = [prompt_pat, pexpect.EOF, pexpect.TIMEOUT]
|
|
output = ''
|
|
result = 0
|
|
if not isinstance(commands, list):
|
|
commands = [commands]
|
|
if isinstance(self.tags, dict) and "screen_length_command" in self.tags:
|
|
commands.insert(0, self.tags["screen_length_command"])
|
|
self.mylog = io.BytesIO()
|
|
self.child.logfile_read = self.mylog
|
|
for c in commands:
|
|
if vars is not None:
|
|
try:
|
|
c = c.format(**vars)
|
|
except KeyError as e:
|
|
self.output = f"Error: Variable {e} not defined in task or inventory"
|
|
self.status = 1
|
|
return self.output
|
|
result = self.child.expect(expects, timeout=eff_cmd)
|
|
# Only set terminal size on devices without a
|
|
# screen_length_command (e.g. Linux/bash servers).
|
|
# Routers already disable pagination via that command.
|
|
# After setwinsize, consume any SIGWINCH re-render
|
|
# prompt (~40ms on bash) with a short timeout.
|
|
if c == commands[0] and (not isinstance(self.tags, dict) or "screen_length_command" not in self.tags):
|
|
try:
|
|
self.child.setwinsize(65535, 65535)
|
|
except Exception: # noqa: BLE001
|
|
try:
|
|
self.child.setwinsize(10000, 10000)
|
|
except Exception as exc: # noqa: BLE001
|
|
_LOGGER.debug("Fallback setwinsize failed in run: %s", exc)
|
|
self.child.expect(expects, timeout=1)
|
|
self.child.sendline(c)
|
|
if result == 2:
|
|
break
|
|
if result != 2:
|
|
result = self.child.expect(expects, timeout=eff_cmd)
|
|
self.child.close()
|
|
output = self._logclean(self.mylog.getvalue().decode("utf-8", errors="replace"), True)
|
|
if logger:
|
|
logger("output", output)
|
|
if folder != '':
|
|
with open(folder + "/" + self.unique + "_" + now + ".txt", "w") as f:
|
|
f.write(output)
|
|
self.output = output
|
|
if result == 2:
|
|
self.status = 2
|
|
else:
|
|
self.status = 0
|
|
return output
|
|
else:
|
|
self.output = connect
|
|
self.status = 1
|
|
if logger:
|
|
logger("error", f"Connection failed: {connect}")
|
|
if folder != '':
|
|
with open(folder + "/" + self.unique + "_" + now + ".txt", "w") as f:
|
|
f.write(connect)
|
|
return connect
|
|
|
|
@MethodHook
|
|
def test(
|
|
self,
|
|
commands,
|
|
expected,
|
|
vars=None,
|
|
*,
|
|
folder='',
|
|
prompt=r'>$|#$|\$$|>.$|#.$|\$.$',
|
|
timeout=None,
|
|
logger=None,
|
|
connect_timeout=None,
|
|
auth_timeout=None,
|
|
_default_cmd_timeout=10,
|
|
unattended=True,
|
|
attempt_tracker=None,
|
|
):
|
|
'''
|
|
Run a command or list of commands on the node, then check if expected value appears on the output after the last command.
|
|
|
|
|
|
### Parameters:
|
|
|
|
- commands (str/list): Commands to run on the node. Should be
|
|
str or a list of str. You can use variables
|
|
as {varname} and defining them in optional
|
|
parameter vars.
|
|
|
|
- expected (str) : Expected text to appear after running
|
|
all the commands on the node.You can use
|
|
variables as {varname} and defining them
|
|
in optional parameter vars.
|
|
|
|
### Optional Parameters:
|
|
|
|
- vars (dict): Dictionary containing the definition of variables
|
|
used in commands and expected parameters.
|
|
Keys: Variable names.
|
|
Values: strings.
|
|
|
|
### Optional Named Parameters:
|
|
|
|
- folder (str): Path where output log should be stored, leave
|
|
empty to not store logs.
|
|
|
|
- prompt (str): Prompt to be expected after a command is finished
|
|
running. Usually linux uses ">" or EOF while
|
|
routers use ">" or "#". The default value should
|
|
work for most nodes. Change it if your connection
|
|
need some special symbol.
|
|
|
|
- timeout (int):Time in seconds for expect to wait for prompt/EOF.
|
|
default 10.
|
|
|
|
### Returns:
|
|
bool: true if expected value is found after running the commands
|
|
false if prompt is found before.
|
|
|
|
'''
|
|
now = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
|
force_ssh = self.protocol == "mosh"
|
|
effective_protocol = "ssh" if force_ssh else self.protocol
|
|
try:
|
|
_, _, eff_cmd, _, _ = resolve_timeouts(
|
|
effective_protocol,
|
|
self.tags,
|
|
getattr(self, "_hop_chain_meta", None) or [],
|
|
timeout=timeout,
|
|
connect_timeout=connect_timeout,
|
|
auth_timeout=auth_timeout,
|
|
_default_cmd_timeout=_default_cmd_timeout,
|
|
)
|
|
except ValueError as e:
|
|
err = f"Connection failed code: 12\nInvalid timeout configuration: {e}"
|
|
self.result = None
|
|
self.output = err
|
|
self.status = 1
|
|
if logger:
|
|
logger("error", f"Connection failed: {err}")
|
|
return err
|
|
|
|
connect = self._connect(
|
|
timeout=timeout,
|
|
logger=logger,
|
|
force_ssh=force_ssh,
|
|
connect_timeout=connect_timeout,
|
|
auth_timeout=auth_timeout,
|
|
_default_cmd_timeout=_default_cmd_timeout,
|
|
unattended=unattended,
|
|
attempt_tracker=attempt_tracker,
|
|
)
|
|
if connect is True:
|
|
if logger:
|
|
port_str = f":{self.port}" if self.port and effective_protocol not in ["ssm", "kubectl", "docker"] else ""
|
|
logger("success", f"Connected to {self.unique} at {self.host}{port_str} via: {effective_protocol}")
|
|
|
|
if isinstance(self.tags, dict) and "prompt" in self.tags:
|
|
prompt = self.tags["prompt"]
|
|
prompt_pat = re.compile(prompt.encode("utf-8"), re.DOTALL)
|
|
expects = [prompt_pat, pexpect.EOF, pexpect.TIMEOUT]
|
|
output = ''
|
|
result = 0
|
|
if not isinstance(commands, list):
|
|
commands = [commands]
|
|
if not isinstance(expected, list):
|
|
expected = [expected]
|
|
if isinstance(self.tags, dict) and "screen_length_command" in self.tags:
|
|
commands.insert(0, self.tags["screen_length_command"])
|
|
self.mylog = io.BytesIO()
|
|
self.child.logfile_read = self.mylog
|
|
for c in commands:
|
|
if vars is not None:
|
|
try:
|
|
c = c.format(**vars)
|
|
except KeyError as e:
|
|
self.output = f"Error: Variable {e} not defined in task or inventory"
|
|
self.status = 1
|
|
return self.output
|
|
result = self.child.expect(expects, timeout=eff_cmd)
|
|
if c == commands[0] and (not isinstance(self.tags, dict) or "screen_length_command" not in self.tags):
|
|
try:
|
|
self.child.setwinsize(65535, 65535)
|
|
except Exception: # noqa: BLE001
|
|
try:
|
|
self.child.setwinsize(10000, 10000)
|
|
except Exception as exc: # noqa: BLE001
|
|
_LOGGER.debug("Fallback setwinsize failed in test: %s", exc)
|
|
self.child.expect(expects, timeout=1)
|
|
self.child.sendline(c)
|
|
if result == 2:
|
|
break
|
|
if result != 2:
|
|
result = self.child.expect(expects, timeout=eff_cmd)
|
|
self.child.close()
|
|
output = self._logclean(self.mylog.getvalue().decode("utf-8", errors="replace"), True)
|
|
if logger:
|
|
logger("output", output)
|
|
if folder != '':
|
|
with open(folder + "/" + self.unique + "_" + now + ".txt", "w") as f:
|
|
f.write(output)
|
|
self.output = output
|
|
if result in [0, 1]:
|
|
# lastcommand = commands[-1]
|
|
# if vars is not None:
|
|
# lastcommand = lastcommand.format(**vars)
|
|
# last_command_index = output.rfind(lastcommand)
|
|
# cleaned_output = output[last_command_index + len(lastcommand):].strip()
|
|
self.result = {}
|
|
for e in expected:
|
|
if vars is not None:
|
|
e = e.format(**vars)
|
|
updatedprompt = re.sub(r'(?<!\\)\$', '', prompt)
|
|
cleaned_output = output
|
|
try:
|
|
newpattern = f".*({updatedprompt}).*{e}.*"
|
|
cleaned_output = re.sub(newpattern, '', cleaned_output)
|
|
except re.error:
|
|
try:
|
|
escaped_e = re.escape(e)
|
|
newpattern = f".*({updatedprompt}).*{escaped_e}.*"
|
|
cleaned_output = re.sub(newpattern, '', cleaned_output)
|
|
except re.error:
|
|
pass
|
|
|
|
if e in cleaned_output:
|
|
self.result[e] = True
|
|
else:
|
|
try:
|
|
if re.search(e, cleaned_output):
|
|
self.result[e] = True
|
|
else:
|
|
self.result[e] = False
|
|
except re.error:
|
|
self.result[e] = False
|
|
self.status = 0
|
|
return self.result
|
|
if result == 2:
|
|
self.result = None
|
|
self.status = 2
|
|
return output
|
|
else:
|
|
self.result = None
|
|
self.output = connect
|
|
self.status = 1
|
|
return connect
|
|
|
|
@staticmethod
|
|
def _inject_ssh_timeouts(raw_opts: str, connect_timeout: int | None) -> str:
|
|
"""Append -o ConnectTimeout and -o ServerAliveInterval=15 unless already present."""
|
|
opts_str = str(raw_opts).strip() if raw_opts else ""
|
|
if connect_timeout is None:
|
|
return opts_str
|
|
ct_val = validate_strict_timeout_int(connect_timeout, "connect_timeout")
|
|
tokens = shlex.split(opts_str) if opts_str else []
|
|
existing_keys = _extract_ssh_o_keys(tokens)
|
|
if "connecttimeout" not in existing_keys and "connecttimeout" in _ALLOWED_SSH_O_KEYS:
|
|
tokens.extend(["-o", f"ConnectTimeout={ct_val}"])
|
|
if (
|
|
"serveraliveinterval" not in existing_keys
|
|
and "serveraliveinterval" in _ALLOWED_SSH_O_KEYS
|
|
):
|
|
tokens.extend(["-o", "ServerAliveInterval=15"])
|
|
return shlex.join(tokens)
|
|
|
|
def _effective_jumphost_opt(self, connect_timeout: int | None = None) -> str:
|
|
"""Return ProxyCommand option, rebuilding hop chain with connect_timeout when set."""
|
|
if (
|
|
connect_timeout is not None
|
|
and getattr(self, "_jumphost_name", "")
|
|
and getattr(self, "config", "") != ""
|
|
):
|
|
chain_res = self._build_jumphost_chain(
|
|
self._jumphost_name,
|
|
self.config,
|
|
include_meta=True,
|
|
connect_timeout=connect_timeout,
|
|
)
|
|
raw_cmd = chain_res[0]
|
|
if raw_cmd:
|
|
escaped = raw_cmd.replace("\\", "\\\\").replace('"', '\\"')
|
|
return f'-o ProxyCommand="{escaped}"'
|
|
return ""
|
|
return self.jumphost
|
|
|
|
@MethodHook
|
|
def _generate_ssh_sftp_cmd(
|
|
self,
|
|
force_ssh=False,
|
|
allow_reverse_tunnel_port=None,
|
|
*,
|
|
connect_timeout=None,
|
|
user_override=None,
|
|
):
|
|
self._validate_base_fields(check_port=True)
|
|
is_from_mosh = (
|
|
self.protocol == "mosh"
|
|
or getattr(self, "_orig_protocol", "") == "mosh"
|
|
)
|
|
if not is_from_mosh:
|
|
self._validate_ssh_options()
|
|
proto = "ssh" if force_ssh else self.protocol
|
|
eff_user = self.user if user_override is None else user_override
|
|
if eff_user:
|
|
u_s = str(eff_user)
|
|
if u_s.startswith("-") or _LEGACY_SSH_UNSAFE_RE.search(u_s):
|
|
raise ValueError(f"Invalid characters in user: '{u_s}'")
|
|
cmd = proto
|
|
if is_from_mosh:
|
|
host_bare, port_s, raw_opts = self._validate_mosh_ssh_fields(
|
|
self.host,
|
|
eff_user,
|
|
self.port,
|
|
self.tags,
|
|
allow_reverse_tunnel_port=allow_reverse_tunnel_port,
|
|
)
|
|
if port_s:
|
|
cmd += (" -p " if proto == "ssh" else " -P ") + port_s
|
|
if raw_opts and proto == "sftp":
|
|
sftp_tokens = []
|
|
toks = shlex.split(raw_opts)
|
|
i = 0
|
|
while i < len(toks):
|
|
t = toks[i]
|
|
if t in ("-X", "-x", "-Y", "-t", "-A", "-a"):
|
|
i += 1
|
|
elif t == "-m":
|
|
i += 2
|
|
elif t.startswith("-m") and len(t) > 2:
|
|
i += 1
|
|
else:
|
|
sftp_tokens.append(t)
|
|
i += 1
|
|
opts = shlex.join(sftp_tokens)
|
|
else:
|
|
opts = str(raw_opts).strip() if raw_opts else ""
|
|
opts = self._inject_ssh_timeouts(opts, connect_timeout)
|
|
if opts:
|
|
cmd += " " + opts
|
|
target_h = f"[{host_bare}]" if (proto == "sftp" and ":" in host_bare) else host_bare
|
|
else:
|
|
if self.port != "" and self.port is not None:
|
|
if proto == "ssh":
|
|
cmd += " -p " + str(self.port)
|
|
elif proto == "sftp":
|
|
cmd += " -P " + str(self.port)
|
|
raw_opts = self.options
|
|
opts = ""
|
|
if raw_opts:
|
|
opts = str(raw_opts).strip()
|
|
if proto == "sftp":
|
|
opts = re.sub(r'(?<!\S)-[XxtTAaNf]\b', '', opts).strip()
|
|
opts = self._inject_ssh_timeouts(opts, connect_timeout)
|
|
if opts:
|
|
cmd += " " + opts
|
|
target_h = self.host
|
|
|
|
jh_opt = self._effective_jumphost_opt(connect_timeout=connect_timeout)
|
|
if jh_opt:
|
|
cmd += " " + jh_opt
|
|
user_host = f"{eff_user}@{target_h}" if eff_user else target_h
|
|
cmd += f" {user_host}"
|
|
return cmd
|
|
|
|
@MethodHook
|
|
def _generate_mosh_cmd(self, *, connect_timeout=None, user_override=None):
|
|
eff_user = self.user if user_override is None else user_override
|
|
if str(self.host).startswith("-"):
|
|
raise ValueError("Host cannot start with '-'")
|
|
if eff_user and str(eff_user).startswith("-"):
|
|
raise ValueError("User cannot start with '-'")
|
|
if not self.host or re.search(r'[\s;|&$`><"\'\\]', str(self.host)):
|
|
raise ValueError(f"Invalid characters in host: '{self.host}'")
|
|
if eff_user and re.search(r'[\s;|&$`><"\'\\]', str(eff_user)):
|
|
raise ValueError(f"Invalid characters in user: '{eff_user}'")
|
|
|
|
cmd_parts = ["mosh"]
|
|
if isinstance(self.tags, dict):
|
|
if self.tags.get("mosh_server"):
|
|
server_bin = str(self.tags["mosh_server"]).strip()
|
|
if server_bin:
|
|
cmd_parts.append(f"--server={shlex.quote(server_bin)}")
|
|
if self.tags.get("mosh_udp_port"):
|
|
udp_port = str(self.tags["mosh_udp_port"]).strip()
|
|
if udp_port:
|
|
if not re.match(r"^\d+(:\d+)?$", udp_port):
|
|
raise ValueError(
|
|
f"Invalid mosh_udp_port '{udp_port}': must be PORT or PORT1:PORT2"
|
|
)
|
|
cmd_parts.append(f"-p {shlex.quote(udp_port)}")
|
|
|
|
ssh_parts = ["ssh"]
|
|
if self.port != "" and self.port is not None:
|
|
port_s = str(self.port).strip()
|
|
if port_s:
|
|
if not re.fullmatch(r"^\d+$", port_s) or not (1 <= int(port_s) <= 65535):
|
|
raise ValueError(f"Invalid TCP SSH port '{self.port}'")
|
|
ssh_parts.append(f"-p {port_s}")
|
|
ssh_opts = ""
|
|
if isinstance(self.tags, dict) and self.tags.get("mosh_ssh_options"):
|
|
ssh_opts = str(self.tags["mosh_ssh_options"]).strip()
|
|
if ssh_opts:
|
|
if not self._is_unsafe_allowed() and re.search(
|
|
r'(?:LocalCommand|ProxyCommand|PermitLocalCommand|KnownHostsCommand)',
|
|
ssh_opts,
|
|
re.IGNORECASE,
|
|
):
|
|
raise ValueError("mosh_ssh_options cannot contain LocalCommand or ProxyCommand")
|
|
try:
|
|
shlex.split(ssh_opts)
|
|
except ValueError as exc:
|
|
raise ValueError("Invalid mosh_ssh_options: unbalanced quotes") from exc
|
|
if connect_timeout is not None:
|
|
ssh_opts = self._inject_ssh_timeouts(ssh_opts, connect_timeout)
|
|
if ssh_opts:
|
|
ssh_parts.append(ssh_opts)
|
|
jh_opt = self._effective_jumphost_opt(connect_timeout=connect_timeout)
|
|
if jh_opt:
|
|
ssh_parts.append(jh_opt)
|
|
|
|
if len(ssh_parts) > 1:
|
|
raw_ssh = " ".join(ssh_parts)
|
|
cmd_parts.append(f"--ssh={shlex.quote(raw_ssh)}")
|
|
|
|
if self.options:
|
|
opts = str(self.options).strip()
|
|
if opts:
|
|
if re.search(r'--(?:ssh|server|client)(?:=|\s|$)', opts):
|
|
raise ValueError("Options cannot override --ssh, --server or --client for mosh")
|
|
if not self._is_unsafe_allowed() and re.search(
|
|
r'(?:LocalCommand|ProxyCommand|PermitLocalCommand|KnownHostsCommand)',
|
|
opts,
|
|
re.IGNORECASE,
|
|
):
|
|
raise ValueError("Options cannot contain LocalCommand or ProxyCommand")
|
|
try:
|
|
shlex.split(opts)
|
|
except ValueError as exc:
|
|
raise ValueError("Invalid options: unbalanced quotes") from exc
|
|
cmd_parts.append(opts)
|
|
|
|
user_host = f"{eff_user}@{self.host}" if eff_user else self.host
|
|
cmd_parts.extend(["--", user_host])
|
|
return " ".join(cmd_parts)
|
|
|
|
@MethodHook
|
|
def _generate_telnet_cmd(self):
|
|
self._validate_base_fields(check_port=True)
|
|
self._validate_options_and_safety()
|
|
cmd = f"telnet {self.host}"
|
|
if self.port:
|
|
cmd += f" {self.port}"
|
|
if self.options:
|
|
cmd += f" {self.options}"
|
|
return cmd
|
|
|
|
@MethodHook
|
|
def _generate_kube_cmd(self):
|
|
self._validate_base_fields(check_port=False)
|
|
self._validate_options_and_safety()
|
|
kube_command = self.tags.get("kube_command", "/bin/bash") if isinstance(self.tags, dict) else "/bin/bash"
|
|
if not self._is_unsafe_allowed() and kube_command:
|
|
k_str = str(kube_command)
|
|
if any(c in k_str for c in ("\n", "\r", "\x00")):
|
|
raise ValueError("kube_command cannot contain newlines or null bytes")
|
|
try:
|
|
shlex.split(k_str)
|
|
except ValueError as e:
|
|
raise ValueError(f"Invalid kube_command: unbalanced quotes ({e})") from e
|
|
cmd = f"kubectl exec {self.options} {self.host} -it --"
|
|
cmd += f" {kube_command}"
|
|
return cmd
|
|
|
|
@MethodHook
|
|
def _generate_docker_cmd(self):
|
|
self._validate_base_fields(check_port=False)
|
|
self._validate_options_and_safety()
|
|
docker_command = self.tags.get("docker_command", "/bin/bash") if isinstance(self.tags, dict) else "/bin/bash"
|
|
if not self._is_unsafe_allowed() and docker_command:
|
|
d_str = str(docker_command)
|
|
if any(c in d_str for c in ("\n", "\r", "\x00")):
|
|
raise ValueError("docker_command cannot contain newlines or null bytes")
|
|
try:
|
|
shlex.split(d_str)
|
|
except ValueError as e:
|
|
raise ValueError(f"Invalid docker_command: unbalanced quotes ({e})") from e
|
|
cmd = f"docker {self.options} exec -it {self.host}"
|
|
cmd += f" {docker_command}"
|
|
return cmd
|
|
|
|
@MethodHook
|
|
def _generate_ssm_cmd(self):
|
|
self._validate_base_fields(check_port=False)
|
|
self._validate_options_and_safety()
|
|
region = self.tags.get("region", "") if isinstance(self.tags, dict) else ""
|
|
profile = self.tags.get("profile", "") if isinstance(self.tags, dict) else ""
|
|
if not self._is_unsafe_allowed():
|
|
for field_name, field_val in [("region", region), ("profile", profile)]:
|
|
if field_val:
|
|
f_str = str(field_val)
|
|
if f_str.startswith("-"):
|
|
raise ValueError(f"SSM tag '{field_name}' cannot start with '-': '{field_val}'")
|
|
if re.search(r'[\s\x00-\x1f\x7f;|&$`><"\']', f_str):
|
|
raise ValueError(f"Invalid characters in SSM tag '{field_name}': '{field_val}'")
|
|
cmd = f"aws ssm start-session --target {self.host}"
|
|
if region:
|
|
cmd += f" --region {region}"
|
|
if profile:
|
|
cmd += f" --profile {profile}"
|
|
if self.options:
|
|
cmd += f" {self.options}"
|
|
return cmd
|
|
|
|
|
|
@MethodHook
|
|
def _get_cmd(
|
|
self,
|
|
force_ssh=False,
|
|
allow_reverse_tunnel_port=None,
|
|
*,
|
|
connect_timeout=None,
|
|
user_override=None,
|
|
):
|
|
if force_ssh or self.protocol in ["ssh", "sftp"]:
|
|
return self._generate_ssh_sftp_cmd(
|
|
force_ssh=force_ssh,
|
|
allow_reverse_tunnel_port=allow_reverse_tunnel_port,
|
|
connect_timeout=connect_timeout,
|
|
user_override=user_override,
|
|
)
|
|
elif self.protocol == "mosh":
|
|
return self._generate_mosh_cmd(
|
|
connect_timeout=connect_timeout,
|
|
user_override=user_override,
|
|
)
|
|
elif self.protocol == "telnet":
|
|
return self._generate_telnet_cmd()
|
|
elif self.protocol == "kubectl":
|
|
return self._generate_kube_cmd()
|
|
elif self.protocol == "docker":
|
|
return self._generate_docker_cmd()
|
|
elif self.protocol == "ssm":
|
|
return self._generate_ssm_cmd()
|
|
elif self.protocol == "local":
|
|
return self.host
|
|
else:
|
|
printer.error(f"Invalid protocol: {self.protocol}")
|
|
sys.exit(1)
|
|
|
|
def _spawn_child(self, args, protocol=None, encoding=None, *, _secrets=()):
|
|
if protocol is None:
|
|
protocol = getattr(self, "protocol", "")
|
|
env = None
|
|
if protocol == "mosh":
|
|
env = os.environ.copy()
|
|
if not any(
|
|
"utf-8" in env.get(k, "").lower() or "utf8" in env.get(k, "").lower()
|
|
for k in ("LC_ALL", "LC_CTYPE", "LANG")
|
|
):
|
|
env["LC_ALL"] = "C.UTF-8"
|
|
env.setdefault("TERM", "xterm-256color")
|
|
spawn_kwargs = {}
|
|
if env is not None:
|
|
spawn_kwargs["env"] = env
|
|
if encoding is not None:
|
|
spawn_kwargs["encoding"] = encoding
|
|
try:
|
|
return pexpect.spawn(args[0], args[1:], **spawn_kwargs), None
|
|
except Exception as exc: # noqa: BLE001
|
|
sanitized = sanitize_exception(
|
|
exc,
|
|
code="SPAWN_ERROR",
|
|
secrets=_secrets,
|
|
context_label="Connection failed",
|
|
)
|
|
return None, str(sanitized)
|
|
|
|
def _extract_auth_budgets(self) -> tuple[int, int | None, int | None, int | None]:
|
|
cfg_dict = (
|
|
self.config.config
|
|
if (self.config and hasattr(self.config, "config") and isinstance(self.config.config, dict))
|
|
else {}
|
|
)
|
|
tags_dict = self.tags if isinstance(self.tags, dict) else {}
|
|
|
|
def _pick(key: str, default_val):
|
|
if key in tags_dict and tags_dict[key] is not None:
|
|
return tags_dict[key]
|
|
if key in cfg_dict and cfg_dict[key] is not None:
|
|
return cfg_dict[key]
|
|
return default_val
|
|
|
|
max_auth_prompts = validate_strict_int_range(
|
|
_pick("max_auth_prompts", DEFAULT_MAX_AUTH_PROMPTS),
|
|
"max_auth_prompts",
|
|
MIN_AUTH_PROMPTS,
|
|
MAX_AUTH_PROMPTS_CAP,
|
|
)
|
|
raw_total = _pick("max_total_auth_attempts", None)
|
|
max_total_auth_attempts = (
|
|
None
|
|
if raw_total is None
|
|
else validate_strict_int_range(
|
|
raw_total,
|
|
"max_total_auth_attempts",
|
|
MIN_TOTAL_AUTH_ATTEMPTS,
|
|
MAX_TOTAL_AUTH_ATTEMPTS_CAP,
|
|
)
|
|
)
|
|
raw_hop = _pick("max_hop_host_attempts", None)
|
|
max_hop_host_attempts = (
|
|
None
|
|
if raw_hop is None
|
|
else validate_strict_int_range(
|
|
raw_hop,
|
|
"max_hop_host_attempts",
|
|
MIN_HOP_HOST_ATTEMPTS,
|
|
MAX_HOP_HOST_ATTEMPTS_CAP,
|
|
)
|
|
)
|
|
raw_global_user = _pick("max_global_user_attempts", None)
|
|
max_global_user_attempts = (
|
|
None
|
|
if raw_global_user is None
|
|
else validate_strict_int_range(
|
|
raw_global_user,
|
|
"max_global_user_attempts",
|
|
1,
|
|
1000,
|
|
)
|
|
)
|
|
return (
|
|
max_auth_prompts,
|
|
max_total_auth_attempts,
|
|
max_hop_host_attempts,
|
|
max_global_user_attempts,
|
|
)
|
|
|
|
@staticmethod
|
|
def _emit_aaa_fallback_log(
|
|
logger,
|
|
*,
|
|
unique: str,
|
|
host: str,
|
|
protocol: str,
|
|
user: str,
|
|
label: str,
|
|
secrets: list[str],
|
|
) -> None:
|
|
raw_msg = (
|
|
f"[AAA-FALLBACK] Activating backup credentials ({label}) "
|
|
f"for node={unique} host={host} proto={protocol} user={user or '<default>'}"
|
|
)
|
|
safe_msg = sanitize_log_message(raw_msg, secrets=secrets)
|
|
if logger:
|
|
try:
|
|
logger("warning", safe_msg)
|
|
return
|
|
except Exception: # noqa: BLE001
|
|
_LOGGER.debug("Fallback logger failed; falling back to stderr")
|
|
with suppress(Exception):
|
|
sys.stderr.write(safe_msg + "\n")
|
|
sys.stderr.flush()
|
|
|
|
def _build_local_decryptor(self):
|
|
decryptor_cache = {}
|
|
|
|
def _decrypt_one(cipher_text: str) -> str:
|
|
if not re.match(r'^b["\'].+["\']$', cipher_text, re.DOTALL):
|
|
return cipher_text
|
|
if "fn" not in decryptor_cache:
|
|
keyfile = getattr(self, "key", None)
|
|
if not keyfile:
|
|
raise SanitizedAuthError("No RSA key configured", code="AUTH_FAILED")
|
|
from Crypto.Cipher import PKCS1_OAEP
|
|
from Crypto.PublicKey import RSA
|
|
|
|
with open(keyfile) as kf:
|
|
rsa_key = RSA.import_key(kf.read())
|
|
decryptor_cache["fn"] = PKCS1_OAEP.new(rsa_key)
|
|
raw_bytes = ast.literal_eval(cipher_text)
|
|
return decryptor_cache["fn"].decrypt(raw_bytes).decode("utf-8")
|
|
|
|
return _decrypt_one
|
|
|
|
def _resolve_runtime_credentials(
|
|
self,
|
|
*,
|
|
allow_fallback: bool,
|
|
unattended: bool,
|
|
) -> tuple[list[RuntimeAuthCandidate], list[list[str]], list[str]]:
|
|
profiles_map = (
|
|
getattr(self.config, "profiles", {})
|
|
if (getattr(self, "config", "") != "" and self.config is not None)
|
|
else {}
|
|
)
|
|
local_decrypt = self._build_local_decryptor()
|
|
passtx_desc = type(self).__dict__.get("_passtx")
|
|
passtx_mocked = "_passtx" in self.__dict__ or not isinstance(passtx_desc, MethodHook)
|
|
|
|
def _decrypt_plain_list(specs: list[str]) -> list[str]:
|
|
if not specs:
|
|
return []
|
|
if passtx_mocked or not unattended:
|
|
return list(self._passtx(specs))
|
|
return [local_decrypt(s) for s in specs]
|
|
|
|
def _expand_spec(spec: str | tuple[str, ...]) -> list[str]:
|
|
items = list(spec) if isinstance(spec, (tuple, list)) else ([spec] if spec != "" else [])
|
|
out: list[str] = []
|
|
for item in items:
|
|
if not isinstance(item, str) or item == "":
|
|
continue
|
|
if is_compose_expression(item):
|
|
variants = parse_compose_expression(
|
|
item,
|
|
profiles=profiles_map,
|
|
decrypt_fn=local_decrypt,
|
|
max_depth=MAX_PROFILE_DEPTH,
|
|
max_variants=MAX_COMPOSE_VARIANTS,
|
|
)
|
|
out.extend(variants)
|
|
else:
|
|
out.extend(_decrypt_plain_list([item]))
|
|
return out
|
|
|
|
candidates_src = list(getattr(self, "_auth_candidates", ()))
|
|
if not candidates_src:
|
|
# Compatibility if a subclass or test mutated self.password directly
|
|
raw_pw_list = self.password if isinstance(self.password, list) else [self.password]
|
|
non_empty = tuple(p for p in raw_pw_list if isinstance(p, str) and p != "")
|
|
if non_empty or self.user:
|
|
candidates_src = [
|
|
AuthCandidate(
|
|
user=str(self.user or ""),
|
|
password_spec=non_empty if len(non_empty) > 1 else (non_empty[0] if non_empty else ""),
|
|
is_fallback=False,
|
|
label="primary",
|
|
)
|
|
]
|
|
|
|
runtime_candidates: list[RuntimeAuthCandidate] = []
|
|
raw_secrets: list[str] = []
|
|
|
|
for cand in candidates_src:
|
|
if cand.is_fallback and not allow_fallback:
|
|
continue
|
|
if isinstance(cand.password_spec, tuple):
|
|
raw_secrets.extend(s for s in cand.password_spec if isinstance(s, str) and s)
|
|
elif isinstance(cand.password_spec, str) and cand.password_spec:
|
|
raw_secrets.append(cand.password_spec)
|
|
|
|
expanded_pws = _expand_spec(cand.password_spec)
|
|
raw_secrets.extend(p for p in expanded_pws if isinstance(p, str) and p)
|
|
if expanded_pws:
|
|
for pw_val in expanded_pws:
|
|
runtime_candidates.append(
|
|
RuntimeAuthCandidate(
|
|
user=cand.user,
|
|
password=pw_val,
|
|
has_password=True,
|
|
is_fallback=cand.is_fallback,
|
|
label=cand.label,
|
|
)
|
|
)
|
|
else:
|
|
runtime_candidates.append(
|
|
RuntimeAuthCandidate(
|
|
user=cand.user,
|
|
password="",
|
|
has_password=False,
|
|
is_fallback=cand.is_fallback,
|
|
label=cand.label,
|
|
)
|
|
)
|
|
|
|
hop_runtime_passwords: list[list[str]] = []
|
|
for hop in getattr(self, "_jumphost_hops", ()):
|
|
h_pws = _expand_spec(hop.get("passwords", ()))
|
|
raw_secrets.extend(p for p in h_pws if isinstance(p, str) and p)
|
|
hop_runtime_passwords.append(h_pws)
|
|
|
|
return runtime_candidates, hop_runtime_passwords, raw_secrets
|
|
|
|
def _disambiguate_prompt(self, line: str) -> tuple[str, int | None]:
|
|
"""Classify password prompt line strictly using exact host boundaries."""
|
|
tags_dict = self.tags if isinstance(self.tags, dict) else {}
|
|
target_names = [str(self.host)]
|
|
if tags_dict.get("fqdn"):
|
|
target_names.append(str(tags_dict["fqdn"]))
|
|
matched_target = any(
|
|
host_exact_re(h).search(line) for h in target_names if h
|
|
)
|
|
|
|
matched_hops: list[int] = []
|
|
for idx, hop in enumerate(getattr(self, "_jumphost_hops", ())):
|
|
h_names = [hop.get("host", "")]
|
|
if hop.get("resolved_fqdn"):
|
|
h_names.append(str(hop["resolved_fqdn"]))
|
|
if any(host_exact_re(h).search(line) for h in h_names if h):
|
|
matched_hops.append(idx)
|
|
|
|
total_matches = len(matched_hops) + (1 if matched_target else 0)
|
|
if total_matches > 1:
|
|
return "conflict", None
|
|
if matched_hops:
|
|
return "hop", matched_hops[0]
|
|
if matched_target:
|
|
return "target", None
|
|
return "ambiguous", None
|
|
|
|
@MethodHook
|
|
def _connect(
|
|
self,
|
|
debug=False,
|
|
timeout=None,
|
|
max_attempts=3,
|
|
logger=None,
|
|
force_ssh=False,
|
|
allow_reverse_tunnel_port=None,
|
|
*,
|
|
connect_timeout=None,
|
|
auth_timeout=None,
|
|
_default_cmd_timeout=10,
|
|
_default_max_total_seconds=None,
|
|
clock=_time_mod.monotonic,
|
|
unattended=False,
|
|
attempt_tracker=None,
|
|
):
|
|
self.__dict__.pop("missingtext", None)
|
|
self.last_fallback_used = False
|
|
effective_protocol = "ssh" if force_ssh else self.protocol
|
|
if effective_protocol == "mosh" and shutil.which("mosh") is None:
|
|
return "Connection failed: mosh binary not found in PATH"
|
|
|
|
hop_meta = getattr(self, "_hop_chain_meta", None) or []
|
|
try:
|
|
validate_strict_timeout_int(
|
|
max_attempts,
|
|
"max_attempts",
|
|
min_val=1,
|
|
max_val=MAX_STAGE_TIMEOUT_CAP,
|
|
)
|
|
eff_ct, eff_at, _eff_cmd, max_ext, max_total = resolve_timeouts(
|
|
effective_protocol,
|
|
self.tags,
|
|
hop_meta,
|
|
timeout=timeout,
|
|
connect_timeout=connect_timeout,
|
|
auth_timeout=auth_timeout,
|
|
_default_cmd_timeout=_default_cmd_timeout,
|
|
_default_max_total_seconds=_default_max_total_seconds,
|
|
)
|
|
except ValueError as e:
|
|
return f"Connection failed code: 12\nInvalid timeout configuration: {e}"
|
|
|
|
try:
|
|
(
|
|
max_auth_prompts,
|
|
max_total_auth_attempts,
|
|
max_hop_host_attempts,
|
|
max_global_user_attempts,
|
|
) = self._extract_auth_budgets()
|
|
except ValueError as exc:
|
|
san = sanitize_exception(
|
|
exc,
|
|
code="AUTH_FAILED",
|
|
context_label="invalid auth budget config",
|
|
)
|
|
return str(san)
|
|
|
|
if effective_protocol == "local":
|
|
cmd = self._get_cmd()
|
|
args = shlex.split(cmd)
|
|
child, err = self._spawn_child(args, protocol=effective_protocol)
|
|
if err:
|
|
return err
|
|
self.child = child
|
|
from pexpect import fdpexpect
|
|
|
|
self.raw_child = fdpexpect.fdspawn(self.child.child_fd)
|
|
if self.logs != '':
|
|
self.logfile = self._logfile()
|
|
return True
|
|
|
|
cfg_dict = (
|
|
self.config.config
|
|
if (self.config and hasattr(self.config, "config") and isinstance(self.config.config, dict))
|
|
else {}
|
|
)
|
|
tags_dict = self.tags if isinstance(self.tags, dict) else {}
|
|
allow_fallback = not (tags_dict.get("allow_fallback") is False)
|
|
disable_insecure_hk = bool(
|
|
getattr(self, "disable_fallback_on_insecure_hostkey", False)
|
|
or tags_dict.get("disable_fallback_on_insecure_hostkey")
|
|
or cfg_dict.get("disable_fallback_on_insecure_hostkey")
|
|
)
|
|
if (
|
|
allow_fallback
|
|
and disable_insecure_hk
|
|
and has_insecure_hostkey_checking(
|
|
self.options,
|
|
self.tags,
|
|
getattr(self, "_jumphost_hops", ()),
|
|
)
|
|
):
|
|
allow_fallback = False
|
|
|
|
raw_secrets: list[str] = []
|
|
try:
|
|
runtime_candidates, hop_runtime_passwords, raw_secrets = self._resolve_runtime_credentials(
|
|
allow_fallback=allow_fallback,
|
|
unattended=unattended,
|
|
)
|
|
except Exception as exc: # noqa: BLE001
|
|
san = sanitize_exception(
|
|
exc,
|
|
code="AUTH_FAILED",
|
|
secrets=raw_secrets,
|
|
context_label="credential resolution failed",
|
|
)
|
|
return str(san)
|
|
|
|
has_configured_passwords = any(c.has_password for c in runtime_candidates)
|
|
use_auth_chain = bool(unattended or has_configured_passwords)
|
|
|
|
if self.logs != '':
|
|
self.logfile = self._logfile()
|
|
default_prompt = r'>$|#$|\$$|>.$|#.$|\$.$'
|
|
prompt = tags_dict.get("prompt", default_prompt) if tags_dict else default_prompt
|
|
password_prompt = (
|
|
r'[pP]assword:|[uU]sername:' if effective_protocol != 'telnet' else r'[pP]assword:'
|
|
)
|
|
prompt_unanchored = "|".join(t.removesuffix("$") for t in prompt.split("|"))
|
|
mosh_prompt = r"(?:" + prompt_unanchored + r")\s*(?:\x1b\[[0-9;?]*[a-zA-Z])*$"
|
|
|
|
def _precompile_user_pat(pat_str: str):
|
|
try:
|
|
pat_str.encode("ascii")
|
|
except UnicodeEncodeError:
|
|
return re.compile(pat_str.encode("utf-8"), re.DOTALL)
|
|
return pat_str
|
|
|
|
user_prompt_pat = _precompile_user_pat(prompt)
|
|
mosh_prompt_pat = _precompile_user_pat(mosh_prompt)
|
|
|
|
expects = {
|
|
"ssh": [
|
|
'yes/no',
|
|
r'(?m:^(?:(?:ssh|sftp):[^\r\n]{1,256} )?Connection refused(?:\r?\n))',
|
|
r'(?im:^(?:(?:ssh|sftp|debug\d+):[^\r\n]{1,256}not supported|Permission denied \([^)\r\n]{1,256}\)\.)(?:\r?\n))',
|
|
r'(?im:^(?:(?:ssh|sftp):[^\r\n]{1,256}Invalid|usage:\s*(?:ssh|sftp)\b[^\r\n]{0,256})(?:\r?\n))',
|
|
'ssh-keygen.*"',
|
|
r'(?im:^(?:(?:ssh|sftp):[^\r\n]{1,256}(?:Operation timed out|Connection timed out)|kex_exchange_identification:[^\r\n]{1,256}timed out|Connection timed out during banner exchange)(?:\r?\n))',
|
|
r'(?im:^(?:ssh|sftp|ssh_exchange_identification):[^\r\n]{1,256}(?:Resource temporarily unavailable|Name or service not known|Temporary failure in name resolution|No address associated with hostname|unavailable)(?:\r?\n))',
|
|
r'(?im:^(?:Connection to [^\r\n]{1,256} closed(?: by remote host)?\.?|Connection closed by (?:remote host|foreign host|authenticating user [^\r\n]{1,128}|[^\r\n]{1,256} port \d{1,5})|(?:kex|ssh)_exchange_identification: Connection closed[^\r\n]{0,256})(?:\r?\n))',
|
|
password_prompt,
|
|
user_prompt_pat,
|
|
'suspend',
|
|
pexpect.EOF,
|
|
pexpect.TIMEOUT,
|
|
r'(?im:^(?:(?:ssh|sftp):[^\r\n]{1,256})?No route to host(?:\r?\n))',
|
|
r'(?im:^(?:ssh|sftp):[^\r\n]{1,256}resolve hostname[^\r\n]{0,256}(?:\r?\n))',
|
|
r'(?im:^(?:(?:ssh|sftp):|Unable to negotiate with )[^\r\n]{1,256}no matching[^\r\n]{0,256}(?:\r?\n))',
|
|
r'(?im:^Bad (?:owner|permissions)[^\r\n]{0,256}(?:\r?\n))',
|
|
],
|
|
"sftp": [
|
|
'yes/no',
|
|
r'(?m:^(?:(?:ssh|sftp):[^\r\n]{1,256} )?Connection refused(?:\r?\n))',
|
|
r'(?im:^(?:(?:ssh|sftp|debug\d+):[^\r\n]{1,256}not supported|Permission denied \([^)\r\n]{1,256}\)\.)(?:\r?\n))',
|
|
r'(?im:^(?:(?:ssh|sftp):[^\r\n]{1,256}Invalid|usage:\s*(?:ssh|sftp)\b[^\r\n]{0,256})(?:\r?\n))',
|
|
'ssh-keygen.*"',
|
|
r'(?im:^(?:(?:ssh|sftp):[^\r\n]{1,256}(?:Operation timed out|Connection timed out)|kex_exchange_identification:[^\r\n]{1,256}timed out|Connection timed out during banner exchange)(?:\r?\n))',
|
|
r'(?im:^(?:ssh|sftp|ssh_exchange_identification):[^\r\n]{1,256}(?:Resource temporarily unavailable|Name or service not known|Temporary failure in name resolution|No address associated with hostname|unavailable)(?:\r?\n))',
|
|
r'(?im:^(?:Connection to [^\r\n]{1,256} closed(?: by remote host)?\.?|Connection closed by (?:remote host|foreign host|authenticating user [^\r\n]{1,128}|[^\r\n]{1,256} port \d{1,5})|(?:kex|ssh)_exchange_identification: Connection closed[^\r\n]{0,256})(?:\r?\n))',
|
|
password_prompt,
|
|
user_prompt_pat,
|
|
'suspend',
|
|
pexpect.EOF,
|
|
pexpect.TIMEOUT,
|
|
r'(?im:^(?:(?:ssh|sftp):[^\r\n]{1,256})?No route to host(?:\r?\n))',
|
|
r'(?im:^(?:ssh|sftp):[^\r\n]{1,256}resolve hostname[^\r\n]{0,256}(?:\r?\n))',
|
|
r'(?im:^(?:(?:ssh|sftp):|Unable to negotiate with )[^\r\n]{1,256}no matching[^\r\n]{0,256}(?:\r?\n))',
|
|
r'(?im:^Bad (?:owner|permissions)[^\r\n]{0,256}(?:\r?\n))',
|
|
],
|
|
"telnet": [
|
|
r'[uU]sername:',
|
|
r'(?im:^(?:telnet:[^\r\n]{1,256} )?(?:Unable to connect to remote host: )?Connection refused(?:\r?\n))',
|
|
r'(?im:^(?:telnet:[^\r\n]{1,256}not supported|[^\r\n]{0,64}Authentication not supported)(?:\r?\n))',
|
|
r'(?im:^(?:telnet:|usage:\s*telnet\b)[^\r\n]{0,256}(?:invalid|unrecognized option)[^\r\n]{0,256}(?:\r?\n))',
|
|
'ssh-keygen.*"',
|
|
r'(?im:^(?:telnet:[^\r\n]{1,256}(?:Operation timed out|Connection timed out)|Connection timed out)(?:\r?\n))',
|
|
r'(?im:^(?:telnet:[^\r\n]{1,256}(?:Name or service not known|Temporary failure in name resolution|unavailable)|[^\r\n]{0,128}: Name or service not known)(?:\r?\n))',
|
|
r'(?im:^(?:Connection closed by (?:foreign host|remote host)|telnet:[^\r\n]{1,256}closed)(?:\r?\n))',
|
|
password_prompt,
|
|
user_prompt_pat,
|
|
'suspend',
|
|
pexpect.EOF,
|
|
pexpect.TIMEOUT,
|
|
r'(?im:^(?:telnet:[^\r\n]{1,256})?No route to host(?:\r?\n))',
|
|
r'(?im:^telnet:[^\r\n]{1,256}resolve hostname[^\r\n]{0,256}(?:\r?\n))',
|
|
r'(?im:^telnet:[^\r\n]{1,256}no matching[^\r\n]{0,256}(?:\r?\n))',
|
|
r'(?im:^Bad (?:owner|permissions)[^\r\n]{0,256}(?:\r?\n))',
|
|
],
|
|
"kubectl": [
|
|
r'[uU]sername:',
|
|
r'[rR]efused',
|
|
r'[Ee]rror',
|
|
'DEPRECATED',
|
|
pexpect.TIMEOUT,
|
|
password_prompt,
|
|
user_prompt_pat,
|
|
pexpect.EOF,
|
|
"expired|invalid",
|
|
],
|
|
"docker": [
|
|
r'[uU]sername:',
|
|
'Cannot',
|
|
r'[Ee]rror',
|
|
'failed',
|
|
'not a docker command',
|
|
'unknown',
|
|
'unable to resolve',
|
|
pexpect.TIMEOUT,
|
|
password_prompt,
|
|
user_prompt_pat,
|
|
pexpect.EOF,
|
|
],
|
|
"ssm": [
|
|
r'[uU]sername:',
|
|
r'(?im:^(?:Cannot (?:perform start-session|connect to instance|start session)[^\r\n]{0,256})(?:\r?\n))',
|
|
r'(?im:^(?:An error occurred \([^)\r\n]{1,128}\)[^\r\n]{0,256}|aws: error:[^\r\n]{1,512}|Error: (?:Target|An error|Unable|The security token|Invalid|AccessDenied|SessionManager)[^\r\n]{1,256})(?:\r?\n))',
|
|
r'(?im:^(?:TargetNotConnected[^\r\n]{0,256}|(?:SessionManagerPlugin|Starting session|An error occurred)[^\r\n]{0,256}failed[^\r\n]{0,256})(?:\r?\n))',
|
|
'SessionManagerPlugin',
|
|
r'(?im:^(?:Unknown(?:Error|ClientException)[^\r\n]{0,256}|An error occurred \([^)\r\n]{0,128}Unknown[^)\r\n]{0,128}\)[^\r\n]{0,256}|aws: [^\r\n]{1,256}invalid choice[^\r\n]{0,256})(?:\r?\n))',
|
|
r'(?im:^(?:Could not resolve host|Unable to locate credentials|Unable to resolve [^\r\n]{1,256})(?:\r?\n))',
|
|
pexpect.TIMEOUT,
|
|
password_prompt,
|
|
user_prompt_pat,
|
|
pexpect.EOF,
|
|
],
|
|
}
|
|
expects["mosh"] = [
|
|
'yes/no',
|
|
r'(?:ssh: connect to host .* port .*: Connection refused|Connection refused)',
|
|
r'(?:(?:ssh|mosh): .*supported|Permission denied \([^)\r\n]{1,256}\)\.)',
|
|
r'(?:(?:ssh|mosh): (?:Invalid|usage)|Unknown option:)',
|
|
'ssh-keygen.*"',
|
|
r'(?:ssh: connect to host .* port .*: Connection timed out|mosh: .* timed out)',
|
|
r'(?:ssh|mosh): .*unavailable',
|
|
r'(?:Connection to .* closed|Connection closed by remote host)',
|
|
password_prompt,
|
|
mosh_prompt_pat,
|
|
'suspend',
|
|
pexpect.EOF,
|
|
pexpect.TIMEOUT,
|
|
"No route to host",
|
|
"resolve hostname",
|
|
r'(?:ssh|mosh): .*no matching',
|
|
r"[bB]ad (owner|permissions)",
|
|
"mosh-server:.*not found|command not found",
|
|
r"(?:Did not find mosh server startup message|(?:mosh: )?Nothing received from server on UDP port \d+\.?|Nothing received from server)",
|
|
"needs a UTF-8 native locale to run|requires a UTF-8 locale|locale.*not available",
|
|
]
|
|
|
|
compiled_expects = compile_expects(expects)
|
|
|
|
error_indices = {
|
|
"ssh": [1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15, 16],
|
|
"sftp": [1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15, 16],
|
|
"telnet": [1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15, 16],
|
|
"kubectl": [1, 2, 3, 4, 8],
|
|
"docker": [1, 2, 3, 4, 5, 6, 7],
|
|
"ssm": [1, 2, 3, 4, 5, 6, 7],
|
|
"mosh": [1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15, 16, 17, 18, 19],
|
|
}
|
|
|
|
eof_indices = {
|
|
"ssh": [8, 9, 10, 11],
|
|
"sftp": [8, 9, 10, 11],
|
|
"telnet": [8, 9, 10, 11],
|
|
"kubectl": [5, 6, 7],
|
|
"docker": [8, 9, 10],
|
|
"ssm": [8, 9, 10],
|
|
"mosh": [8, 9, 10, 11],
|
|
}
|
|
|
|
initial_indices = {
|
|
"ssh": [0],
|
|
"sftp": [0],
|
|
"telnet": [0],
|
|
"kubectl": [0],
|
|
"docker": [0],
|
|
"ssm": [0],
|
|
"mosh": [0],
|
|
}
|
|
|
|
def _to_bytes(val) -> bytes:
|
|
if val is None:
|
|
return b""
|
|
if isinstance(val, bytes):
|
|
return val
|
|
return str(val).encode("utf-8", errors="replace")
|
|
|
|
def _to_str(val) -> str:
|
|
if val is None:
|
|
return ""
|
|
if isinstance(val, bytes):
|
|
return val.decode("utf-8", errors="replace")
|
|
return str(val)
|
|
|
|
tracker = (
|
|
attempt_tracker
|
|
if attempt_tracker is not None
|
|
else AuthAttemptTracker(
|
|
default_max_host_attempts=DEFAULT_MAX_TOTAL_AUTH_ATTEMPTS,
|
|
default_max_hop_attempts=DEFAULT_MAX_HOP_HOST_ATTEMPTS,
|
|
clock=clock,
|
|
)
|
|
)
|
|
deadline = clock() + max_total
|
|
budget_warned = False
|
|
proto_expects = compiled_expects[effective_protocol]
|
|
pw_prompt_idx = 5 if effective_protocol == "kubectl" else 8
|
|
suspend_idx = 10 if effective_protocol in ("ssh", "sftp", "telnet", "mosh") else None
|
|
stream_limit = MAX_AUTH_STREAM_BYTES if unattended else MAX_PRE_AUTH_BUFFER_BYTES
|
|
jumphost_hops = getattr(self, "_jumphost_hops", ())
|
|
|
|
st: dict[str, object] = {
|
|
"cand_idx": 0,
|
|
"connect_retries": 0,
|
|
"eof_fallback_advances": 0,
|
|
"target_creds_sent": False,
|
|
"cur_cand": None,
|
|
"cur_user": "",
|
|
"handshake_progressed": False,
|
|
"prompts_this_conn": 0,
|
|
"cand_sent_pw_this_conn": False,
|
|
"logged_fallback_this_cand": False,
|
|
"stage": "connect",
|
|
"last_observed_bytes_len": 0,
|
|
}
|
|
|
|
def _handle_username_prompt(child_obj) -> tuple[bool, str | None]:
|
|
if not use_auth_chain:
|
|
manual_user = (
|
|
(runtime_candidates[0].user if runtime_candidates else "")
|
|
or str(self.user or "")
|
|
)
|
|
if manual_user:
|
|
child_obj.sendline(manual_user)
|
|
st["last_observed_bytes_len"] = 0
|
|
return False, None
|
|
self.missingtext = True
|
|
return True, None
|
|
|
|
cand_idx_cur = int(st["cand_idx"])
|
|
prompts_cur = int(st["prompts_this_conn"])
|
|
cur_cand = st["cur_cand"]
|
|
|
|
if st["cand_sent_pw_this_conn"]:
|
|
if (cand_idx_cur + 1) >= len(runtime_candidates):
|
|
return False, str(sanitize_exception(None, code="AUTH_FAILED", secrets=raw_secrets))
|
|
if prompts_cur >= max_auth_prompts:
|
|
st["cand_idx"] = cand_idx_cur + 1
|
|
return False, "__NEXT__:next_candidate"
|
|
cand_idx_cur += 1
|
|
st["cand_idx"] = cand_idx_cur
|
|
cur_cand = runtime_candidates[cand_idx_cur]
|
|
st["cur_cand"] = cur_cand
|
|
st["cur_user"] = cur_cand.user
|
|
st["cand_sent_pw_this_conn"] = False
|
|
st["logged_fallback_this_cand"] = False
|
|
|
|
u_to_send = cur_cand.user if cur_cand is not None else ""
|
|
if not u_to_send:
|
|
if (cand_idx_cur + 1) < len(runtime_candidates) and runtime_candidates[cand_idx_cur + 1].user:
|
|
cand_idx_cur += 1
|
|
st["cand_idx"] = cand_idx_cur
|
|
cur_cand = runtime_candidates[cand_idx_cur]
|
|
st["cur_cand"] = cur_cand
|
|
st["cur_user"] = cur_cand.user
|
|
u_to_send = cur_cand.user
|
|
st["logged_fallback_this_cand"] = False
|
|
else:
|
|
return False, str(sanitize_exception(None, code="NO_CREDS", secrets=raw_secrets))
|
|
|
|
if not tracker.try_acquire_user(u_to_send, limit=max_global_user_attempts):
|
|
return False, str(
|
|
sanitize_exception(None, code="USER_BUDGET_EXHAUSTED", secrets=raw_secrets)
|
|
)
|
|
|
|
if cur_cand is not None and cur_cand.is_fallback and not st["logged_fallback_this_cand"]:
|
|
self._emit_aaa_fallback_log(
|
|
logger,
|
|
unique=str(self.unique),
|
|
host=str(self.host),
|
|
protocol=effective_protocol,
|
|
user=u_to_send,
|
|
label=cur_cand.label,
|
|
secrets=raw_secrets,
|
|
)
|
|
self.last_fallback_used = True
|
|
st["logged_fallback_this_cand"] = True
|
|
|
|
child_obj.sendline(u_to_send)
|
|
st["handshake_progressed"] = True
|
|
st["stage"] = "auth"
|
|
st["last_observed_bytes_len"] = 0
|
|
return False, None
|
|
|
|
def _handle_target_password_prompt(child_obj) -> tuple[bool, str | None]:
|
|
cand_idx_cur = int(st["cand_idx"])
|
|
prompts_cur = int(st["prompts_this_conn"])
|
|
cur_cand = st["cur_cand"]
|
|
cur_user = str(st["cur_user"])
|
|
|
|
if not use_auth_chain:
|
|
self.missingtext = True
|
|
return True, None
|
|
|
|
if st["cand_sent_pw_this_conn"]:
|
|
if (cand_idx_cur + 1) >= len(runtime_candidates):
|
|
return False, str(sanitize_exception(None, code="AUTH_FAILED", secrets=raw_secrets))
|
|
nxt = runtime_candidates[cand_idx_cur + 1]
|
|
if nxt.user == cur_user and prompts_cur < max_auth_prompts:
|
|
cand_idx_cur += 1
|
|
st["cand_idx"] = cand_idx_cur
|
|
cur_cand = nxt
|
|
st["cur_cand"] = cur_cand
|
|
st["cand_sent_pw_this_conn"] = False
|
|
st["logged_fallback_this_cand"] = False
|
|
else:
|
|
st["cand_idx"] = cand_idx_cur + 1
|
|
return False, "__NEXT__:next_candidate"
|
|
|
|
if cur_cand is None or not cur_cand.has_password:
|
|
if (cand_idx_cur + 1) < len(runtime_candidates):
|
|
nxt = runtime_candidates[cand_idx_cur + 1]
|
|
if nxt.user == cur_user and prompts_cur < max_auth_prompts:
|
|
cand_idx_cur += 1
|
|
st["cand_idx"] = cand_idx_cur
|
|
cur_cand = nxt
|
|
st["cur_cand"] = cur_cand
|
|
st["cand_sent_pw_this_conn"] = False
|
|
st["logged_fallback_this_cand"] = False
|
|
else:
|
|
st["cand_idx"] = cand_idx_cur + 1
|
|
return False, "__NEXT__:next_candidate"
|
|
else:
|
|
return False, str(sanitize_exception(None, code="NO_CREDS", secrets=raw_secrets))
|
|
|
|
if not cur_cand.has_password:
|
|
return False, str(sanitize_exception(None, code="NO_CREDS", secrets=raw_secrets))
|
|
|
|
if prompts_cur >= max_auth_prompts:
|
|
if (cand_idx_cur + 1) < len(runtime_candidates):
|
|
st["cand_idx"] = cand_idx_cur + 1
|
|
return False, "__NEXT__:next_candidate"
|
|
return False, str(sanitize_exception(None, code="AUTH_FAILED", secrets=raw_secrets))
|
|
|
|
eff_u = cur_cand.user or str(self.user or "")
|
|
acquired, budget_err_code = tracker.try_acquire_target_attempt(
|
|
host=str(self.host),
|
|
port=str(self.port or ""),
|
|
protocol=effective_protocol,
|
|
user=eff_u,
|
|
host_limit=max_total_auth_attempts,
|
|
user_limit=max_global_user_attempts,
|
|
)
|
|
if not acquired:
|
|
return False, str(
|
|
sanitize_exception(
|
|
None,
|
|
code=budget_err_code or "HOST_BUDGET_EXHAUSTED",
|
|
secrets=raw_secrets,
|
|
)
|
|
)
|
|
|
|
if cur_cand.is_fallback and not st["logged_fallback_this_cand"]:
|
|
self._emit_aaa_fallback_log(
|
|
logger,
|
|
unique=str(self.unique),
|
|
host=str(self.host),
|
|
protocol=effective_protocol,
|
|
user=eff_u,
|
|
label=cur_cand.label,
|
|
secrets=raw_secrets,
|
|
)
|
|
self.last_fallback_used = True
|
|
st["logged_fallback_this_cand"] = True
|
|
|
|
child_obj.sendline(cur_cand.password)
|
|
st["cand_sent_pw_this_conn"] = True
|
|
st["target_creds_sent"] = True
|
|
st["prompts_this_conn"] = prompts_cur + 1
|
|
st["handshake_progressed"] = True
|
|
st["stage"] = "auth"
|
|
st["last_observed_bytes_len"] = 0
|
|
return False, None
|
|
|
|
while True:
|
|
remaining = deadline - clock()
|
|
if remaining <= 0:
|
|
return _redact_known_secrets(
|
|
f"Connection failed code: 12\nConnection timeout (global deadline {max_total}s exceeded)",
|
|
raw_secrets,
|
|
)
|
|
|
|
cand_idx_val = int(st["cand_idx"])
|
|
cur_cand = runtime_candidates[cand_idx_val] if cand_idx_val < len(runtime_candidates) else None
|
|
cur_user = cur_cand.user if cur_cand is not None else str(self.user or "")
|
|
st["cur_cand"] = cur_cand
|
|
st["cur_user"] = cur_user
|
|
user_override = (
|
|
cur_user
|
|
if (cand_idx_val > 0 or (cur_user and cur_user != str(self.user or "")))
|
|
else None
|
|
)
|
|
|
|
cmd = self._get_cmd(
|
|
force_ssh=force_ssh,
|
|
allow_reverse_tunnel_port=allow_reverse_tunnel_port,
|
|
connect_timeout=eff_ct,
|
|
user_override=user_override,
|
|
)
|
|
|
|
st["handshake_progressed"] = False
|
|
args = shlex.split(cmd)
|
|
child, err = self._spawn_child(
|
|
args,
|
|
protocol=effective_protocol,
|
|
_secrets=raw_secrets,
|
|
)
|
|
if err:
|
|
return _redact_known_secrets(err, raw_secrets)
|
|
|
|
connected_ok = False
|
|
local_mylog = None
|
|
next_action: str | None = None
|
|
hop_pw_idx = [0] * len(jumphost_hops)
|
|
hop_cursor = 0
|
|
st["prompts_this_conn"] = 0
|
|
st["cand_sent_pw_this_conn"] = False
|
|
st["logged_fallback_this_cand"] = False
|
|
transport_error_seen = False
|
|
|
|
try:
|
|
child.logfile = None
|
|
child.logfile_send = None
|
|
if unattended:
|
|
child.logfile_read = None
|
|
|
|
if isinstance(self.tags, dict) and self.tags.get("console"):
|
|
child.sendline()
|
|
if debug:
|
|
if logger:
|
|
logger("debug", f"Command:\n{_redact_known_secrets(cmd, raw_secrets)}")
|
|
local_mylog = io.BytesIO()
|
|
self.mylog = local_mylog
|
|
redacted_cmd = _redact_known_secrets(cmd, raw_secrets)
|
|
local_mylog.write(f"[i] [DEBUG] Command:\r\n {redacted_cmd}\r\n".encode())
|
|
if not unattended:
|
|
child.logfile_read = local_mylog
|
|
|
|
st["stage"] = "connect"
|
|
progress_extensions = 0
|
|
st["last_observed_bytes_len"] = 0
|
|
|
|
while True:
|
|
remaining = deadline - clock()
|
|
if remaining <= 0:
|
|
return _redact_known_secrets(
|
|
f"Connection failed code: 12\nConnection timeout (global deadline {max_total}s exceeded)",
|
|
raw_secrets,
|
|
)
|
|
stage_timeout = eff_ct if st["stage"] == "connect" else eff_at
|
|
expect_timeout = max(0.001, min(stage_timeout, remaining))
|
|
results = child.expect(proto_expects, timeout=expect_timeout)
|
|
results_value = proto_expects[results]
|
|
before_bytes = _to_bytes(getattr(child, "before", b""))
|
|
after_bytes = _to_bytes(getattr(child, "after", b""))
|
|
|
|
if len(before_bytes) > stream_limit:
|
|
return _redact_known_secrets(
|
|
f"Connection failed code: 12\nPre-auth buffer exceeded maximum size ({stream_limit} bytes)",
|
|
raw_secrets,
|
|
)
|
|
|
|
if results in initial_indices[effective_protocol]:
|
|
if effective_protocol in ["ssh", "sftp", "mosh"]:
|
|
child.sendline('yes')
|
|
st["handshake_progressed"] = True
|
|
st["last_observed_bytes_len"] = 0
|
|
elif effective_protocol in ["telnet", "kubectl", "docker", "ssm"]:
|
|
should_break, msg_or_action = _handle_username_prompt(child)
|
|
if msg_or_action == "__NEXT__:next_candidate":
|
|
next_action = "next_candidate"
|
|
break
|
|
if msg_or_action is not None:
|
|
return msg_or_action
|
|
if should_break:
|
|
break
|
|
|
|
elif results in error_indices[effective_protocol]:
|
|
if results_value == pexpect.TIMEOUT:
|
|
cur_len = len(before_bytes)
|
|
bytes_grew = cur_len > int(st["last_observed_bytes_len"])
|
|
st["last_observed_bytes_len"] = cur_len
|
|
remaining = deadline - clock()
|
|
can_extend = (
|
|
bytes_grew
|
|
and progress_extensions < max_ext
|
|
and remaining > 0
|
|
and not (effective_protocol == "mosh" and st["handshake_progressed"])
|
|
)
|
|
if can_extend:
|
|
progress_extensions += 1
|
|
if not budget_warned:
|
|
budget_warned = True
|
|
if logger:
|
|
logger(
|
|
"warning",
|
|
f"Slow {effective_protocol} handshake making progress; extending connect wait (deadline {max_total}s)",
|
|
)
|
|
continue
|
|
|
|
if (
|
|
use_auth_chain
|
|
and st["stage"] == "auth"
|
|
and st["target_creds_sent"]
|
|
and (int(st["cand_idx"]) + 1) < len(runtime_candidates)
|
|
and remaining > 0
|
|
):
|
|
st["cand_idx"] = int(st["cand_idx"]) + 1
|
|
next_action = "next_candidate"
|
|
break
|
|
|
|
can_retry = (
|
|
not st["target_creds_sent"]
|
|
and (int(st["connect_retries"]) + 1) < max_attempts
|
|
and remaining > 0
|
|
and not (effective_protocol == "mosh" and st["handshake_progressed"])
|
|
)
|
|
if can_retry:
|
|
st["connect_retries"] = int(st["connect_retries"]) + 1
|
|
next_action = "retry_connect"
|
|
break
|
|
|
|
before_str = _to_str(before_bytes)
|
|
if effective_protocol == "mosh":
|
|
before_str = self._logclean(before_str, True)
|
|
if remaining <= 0:
|
|
after_str = f"Connection timeout (global deadline {max_total}s exceeded)"
|
|
else:
|
|
after_str = "Connection timeout"
|
|
return _redact_known_secrets(
|
|
f"Connection failed code: {results}\n{before_str.lstrip()}{after_str}".rstrip(),
|
|
raw_secrets,
|
|
)
|
|
|
|
is_perm_denied = (
|
|
effective_protocol in ("ssh", "sftp", "mosh")
|
|
and results == 2
|
|
and b"permission denied" in (before_bytes + after_bytes).lower()
|
|
)
|
|
if (
|
|
use_auth_chain
|
|
and is_perm_denied
|
|
and (int(st["cand_idx"]) + 1) < len(runtime_candidates)
|
|
and remaining > 0
|
|
):
|
|
st["cand_idx"] = int(st["cand_idx"]) + 1
|
|
next_action = "next_candidate"
|
|
break
|
|
|
|
transport_error_seen = not is_perm_denied
|
|
after = _to_str(after_bytes)
|
|
before = _to_str(before_bytes)
|
|
readline_val = _to_str(child.readline())
|
|
if effective_protocol == "mosh":
|
|
after = self._logclean(after, True)
|
|
before = self._logclean(before, True)
|
|
readline_val = self._logclean(readline_val, True)
|
|
if (
|
|
effective_protocol == "mosh"
|
|
and results == 18
|
|
and ("Nothing received from server" in after or "Nothing received from server" in before)
|
|
):
|
|
err_msg = (
|
|
after.strip()
|
|
if "Nothing received from server" in after
|
|
else before.strip()
|
|
)
|
|
return _redact_known_secrets(
|
|
f"Connection failed code: {results}\n{err_msg} — UDP traffic is blocked or unreachable. Verify firewall rules for UDP ports (60000-61000 or mosh_udp_port).",
|
|
raw_secrets,
|
|
)
|
|
return _redact_known_secrets(
|
|
f"Connection failed code: {results}\n{before.lstrip()}{after}{readline_val}".rstrip(),
|
|
raw_secrets,
|
|
)
|
|
|
|
elif results in eof_indices[effective_protocol]:
|
|
if results == pw_prompt_idx:
|
|
after_lower = after_bytes.lower()
|
|
if b"username" in after_lower and b"password" not in after_lower:
|
|
should_break, msg_or_action = _handle_username_prompt(child)
|
|
if msg_or_action == "__NEXT__:next_candidate":
|
|
next_action = "next_candidate"
|
|
break
|
|
if msg_or_action is not None:
|
|
return msg_or_action
|
|
if should_break:
|
|
break
|
|
continue
|
|
|
|
if jumphost_hops:
|
|
prompt_line = normalize_prompt_line(before_bytes, after_bytes)
|
|
kind, hop_idx = self._disambiguate_prompt(prompt_line)
|
|
if kind == "conflict":
|
|
return str(
|
|
sanitize_exception(None, code="JUMP_AUTH_FAILED", secrets=raw_secrets)
|
|
)
|
|
if kind == "hop":
|
|
assert hop_idx is not None
|
|
if st["target_creds_sent"] or hop_idx < hop_cursor:
|
|
return str(
|
|
sanitize_exception(None, code="JUMP_AUTH_FAILED", secrets=raw_secrets)
|
|
)
|
|
hop_cursor = hop_idx
|
|
hop_obj = jumphost_hops[hop_idx]
|
|
hop_pws = hop_runtime_passwords[hop_idx]
|
|
if hop_pw_idx[hop_idx] >= len(hop_pws):
|
|
return str(
|
|
sanitize_exception(None, code="JUMP_AUTH_FAILED", secrets=raw_secrets)
|
|
)
|
|
if not tracker.try_acquire_hop(
|
|
hop_obj["host"],
|
|
hop_obj["port"],
|
|
hop_obj["protocol"],
|
|
limit=max_hop_host_attempts,
|
|
):
|
|
return str(
|
|
sanitize_exception(None, code="HOST_BUDGET_EXHAUSTED", secrets=raw_secrets)
|
|
)
|
|
child.sendline(hop_pws[hop_pw_idx[hop_idx]])
|
|
hop_pw_idx[hop_idx] += 1
|
|
st["handshake_progressed"] = True
|
|
st["last_observed_bytes_len"] = 0
|
|
continue
|
|
if kind == "target":
|
|
hop_cursor = len(jumphost_hops)
|
|
else:
|
|
no_pending_hop_pws = all(
|
|
hop_pw_idx[i] >= len(hop_runtime_passwords[i])
|
|
for i in range(hop_cursor, len(jumphost_hops))
|
|
)
|
|
if (
|
|
st["target_creds_sent"]
|
|
or hop_cursor >= len(jumphost_hops)
|
|
or no_pending_hop_pws
|
|
):
|
|
hop_cursor = len(jumphost_hops)
|
|
else:
|
|
allow_ambig = bool(
|
|
tags_dict.get("allow_ambiguous_single_hop_prompt", False)
|
|
)
|
|
if len(jumphost_hops) > 1 or not allow_ambig:
|
|
return str(
|
|
sanitize_exception(
|
|
None,
|
|
code="JUMP_AUTH_FAILED",
|
|
secrets=raw_secrets,
|
|
)
|
|
)
|
|
hop_obj = jumphost_hops[hop_cursor]
|
|
if not tracker.try_acquire_hop(
|
|
hop_obj["host"],
|
|
hop_obj["port"],
|
|
hop_obj["protocol"],
|
|
limit=max_hop_host_attempts,
|
|
):
|
|
return str(
|
|
sanitize_exception(
|
|
None,
|
|
code="HOST_BUDGET_EXHAUSTED",
|
|
secrets=raw_secrets,
|
|
)
|
|
)
|
|
child.sendline(
|
|
hop_runtime_passwords[hop_cursor][hop_pw_idx[hop_cursor]]
|
|
)
|
|
hop_pw_idx[hop_cursor] += 1
|
|
st["handshake_progressed"] = True
|
|
st["last_observed_bytes_len"] = 0
|
|
continue
|
|
|
|
should_break, msg_or_action = _handle_target_password_prompt(child)
|
|
if msg_or_action == "__NEXT__:next_candidate":
|
|
next_action = "next_candidate"
|
|
break
|
|
if msg_or_action is not None:
|
|
return msg_or_action
|
|
if should_break:
|
|
break
|
|
|
|
elif suspend_idx is not None and results == suspend_idx:
|
|
child.sendline("\r")
|
|
sleep(2)
|
|
st["last_observed_bytes_len"] = 0
|
|
elif results_value == pexpect.EOF:
|
|
norm_buf = (
|
|
before_bytes.replace(b"\r\n", b"\n").replace(b"\r", b"\n") + b"\n"
|
|
)
|
|
eof_perm_denied = False
|
|
for err_idx in error_indices[effective_protocol]:
|
|
err_pat = proto_expects[err_idx]
|
|
if isinstance(err_pat, re.Pattern):
|
|
m = err_pat.search(norm_buf)
|
|
if m:
|
|
is_pd = (
|
|
effective_protocol in ("ssh", "sftp", "mosh")
|
|
and err_idx == 2
|
|
and b"permission denied" in m.group(0).lower()
|
|
)
|
|
if (
|
|
use_auth_chain
|
|
and is_pd
|
|
and (int(st["cand_idx"]) + 1) < len(runtime_candidates)
|
|
and remaining > 0
|
|
):
|
|
eof_perm_denied = True
|
|
break
|
|
transport_error_seen = not is_pd
|
|
b_str = _to_str(norm_buf[: m.start()])
|
|
a_str = _to_str(m.group(0))
|
|
if effective_protocol == "mosh":
|
|
b_str = self._logclean(b_str, True)
|
|
a_str = self._logclean(a_str, True)
|
|
return _redact_known_secrets(
|
|
f"Connection failed code: {err_idx}\n{b_str.lstrip()}{a_str}".rstrip(),
|
|
raw_secrets,
|
|
)
|
|
if eof_perm_denied:
|
|
st["cand_idx"] = int(st["cand_idx"]) + 1
|
|
next_action = "next_candidate"
|
|
break
|
|
|
|
if use_auth_chain and not transport_error_seen:
|
|
if st["target_creds_sent"]:
|
|
if (int(st["cand_idx"]) + 1) < len(runtime_candidates):
|
|
eof_adv = int(st["eof_fallback_advances"])
|
|
if eof_adv < MAX_EOF_FALLBACK_ADVANCES:
|
|
st["eof_fallback_advances"] = eof_adv + 1
|
|
st["cand_idx"] = int(st["cand_idx"]) + 1
|
|
next_action = "next_candidate"
|
|
break
|
|
return str(
|
|
sanitize_exception(
|
|
None,
|
|
code="EOF_BUDGET_EXHAUSTED",
|
|
secrets=raw_secrets,
|
|
)
|
|
)
|
|
return str(
|
|
sanitize_exception(
|
|
None,
|
|
code="AUTH_FAILED",
|
|
secrets=raw_secrets,
|
|
)
|
|
)
|
|
if effective_protocol != "local":
|
|
return str(
|
|
sanitize_exception(
|
|
None,
|
|
code="TRANSPORT_ERROR",
|
|
secrets=raw_secrets,
|
|
)
|
|
)
|
|
|
|
try:
|
|
child.sendline()
|
|
except Exception as exc: # noqa: BLE001
|
|
_LOGGER.debug("EOF fallback sendline ignored: %s", type(exc).__name__)
|
|
break
|
|
else:
|
|
cur_cand_now = st["cur_cand"]
|
|
if (
|
|
cur_cand_now is not None
|
|
and cur_cand_now.is_fallback
|
|
and not st["logged_fallback_this_cand"]
|
|
):
|
|
self._emit_aaa_fallback_log(
|
|
logger,
|
|
unique=str(self.unique),
|
|
host=str(self.host),
|
|
protocol=effective_protocol,
|
|
user=str(st["cur_user"]),
|
|
label=cur_cand_now.label,
|
|
secrets=raw_secrets,
|
|
)
|
|
self.last_fallback_used = True
|
|
st["logged_fallback_this_cand"] = True
|
|
child.sendline()
|
|
break
|
|
|
|
if next_action in ("retry_connect", "next_candidate"):
|
|
continue
|
|
|
|
if isinstance(self.tags, dict) and self.tags.get("post_connect_commands"):
|
|
cmds = self.tags.get("post_connect_commands")
|
|
post_commands = [cmds] if isinstance(cmds, str) else cmds
|
|
for command in post_commands:
|
|
child.sendline(command)
|
|
sleep(1)
|
|
child.readline(0)
|
|
self.child = child
|
|
from pexpect import fdpexpect
|
|
|
|
self.raw_child = fdpexpect.fdspawn(self.child.child_fd)
|
|
connected_ok = True
|
|
return True
|
|
except Exception as exc: # noqa: BLE001
|
|
san = sanitize_exception(
|
|
exc,
|
|
code="AUTH_FAILED",
|
|
secrets=raw_secrets,
|
|
context_label="unexpected connection error",
|
|
)
|
|
return str(san)
|
|
finally:
|
|
if not connected_ok:
|
|
try:
|
|
child.logfile_read = None
|
|
except Exception as exc: # noqa: BLE001
|
|
_LOGGER.debug("Cleanup logfile_read reset failed: %s", type(exc).__name__)
|
|
try:
|
|
child.terminate(force=True)
|
|
except Exception as exc: # noqa: BLE001
|
|
_LOGGER.debug("Cleanup terminate failed: %s", type(exc).__name__)
|
|
if local_mylog is not None and not getattr(local_mylog, "closed", False):
|
|
try:
|
|
local_mylog.close()
|
|
except Exception as exc: # noqa: BLE001
|
|
_LOGGER.debug("Cleanup local_mylog close failed: %s", type(exc).__name__)
|
|
|
|
@ClassHook
|
|
class nodes:
|
|
''' This class generates a nodes object. Contains a list of node class objects and methods to run multiple tasks on nodes simultaneously.
|
|
|
|
### Attributes:
|
|
|
|
- nodelist (list): List of node class objects passed to the init
|
|
function.
|
|
|
|
- output (dict): Dictionary formed by nodes unique as keys,
|
|
output of the commands you ran on the node as
|
|
value. Created after running methods run or test.
|
|
|
|
- result (dict): Dictionary formed by nodes unique as keys, value
|
|
is True if expected value is found after running
|
|
the commands, False if prompt is found before.
|
|
Created after running method test.
|
|
|
|
- status (dict): Dictionary formed by nodes unique as keys, value:
|
|
0 if method run or test ended successfully.
|
|
1 if connection failed.
|
|
2 if expect timeouts without prompt or EOF.
|
|
|
|
- <unique> (obj): For each item in nodelist, there is an attribute
|
|
generated with the node unique.
|
|
'''
|
|
|
|
def __init__(self, nodes: dict, config = ''):
|
|
'''
|
|
### Parameters:
|
|
|
|
- nodes (dict): Dictionary formed by node information:
|
|
Keys: Unique name for each node.
|
|
Mandatory Subkeys: host(str).
|
|
Optional Subkeys: options(str), logs(str), password(str),
|
|
port(str), protocol(str), user(str).
|
|
For reference on subkeys check node class.
|
|
|
|
### Optional Parameters:
|
|
|
|
- config (obj): Pass the object created with class configfile with key
|
|
for decryption and extra configuration if you are using
|
|
connection manager.
|
|
'''
|
|
self.nodelist = []
|
|
self.config = config
|
|
self.fallback_used = {}
|
|
for n, node_cfg in nodes.items():
|
|
this = node(n, **node_cfg, config=config)
|
|
self.nodelist.append(this)
|
|
setattr(self, n, this)
|
|
|
|
@MethodHook
|
|
def _splitlist(self, lst, n):
|
|
#split a list in lists of n members.
|
|
for i in range(0, len(lst), n):
|
|
yield lst[i:i + n]
|
|
|
|
@MethodHook
|
|
def run(
|
|
self,
|
|
commands,
|
|
vars=None,
|
|
*,
|
|
folder=None,
|
|
prompt=None,
|
|
stdout=None,
|
|
parallel=10,
|
|
timeout=None,
|
|
on_complete=None,
|
|
logger=None,
|
|
connect_timeout=None,
|
|
auth_timeout=None,
|
|
_default_cmd_timeout=10,
|
|
unattended=True,
|
|
attempt_tracker=None,
|
|
):
|
|
'''
|
|
Run a command or list of commands on all the nodes in nodelist.
|
|
|
|
|
|
### Parameters:
|
|
|
|
- commands (str/list): Commands to run on the nodes. Should be str or
|
|
list of str. You can use variables as {varname}
|
|
and defining them in optional parameter vars.
|
|
|
|
### Optional Parameters:
|
|
|
|
- vars (dict): Dictionary containing the definition of variables for
|
|
each node, used in commands parameter.
|
|
Keys should be formed by nodes unique names. Use
|
|
special key name __global__ for global variables.
|
|
Subkeys: Variable names.
|
|
Values: strings.
|
|
|
|
### Optional Named Parameters:
|
|
|
|
- folder (str): Path where output log should be stored, leave empty
|
|
to disable logging.
|
|
|
|
- prompt (str): Prompt to be expected after a command is finished
|
|
running. Usually linux uses ">" or EOF while routers
|
|
use ">" or "#". The default value should work for
|
|
most nodes. Change it if your connection need some
|
|
special symbol.
|
|
|
|
- stdout (bool): Set True to send the command output to stdout.
|
|
Default False.
|
|
|
|
- parallel (int): Number of nodes to run the commands simultaneously.
|
|
Default is 10, if there are more nodes that this
|
|
value, nodes are groups in groups with max this
|
|
number of members.
|
|
|
|
- timeout (int): Time in seconds for expect to wait for prompt/EOF.
|
|
default 10.
|
|
|
|
- on_complete (callable): Optional callback called when each node
|
|
finishes. Receives (unique, output, status).
|
|
Called from the node's thread so it must
|
|
be thread-safe.
|
|
|
|
###Returns:
|
|
|
|
dict: Dictionary formed by nodes unique as keys, Output of the
|
|
commands you ran on the node as value.
|
|
|
|
'''
|
|
args = {}
|
|
nodesargs = {}
|
|
args["commands"] = commands
|
|
if folder is not None:
|
|
args["folder"] = folder
|
|
Path(folder).mkdir(parents=True, exist_ok=True)
|
|
if prompt is not None:
|
|
args["prompt"] = prompt
|
|
if stdout is not None and on_complete is None:
|
|
args["stdout"] = stdout
|
|
if timeout is not None:
|
|
args["timeout"] = timeout
|
|
if connect_timeout is not None:
|
|
args["connect_timeout"] = connect_timeout
|
|
if auth_timeout is not None:
|
|
args["auth_timeout"] = auth_timeout
|
|
if _default_cmd_timeout != 10:
|
|
args["_default_cmd_timeout"] = _default_cmd_timeout
|
|
if unattended is not True:
|
|
args["unattended"] = unattended
|
|
output = {}
|
|
status = {}
|
|
fallback_used = {}
|
|
tasks = []
|
|
|
|
def _run_node(node_obj, node_args, callback):
|
|
"""Wrapper that runs a node and fires the callback on completion."""
|
|
node_obj.run(**node_args)
|
|
if callback:
|
|
callback(node_obj.unique, node_obj.output, node_obj.status)
|
|
|
|
for n in self.nodelist:
|
|
nodesargs[n.unique] = deepcopy(args)
|
|
if attempt_tracker is not None:
|
|
nodesargs[n.unique]["attempt_tracker"] = attempt_tracker
|
|
if vars is not None:
|
|
nodesargs[n.unique]["vars"] = {}
|
|
if "__global__" in vars:
|
|
nodesargs[n.unique]["vars"].update(vars["__global__"])
|
|
for var_key, var_val in vars.items():
|
|
if var_key == "__global__":
|
|
continue
|
|
try:
|
|
if re.search(var_key, n.unique, re.IGNORECASE):
|
|
nodesargs[n.unique]["vars"].update(var_val)
|
|
except re.error:
|
|
if var_key == n.unique:
|
|
nodesargs[n.unique]["vars"].update(var_val)
|
|
|
|
# Pass the logger to the node
|
|
nodesargs[n.unique]["logger"] = logger
|
|
|
|
if on_complete:
|
|
tasks.append(threading.Thread(target=_run_node, args=(n, nodesargs[n.unique], on_complete)))
|
|
else:
|
|
tasks.append(threading.Thread(target=n.run, kwargs=nodesargs[n.unique]))
|
|
|
|
taskslist = list(self._splitlist(tasks, parallel))
|
|
|
|
for t in taskslist:
|
|
for i in t:
|
|
i.start()
|
|
for i in t:
|
|
i.join()
|
|
for i in self.nodelist:
|
|
output[i.unique] = i.output
|
|
status[i.unique] = i.status
|
|
fallback_used[i.unique] = bool(getattr(i, "last_fallback_used", False))
|
|
self.output = output
|
|
self.status = status
|
|
self.fallback_used = fallback_used
|
|
return output
|
|
|
|
@MethodHook
|
|
def test(
|
|
self,
|
|
commands,
|
|
expected,
|
|
vars=None,
|
|
*,
|
|
folder=None,
|
|
prompt=None,
|
|
parallel=10,
|
|
timeout=None,
|
|
on_complete=None,
|
|
logger=None,
|
|
connect_timeout=None,
|
|
auth_timeout=None,
|
|
_default_cmd_timeout=10,
|
|
unattended=True,
|
|
attempt_tracker=None,
|
|
):
|
|
'''
|
|
Run a command or list of commands on all the nodes in nodelist, then check if expected value appears on the output after the last command.
|
|
|
|
|
|
### Parameters:
|
|
|
|
- commands (str/list): Commands to run on the node. Should be str or
|
|
list of str.
|
|
|
|
- expected (str) : Expected text to appear after running all the
|
|
commands on the node.
|
|
|
|
### Optional Parameters:
|
|
|
|
- vars (dict): Dictionary containing the definition of variables for
|
|
each node, used in commands and expected parameters.
|
|
Keys should be formed by nodes unique names. Use
|
|
special key name __global__ for global variables.
|
|
Subkeys: Variable names.
|
|
Values: strings.
|
|
|
|
### Optional Named Parameters:
|
|
|
|
- prompt (str): Prompt to be expected after a command is finished
|
|
running. Usually linux uses ">" or EOF while
|
|
routers use ">" or "#". The default value should
|
|
work for most nodes. Change it if your connection
|
|
need some special symbol.
|
|
|
|
|
|
- parallel (int): Number of nodes to run the commands simultaneously.
|
|
Default is 10, if there are more nodes that this
|
|
value, nodes are groups in groups with max this
|
|
number of members.
|
|
|
|
- timeout (int): Time in seconds for expect to wait for prompt/EOF.
|
|
default 10.
|
|
|
|
- on_complete (callable): Optional callback called when each node
|
|
finishes. Receives (unique, output, status).
|
|
Called from the node's thread so it must
|
|
be thread-safe.
|
|
|
|
### Returns:
|
|
|
|
dict: Dictionary formed by nodes unique as keys, value is True if
|
|
expected value is found after running the commands, False
|
|
if prompt is found before.
|
|
|
|
'''
|
|
args = {}
|
|
nodesargs = {}
|
|
args["commands"] = commands
|
|
args["expected"] = expected
|
|
if folder is not None:
|
|
args["folder"] = folder
|
|
Path(folder).mkdir(parents=True, exist_ok=True)
|
|
if prompt is not None:
|
|
args["prompt"] = prompt
|
|
if timeout is not None:
|
|
args["timeout"] = timeout
|
|
if connect_timeout is not None:
|
|
args["connect_timeout"] = connect_timeout
|
|
if auth_timeout is not None:
|
|
args["auth_timeout"] = auth_timeout
|
|
if _default_cmd_timeout != 10:
|
|
args["_default_cmd_timeout"] = _default_cmd_timeout
|
|
if unattended is not True:
|
|
args["unattended"] = unattended
|
|
output = {}
|
|
result = {}
|
|
status = {}
|
|
fallback_used = {}
|
|
tasks = []
|
|
|
|
def _test_node(node_obj, node_args, callback):
|
|
"""Wrapper that runs a node test and fires the callback on completion."""
|
|
node_obj.test(**node_args)
|
|
if callback:
|
|
callback(node_obj.unique, node_obj.output, node_obj.status, node_obj.result)
|
|
|
|
for n in self.nodelist:
|
|
nodesargs[n.unique] = deepcopy(args)
|
|
if attempt_tracker is not None:
|
|
nodesargs[n.unique]["attempt_tracker"] = attempt_tracker
|
|
if vars is not None:
|
|
nodesargs[n.unique]["vars"] = {}
|
|
if "__global__" in vars:
|
|
nodesargs[n.unique]["vars"].update(vars["__global__"])
|
|
for var_key, var_val in vars.items():
|
|
if var_key == "__global__":
|
|
continue
|
|
try:
|
|
if re.search(var_key, n.unique, re.IGNORECASE):
|
|
nodesargs[n.unique]["vars"].update(var_val)
|
|
except re.error:
|
|
if var_key == n.unique:
|
|
nodesargs[n.unique]["vars"].update(var_val)
|
|
nodesargs[n.unique]["logger"] = logger
|
|
|
|
if on_complete:
|
|
tasks.append(threading.Thread(target=_test_node, args=(n, nodesargs[n.unique], on_complete)))
|
|
else:
|
|
tasks.append(threading.Thread(target=n.test, kwargs=nodesargs[n.unique]))
|
|
|
|
taskslist = list(self._splitlist(tasks, parallel))
|
|
for t in taskslist:
|
|
for i in t:
|
|
i.start()
|
|
for i in t:
|
|
i.join()
|
|
for i in self.nodelist:
|
|
result[i.unique] = i.result
|
|
output[i.unique] = i.output
|
|
status[i.unique] = i.status
|
|
fallback_used[i.unique] = bool(getattr(i, "last_fallback_used", False))
|
|
self.output = output
|
|
self.result = result
|
|
self.status = status
|
|
self.fallback_used = fallback_used
|
|
return result
|
|
|
|
# script
|