262 lines
9.8 KiB
Bash
Executable File
262 lines
9.8 KiB
Bash
Executable File
#!/bin/sh
|
|
set -eu
|
|
umask 077
|
|
|
|
ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
|
|
source_target=${PRODUCTION_SSH_TARGET:-whoneedhelp}
|
|
monitor_target=${MONITOR_SSH_TARGET:-buyvm-maya}
|
|
production_env=${PRODUCTION_ENV_PATH:-/srv/who_need_help-production/.env}
|
|
remote_root=${MONITOR_REMOTE_ROOT:-}
|
|
remote_config=${MONITOR_REMOTE_CONFIG:-}
|
|
remote_state=${MONITOR_REMOTE_STATE:-}
|
|
monitor_smtp_values_file=${MONITOR_SMTP_VALUES_FILE:-}
|
|
monitor_install_work_root=${MONITOR_INSTALL_WORK_ROOT:-}
|
|
monitor_url=${MONITOR_URL:-https://whoneedhelp.com/healthz/ready}
|
|
metrics_url=${MONITOR_METRICS_URL:-https://whoneedhelp.com/metrics}
|
|
monitor_calendar=${MONITOR_ON_CALENDAR:-'*:0/1'}
|
|
health_timeout=${MONITOR_HEALTH_TIMEOUT_SECONDS:-3}
|
|
metrics_timeout=${MONITOR_METRICS_TIMEOUT_SECONDS:-3}
|
|
backup_heartbeat_path=${MONITOR_BACKUP_HEARTBEAT_PATH:-}
|
|
backup_max_age=${MONITOR_BACKUP_MAX_AGE_SECONDS:-}
|
|
|
|
for command in awk install mktemp python3 realpath scp ssh stat; do
|
|
command -v "$command" >/dev/null 2>&1 || {
|
|
echo "Required command is unavailable: $command" >&2
|
|
exit 2
|
|
}
|
|
done
|
|
|
|
if [ -z "$monitor_install_work_root" ]; then
|
|
monitor_install_work_root=$ROOT/tmp/production-operations
|
|
install -d -m 700 "$monitor_install_work_root"
|
|
elif [ ! -d "$monitor_install_work_root" ]; then
|
|
echo "MONITOR_INSTALL_WORK_ROOT must be an existing directory when supplied." >&2
|
|
exit 2
|
|
fi
|
|
monitor_install_work_root=$(realpath "$monitor_install_work_root")
|
|
if [ ! -w "$monitor_install_work_root" ]; then
|
|
echo "MONITOR_INSTALL_WORK_ROOT must be an existing writable directory." >&2
|
|
exit 2
|
|
fi
|
|
|
|
if [ -n "$monitor_smtp_values_file" ]; then
|
|
if [ ! -f "$monitor_smtp_values_file" ]; then
|
|
echo "MONITOR_SMTP_VALUES_FILE must be an existing regular file." >&2
|
|
exit 2
|
|
fi
|
|
monitor_smtp_values_file=$(realpath "$monitor_smtp_values_file")
|
|
case "$(stat -c '%a' "$monitor_smtp_values_file")" in
|
|
400 | 600) ;;
|
|
*)
|
|
echo "MONITOR_SMTP_VALUES_FILE must have mode 0400 or 0600." >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
if [ -n "$monitor_smtp_values_file" ]; then
|
|
smtp_source=override
|
|
else
|
|
smtp_source=application
|
|
fi
|
|
|
|
source_host=$(ssh -G "$source_target" | awk '$1 == "hostname" {print $2; exit}')
|
|
monitor_host=$(ssh -G "$monitor_target" | awk '$1 == "hostname" {print $2; exit}')
|
|
if [ -z "$source_host" ] || [ -z "$monitor_host" ] || [ "$source_host" = "$monitor_host" ]; then
|
|
echo "The external monitor must resolve and run on a host other than production." >&2
|
|
exit 2
|
|
fi
|
|
|
|
case "$health_timeout" in
|
|
'' | *[!0-9]*) echo "MONITOR_HEALTH_TIMEOUT_SECONDS must be a positive integer." >&2; exit 2 ;;
|
|
0) echo "MONITOR_HEALTH_TIMEOUT_SECONDS must be a positive integer." >&2; exit 2 ;;
|
|
esac
|
|
|
|
case "$metrics_timeout" in
|
|
'' | *[!0-9]*) echo "MONITOR_METRICS_TIMEOUT_SECONDS must be a positive integer." >&2; exit 2 ;;
|
|
0) echo "MONITOR_METRICS_TIMEOUT_SECONDS must be a positive integer." >&2; exit 2 ;;
|
|
esac
|
|
|
|
if ! monitor_identity=$(ssh -o BatchMode=yes "$monitor_target" \
|
|
'printf "%s\n%s\n%s\n" "$(id -un)" "$HOME" "$(loginctl show-user "$(id -un)" --property=Linger --value)"'); then
|
|
echo "Unable to verify systemd user lingering on the external monitor host." >&2
|
|
exit 2
|
|
fi
|
|
monitor_user=$(printf '%s\n' "$monitor_identity" | awk 'NR == 1 {print; exit}')
|
|
monitor_home=$(printf '%s\n' "$monitor_identity" | awk 'NR == 2 {print; exit}')
|
|
monitor_linger=$(printf '%s\n' "$monitor_identity" | awk 'NR == 3 {print; exit}')
|
|
if [ -z "$monitor_user" ] || [ -z "$monitor_home" ] || [ "$monitor_linger" != yes ]; then
|
|
echo "External monitor installation requires systemd user lingering for '$monitor_user' on '$monitor_target'." >&2
|
|
echo "An administrator must run: sudo loginctl enable-linger '$monitor_user'" >&2
|
|
echo "Verify with: loginctl show-user '$monitor_user' --property=Linger --value" >&2
|
|
exit 2
|
|
fi
|
|
|
|
remote_root=${remote_root:-$monitor_home/.local/lib/who-need-help}
|
|
remote_config=${remote_config:-$monitor_home/.config/who-need-help/monitor.json}
|
|
remote_state=${remote_state:-$monitor_home/.local/state/who-need-help/monitor.json}
|
|
backup_heartbeat_path=${backup_heartbeat_path:-$monitor_home/.local/state/who-need-help/production-backup.json}
|
|
remote_systemd_dir=$monitor_home/.config/systemd/user
|
|
|
|
if [ -n "$backup_max_age" ]; then
|
|
case "$backup_max_age" in
|
|
*[!0-9]*) echo "MONITOR_BACKUP_MAX_AGE_SECONDS must be a positive integer." >&2; exit 2 ;;
|
|
0) echo "MONITOR_BACKUP_MAX_AGE_SECONDS must be a positive integer." >&2; exit 2 ;;
|
|
esac
|
|
if [ "$backup_heartbeat_path" != "$monitor_home/.local/state/who-need-help/production-backup.json" ]; then
|
|
echo "MONITOR_BACKUP_HEARTBEAT_PATH is outside the reviewed monitor state path." >&2
|
|
exit 2
|
|
fi
|
|
fi
|
|
|
|
work_dir=$(mktemp -d "$monitor_install_work_root/monitor-install.XXXXXX")
|
|
config="$work_dir/monitor.json"
|
|
service="$work_dir/who-need-help-production-monitor.service"
|
|
timer="$work_dir/who-need-help-production-monitor.timer"
|
|
|
|
cleanup() {
|
|
trap - 0 HUP INT TERM
|
|
rm -rf "$work_dir"
|
|
}
|
|
trap cleanup 0 HUP INT TERM
|
|
|
|
ssh -o BatchMode=yes "$source_target" \
|
|
"python3 - '$production_env' '$monitor_url' '$metrics_url' '$health_timeout' '$metrics_timeout' '$smtp_source' '$backup_heartbeat_path' '$backup_max_age'" >"$config" <<'PY'
|
|
import json
|
|
import shlex
|
|
import sys
|
|
|
|
(
|
|
path,
|
|
health_url,
|
|
metrics_url,
|
|
health_timeout,
|
|
metrics_timeout,
|
|
smtp_source,
|
|
backup_heartbeat_path,
|
|
backup_max_age,
|
|
) = sys.argv[1:]
|
|
smtp_keys = {
|
|
"SMTP_RELAY",
|
|
"SMTP_PORT",
|
|
"SMTP_USERNAME",
|
|
"SMTP_PASSWORD",
|
|
"SMTP_TLS",
|
|
"SMTP_SSL",
|
|
"EMAIL_FROM_ADDRESS",
|
|
"EMAIL_FROM_NAME",
|
|
"SUPPORT_INBOX_ADDRESS",
|
|
}
|
|
wanted = {"METRICS_TOKEN"}
|
|
if smtp_source == "application":
|
|
wanted.update(smtp_keys)
|
|
elif smtp_source != "override":
|
|
raise SystemExit("Unknown monitor SMTP source")
|
|
|
|
values = {}
|
|
with open(path, encoding="utf-8") as handle:
|
|
for raw_line in handle:
|
|
line = raw_line.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
key, value = line.split("=", 1)
|
|
if key not in wanted:
|
|
continue
|
|
parsed = shlex.split(value, comments=False, posix=True)
|
|
values[key] = parsed[0] if parsed else ""
|
|
|
|
missing = sorted(key for key in wanted if not values.get(key))
|
|
if missing:
|
|
raise SystemExit("Missing production monitor settings: " + ", ".join(missing))
|
|
|
|
smtp = {}
|
|
if smtp_source == "application":
|
|
smtp = {
|
|
"from_address": values["EMAIL_FROM_ADDRESS"],
|
|
"from_name": values["EMAIL_FROM_NAME"],
|
|
"implicit_ssl": values["SMTP_SSL"].lower() == "true",
|
|
"password": values["SMTP_PASSWORD"],
|
|
"port": int(values["SMTP_PORT"]),
|
|
"recipient": values["SUPPORT_INBOX_ADDRESS"],
|
|
"relay": values["SMTP_RELAY"],
|
|
"starttls": values["SMTP_TLS"].lower() == "always",
|
|
"username": values["SMTP_USERNAME"],
|
|
}
|
|
|
|
payload = {
|
|
"health_timeout_seconds": int(health_timeout),
|
|
"health_url": health_url,
|
|
"metrics_timeout_seconds": int(metrics_timeout),
|
|
"metrics_token": values["METRICS_TOKEN"],
|
|
"metrics_url": metrics_url,
|
|
"smtp": smtp,
|
|
}
|
|
if backup_max_age:
|
|
payload["backup"] = {
|
|
"heartbeat_path": backup_heartbeat_path,
|
|
"max_age_seconds": int(backup_max_age),
|
|
}
|
|
json.dump(payload, sys.stdout, ensure_ascii=False, indent=2, sort_keys=True)
|
|
sys.stdout.write("\n")
|
|
PY
|
|
chmod 600 "$config"
|
|
|
|
if [ -n "$monitor_smtp_values_file" ]; then
|
|
"$ROOT/scripts/override-production-monitor-smtp.py" \
|
|
"$config" "$monitor_smtp_values_file"
|
|
fi
|
|
|
|
cat >"$service" <<EOF
|
|
[Unit]
|
|
Description=Who Need Help independent production operations monitor
|
|
Wants=network-online.target
|
|
After=network-online.target
|
|
|
|
[Service]
|
|
Type=oneshot
|
|
ExecStart=/usr/bin/python3 $remote_root/production-external-monitor.py --config $remote_config --state $remote_state check
|
|
EOF
|
|
|
|
cat >"$timer" <<EOF
|
|
[Unit]
|
|
Description=Run the Who Need Help independent production operations monitor
|
|
|
|
[Timer]
|
|
OnCalendar=$monitor_calendar
|
|
Persistent=true
|
|
Unit=who-need-help-production-monitor.service
|
|
|
|
[Install]
|
|
WantedBy=timers.target
|
|
EOF
|
|
|
|
ssh -o BatchMode=yes "$monitor_target" \
|
|
"install -d -m 700 '$remote_root' '$monitor_home/.config/who-need-help' '$monitor_home/.local/state/who-need-help' '$remote_systemd_dir'"
|
|
scp -q "$ROOT/scripts/production-external-monitor.py" \
|
|
"$monitor_target:$remote_root/production-external-monitor.py"
|
|
scp -q "$config" "$monitor_target:$remote_config"
|
|
scp -q "$service" "$monitor_target:$remote_systemd_dir/who-need-help-production-monitor.service"
|
|
scp -q "$timer" "$monitor_target:$remote_systemd_dir/who-need-help-production-monitor.timer"
|
|
|
|
ssh -o BatchMode=yes "$monitor_target" \
|
|
"chmod 700 '$remote_root/production-external-monitor.py'; chmod 600 '$remote_config' '$remote_systemd_dir/who-need-help-production-monitor.service' '$remote_systemd_dir/who-need-help-production-monitor.timer'; systemctl --user daemon-reload; systemctl --user start who-need-help-production-monitor.service; systemctl --user enable --now who-need-help-production-monitor.timer"
|
|
|
|
printf 'External monitor installed on %s (%s).\n' "$monitor_target" "$monitor_host"
|
|
printf 'Persistent systemd user manager verified for %s (linger=yes).\n' "$monitor_user"
|
|
printf 'Health URL: %s\n' "$monitor_url"
|
|
printf 'Metrics URL: %s\n' "$metrics_url"
|
|
printf 'Schedule: %s; health timeout: %ss; metrics timeout: %ss.\n' \
|
|
"$monitor_calendar" "$health_timeout" "$metrics_timeout"
|
|
if [ -n "$backup_max_age" ]; then
|
|
printf 'Backup freshness: heartbeat %s; operator-selected maximum age %ss.\n' \
|
|
"$backup_heartbeat_path" "$backup_max_age"
|
|
else
|
|
echo "Backup freshness monitoring is not enabled because no operator-selected maximum age was supplied."
|
|
fi
|
|
echo "The SMTP and metrics credentials are stored only in a mode-0600 configuration on the external monitor host."
|
|
if [ -n "$monitor_smtp_values_file" ]; then
|
|
echo "The external monitor uses the independently supplied SMTP credential set."
|
|
else
|
|
echo "The external monitor currently mirrors the production application SMTP credential set."
|
|
fi
|