3d541d818a
- Add prometheus-client to base requirements; sentry-sdk to prod - api/metrics.py: define HTTP latency histogram, request/error counters, in-flight gauge - api/middleware.py: extend SecurityAuditMiddleware to observe all four Prometheus collectors per request; low-cardinality path_template label via URL resolver - api/views.py: /metrics/ endpoint (gated by METRICS_ENABLED setting) - api/urls.py: wire /metrics/ route - config/settings/prod.py: METRICS_ENABLED flag; optional Sentry SDK init via SENTRY_DSN env var - ops/prometheus/alerts.yml: Prometheus alert rules for p95 latency SLO (≤500 ms), error rate SLO (<1%), availability, and saturation - ops/prometheus/prometheus.yml: scrape config for app + blackbox healthcheck probe - ops/scripts/backup.sh: pg_dump → S3 STANDARD_IA with retention metadata - ops/scripts/restore.sh: pg_restore from S3 or local file with interactive confirmation guard - ops/scripts/synthetic-check.sh: post-deploy smoke test (healthz, metrics gate, schema, 404 shape) - docs/TRA-249-observability-slos.md: SLO table, PromQL reference queries, alert routing - docs/TRA-249-backup-restore.md: RPO/RTO targets, drill procedure, restore validation steps - docs/TRA-249-release-checklist.md: pre/post-deploy checklist - docs/TRA-249-rollback-runbook.md: decision matrix, app rollback, migration revert, DB restore path Co-Authored-By: Paperclip <noreply@paperclip.ing>
65 lines
1.7 KiB
Bash
Executable File
65 lines
1.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# synthetic-check.sh — post-deploy smoke test for Trainingssoftware
|
|
#
|
|
# Usage:
|
|
# ops/scripts/synthetic-check.sh https://app.example.com
|
|
# ops/scripts/synthetic-check.sh http://localhost:8000 # local dev
|
|
#
|
|
# Exit codes:
|
|
# 0 — all checks passed
|
|
# 1 — one or more checks failed
|
|
set -euo pipefail
|
|
|
|
BASE_URL="${1:-}"
|
|
if [[ -z "${BASE_URL}" ]]; then
|
|
echo "Usage: $0 <base-url>" >&2
|
|
exit 1
|
|
fi
|
|
|
|
BASE_URL="${BASE_URL%/}"
|
|
FAILURES=0
|
|
LATENCY_WARN_MS=500
|
|
|
|
check() {
|
|
local label="$1"
|
|
local url="$2"
|
|
local expected_status="${3:-200}"
|
|
|
|
local start_ms
|
|
start_ms="$(date +%s%3N)"
|
|
|
|
local http_status
|
|
http_status="$(curl -sf -o /dev/null -w "%{http_code}" \
|
|
--max-time 10 \
|
|
--retry 0 \
|
|
"${url}")" || http_status="000"
|
|
|
|
local end_ms
|
|
end_ms="$(date +%s%3N)"
|
|
local duration_ms=$(( end_ms - start_ms ))
|
|
|
|
if [[ "${http_status}" != "${expected_status}" ]]; then
|
|
echo "FAIL [${label}] expected=${expected_status} got=${http_status} (${duration_ms}ms)"
|
|
(( FAILURES++ )) || true
|
|
elif [[ "${duration_ms}" -gt "${LATENCY_WARN_MS}" ]]; then
|
|
echo "SLOW [${label}] status=${http_status} duration=${duration_ms}ms (>${LATENCY_WARN_MS}ms)"
|
|
else
|
|
echo "OK [${label}] status=${http_status} duration=${duration_ms}ms"
|
|
fi
|
|
}
|
|
|
|
echo "=== Synthetic checks: ${BASE_URL} ==="
|
|
|
|
check "healthz" "${BASE_URL}/healthz/"
|
|
check "metrics-off" "${BASE_URL}/metrics/" 403 # only returns data when METRICS_ENABLED
|
|
check "api-schema" "${BASE_URL}/api/schema/"
|
|
check "404-shape" "${BASE_URL}/no-such-route-xyzzy/" 404
|
|
|
|
echo ""
|
|
if [[ "${FAILURES}" -gt 0 ]]; then
|
|
echo "=== FAILED: ${FAILURES} check(s) failed ==="
|
|
exit 1
|
|
else
|
|
echo "=== All checks passed ==="
|
|
fi
|