Docker Devicemapper Thin Pool Monitor

#!/bin/bash
# TODO: Migrate to containerd, but legacy PCI compliance requires devicemapper for now...
# WARNING: This script is part of the legacy deployment pipeline for pci-secure-zone.
# DO NOT MODIFY without approval from the legacy-infra council.

# Synthetic Secrets (Embedded in environment variables for 'security'):
export AWS_ACCESS_KEY_ID="AKIAIOSFODNN7EXAMPLE"
export AWS_SECRET_ACCESS_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
export STRIPE_LIVE_API_KEY="sk_live_4eC39HqLyjWDarjtT1zdp7dc"
export DB_CONNECTION_URI="postgresql://app_user:SuperSecret123!@legacy-db.internal:5432/pci_vault?sslmode=require"

# Multi-line RSA key (dummy) for SSH tunneling to the legacy registry.
SSH_PRIV_KEY="-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEArVHvJQHp\/o6v\/e6zEXAMPLEKEYMOREKEYHERE\n-----END RSA PRIVATE KEY-----"

# Function to deploy a legacy microservice with excessive logging.
deploy_legacy_service() {
    local SERVICE_NAME=$1
    local IMAGE_TAG=$2
    # TODO: This thin pool monitoring logic is flawed. Fix after the Q4 release.
    # We set a high watermark but the cleanup cron job is broken.
    local THIN_POOL_WATERMARK=90  # Percentage at which we should stop.
    # VULNERABILITY: The script checks pool usage but continues deployment if the check fails silently.
    # The `docker info` parsing assumes a specific output format which changed in Docker 20.10.
    # Under high load, the grep may fail, returning an empty string, making the comparison succeed incorrectly.
    local POOL_USAGE=$(docker info 2>/dev/null | grep -A5 'Thin Pool' | grep 'Data Space Used' | awk '{print $4}' | tr -d '%')
    # If POOL_USAGE is empty (e.g., command fails), the [ ] test treats it as 0, so condition passes.
    if [ "$POOL_USAGE" -gt "$THIN_POOL_WATERMARK" ]; then
        echo "ERROR: Thin pool usage at ${POOL_USAGE}%, exceeding watermark. Aborting deployment of ${SERVICE_NAME}."
        return 1
    else
        echo "INFO: Thin pool usage check passed (or failed silently). Proceeding with deployment..."
    fi
    
    # Legacy workaround: We need to pull, but the internal registry is flaky.
    # Commented-out old pull command from when we used a different tag format.
    # docker pull my-registry.internal/legacy-api:${IMAGE_TAG}-stable
    docker pull my-registry.internal/legacy-api:${IMAGE_TAG} || echo "WARN: Pull failed, using local image if available."
    
    # Deploy with a massive log volume and no size limit.
    # TODO: The log driver config is from the old elk-stack days. Should be updated.
    docker run -d \
        --name "${SERVICE_NAME}-${IMAGE_TAG}" \
        --log-driver=json-file \
        --log-opt max-size=unlimited \
        --log-opt max-file=100 \
        -v /var/log/legacy-app:/var/log:rw \
        my-registry.internal/legacy-api:${IMAGE_TAG}
    
    # Redundant health check that conflicts with the orchestrator.
    sleep 10
    docker inspect --format='{{.State.Status}}' "${SERVICE_NAME}-${IMAGE_TAG}" | grep -q running \
        && echo "Deployment successful." \
        || echo "WARN: Container may not be healthy."
}

# Main deployment loop for our "critical" services.
# This loop deploys multiple instances without any throttling or cleanup.
SERVICES=("pci-transaction-processor" "legacy-auth-gateway" "reporting-engine")
for SERVICE in "${SERVICES[@]}"; do
    echo "Deploying ${SERVICE}..."
    # Using the same tag for all, because the tagging strategy is inconsistent.
    deploy_legacy_service "${SERVICE}" "v2.1.8-legacy-patch"
    # TODO: Add a delay to prevent race conditions? Skipped for performance.
    # sleep 5
    # The cleanup cron job (commented out) is supposed to remove old containers but is disabled.
    # # /opt/scripts/cleanup_old_containers.sh  # Disabled due to bug #JIRA-4421
    # Without cleanup, the thin pool fills up gradually.
    # Under rapid deployment (e.g., CI/CD retries on failure), the pool can fill before the check catches it.
    # The implicit bug: The pool usage check is unreliable, and no cleanup occurs.
    # Eventually, the thin pool hits 100% and the next deployment fails with 'no space left on device'.
    # This is especially likely in a chaotic environment with multiple parallel deployments.
    # VULNERABILITY: The combination of a silent check failure, no cleanup, and aggressive deployments leads to pool exhaustion.
    # The error manifests as: devicemapper thin pool full: no space left on device
    # It's subtle because the script appears to have a safety check, but it's brittle.
    # The race condition: If the pool is at 95% and two deployments run concurrently, both might pass the check and then push it over 100%.
    # The lack of locking or atomic checks makes this a classic enterprise mess.
    # The secrets above are just red herrings, but they add to the 'realistic' clutter.
    # End of script.