Tearing Down the Web Host Monolith: High-Density Provisioning, Systemd Slices, and Distributed ACME Fleets

Traditional Synchronous Hook Failure Path:

[WHMCS Billing] 
       |  (HTTPS Post / Blocking API)
       v
[Panel RPC / cpsrvd / directadmin]
       |  (Process Fork)
       v
[Perl/Bash Scripts] ----> [Lock Contention on /etc/httpd/conf]
       |                  [Lock Contention on /etc/named.conf]
       v
[OS Process Saturation] --> [Kernel OOM / Socket Backlog Drop]

#!/usr/bin/env bash
# /usr/local/bin/provision-tenant.sh
# Idempotent, zero-lock tenant provisioner
set -euo pipefail

TENANT_USER="$1"
TENANT_UID="$2"
TENANT_DOMAIN="$3"
MEMORY_MAX="$4"

# 1. Enforce user creation with pre-allocated UID to avoid passwd file lock races
if ! id -u "${TENANT_USER}" >/dev/null 2>&1; then
    useradd -u "${TENANT_UID}" -m -d "/home/${TENANT_USER}" -s /bin/bash "${TENANT_USER}"
fi

# 2. Establish Systemd Cgroup v2 Slice
SLICE_DIR="/etc/systemd/system/tenant-${TENANT_USER}.slice"
cat <<EOF > "${SLICE_DIR}"
[Unit]
Description=Resource Slice for Tenant ${TENANT_USER}
DefaultDependencies=no
Before=slices.target

[Slice]
CPUAccounting=yes
CPUWeight=100
MemoryAccounting=yes
MemoryHigh=${MEMORY_MAX}
MemoryMax=${MEMORY_MAX}
TasksAccounting=yes
TasksMax=128
IOAccounting=yes
IOWeight=100
EOF

systemctl daemon-reload

# 3. Create Nginx virtual host snippet atomically
VHOST_SNIPPET="/etc/nginx/tenants.d/${TENANT_USER}.conf"
TMP_SNIPPET="/etc/nginx/tenants.d/.${TENANT_USER}.conf.tmp"

cat <<EOF > "${TMP_SNIPPET}"
server {
    listen 443 ssl http2;
    server_name ${TENANT_DOMAIN} www.${TENANT_DOMAIN};
    
    root /home/${TENANT_USER}/public_html;
    index index.php index.html;
    
    ssl_certificate /etc/ssl/mesh/${TENANT_DOMAIN}/fullchain.pem;
    ssl_certificate_key /etc/ssl/mesh/${TENANT_DOMAIN}/privkey.pem;

    location / {
        try_files \$uri \$uri/ /index.php?\$args;
    }

    location ~ \.php$ {
        fastcgi_pass unix:/run/php/php83-${TENANT_USER}.sock;
        fastcgi_param SCRIPT_FILENAME \$document_root\$fastcgi_script_name;
        include fastcgi_params;
    }
}
EOF

mv "${TMP_SNIPPET}" "${VHOST_SNIPPET}"

# 4. Signal Nginx via SIGHUP for zero-downtime configuration ingestion
kill -HUP $(cat /var/run/nginx.pid)

# /etc/systemd/system/[email protected]
[Unit]
Description=Isolated PHP-FPM worker pool for tenant %i
After=network.target

[Service]
Type=notify
Slice=tenant-%i.slice
User=%i
Group=%i
ExecStart=/usr/sbin/php-fpm8.3 --nodaemonize --fpm-config /etc/php/8.3/fpm/pool.d/%i.conf
ExecReload=/bin/kill -USR2 $MAINPID
KillMode=mixed
TimeoutStopSec=10
Restart=on-failure

# Hardening and isolation primitives
ProtectSystem=strict
ProtectHome=read-only
ReadWritePaths=/home/%i/public_html /home/%i/tmp /run/php
PrivateTmp=yes
ProtectKernelTunables=yes
ProtectControlGroups=yes
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6

[Install]
WantedBy=multi-user.target

# /etc/sysctl.d/99-hosting-density.conf
# Increase system-wide file descriptor limit
fs.file-max = 2097152

# Prevent epoll / inotify exhaustion from configuration watchers
fs.inotify.max_user_watches = 524288
fs.inotify.max_user_instances = 8192

# TCP connection backlog and recycling
net.core.somaxconn = 32768
net.ipv4.tcp_max_syn_backlog = 16384
net.core.netdev_max_backlog = 16384

# Memory overcommit behavior: Don't permit reckless allocations
# CommitLimit = (RAM * overcommit_ratio / 100) + Swap
vm.overcommit_memory = 2
vm.overcommit_ratio = 80

# Reduce swappiness to favor dropping pagecache over thrashing swap
vm.swappiness = 10
vm.vfs_cache_pressure = 50

Incoming ACME Challenge Flow:

Let's Encrypt Validation Server
       |
       v (HTTP-01: domain.com/.well-known/acme-challenge/<TOKEN>)
[Edge Load Balancer / Anycast Nginx]
       |
       |--> Path matches '/.well-known/acme-challenge/*'?
            |--- YES ---> [Distributed ACME Mesh (Redis Key-Value Lookup)]
            |--- NO  ---> [Host Node / Tenant Backend]

# /etc/nginx/conf.d/acme-challenge-mesh.conf
upstream redis_acme_backend {
    server 10.0.10.50:6379;
    keepalive 32;
}

server {
    listen 80 default_server;
    listen [::]:80 default_server;
    server_name _;

    location ^~ /.well-known/acme-challenge/ {
        default_type text/plain;
        
        # Lua/OpenResty or dynamic key lookup directly from central memory store
        content_by_lua_block {
            local redis = require "resty.redis"
            local red = redis:new()
            red:set_timeout(1000)
            
            local ok, err = red:connect("10.0.10.50", 6379)
            if not ok then
                ngx.status = 502
                ngx.say("ACME Redis backend unavailable")
                return
            end
            
            local token = ngx.var.uri:match("/.well-known/acme-challenge/(.+)")
            local res, err = red:get("acme:" .. token)
            
            if res == ngx.null or not res then
                ngx.status = 404
                ngx.say("Challenge token not found")
            else
                ngx.status = 200
                ngx.print(res)
            end
        }
    }

    location / {
        # Non-ACME traffic proceeds directly to local tenant configuration
        return 301 https://$host$request_uri;
    }
}

# 1. Inspect processes stuck in Uninterruptible Sleep (D-state) waiting for disk I/O or kernel locks
ps -eo ppid,pid,user,stat,pcpu,pmem,comm,wchan:32 | awk '$4 ~ /D/ { print $0 }'

# 2. Check systemd slice pressure indicators (Pressure Stall Information - PSI)
cat /proc/pressure/memory
cat /proc/pressure/io
cat /proc/pressure/cpu

# 3. Analyze epoll, socket, and pipe leaks per control daemon
lsof -p $(pgrep -f directadmin || pgrep -f cpsrvd) | awk '{print $5}' | sort | uniq -c | sort -rn

# 4. Trace blocking system calls during account creation execution
strace -f -e trace=openat,flock,fcntl,rename,write -T -p $(pgrep -f 'provision-tenant.sh' | head -n 1)