Cron Job Monitoring: The Complete Developer's Guide

# This backup job runs every night at 2 AM
0 2 * * * /usr/local/bin/backup-database.sh

# What the job does:
# 1. Dumps the database
# 2. Compresses the dump
# 3. Uploads to S3
# 4. Cleans up local files

# What goes wrong:
# - AWS credentials expire → Upload fails, exit code 0 (script continues)
# - Disk is full → Dump fails, script exits silently
# - Network issues → Upload times out, no retry logic
# - Script has a bug → Stops at step 2, returns 0

# Result: No backups for weeks, discovered only during disaster recovery

# Process pending payments every 15 minutes
*/15 * * * * /app/bin/process-payments

# What goes wrong:
# - Database connection pool exhausted → Job hangs indefinitely
# - Payment gateway API changes → All transactions fail
# - Server memory issue → Process killed by OOM, no logging
# - Timezone bug → Job runs at wrong time, misses payment windows

# Result: Thousands of failed payments, angry customers, revenue loss

# Generate daily sales report at 6 AM
0 6 * * * /usr/local/bin/generate-sales-report.sh

# What goes wrong:
# - Report server migrated, crontab not updated → Never runs
# - Dependencies updated → Python script breaks with import error
# - Email server down → Report generated but never sent
# - Report takes 4 hours instead of 30 minutes → Still running when next job starts

# Result: Executives don't get reports, business decisions delayed

# Cron logs look like this:
Nov 15 02:00:01 server CRON[12345]: (root) CMD (/usr/local/bin/backup.sh)

# That's it. No indication of:
# - Did the script succeed?
# - How long did it take?
# - What was the output?
# - Were there errors?

# This security audit log cleanup should run daily
0 3 * * * /usr/local/bin/rotate-audit-logs.sh

# When it fails:
# - Compliance violations (logs not retained properly)
# - Storage costs spike (logs never archived)
# - Security events lost (old logs overwritten)
# - Audit failures during inspections

# You think this runs every day at midnight:
0 0 * * * /home/user/backup.sh

# But it never runs because:
# - backup.sh doesn't have execute permissions
# - User 'user' was deleted
# - Cron daemon crashed and wasn't restarted

#!/bin/bash
# backup-database.sh

# This will exit with code 1 if pg_dump fails
pg_dump mydb > /backups/db-$(date +%Y%m%d).sql

if [ $? -ne 0 ]; then
    # Even with this check, cron won't alert you
    echo "Backup failed!" >&2
    exit 1
fi

# Upload to S3
aws s3 cp /backups/db-$(date +%Y%m%d).sql s3://my-bucket/

#!/usr/bin/env python3
# data-sync.py

import time
import psycopg2

def sync_data():
    # This query locks and never returns
    conn = psycopg2.connect("dbname=mydb")
    cursor = conn.cursor()
    
    # Query has a table lock, waits forever
    cursor.execute("""
        SELECT * FROM large_table 
        WHERE status = 'pending' 
        FOR UPDATE
    """)
    
    # Never reaches here
    process_rows(cursor.fetchall())

if __name__ == "__main__":
    sync_data()

#!/bin/bash
# process-orders.sh

# Processes new orders from database
ORDER_COUNT=$(psql -t -c "SELECT COUNT(*) FROM orders WHERE status='new'")

# Bug: Variable is empty due to connection error
# But script continues anyway
for order_id in $(psql -t -c "SELECT id FROM orders WHERE status='new'"); do
    process_order $order_id
done

# Returns 0 even though no orders were processed!
exit 0

#!/bin/bash
# generate-reports.sh

# Generates CSV reports for all customers
for customer_id in $(get_customer_ids); do
    # Each report is 500MB, stored in /tmp
    generate_report $customer_id > /tmp/report-${customer_id}.csv
done

# Problem: /tmp fills up after 10 reports
# Remaining 990 customers get no reports
# Script exits 0 because loop completed

# Your existing cron job
0 2 * * * /usr/local/bin/backup-database.sh

# Enhanced with Seiri monitoring
0 2 * * * curl -fsS -m 10 --retry 5 https://cloud.seiri.app/webhook/<cid>:<unique-id>/start && /usr/local/bin/backup-database.sh && curl -fsS -m 10 --retry 5 https://cloud.seiri.app/webhook/<cid>:<unique-id>/success || curl -fsS -m 10 --retry 5 https://cloud.seiri.app/webhook/<cid>:<unique-id>/fail

# Before job starts - "I'm about to run"
curl -fsS -m 10 --retry 5 https://cloud.seiri.app/webhook/<cid>:<unique-id>/start

# Run the actual job
/usr/local/bin/backup-database.sh

# After success - "I completed successfully"
curl -fsS -m 10 --retry 5 https://cloud.seiri.app/webhook/<cid>:<unique-id>/success

# On failure - "I failed"
curl -fsS -m 10 --retry 5 https://cloud.seiri.app/webhook/<cid>:<unique-id>/fail

#!/bin/bash
# cron-wrapper.sh - Universal cron job wrapper with monitoring

set -euo pipefail

# Configuration
SEIRI_PING_URL="${SEIRI_PING_URL:-}"
JOB_NAME="${1:-unknown-job}"
shift
JOB_COMMAND="$@"

# Validate configuration
if [ -z "$SEIRI_PING_URL" ]; then
    echo "Error: SEIRI_PING_URL not set" >&2
    exit 1
fi

# Function to send ping with retry logic
send_ping() {
    local status="$1"
    local max_attempts=3
    local attempt=1
    
    while [ $attempt -le $max_attempts ]; do
        if curl -fsS -m 10 "${SEIRI_PING_URL}/${status}" 2>/dev/null; then
            return 0
        fi
        echo "Ping attempt $attempt failed, retrying..." >&2
        sleep 2
        attempt=$((attempt + 1))
    done
    
    echo "Warning: Failed to send $status ping after $max_attempts attempts" >&2
    return 1
}

# Send start ping
send_ping "start"

# Record start time
START_TIME=$(date +%s)

# Execute the job and capture output
JOB_OUTPUT=$(mktemp)
JOB_EXIT_CODE=0

if $JOB_COMMAND > "$JOB_OUTPUT" 2>&1; then
    JOB_EXIT_CODE=0
    PING_STATUS="success"
else
    JOB_EXIT_CODE=$?
    PING_STATUS="fail"
fi

# Calculate duration
END_TIME=$(date +%s)
DURATION=$((END_TIME - START_TIME))

# Send completion ping with metadata
send_ping "${PING_STATUS}"

# Optional: Send detailed metrics
curl -fsS -X POST "${SEIRI_PING_URL}" \
    -H "Content-Type: application/json" \
    -d @- <<EOF 2>/dev/null || true
{
    "job_name": "${JOB_NAME}",
    "exit_code": ${JOB_EXIT_CODE},
    "duration_seconds": ${DURATION},
    "output_lines": $(wc -l < "$JOB_OUTPUT"),
    "timestamp": $(date +%s)
}
EOF

# If job failed, output the logs
if [ $JOB_EXIT_CODE -ne 0 ]; then
    echo "Job ${JOB_NAME} failed with exit code ${JOB_EXIT_CODE}" >&2
    echo "Job output:" >&2
    cat "$JOB_OUTPUT" >&2
fi

# Cleanup
rm -f "$JOB_OUTPUT"

exit $JOB_EXIT_CODE

# Set the Seiri ping URL
SEIRI_PING_URL=https://cloud.seiri.app/webhook/<cid>:<unique-id>

# Use the wrapper for all jobs
0 2 * * * /usr/local/bin/cron-wrapper.sh "database-backup" /usr/local/bin/backup-database.sh
0 6 * * * /usr/local/bin/cron-wrapper.sh "sales-report" /usr/local/bin/generate-sales-report.sh
*/15 * * * * /usr/local/bin/cron-wrapper.sh "process-payments" /app/bin/process-payments

#!/usr/bin/env python3
"""
cron_monitor.py - Python decorator for cron job monitoring
"""

import os
import sys
import time
import requests
import functools
import traceback
from typing import Callable, Any

class CronMonitor:
    """Monitor cron jobs with Seiri heartbeat pings"""
    
    def __init__(self, ping_url: str, job_name: str):
        self.ping_url = ping_url
        self.job_name = job_name
        self.session = requests.Session()
        self.session.headers.update({'User-Agent': 'Seiri-Cron-Monitor/1.0'})
    
    def send_ping(self, status: str, retry: int = 3) -> bool:
        """Send ping with retry logic"""
        url = f"{self.ping_url}/{status}"
        
        for attempt in range(retry):
            try:
                response = self.session.get(url, timeout=10)
                response.raise_for_status()
                return True
            except requests.RequestException as e:
                if attempt == retry - 1:
                    print(f"Failed to send {status} ping: {e}", file=sys.stderr)
                    return False
                time.sleep(2 ** attempt)  # Exponential backoff
        
        return False
    
    def send_metrics(self, metrics: dict) -> bool:
        """Send detailed metrics via POST"""
        try:
            response = self.session.post(
                self.ping_url,
                json=metrics,
                timeout=10
            )
            response.raise_for_status()
            return True
        except requests.RequestException as e:
            print(f"Failed to send metrics: {e}", file=sys.stderr)
            return False
    
    def __call__(self, func: Callable) -> Callable:
        """Decorator to wrap cron job functions"""
        
        @functools.wraps(func)
        def wrapper(*args, **kwargs) -> Any:
            # Send start ping
            self.send_ping('start')
            
            start_time = time.time()
            exit_code = 0
            error_message = None
            result = None
            
            try:
                # Execute the job
                result = func(*args, **kwargs)
                status = 'success'
                
            except Exception as e:
                # Job failed with exception
                status = 'fail'
                exit_code = 1
                error_message = str(e)
                
                # Log the full traceback
                print(f"Job {self.job_name} failed:", file=sys.stderr)
                traceback.print_exc()
                
                raise  # Re-raise the exception
                
            finally:
                # Calculate duration
                duration = time.time() - start_time
                
                # Send completion ping
                self.send_ping(status)
                
                # Send detailed metrics
                metrics = {
                    'job_name': self.job_name,
                    'status': status,
                    'exit_code': exit_code,
                    'duration_seconds': round(duration, 2),
                    'timestamp': int(time.time())
                }
                
                if error_message:
                    metrics['error_message'] = error_message
                
                self.send_metrics(metrics)
            
            return result
        
        return wrapper


# Example usage
SEIRI_PING_URL = os.getenv('SEIRI_PING_URL')

if not SEIRI_PING_URL:
    raise ValueError("SEIRI_PING_URL environment variable not set")

@CronMonitor(SEIRI_PING_URL, 'database-backup')
def backup_database():
    """Daily database backup job"""
    import subprocess
    
    print("Starting database backup...")
    
    # Run pg_dump
    result = subprocess.run(
        ['pg_dump', '-h', 'localhost', '-U', 'postgres', 'mydb'],
        capture_output=True,
        check=True
    )
    
    # Save to file
    backup_file = f"/backups/db-{time.strftime('%Y%m%d')}.sql"
    with open(backup_file, 'wb') as f:
        f.write(result.stdout)
    
    print(f"Backup saved to {backup_file}")
    
    # Upload to S3
    subprocess.run(
        ['aws', 's3', 'cp', backup_file, 's3://my-bucket/backups/'],
        check=True
    )
    
    print("Backup uploaded to S3")
    
    return backup_file


@CronMonitor(SEIRI_PING_URL, 'process-orders')
def process_pending_orders():
    """Process pending orders from database"""
    import psycopg2
    
    conn = psycopg2.connect(
        host='localhost',
        database='mydb',
        user='postgres'
    )
    
    cursor = conn.cursor()
    
    # Get pending orders
    cursor.execute("SELECT id FROM orders WHERE status = 'pending' LIMIT 100")
    order_ids = [row[0] for row in cursor.fetchall()]
    
    if not order_ids:
        print("No pending orders to process")
        return 0
    
    processed_count = 0
    failed_count = 0
    
    for order_id in order_ids:
        try:
            # Process each order
            cursor.execute("""
                UPDATE orders 
                SET status = 'processed', processed_at = NOW() 
                WHERE id = %s
            """, (order_id,))
            conn.commit()
            processed_count += 1
            
        except Exception as e:
            print(f"Failed to process order {order_id}: {e}", file=sys.stderr)
            conn.rollback()
            failed_count += 1
    
    cursor.close()
    conn.close()
    
    print(f"Processed {processed_count} orders, {failed_count} failed")
    
    # Raise exception if too many failures
    if failed_count > processed_count * 0.1:  # More than 10% failed
        raise Exception(f"Too many order processing failures: {failed_count}/{len(order_ids)}")
    
    return processed_count


if __name__ == '__main__':
    # This script can be called directly from cron
    import sys
    
    if len(sys.argv) < 2:
        print("Usage: cron_jobs.py <job_name>", file=sys.stderr)
        sys.exit(1)
    
    job_name = sys.argv[1]
    
    if job_name == 'backup':
        backup_database()
    elif job_name == 'process-orders':
        process_pending_orders()
    else:
        print(f"Unknown job: {job_name}", file=sys.stderr)
        sys.exit(1)

# Export Seiri ping URL
SEIRI_PING_URL=https://cloud.seiri.app/webhook/<cid>:<unique-id>

# Run different jobs
0 2 * * * /usr/local/bin/cron_jobs.py backup
*/15 * * * * /usr/local/bin/cron_jobs.py process-orders

#!/usr/bin/env node
/**
 * cron-monitor.js - Node.js cron job monitoring
 */

const axios = require('axios');
const { performance } = require('perf_hooks');

class CronMonitor {
    constructor(pingUrl, jobName) {
        this.pingUrl = pingUrl;
        this.jobName = jobName;
        this.client = axios.create({
            timeout: 10000,
            headers: { 'User-Agent': 'Seiri-Cron-Monitor/1.0' }
        });
    }
    
    async sendPing(status, retries = 3) {
        const url = `${this.pingUrl}/${status}`;
        
        for (let attempt = 0; attempt < retries; attempt++) {
            try {
                await this.client.get(url);
                return true;
            } catch (error) {
                if (attempt === retries - 1) {
                    console.error(`Failed to send ${status} ping:`, error.message);
                    return false;
                }
                await new Promise(resolve => setTimeout(resolve, Math.pow(2, attempt) * 1000));
            }
        }
        
        return false;
    }
    
    async sendMetrics(metrics) {
        try {
            await this.client.post(this.pingUrl, metrics);
            return true;
        } catch (error) {
            console.error('Failed to send metrics:', error.message);
            return false;
        }
    }
    
    async wrap(jobFunction) {
        // Send start ping
        await this.sendPing('start');
        
        const startTime = performance.now();
        let status = 'success';
        let exitCode = 0;
        let errorMessage = null;
        let result = null;
        
        try {
            // Execute the job
            result = await jobFunction();
            
        } catch (error) {
            // Job failed
            status = 'fail';
            exitCode = 1;
            errorMessage = error.message;
            
            console.error(`Job ${this.jobName} failed:`, error);
            throw error;
            
        } finally {
            // Calculate duration
            const duration = (performance.now() - startTime) / 1000;
            
            // Send completion ping
            await this.sendPing(status);
            
            // Send detailed metrics
            const metrics = {
                job_name: this.jobName,
                status,
                exit_code: exitCode,
                duration_seconds: Math.round(duration * 100) / 100,
                timestamp: Math.floor(Date.now() / 1000)
            };
            
            if (errorMessage) {
                metrics.error_message = errorMessage;
            }
            
            await this.sendMetrics(metrics);
        }
        
        return result;
    }
}

// Example usage
const SEIRI_PING_URL = process.env.SEIRI_PING_URL;

if (!SEIRI_PING_URL) {
    console.error('Error: SEIRI_PING_URL environment variable not set');
    process.exit(1);
}

// Database backup job
async function backupDatabase() {
    const { exec } = require('child_process');
    const { promisify } = require('util');
    const execAsync = promisify(exec);
    
    console.log('Starting database backup...');
    
    const date = new Date().toISOString().split('T')[0].replace(/-/g, '');
    const backupFile = `/backups/db-${date}.sql`;
    
    // Run pg_dump
    await execAsync(`pg_dump -h localhost -U postgres mydb > ${backupFile}`);
    console.log(`Backup saved to ${backupFile}`);
    
    // Upload to S3
    await execAsync(`aws s3 cp ${backupFile} s3://my-bucket/backups/`);
    console.log('Backup uploaded to S3');
    
    return backupFile;
}

// Process orders job
async function processOrders() {
    const { Client } = require('pg');
    
    const client = new Client({
        host: 'localhost',
        database: 'mydb',
        user: 'postgres'
    });
    
    await client.connect();
    
    try {
        // Get pending orders
        const result = await client.query(
            "SELECT id FROM orders WHERE status = 'pending' LIMIT 100"
        );
        
        if (result.rows.length === 0) {
            console.log('No pending orders to process');
            return 0;
        }
        
        let processedCount = 0;
        let failedCount = 0;
        
        for (const row of result.rows) {
            try {
                await client.query(
                    "UPDATE orders SET status = 'processed', processed_at = NOW() WHERE id = $1",
                    [row.id]
                );
                processedCount++;
            } catch (error) {
                console.error(`Failed to process order ${row.id}:`, error.message);
                failedCount++;
            }
        }
        
        console.log(`Processed ${processedCount} orders, ${failedCount} failed`);
        
        // Fail if too many errors
        if (failedCount > result.rows.length * 0.1) {
            throw new Error(`Too many failures: ${failedCount}/${result.rows.length}`);
        }
        
        return processedCount;
        
    } finally {
        await client.end();
    }
}

// Main execution
async function main() {
    const jobName = process.argv[2];
    
    if (!jobName) {
        console.error('Usage: node cron-monitor.js <job_name>');
        process.exit(1);
    }
    
    let monitor;
    let jobFunction;
    
    if (jobName === 'backup') {
        monitor = new CronMonitor(SEIRI_PING_URL, 'database-backup');
        jobFunction = backupDatabase;
    } else if (jobName === 'process-orders') {
        monitor = new CronMonitor(SEIRI_PING_URL, 'process-orders');
        jobFunction = processOrders;
    } else {
        console.error(`Unknown job: ${jobName}`);
        process.exit(1);
    }
    
    try {
        await monitor.wrap(jobFunction);
        process.exit(0);
    } catch (error) {
        process.exit(1);
    }
}

if (require.main === module) {
    main();
}

module.exports = { CronMonitor };

SEIRI_PING_URL=https://cloud.seiri.app/webhook/<cid>:<unique-id>

0 2 * * * /usr/local/bin/node /app/cron-monitor.js backup
*/15 * * * * /usr/local/bin/node /app/cron-monitor.js process-orders

<?php
/**
 * CronMonitor.php - PHP cron job monitoring
 */

class CronMonitor {
    private $pingUrl;
    private $jobName;
    private $timeout = 10;
    private $retries = 3;
    
    public function __construct($pingUrl, $jobName) {
        $this->pingUrl = $pingUrl;
        $this->jobName = $jobName;
    }
    
    private function sendPing($status) {
        $url = $this->pingUrl . '/' . $status;
        
        for ($attempt = 0; $attempt < $this->retries; $attempt++) {
            $ch = curl_init();
            curl_setopt($ch, CURLOPT_URL, $url);
            curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
            curl_setopt($ch, CURLOPT_TIMEOUT, $this->timeout);
            curl_setopt($ch, CURLOPT_USERAGENT, 'Seiri-Cron-Monitor/1.0');
            
            $result = curl_exec($ch);
            $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
            curl_close($ch);
            
            if ($httpCode >= 200 && $httpCode < 300) {
                return true;
            }
            
            if ($attempt < $this->retries - 1) {
                sleep(pow(2, $attempt));
            }
        }
        
        error_log("Failed to send $status ping after {$this->retries} attempts");
        return false;
    }
    
    private function sendMetrics($metrics) {
        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $this->pingUrl);
        curl_setopt($ch, CURLOPT_POST, true);
        curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($metrics));
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($ch, CURLOPT_TIMEOUT, $this->timeout);
        curl_setopt($ch, CURLOPT_HTTPHEADER, [
            'Content-Type: application/json',
            'User-Agent: Seiri-Cron-Monitor/1.0'
        ]);
        
        $result = curl_exec($ch);
        $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
        curl_close($ch);
        
        if ($httpCode < 200 || $httpCode >= 300) {
            error_log("Failed to send metrics: HTTP $httpCode");
            return false;
        }
        
        return true;
    }
    
    public function wrap(callable $jobFunction) {
        // Send start ping
        $this->sendPing('start');
        
        $startTime = microtime(true);
        $status = 'success';
        $exitCode = 0;
        $errorMessage = null;
        $result = null;
        
        try {
            // Execute the job
            $result = $jobFunction();
            
        } catch (Exception $e) {
            // Job failed
            $status = 'fail';
            $exitCode = 1;
            $errorMessage = $e->getMessage();
            
            error_log("Job {$this->jobName} failed: " . $e->getMessage());
            error_log($e->getTraceAsString());
            
            throw $e;
            
        } finally {
            // Calculate duration
            $duration = microtime(true) - $startTime;
            
            // Send completion ping
            $this->sendPing($status);
            
            // Send detailed metrics
            $metrics = [
                'job_name' => $this->jobName,
                'status' => $status,
                'exit_code' => $exitCode,
                'duration_seconds' => round($duration, 2),
                'timestamp' => time()
            ];
            
            if ($errorMessage !== null) {
                $metrics['error_message'] = $errorMessage;
            }
            
            $this->sendMetrics($metrics);
        }
        
        return $result;
    }
}

// Example usage
$SEIRI_PING_URL = getenv('SEIRI_PING_URL');

if (!$SEIRI_PING_URL) {
    fwrite(STDERR, "Error: SEIRI_PING_URL environment variable not set\n");
    exit(1);
}

// Database backup job
function backupDatabase() {
    echo "Starting database backup...\n";
    
    $date = date('Ymd');
    $backupFile = "/backups/db-{$date}.sql";
    
    // Run mysqldump
    $command = "mysqldump -h localhost -u root mydb > " . escapeshellarg($backupFile);
    exec($command, $output, $returnCode);
    
    if ($returnCode !== 0) {
        throw new Exception("mysqldump failed with exit code $returnCode");
    }
    
    echo "Backup saved to $backupFile\n";
    
    // Upload to S3
    $command = "aws s3 cp " . escapeshellarg($backupFile) . " s3://my-bucket/backups/";
    exec($command, $output, $returnCode);
    
    if ($returnCode !== 0) {
        throw new Exception("S3 upload failed with exit code $returnCode");
    }
    
    echo "Backup uploaded to S3\n";
    
    return $backupFile;
}

// Process orders job
function processOrders() {
    $pdo = new PDO('mysql:host=localhost;dbname=mydb', 'root', '');
    
    // Get pending orders
    $stmt = $pdo->query("SELECT id FROM orders WHERE status = 'pending' LIMIT 100");
    $orders = $stmt->fetchAll(PDO::FETCH_COLUMN);
    
    if (empty($orders)) {
        echo "No pending orders to process\n";
        return 0;
    }
    
    $processedCount = 0;
    $failedCount = 0;
    
    foreach ($orders as $orderId) {
        try {
            $stmt = $pdo->prepare(
                "UPDATE orders SET status = 'processed', processed_at = NOW() WHERE id = ?"
            );
            $stmt->execute([$orderId]);
            $processedCount++;
            
        } catch (Exception $e) {
            fwrite(STDERR, "Failed to process order $orderId: " . $e->getMessage() . "\n");
            $failedCount++;
        }
    }
    
    echo "Processed $processedCount orders, $failedCount failed\n";
    
    // Fail if too many errors
    if ($failedCount > count($orders) * 0.1) {
        throw new Exception("Too many failures: $failedCount/" . count($orders));
    }
    
    return $processedCount;
}

// Main execution
if (php_sapi_name() === 'cli') {
    if ($argc < 2) {
        fwrite(STDERR, "Usage: php cron-monitor.php <job_name>\n");
        exit(1);
    }
    
    $jobName = $argv[1];
    
    try {
        if ($jobName === 'backup') {
            $monitor = new CronMonitor($SEIRI_PING_URL, 'database-backup');
            $monitor->wrap('backupDatabase');
        } elseif ($jobName === 'process-orders') {
            $monitor = new CronMonitor($SEIRI_PING_URL, 'process-orders');
            $monitor->wrap('processOrders');
        } else {
            fwrite(STDERR, "Unknown job: $jobName\n");
            exit(1);
        }
        
        exit(0);
        
    } catch (Exception $e) {
        exit(1);
    }
}
?>

SEIRI_PING_URL=https://cloud.seiri.app/webhook/<cid>:<unique-id>

0 2 * * * php /app/cron-monitor.php backup
*/15 * * * * php /app/cron-monitor.php process-orders

import time
import statistics

class ExecutionTimeMonitor:
    """Track execution time trends"""
    
    def __init__(self, ping_url, job_name):
        self.ping_url = ping_url
        self.job_name = job_name
        self.history_file = f'/var/log/cron/{job_name}-times.log'
    
    def record_execution(self, duration_seconds):
        """Record execution time"""
        with open(self.history_file, 'a') as f:
            f.write(f"{int(time.time())},{duration_seconds}\n")
        
        # Keep only last 100 executions
        self.trim_history()
    
    def trim_history(self, keep=100):
        """Keep only recent executions"""
        try:
            with open(self.history_file, 'r') as f:
                lines = f.readlines()
            
            if len(lines) > keep:
                with open(self.history_file, 'w') as f:
                    f.writelines(lines[-keep:])
        except FileNotFoundError:
            pass
    
    def get_statistics(self):
        """Calculate execution time statistics"""
        try:
            with open(self.history_file, 'r') as f:
                times = [float(line.split(',')[1]) for line in f]
            
            if not times:
                return None
            
            return {
                'mean': statistics.mean(times),
                'median': statistics.median(times),
                'stdev': statistics.stdev(times) if len(times) > 1 else 0,
                'min': min(times),
                'max': max(times),
                'recent': times[-1],
                'count': len(times)
            }
        except (FileNotFoundError, ValueError):
            return None
    
    def check_for_anomaly(self, current_duration):
        """Detect if current execution is anomalously slow"""
        stats = self.get_statistics()
        
        if not stats or stats['count'] < 10:
            return False  # Not enough data
        
        # If current duration is more than 3 standard deviations from mean
        threshold = stats['mean'] + (3 * stats['stdev'])
        
        return current_duration > threshold


# Usage example
@CronMonitor(SEIRI_PING_URL, 'data-sync')
def sync_data():
    time_monitor = ExecutionTimeMonitor(SEIRI_PING_URL, 'data-sync')
    
    start = time.time()
    
    # Do the actual work
    perform_sync()
    
    duration = time.time() - start
    
    # Record execution time
    time_monitor.record_execution(duration)
    
    # Check for anomaly
    if time_monitor.check_for_anomaly(duration):
        stats = time_monitor.get_statistics()
        print(f"WARNING: Job took {duration:.1f}s (mean: {stats['mean']:.1f}s)", 
              file=sys.stderr)
        
        # Send alert metric
        requests.post(SEIRI_PING_URL, json={
            'alert': 'slow_execution',
            'current_duration': duration,
            'mean_duration': stats['mean'],
            'deviation': duration - stats['mean']
        })

#!/bin/bash
# validate-backup.sh - Validate backup job results

SEIRI_PING_URL="https://cloud.seiri.app/webhook/<cid>:<unique-id>"
BACKUP_DIR="/backups"
MIN_SIZE_MB=100  # Minimum expected backup size
MAX_AGE_HOURS=2  # Maximum age for latest backup

# Send start ping
curl -fsS "${SEIRI_PING_URL}/start"

# Find latest backup
LATEST_BACKUP=$(find "$BACKUP_DIR" -name "db-*.sql" -type f -printf '%T@ %p\n' | sort -n | tail -1 | cut -d' ' -f2)

if [ -z "$LATEST_BACKUP" ]; then
    echo "ERROR: No backup files found" >&2
    curl -fsS "${SEIRI_PING_URL}/fail"
    exit 1
fi

# Check backup age
BACKUP_AGE_HOURS=$(( ($(date +%s) - $(stat -c %Y "$LATEST_BACKUP")) / 3600 ))

if [ $BACKUP_AGE_HOURS -gt $MAX_AGE_HOURS ]; then
    echo "ERROR: Latest backup is ${BACKUP_AGE_HOURS} hours old (max: ${MAX_AGE_HOURS})" >&2
    curl -fsS "${SEIRI_PING_URL}/fail"
    exit 1
fi

# Check backup size
BACKUP_SIZE_MB=$(du -m "$LATEST_BACKUP" | cut -f1)

if [ $BACKUP_SIZE_MB -lt $MIN_SIZE_MB ]; then
    echo "ERROR: Backup is only ${BACKUP_SIZE_MB}MB (min: ${MIN_SIZE_MB}MB)" >&2
    curl -fsS "${SEIRI_PING_URL}/fail"
    exit 1
fi

# Validate SQL syntax (quick check)
if ! head -100 "$LATEST_BACKUP" | grep -q "CREATE TABLE"; then
    echo "ERROR: Backup doesn't appear to be valid SQL" >&2
    curl -fsS "${SEIRI_PING_URL}/fail"
    exit 1
fi

# All validations passed
echo "Backup validation passed: ${BACKUP_SIZE_MB}MB, ${BACKUP_AGE_HOURS}h old"

curl -fsS "${SEIRI_PING_URL}/success"

# Send detailed metrics
curl -fsS -X POST "${SEIRI_PING_URL}" \
    -H "Content-Type: application/json" \
    -d "{
        \"backup_size_mb\": ${BACKUP_SIZE_MB},
        \"backup_age_hours\": ${BACKUP_AGE_HOURS},
        \"validation_passed\": true
    }"

import requests
import time

class DependencyChain:
    """Monitor dependent cron jobs"""
    
    def __init__(self, ping_url):
        self.ping_url = ping_url
    
    def wait_for_upstream(self, upstream_job, timeout_seconds=3600, check_interval=60):
        """Wait for upstream job to complete"""
        start_time = time.time()
        
        while time.time() - start_time < timeout_seconds:
            # Check if upstream job completed successfully
            # This assumes Seiri provides a status check endpoint
            try:
                response = requests.get(
                    f"{self.ping_url}/../{upstream_job}/status",
                    timeout=10
                )
                
                if response.status_code == 200:
                    data = response.json()
                    
                    if data.get('last_status') == 'success':
                        # Check recency (within last hour)
                        last_run = data.get('last_run_timestamp', 0)
                        if time.time() - last_run < 3600:
                            return True
            
            except requests.RequestException:
                pass
            
            # Wait before checking again
            time.sleep(check_interval)
        
        # Timeout reached
        raise TimeoutError(f"Upstream job {upstream_job} did not complete within {timeout_seconds}s")


# Example: Data pipeline with dependencies
@CronMonitor(SEIRI_PING_URL, 'extract-data')
def extract_data():
    """Step 1: Extract data from source"""
    # Extract logic here
    pass


@CronMonitor(SEIRI_PING_URL, 'transform-data')
def transform_data():
    """Step 2: Transform extracted data (depends on extract)"""
    chain = DependencyChain(SEIRI_PING_URL)
    
    # Wait for extract job to complete
    chain.wait_for_upstream('extract-data', timeout_seconds=1800)
    
    # Now safe to transform
    # Transform logic here
    pass


@CronMonitor(SEIRI_PING_URL, 'load-data')
def load_data():
    """Step 3: Load transformed data (depends on transform)"""
    chain = DependencyChain(SEIRI_PING_URL)
    
    # Wait for transform job to complete
    chain.wait_for_upstream('transform-data', timeout_seconds=1800)
    
    # Now safe to load
    # Load logic here
    pass

# Extract runs at 1 AM
0 1 * * * /app/pipeline.py extract

# Transform runs at 2 AM (after extract)
0 2 * * * /app/pipeline.py transform

# Load runs at 3 AM (after transform)
0 3 * * * /app/pipeline.py load

class GracePeriodConfig:
    """Configure appropriate grace periods for different job types"""
    
    CONFIGS = {
        'database-backup': {
            'expected_duration_minutes': 30,
            'grace_period_minutes': 15,  # 15 minutes past expected
            'schedule_cron': '0 2 * * *',  # Daily at 2 AM
            'criticality': 'high'
        },
        'log-rotation': {
            'expected_duration_minutes': 2,
            'grace_period_minutes': 5,
            'schedule_cron': '0 0 * * *',  # Daily at midnight
            'criticality': 'medium'
        },
        'send-reports': {
            'expected_duration_minutes': 10,
            'grace_period_minutes': 20,
            'schedule_cron': '0 6 * * 1',  # Weekly on Monday at 6 AM
            'criticality': 'medium'
        },
        'process-payments': {
            'expected_duration_minutes': 5,
            'grace_period_minutes': 3,
            'schedule_cron': '*/15 * * * *',  # Every 15 minutes
            'criticality': 'critical'
        }
    }
    
    @classmethod
    def get_config(cls, job_name):
        """Get configuration for a job"""
        return cls.CONFIGS.get(job_name, {
            'expected_duration_minutes': 60,
            'grace_period_minutes': 30,
            'criticality': 'low'
        })
    
    @classmethod
    def should_alert(cls, job_name, duration_minutes):
        """Determine if duration warrants an alert"""
        config = cls.get_config(job_name)
        threshold = config['expected_duration_minutes'] + config['grace_period_minutes']
        
        return duration_minutes > threshold

#!/bin/bash
# monitor-cron.sh - Monitor cron daemon health

SEIRI_PING_URL="https://cloud.seiri.app/ping/cron-daemon-health"

# Check if cron daemon is running
if ! pgrep -x cron > /dev/null && ! pgrep -x crond > /dev/null; then
    echo "ERROR: Cron daemon not running" >&2
    curl -fsS "${SEIRI_PING_URL}/fail"
    
    # Attempt to restart cron
    if command -v systemctl > /dev/null; then
        systemctl restart cron || systemctl restart crond
    else
        service cron restart || service crond restart
    fi
    
    exit 1
fi

# Check cron logs for recent activity
if [ -f /var/log/cron ]; then
    RECENT_JOBS=$(grep "CMD" /var/log/cron | tail -10)
    
    if [ -z "$RECENT_JOBS" ]; then
        echo "WARNING: No recent cron activity in logs" >&2
    fi
fi

# All checks passed
curl -fsS "${SEIRI_PING_URL}/success"

0 * * * * /usr/local/bin/monitor-cron.sh

# Monitor-ScheduledTask.ps1
param(
    [Parameter(Mandatory=$true)]
    [string]$TaskName,
    
    [Parameter(Mandatory=$true)]
    [string]$SeiriPingUrl
)

# Send start ping
Invoke-RestMethod -Uri "$SeiriPingUrl/start" -Method Get -TimeoutSec 10

try {
    # Get task information
    $task = Get-ScheduledTask -TaskName $TaskName -ErrorAction Stop
    $taskInfo = Get-ScheduledTaskInfo -TaskName $TaskName
    
    # Check if task is enabled
    if ($task.State -ne 'Ready') {
        throw "Task is in state: $($task.State)"
    }
    
    # Check last run result
    if ($taskInfo.LastTaskResult -ne 0) {
        throw "Last run failed with code: $($taskInfo.LastTaskResult)"
    }
    
    # Check if task ran recently (within expected window)
    $expectedIntervalHours = 24  # Adjust based on task schedule
    $hoursSinceRun = (Get-Date) - $taskInfo.LastRunTime | Select-Object -ExpandProperty TotalHours
    
    if ($hoursSinceRun -gt ($expectedIntervalHours + 2)) {
        throw "Task hasn't run in $hoursSinceRun hours (expected: $expectedIntervalHours)"
    }
    
    # Send success ping with metrics
    $metrics = @{
        task_name = $TaskName
        last_run_time = $taskInfo.LastRunTime.ToString('o')
        last_result = $taskInfo.LastTaskResult
        next_run_time = $taskInfo.NextRunTime.ToString('o')
        state = $task.State
    } | ConvertTo-Json
    
    Invoke-RestMethod -Uri "$SeiriPingUrl/success" -Method Get -TimeoutSec 10
    Invoke-RestMethod -Uri $SeiriPingUrl -Method Post -Body $metrics -ContentType "application/json" -TimeoutSec 10
    
    exit 0
    
} catch {
    Write-Error "Task monitoring failed: $_"
    
    # Send failure ping
    Invoke-RestMethod -Uri "$SeiriPingUrl/fail" -Method Get -TimeoutSec 10
    
    exit 1
}

# Instead of running your script directly:
C:\Scripts\backup.ps1

# Run it through the monitor:
powershell.exe -File C:\Scripts\Monitor-ScheduledTask.ps1 -TaskName "Database Backup" -SeiriPingUrl "https://cloud.seiri.app/ping/your-id" && C:\Scripts\backup.ps1

apiVersion: batch/v1
kind: CronJob
metadata:
  name: database-backup
spec:
  schedule: "0 2 * * *"
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: backup
            image: postgres:14
            env:
            - name: SEIRI_PING_URL
              valueFrom:
                secretKeyRef:
                  name: seiri-credentials
                  key: ping-url
            command:
            - /bin/sh
            - -c
            - |
              # Send start ping
              wget -qO- "${SEIRI_PING_URL}/start"
              
              # Run backup
              if pg_dump -h postgres -U postgres mydb > /backup/db-$(date +%Y%m%d).sql; then
                # Success
                wget -qO- "${SEIRI_PING_URL}/success"
                exit 0
              else
                # Failure
                wget -qO- "${SEIRI_PING_URL}/fail"
                exit 1
              fi
            volumeMounts:
            - name: backup-storage
              mountPath: /backup
          volumes:
          - name: backup-storage
            persistentVolumeClaim:
              claimName: backup-pvc
          restartPolicy: OnFailure

import json
import os
import urllib3

http = urllib3.PoolManager()

SEIRI_PING_URL = os.environ['SEIRI_PING_URL']

def send_ping(status):
    """Send ping to Seiri"""
    try:
        http.request('GET', f"{SEIRI_PING_URL}/{status}", timeout=10)
    except Exception as e:
        print(f"Failed to send ping: {e}")

def lambda_handler(event, context):
    """AWS Lambda handler with monitoring"""
    
    # Send start ping
    send_ping('start')
    
    try:
        # Your actual Lambda logic here
        result = perform_scheduled_task()
        
        # Send success ping
        send_ping('success')
        
        return {
            'statusCode': 200,
            'body': json.dumps('Success')
        }
        
    except Exception as e:
        print(f"Error: {e}")
        
        # Send failure ping
        send_ping('fail')
        
        return {
            'statusCode': 500,
            'body': json.dumps(f'Error: {str(e)}')
        }

def perform_scheduled_task():
    """Your scheduled task logic"""
    # Task implementation
    return {'processed': 100}

from flask import Flask, request
import requests
import os

app = Flask(__name__)

SEIRI_PING_URL = os.environ['SEIRI_PING_URL']

@app.route('/scheduled-task', methods=['POST'])
def scheduled_task():
    """Endpoint triggered by Cloud Scheduler"""
    
    # Send start ping
    requests.get(f"{SEIRI_PING_URL}/start", timeout=10)
    
    try:
        # Your task logic
        result = perform_task()
        
        # Send success ping
        requests.get(f"{SEIRI_PING_URL}/success", timeout=10)
        
        return 'Success', 200
        
    except Exception as e:
        # Send failure ping
        requests.get(f"{SEIRI_PING_URL}/fail", timeout=10)
        
        return f'Error: {str(e)}', 500

def perform_task():
    """Your scheduled task"""
    pass

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8080)

class DeadManSwitch:
    """Implement dead man's switch pattern"""
    
    def __init__(self, ping_url, interval_minutes):
        self.ping_url = ping_url
        self.interval_minutes = interval_minutes
    
    def send_heartbeat(self):
        """Send heartbeat ping"""
        try:
            response = requests.get(
                self.ping_url,
                timeout=10,
                headers={'User-Agent': 'DeadManSwitch/1.0'}
            )
            response.raise_for_status()
            return True
        except Exception as e:
            print(f"Heartbeat failed: {e}", file=sys.stderr)
            return False
    
    def get_next_expected(self):
        """Calculate when next heartbeat is expected"""
        return time.time() + (self.interval_minutes * 60)


# For a job that runs every hour
@CronMonitor(SEIRI_PING_URL, 'hourly-sync')
def hourly_sync_job():
    """Job that runs every hour"""
    # Job logic here
    pass

# Crontab:
# 0 * * * * /app/jobs.py hourly-sync
# 
# Seiri is configured to expect heartbeats every hour with 10-minute grace period
# If no ping received in 70 minutes → Alert