Performance Metrics

Application Performance Metrics

API Performance

# Example Prometheus metrics
http_requests_total{method="GET", endpoint="/api/agents", status="200"} 15234
http_requests_total{method="POST", endpoint="/api/tasks", status="201"} 542
http_request_duration_seconds_bucket{endpoint="/api/agents", le="0.1"} 12890
http_request_duration_seconds_bucket{endpoint="/api/agents", le="0.5"} 15123
http_request_duration_seconds_bucket{endpoint="/api/agents", le="1.0"} 15200
                                

Implementation Example

# Python FastAPI metrics implementation
from prometheus_client import Counter, Histogram, Gauge
import time
from functools import wraps

# Define metrics
REQUEST_COUNT = Counter(
    'http_requests_total',
    'Total HTTP requests',
    ['method', 'endpoint', 'status_code']
)

REQUEST_DURATION = Histogram(
    'http_request_duration_seconds',
    'HTTP request latency',
    ['endpoint'],
    buckets=[0.01, 0.05, 0.1, 0.5, 1.0, 2.5, 5.0, 10.0]
)

ACTIVE_CONNECTIONS = Gauge(
    'websocket_connections_active',
    'Active WebSocket connections',
    ['room']
)

# Middleware for automatic metrics collection
@app.middleware("http")
async def metrics_middleware(request: Request, call_next):
    start_time = time.time()

    response = await call_next(request)

    # Record metrics
    endpoint = request.url.path
    method = request.method
    status_code = str(response.status_code)
    duration = time.time() - start_time

    REQUEST_COUNT.labels(
        method=method,
        endpoint=endpoint,
        status_code=status_code
    ).inc()

    REQUEST_DURATION.labels(endpoint=endpoint).observe(duration)

    return response

# Business metrics decorator
def track_task_execution(task_type: str):
    def decorator(func):
        @wraps(func)
        async def wrapper(*args, **kwargs):
            start_time = time.time()
            try:
                result = await func(*args, **kwargs)
                TASK_SUCCESS_RATE.labels(task_type=task_type).inc()
                return result
            except Exception as e:
                TASK_FAILURES.labels(
                    task_type=task_type,
                    error_type=type(e).__name__
                ).inc()
                raise
            finally:
                duration = time.time() - start_time
                TASK_DURATION.labels(task_type=task_type).observe(duration)
        return wrapper
    return decorator
                        

Infrastructure Metrics

System Resource Metrics

Agent Metrics Collection

# Agent-side metrics collection
class AgentMetricsCollector:
    def __init__(self):
        self.metrics_buffer = []
        self.collection_interval = 30  # seconds
        self.max_buffer_size = 1000

    async def collect_system_metrics(self):
        """Collect system-level metrics"""
        metrics = {
            'timestamp': datetime.utcnow().isoformat(),
            'cpu': {
                'usage_percent': psutil.cpu_percent(interval=1),
                'cores': psutil.cpu_count(),
                'load_avg': os.getloadavg() if hasattr(os, 'getloadavg') else None
            },
            'memory': {
                'total': psutil.virtual_memory().total,
                'available': psutil.virtual_memory().available,
                'used': psutil.virtual_memory().used,
                'percent': psutil.virtual_memory().percent
            },
            'disk': {
                dev.device: {
                    'total': psutil.disk_usage(dev.mountpoint).total,
                    'used': psutil.disk_usage(dev.mountpoint).used,
                    'free': psutil.disk_usage(dev.mountpoint).free
                }
                for dev in psutil.disk_partitions()
            },
            'network': {
                interface: {
                    'bytes_sent': stats.bytes_sent,
                    'bytes_recv': stats.bytes_recv,
                    'packets_sent': stats.packets_sent,
                    'packets_recv': stats.packets_recv
                }
                for interface, stats in psutil.net_io_counters(pernic=True).items()
            }
        }
        return metrics

    async def collect_application_metrics(self):
        """Collect application-specific metrics"""
        return {
            'agent': {
                'version': self.agent_version,
                'uptime_seconds': time.time() - self.start_time,
                'tasks_completed': self.tasks_completed,
                'last_heartbeat': self.last_heartbeat.isoformat()
            },
            'services': await self.get_service_metrics(),
            'packages': await self.get_package_metrics()
        }

    async def send_metrics(self):
        """Send collected metrics to server"""
        if self.metrics_buffer:
            try:
                await self.api_client.post('/api/metrics', {
                    'agent_id': self.agent_id,
                    'metrics': self.metrics_buffer
                })
                self.metrics_buffer.clear()
            except Exception as e:
                logger.error(f"Failed to send metrics: {e}")
                # Implement exponential backoff retry
                await self.schedule_retry()
                        

Database Performance Metrics

Connection Pool Metrics

• db_connections_active: Active database connections
• db_connections_idle: Idle connections in pool
• db_connection_wait_time_seconds: Time waiting for connection
• db_pool_exhausted_total: Pool exhaustion events

Query Performance Metrics

• db_query_duration_seconds: Query execution time by type
• db_queries_total: Query count by operation (SELECT, INSERT, etc.)
• db_slow_queries_total: Queries exceeding threshold
• db_deadlocks_total: Database deadlock occurrences

Database Monitoring Implementation

# SQLAlchemy metrics integration
from sqlalchemy import event
from sqlalchemy.engine import Engine
import time

# Query timing metrics
QUERY_DURATION = Histogram(
    'db_query_duration_seconds',
    'Database query execution time',
    ['operation', 'table']
)

QUERY_COUNT = Counter(
    'db_queries_total',
    'Total database queries',
    ['operation', 'table', 'status']
)

@event.listens_for(Engine, "before_cursor_execute")
def receive_before_cursor_execute(conn, cursor, statement, parameters, context, executemany):
    context._query_start_time = time.time()

@event.listens_for(Engine, "after_cursor_execute")
def receive_after_cursor_execute(conn, cursor, statement, parameters, context, executemany):
    duration = time.time() - context._query_start_time

    # Parse SQL to extract operation and table
    operation = statement.strip().split()[0].upper()
    table = extract_table_name(statement)

    QUERY_DURATION.labels(operation=operation, table=table).observe(duration)
    QUERY_COUNT.labels(operation=operation, table=table, status='success').inc()

# Connection pool monitoring
class MonitoredConnectionPool:
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.active_connections = Gauge(
            'db_connections_active',
            'Active database connections'
        )
        self.pool_size = Gauge(
            'db_pool_size',
            'Database connection pool size'
        )

    def _do_get(self):
        start_time = time.time()
        try:
            conn = super()._do_get()
            self.active_connections.inc()
            return conn
        finally:
            wait_time = time.time() - start_time
            if wait_time > 0.1:  # Log slow connection acquisition
                DB_CONNECTION_WAIT_TIME.observe(wait_time)

    def _do_return_conn(self, conn):
        super()._do_return_conn(conn)
        self.active_connections.dec()
                        

User Experience Metrics

Frontend Performance

• page_load_time_seconds: Page load duration by route
• api_call_duration_seconds: Client-side API call timing
• websocket_latency_seconds: Real-time update latency
• ui_interactions_total: User interaction counts by type

Client-side Metrics Collection

// JavaScript client-side metrics
class ClientMetrics {
    constructor() {
        this.metrics = [];
        this.batchSize = 50;
        this.flushInterval = 30000; // 30 seconds
        this.startFlushTimer();
    }

    // Performance timing
    recordPageLoad(route) {
        const navigation = performance.getEntriesByType('navigation')[0];
        this.addMetric({
            name: 'page_load_time_seconds',
            value: navigation.loadEventEnd / 1000,
            labels: { route: route },
            timestamp: Date.now()
        });
    }

    // API call timing
    recordApiCall(endpoint, method, duration, status) {
        this.addMetric({
            name: 'api_call_duration_seconds',
            value: duration / 1000,
            labels: {
                endpoint: endpoint,
                method: method,
                status: status.toString()
            },
            timestamp: Date.now()
        });
    }

    // WebSocket latency
    recordWebSocketLatency(messageType, latency) {
        this.addMetric({
            name: 'websocket_latency_seconds',
            value: latency / 1000,
            labels: { message_type: messageType },
            timestamp: Date.now()
        });
    }

    // User interactions
    recordInteraction(action, component) {
        this.addMetric({
            name: 'ui_interactions_total',
            value: 1,
            labels: {
                action: action,
                component: component
            },
            timestamp: Date.now()
        });
    }

    addMetric(metric) {
        this.metrics.push(metric);
        if (this.metrics.length >= this.batchSize) {
            this.flush();
        }
    }

    async flush() {
        if (this.metrics.length === 0) return;

        const batch = this.metrics.splice(0);
        try {
            await fetch('/api/metrics/client', {
                method: 'POST',
                headers: { 'Content-Type': 'application/json' },
                body: JSON.stringify({ metrics: batch })
            });
        } catch (error) {
            console.error('Failed to send client metrics:', error);
            // Could implement retry logic here
        }
    }

    startFlushTimer() {
        setInterval(() => this.flush(), this.flushInterval);
    }
}

// Integration with React components
const useMetrics = () => {
    const metrics = useRef(new ClientMetrics());

    const recordPageView = useCallback((route) => {
        metrics.current.recordPageLoad(route);
    }, []);

    const recordApiCall = useCallback((endpoint, method, duration, status) => {
        metrics.current.recordApiCall(endpoint, method, duration, status);
    }, []);

    return { recordPageView, recordApiCall };
};