Add MSP debugging tools

2025-07-25 01:05:15 +03:00 · 2025-06-12 18:14:55 +02:00 · 2025-06-12 18:14:55 +02:00 · 07ca99ad0d
commit 07ca99ad0d
parent 1a30f95176
11 changed files with 3051 additions and 92 deletions
--- a/src/js/msp/msp_queue_monitor.js
+++ b/src/js/msp/msp_queue_monitor.js
@ -0,0 +1,549 @@
+/**
+ * MSP Queue Monitor - Real-time monitoring of MSP message queue
+ * Provides insights into queue health, performance metrics, and potential issues
+ */
+
+export class MSPQueueMonitor {
+    constructor(mspInstance) {
+        this.msp = mspInstance;
+        this.isMonitoring = false;
+        this.metrics = {
+            totalRequests: 0,
+            completedRequests: 0,
+            failedRequests: 0,
+            timeouts: 0,
+            duplicates: 0,
+            avgResponseTime: 0,
+            maxResponseTime: 0,
+            queuePeakSize: 0,
+            requestsByCode: new Map(),
+            responseTimes: [],
+            errorsByType: new Map(),
+        };
+
+        this.alerts = {
+            queueFull: false,
+            highTimeout: false,
+            slowResponses: false,
+            memoryLeak: false,
+        };
+
+        this.thresholds = {
+            maxQueueSize: 40, // Alert when queue > 80% of MAX_QUEUE_SIZE
+            maxAvgResponseTime: 2000, // Alert when avg response > 2s
+            maxTimeoutRate: 0.1, // Alert when timeout rate > 10%
+            memoryLeakThreshold: 100, // Alert when callbacks grow beyond expected
+        };
+
+        this.monitoringInterval = null;
+        this.listeners = [];
+
+        // Hook into MSP methods to collect metrics
+        this._hookMSPMethods();
+    }
+
+    /**
+     * Hook into MSP methods to collect real-time metrics
+     */
+    _hookMSPMethods() {
+        // Store original methods
+        this.originalSendMessage = this.msp.send_message.bind(this.msp);
+        this.originalDispatchMessage = this.msp._dispatch_message.bind(this.msp);
+        this.originalRemoveRequest = this.msp._removeRequestFromCallbacks?.bind(this.msp);
+
+        // Override send_message to track requests
+        this.msp.send_message = (...args) => {
+            this._trackRequestStart(args[0], args[1]);
+            return this.originalSendMessage(...args);
+        };
+
+        // Override _dispatch_message to track responses
+        this.msp._dispatch_message = (...args) => {
+            this._trackResponse();
+            return this.originalDispatchMessage(...args);
+        };
+
+        // Override _removeRequestFromCallbacks to track completions
+        if (this.originalRemoveRequest) {
+            this.msp._removeRequestFromCallbacks = (requestObj) => {
+                this._trackRequestCompletion(requestObj);
+                return this.originalRemoveRequest(requestObj);
+            };
+        }
+    }
+
+    /**
+     * Track when a request starts
+     */
+    _trackRequestStart(code, data) {
+        this.metrics.totalRequests++;
+
+        // Track requests by code
+        const count = this.metrics.requestsByCode.get(code) || 0;
+        this.metrics.requestsByCode.set(code, count + 1);
+
+        // Check for queue size peaks
+        const currentQueueSize = this.msp.callbacks.length;
+        if (currentQueueSize > this.metrics.queuePeakSize) {
+            this.metrics.queuePeakSize = currentQueueSize;
+        }
+
+        this._checkAlerts();
+    }
+
+    /**
+     * Track when a response is received
+     */
+    _trackResponse() {
+        // This will be called for both successful and failed responses
+        // More detailed tracking happens in _trackRequestCompletion
+    }
+
+    /**
+     * Track when a request is completed (success or failure)
+     */
+    _trackRequestCompletion(requestObj) {
+        if (!requestObj) return;
+
+        const responseTime = performance.now() - requestObj.start;
+        this.metrics.responseTimes.push(responseTime);
+
+        // Keep only last 100 response times for rolling average
+        if (this.metrics.responseTimes.length > 100) {
+            this.metrics.responseTimes.shift();
+        }
+
+        // Update max response time
+        if (responseTime > this.metrics.maxResponseTime) {
+            this.metrics.maxResponseTime = responseTime;
+        }
+
+        // Calculate average response time
+        this.metrics.avgResponseTime =
+            this.metrics.responseTimes.reduce((a, b) => a + b, 0) / this.metrics.responseTimes.length;
+
+        // Track completion type
+        if (requestObj.attempts > 1) {
+            this.metrics.timeouts += requestObj.attempts - 1;
+        }
+
+        if (requestObj.success === false) {
+            this.metrics.failedRequests++;
+
+            // Track error types
+            const errorType = requestObj.errorType || "unknown";
+            const errorCount = this.metrics.errorsByType.get(errorType) || 0;
+            this.metrics.errorsByType.set(errorType, errorCount + 1);
+        } else {
+            this.metrics.completedRequests++;
+        }
+
+        this._checkAlerts();
+    }
+
+    /**
+     * Check for alert conditions
+     */
+    _checkAlerts() {
+        const queueSize = this.msp.callbacks.length;
+        const maxQueueSize = this.msp.MAX_QUEUE_SIZE || 50;
+
+        // Queue full alert
+        const wasQueueFull = this.alerts.queueFull;
+        this.alerts.queueFull = queueSize > this.thresholds.maxQueueSize;
+
+        // High timeout rate alert
+        const timeoutRate = this.metrics.totalRequests > 0 ? this.metrics.timeouts / this.metrics.totalRequests : 0;
+        const wasHighTimeout = this.alerts.highTimeout;
+        this.alerts.highTimeout = timeoutRate > this.thresholds.maxTimeoutRate;
+
+        // Slow responses alert
+        const wasSlowResponses = this.alerts.slowResponses;
+        this.alerts.slowResponses = this.metrics.avgResponseTime > this.thresholds.maxAvgResponseTime;
+
+        // Memory leak detection (callbacks not being cleaned up)
+        const wasMemoryLeak = this.alerts.memoryLeak;
+        this.alerts.memoryLeak = queueSize > this.thresholds.memoryLeakThreshold;
+
+        // Debug logging for alert changes (only when alerts become active)
+        if (this.alerts.queueFull !== wasQueueFull && this.alerts.queueFull) {
+            console.warn(`🚨 Queue Full Alert: size ${queueSize}/${this.thresholds.maxQueueSize}`);
+        }
+        if (this.alerts.highTimeout !== wasHighTimeout && this.alerts.highTimeout) {
+            console.warn(`⏱️ High Timeout Alert: rate ${(timeoutRate * 100).toFixed(1)}%`);
+        }
+        if (this.alerts.slowResponses !== wasSlowResponses && this.alerts.slowResponses) {
+            console.warn(`🐌 Slow Response Alert: avg ${this.metrics.avgResponseTime}ms`);
+        }
+        if (this.alerts.memoryLeak !== wasMemoryLeak && this.alerts.memoryLeak) {
+            console.warn(`💾 Memory Leak Alert: callbacks ${queueSize}`);
+        }
+
+        // Notify listeners of alerts
+        this._notifyListeners();
+    }
+
+    /**
+     * Start monitoring
+     */
+    startMonitoring(intervalMs = 1000) {
+        if (this.isMonitoring) return;
+
+        this.isMonitoring = true;
+        this.monitoringInterval = setInterval(() => {
+            this._collectMetrics();
+            this._notifyListeners();
+        }, intervalMs);
+
+        console.log("MSP Queue Monitor started");
+    }
+
+    /**
+     * Stop monitoring
+     */
+    stopMonitoring() {
+        if (!this.isMonitoring) return;
+
+        this.isMonitoring = false;
+        if (this.monitoringInterval) {
+            clearInterval(this.monitoringInterval);
+            this.monitoringInterval = null;
+        }
+
+        console.log("MSP Queue Monitor stopped");
+    }
+
+    /**
+     * Collect current metrics snapshot
+     */
+    _collectMetrics() {
+        // Update current queue size
+        this.currentQueueSize = this.msp.callbacks.length;
+
+        // Calculate success rate
+        this.metrics.successRate =
+            this.metrics.totalRequests > 0 ? this.metrics.completedRequests / this.metrics.totalRequests : 0;
+
+        // Calculate timeout rate
+        this.metrics.timeoutRate =
+            this.metrics.totalRequests > 0 ? this.metrics.timeouts / this.metrics.totalRequests : 0;
+    }
+
+    /**
+     * Get current status report
+     */
+    getStatus() {
+        return {
+            isMonitoring: this.isMonitoring,
+            currentQueueSize: this.msp.callbacks.length,
+            maxQueueSize: this.msp.MAX_QUEUE_SIZE || 50,
+            metrics: { ...this.metrics },
+            alerts: { ...this.alerts },
+            queueContents: this.msp.callbacks.map((req) => ({
+                code: req.code,
+                attempts: req.attempts || 0,
+                age: performance.now() - req.start,
+                hasTimer: !!req.timer,
+            })),
+        };
+    }
+
+    /**
+     * Get detailed queue analysis
+     */
+    analyzeQueue() {
+        const callbacks = this.msp.callbacks;
+        const now = performance.now();
+
+        const analysis = {
+            totalItems: callbacks.length,
+            byCode: {},
+            ageDistribution: {
+                fresh: 0, // < 1s
+                recent: 0, // 1-5s
+                stale: 0, // 5-10s
+                ancient: 0, // > 10s
+            },
+            retryDistribution: {
+                firstAttempt: 0,
+                retrying: 0,
+                multipleRetries: 0,
+            },
+            potentialIssues: [],
+        };
+
+        callbacks.forEach((req) => {
+            // Group by code
+            if (!analysis.byCode[req.code]) {
+                analysis.byCode[req.code] = 0;
+            }
+            analysis.byCode[req.code]++;
+
+            // Age analysis
+            const age = now - req.start;
+            if (age < 1000) analysis.ageDistribution.fresh++;
+            else if (age < 5000) analysis.ageDistribution.recent++;
+            else if (age < 10000) analysis.ageDistribution.stale++;
+            else analysis.ageDistribution.ancient++;
+
+            // Retry analysis
+            const attempts = req.attempts || 0;
+            if (attempts === 0) analysis.retryDistribution.firstAttempt++;
+            else if (attempts === 1) analysis.retryDistribution.retrying++;
+            else analysis.retryDistribution.multipleRetries++;
+
+            // Identify potential issues
+            if (age > 10000) {
+                analysis.potentialIssues.push(`Ancient request: code ${req.code}, age ${Math.round(age / 1000)}s`);
+            }
+            if (attempts > 5) {
+                analysis.potentialIssues.push(`High retry count: code ${req.code}, attempts ${attempts}`);
+            }
+            if (!req.timer) {
+                analysis.potentialIssues.push(`Missing timer: code ${req.code}`);
+            }
+        });
+
+        return analysis;
+    }
+
+    /**
+     * Add a listener for monitoring events
+     */
+    addListener(callback) {
+        this.listeners.push(callback);
+    }
+
+    /**
+     * Remove a listener
+     */
+    removeListener(callback) {
+        const index = this.listeners.indexOf(callback);
+        if (index > -1) {
+            this.listeners.splice(index, 1);
+        }
+    }
+
+    /**
+     * Notify all listeners
+     */
+    _notifyListeners() {
+        const status = this.getStatus();
+        this.listeners.forEach((listener) => {
+            try {
+                listener(status);
+            } catch (error) {
+                console.error("Error in MSP monitor listener:", error);
+            }
+        });
+    }
+
+    /**
+     * Reset metrics
+     */
+    resetMetrics() {
+        this.metrics = {
+            totalRequests: 0,
+            completedRequests: 0,
+            failedRequests: 0,
+            timeouts: 0,
+            duplicates: 0,
+            avgResponseTime: 0,
+            maxResponseTime: 0,
+            queuePeakSize: 0,
+            requestsByCode: new Map(),
+            responseTimes: [],
+            errorsByType: new Map(),
+        };
+
+        this.alerts = {
+            queueFull: false,
+            highTimeout: false,
+            slowResponses: false,
+            memoryLeak: false,
+        };
+    }
+
+    /**
+     * Generate a detailed report
+     */
+    generateReport() {
+        const status = this.getStatus();
+        const analysis = this.analyzeQueue();
+
+        return {
+            timestamp: new Date().toISOString(),
+            summary: {
+                queueHealth: this._assessQueueHealth(),
+                performanceGrade: this._calculatePerformanceGrade(),
+                recommendations: this._generateRecommendations(),
+            },
+            status,
+            analysis,
+            rawMetrics: this.metrics,
+        };
+    }
+
+    /**
+     * Assess overall queue health
+     */
+    _assessQueueHealth() {
+        const alerts = Object.values(this.alerts);
+        const activeAlerts = alerts.filter((alert) => alert).length;
+
+        if (activeAlerts === 0) return "HEALTHY";
+        if (activeAlerts <= 2) return "WARNING";
+        return "CRITICAL";
+    }
+
+    /**
+     * Calculate performance grade
+     */
+    _calculatePerformanceGrade() {
+        let score = 100;
+
+        // Deduct for high timeout rate
+        if (this.metrics.timeoutRate > 0.1) score -= 30;
+        else if (this.metrics.timeoutRate > 0.05) score -= 15;
+
+        // Deduct for slow responses
+        if (this.metrics.avgResponseTime > 2000) score -= 25;
+        else if (this.metrics.avgResponseTime > 1000) score -= 10;
+
+        // Deduct for queue size issues
+        const queueRatio = this.currentQueueSize / (this.msp.MAX_QUEUE_SIZE || 50);
+        if (queueRatio > 0.8) score -= 20;
+        else if (queueRatio > 0.6) score -= 10;
+
+        // Deduct for failed requests
+        const failureRate =
+            this.metrics.totalRequests > 0 ? this.metrics.failedRequests / this.metrics.totalRequests : 0;
+        if (failureRate > 0.05) score -= 15;
+
+        if (score >= 90) return "A";
+        if (score >= 80) return "B";
+        if (score >= 70) return "C";
+        if (score >= 60) return "D";
+        return "F";
+    }
+
+    /**
+     * Generate recommendations
+     */
+    _generateRecommendations() {
+        const recommendations = [];
+
+        if (this.alerts.queueFull) {
+            recommendations.push(
+                "Queue is near capacity. Consider implementing request prioritization or increasing queue size.",
+            );
+        }
+
+        if (this.alerts.highTimeout) {
+            recommendations.push(
+                "High timeout rate detected. Check serial connection stability or increase timeout values.",
+            );
+        }
+
+        if (this.alerts.slowResponses) {
+            recommendations.push(
+                "Slow response times detected. Investigate flight controller performance or reduce request frequency.",
+            );
+        }
+
+        if (this.alerts.memoryLeak) {
+            recommendations.push(
+                "Potential memory leak detected. Check that all requests are being properly cleaned up.",
+            );
+        }
+
+        if (this.metrics.maxResponseTime > 5000) {
+            recommendations.push(
+                "Some requests are taking very long to complete. Consider implementing request timeouts.",
+            );
+        }
+
+        return recommendations;
+    }
+
+    /**
+     * Test the alert system by manually triggering alerts
+     */
+    triggerTestAlerts() {
+        console.log("🧪 Triggering test alerts...");
+
+        // Store original alerts
+        const originalAlerts = { ...this.alerts };
+
+        // Trigger all alerts
+        this.alerts.queueFull = true;
+        this.alerts.highTimeout = true;
+        this.alerts.slowResponses = true;
+        this.alerts.memoryLeak = true;
+
+        console.log("🚨 Test alerts triggered:", this.alerts);
+
+        // Notify listeners immediately
+        this._notifyListeners();
+
+        // Reset after 10 seconds
+        setTimeout(() => {
+            this.alerts = originalAlerts;
+            console.log("✅ Test alerts reset");
+            this._notifyListeners();
+        }, 10000);
+
+        return this.alerts;
+    }
+
+    /**
+     * Lower alert thresholds for testing
+     */
+    setTestThresholds() {
+        console.log("🎯 Setting test thresholds for easier alert triggering...");
+        this.thresholds = {
+            maxQueueSize: 1, // Alert when queue > 1
+            maxAvgResponseTime: 100, // Alert when avg response > 100ms
+            maxTimeoutRate: 0.01, // Alert when timeout rate > 1%
+            memoryLeakThreshold: 5, // Alert when callbacks > 5
+        };
+        console.log("New thresholds:", this.thresholds);
+    }
+
+    /**
+     * Reset to normal thresholds
+     */
+    setNormalThresholds() {
+        console.log("🔧 Resetting to normal thresholds...");
+        this.thresholds = {
+            maxQueueSize: 40, // Alert when queue > 80% of MAX_QUEUE_SIZE
+            maxAvgResponseTime: 2000, // Alert when avg response > 2s
+            maxTimeoutRate: 0.1, // Alert when timeout rate > 10%
+            memoryLeakThreshold: 100, // Alert when callbacks grow beyond expected
+        };
+        console.log("Normal thresholds restored:", this.thresholds);
+    }
+
+    /**
+     * Cleanup and restore original MSP methods
+     */
+    destroy() {
+        this.stopMonitoring();
+
+        // Restore original methods
+        if (this.originalSendMessage) {
+            this.msp.send_message = this.originalSendMessage;
+        }
+        if (this.originalDispatchMessage) {
+            this.msp._dispatch_message = this.originalDispatchMessage;
+        }
+        if (this.originalRemoveRequest) {
+            this.msp._removeRequestFromCallbacks = this.originalRemoveRequest;
+        }
+
+        this.listeners = [];
+    }
+}
+
+// Export singleton instance for easy use
+export const mspQueueMonitor = new MSPQueueMonitor(window.MSP);