unified_telemetry_system.py

"""TNFR Unified Telemetry System - Consolidated Metrics and Event Collection.

CONSOLIDATION ACHIEVEMENT: This module unifies all TNFR telemetry implementations
under a single coherent interface following nodal equation dynamics principles.

Unified Architecture:
- Consolidates telemetry/emit.py core telemetry system
- Merges factorization failure telemetry functionality
- Integrates cache telemetry and performance monitoring
- Unifies event collection across all TNFR modules
- Consistent structured logging with correlation IDs

Theoretical Foundation:
Telemetry captures the observable manifestation of nodal equation ∂EPI/∂t = νf · ΔNFR(t)
through structural field measurements (Φ_s, |∇φ|, K_φ, ξ_C) and system state evolution
without perturbing the underlying TNFR dynamics.

Consolidated Features:
1. Structural Telemetry: Tetrad field measurements and coherence metrics
2. Performance Telemetry: Operation timing, memory usage, and throughput
3. Failure Telemetry: Error analysis and system degradation tracking
4. Event Correlation: Unified correlation IDs across all telemetry streams
5. Batched Emission: Efficient buffering and periodic flushing
6. Structured Storage: JSONL format with metadata for analysis

Consolidates:
- src/tnfr/telemetry/emit.py (TelemetryEmitter)
- factorization-lab/tnfr_factorization/failure_telemetry.py (FailureTelemetryManager)
- src/tnfr/telemetry/cache_metrics.py (cache telemetry)
- Various scattered telemetry implementations across modules

Status: UNIFIED TELEMETRY CONSOLIDATION - All telemetry centralized
"""

from __future__ import annotations

import json
import logging
import threading
import time
import uuid
from collections import deque
from dataclasses import asdict, dataclass, field
from pathlib import Path
from typing import Any

# Unified configuration and backend integration
from ..config import get_config

logger = logging.getLogger(__name__)


@dataclass
class TelemetryConfiguration:
    """Configuration for unified telemetry system."""

    # Collection settings
    enable_telemetry: bool = True
    enable_structural_telemetry: bool = True
    enable_performance_telemetry: bool = True
    enable_failure_telemetry: bool = True

    # Batching and storage
    batch_size: int = 100
    flush_interval_seconds: float = 30.0
    max_memory_buffer_mb: float = 50.0

    # Output configuration
    output_directory: Path = Path("results/telemetry")
    file_format: str = "jsonl"  # "jsonl", "parquet", "csv"
    enable_correlation_tracking: bool = True

    # Performance tuning
    async_emission: bool = True
    compression_enabled: bool = True

    # Filtering
    min_event_level: str = "INFO"  # "DEBUG", "INFO", "WARNING", "ERROR"
    event_type_filters: list[str] = field(default_factory=list)


@dataclass
class StructuralTelemetryEvent:
    """Telemetry event for TNFR structural measurements."""

    # Event metadata
    event_id: str
    correlation_id: str
    timestamp: float
    node_id: str | None = None

    # Structural field tetrad (audit 2026: π genuine; γ/e/φ overlay)
    phi_s: float | None = None  # Structural potential
    phase_gradient: float | None = None  # |∇φ|
    phase_curvature: float | None = None  # K_φ
    coherence_length: float | None = None  # ξ_C

    # Core TNFR metrics
    coherence: float | None = None  # C(t)
    sense_index: float | None = None  # Si
    delta_nfr: float | None = None  # ΔNFR
    vf: float | None = None  # νf
    phase: float | None = None  # φ/θ
    epi: float | None = None  # EPI

    # System state
    operator_sequence: list[str] | None = None
    system_status: str = "normal"

    # Additional context
    metadata: dict[str, Any] = field(default_factory=dict)


@dataclass
class PerformanceTelemetryEvent:
    """Telemetry event for performance monitoring."""

    # Event metadata
    event_id: str
    correlation_id: str
    timestamp: float

    # Performance metrics
    operation_name: str
    duration_ms: float
    memory_usage_mb: float = 0.0
    cpu_usage_percent: float = 0.0
    gpu_usage_percent: float = 0.0

    # Throughput metrics
    operations_per_second: float | None = None
    data_throughput_mbps: float | None = None

    # Resource utilization
    backend_used: str = "unknown"
    device_used: str | None = None
    cache_hit_rate: float | None = None

    # Quality metrics
    success: bool = True
    error_message: str | None = None

    # Context
    metadata: dict[str, Any] = field(default_factory=dict)


@dataclass
class FailureTelemetryEvent:
    """Telemetry event for failure analysis."""

    # Event metadata
    event_id: str
    correlation_id: str
    timestamp: float

    # Failure details
    failure_type: str  # "computational", "memory", "convergence", "validation"
    error_code: str | None = None
    error_message: str = ""
    stack_trace: str | None = None

    # System context at failure
    system_state: dict[str, Any] = field(default_factory=dict)
    operation_context: dict[str, Any] = field(default_factory=dict)

    # Recovery information
    recovery_attempted: bool = False
    recovery_successful: bool = False
    fallback_used: str | None = None

    # Impact assessment
    severity: str = "medium"  # "low", "medium", "high", "critical"
    affected_operations: list[str] = field(default_factory=list)

    # Additional context
    metadata: dict[str, Any] = field(default_factory=dict)


class TNFRUnifiedTelemetrySystem:
    """Unified Telemetry System - Consolidated Metrics and Event Collection.

    ARCHITECTURE: This system consolidates all TNFR telemetry implementations
    under a unified interface with intelligent routing, batching, and storage.

    Consolidates:
    - TelemetryEmitter from telemetry/emit.py
    - FailureTelemetryManager from factorization failure telemetry
    - Cache telemetry from utils/cache.py
    - Performance monitoring across all modules
    - Event correlation and structured logging

    Usage:
        # Single entry point for all telemetry
        telemetry = TNFRUnifiedTelemetrySystem()

        # Structural measurements
        telemetry.emit_structural_event(
            phi_s=structural_potential,
            coherence=coherence_value,
            correlation_id=session_id
        )

        # Performance monitoring
        telemetry.emit_performance_event(
            operation_name="delta_nfr_computation",
            duration_ms=computation_time,
            backend_used="torch"
        )

        # Failure tracking
        telemetry.emit_failure_event(
            failure_type="memory",
            error_message="GPU out of memory",
            system_state=current_state
        )

    Benefits:
        - Eliminates telemetry redundancy across codebase
        - Unified correlation tracking for analysis
        - Consistent structured storage format
        - Automatic batching and performance optimization
        - Integrated with unified config system
    """

    def __init__(self, config: TelemetryConfiguration | None = None):
        """Initialize unified telemetry system."""
        self.config = config or TelemetryConfiguration()

        # Event buffers for batching
        self._structural_buffer: deque = deque()
        self._performance_buffer: deque = deque()
        self._failure_buffer: deque = deque()

        # Threading for async emission
        self._flush_lock = threading.Lock()
        self._flush_timer: threading.Timer | None = None

        # Correlation tracking
        self._active_correlations: dict[str, dict[str, Any]] = {}
        self._correlation_counters: dict[str, int] = {}

        # Performance statistics
        self._emission_stats = {
            "total_events": 0,
            "structural_events": 0,
            "performance_events": 0,
            "failure_events": 0,
            "flush_count": 0,
            "bytes_emitted": 0,
        }

        # Integration with unified systems
        self.global_config = get_config()

        # Ensure output directory exists
        self.config.output_directory.mkdir(parents=True, exist_ok=True)

        # Start periodic flushing if enabled
        if self.config.async_emission:
            self._schedule_flush()

        logger.info(
            f"Initialized unified telemetry system: {self.config.output_directory}"
        )

    def emit_structural_event(
        self,
        correlation_id: str | None = None,
        node_id: str | None = None,
        **kwargs: Any,
    ) -> str:
        """Emit structural telemetry event with TNFR field measurements.

        Parameters
        ----------
        correlation_id : str, optional
            Correlation ID for event tracking (auto-generated if not provided)
        node_id : str, optional
            Node identifier for the measurement
        **kwargs
            Structural field values (phi_s, phase_gradient, coherence, etc.)

        Returns
        -------
        str
            Event ID for the emitted event
        """
        if not self.config.enable_structural_telemetry:
            return ""

        event_id = str(uuid.uuid4())
        correlation_id = correlation_id or self._generate_correlation_id("structural")

        event = StructuralTelemetryEvent(
            event_id=event_id,
            correlation_id=correlation_id,
            timestamp=time.time(),
            node_id=node_id,
            **kwargs,
        )

        self._structural_buffer.append(event)
        self._emission_stats["structural_events"] += 1
        self._emission_stats["total_events"] += 1

        # Track correlation
        self._update_correlation_tracking(correlation_id, "structural", event_id)

        # Flush if buffer is full
        if len(self._structural_buffer) >= self.config.batch_size:
            self._flush_structural_events()

        return event_id

    def emit_performance_event(
        self,
        operation_name: str,
        duration_ms: float,
        correlation_id: str | None = None,
        **kwargs: Any,
    ) -> str:
        """Emit performance telemetry event for operation monitoring.

        Parameters
        ----------
        operation_name : str
            Name of the operation being monitored
        duration_ms : float
            Operation duration in milliseconds
        correlation_id : str, optional
            Correlation ID for event tracking
        **kwargs
            Additional performance metrics

        Returns
        -------
        str
            Event ID for the emitted event
        """
        if not self.config.enable_performance_telemetry:
            return ""

        event_id = str(uuid.uuid4())
        correlation_id = correlation_id or self._generate_correlation_id("performance")

        event = PerformanceTelemetryEvent(
            event_id=event_id,
            correlation_id=correlation_id,
            timestamp=time.time(),
            operation_name=operation_name,
            duration_ms=duration_ms,
            **kwargs,
        )

        self._performance_buffer.append(event)
        self._emission_stats["performance_events"] += 1
        self._emission_stats["total_events"] += 1

        # Track correlation
        self._update_correlation_tracking(correlation_id, "performance", event_id)

        # Flush if buffer is full
        if len(self._performance_buffer) >= self.config.batch_size:
            self._flush_performance_events()

        return event_id

    def emit_failure_event(
        self,
        failure_type: str,
        error_message: str = "",
        correlation_id: str | None = None,
        **kwargs: Any,
    ) -> str:
        """Emit failure telemetry event for error analysis.

        Parameters
        ----------
        failure_type : str
            type of failure (computational, memory, convergence, etc.)
        error_message : str
            Description of the failure
        correlation_id : str, optional
            Correlation ID for event tracking
        **kwargs
            Additional failure context and metadata

        Returns
        -------
        str
            Event ID for the emitted event
        """
        if not self.config.enable_failure_telemetry:
            return ""

        event_id = str(uuid.uuid4())
        correlation_id = correlation_id or self._generate_correlation_id("failure")

        event = FailureTelemetryEvent(
            event_id=event_id,
            correlation_id=correlation_id,
            timestamp=time.time(),
            failure_type=failure_type,
            error_message=error_message,
            **kwargs,
        )

        self._failure_buffer.append(event)
        self._emission_stats["failure_events"] += 1
        self._emission_stats["total_events"] += 1

        # Track correlation
        self._update_correlation_tracking(correlation_id, "failure", event_id)

        # Immediate flush for failures (they're important)
        self._flush_failure_events()

        return event_id

    def start_correlation(
        self, correlation_name: str, context: dict[str, Any] | None = None
    ) -> str:
        """Start a new correlation session for tracking related events.

        Parameters
        ----------
        correlation_name : str
            Human-readable name for the correlation
        context : dict, optional
            Initial context for the correlation

        Returns
        -------
        str
            Correlation ID for tracking events
        """
        correlation_id = self._generate_correlation_id(correlation_name)

        self._active_correlations[correlation_id] = {
            "name": correlation_name,
            "start_time": time.time(),
            "context": context or {},
            "event_count": 0,
            "event_types": set(),
        }

        return correlation_id

    def end_correlation(self, correlation_id: str) -> dict[str, Any]:
        """End a correlation session and return summary.

        Parameters
        ----------
        correlation_id : str
            Correlation ID to end

        Returns
        -------
        dict
            Correlation summary with event statistics
        """
        if correlation_id not in self._active_correlations:
            return {"error": "correlation_not_found"}

        correlation = self._active_correlations.pop(correlation_id)
        correlation["end_time"] = time.time()
        correlation["duration"] = correlation["end_time"] - correlation["start_time"]

        return correlation

    def flush_all(self) -> None:
        """Flush all telemetry buffers to storage immediately."""
        with self._flush_lock:
            self._flush_structural_events()
            self._flush_performance_events()
            self._flush_failure_events()
            self._emission_stats["flush_count"] += 1

    def _generate_correlation_id(self, prefix: str) -> str:
        """Generate a unique correlation ID with prefix."""
        counter = self._correlation_counters.get(prefix, 0) + 1
        self._correlation_counters[prefix] = counter

        return f"{prefix}_{int(time.time())}_{counter}_{str(uuid.uuid4())[:8]}"

    def _update_correlation_tracking(
        self, correlation_id: str, event_type: str, event_id: str
    ) -> None:
        """Update correlation tracking with new event."""
        if correlation_id in self._active_correlations:
            correlation = self._active_correlations[correlation_id]
            correlation["event_count"] += 1
            correlation["event_types"].add(event_type)
            correlation["last_event_id"] = event_id
            correlation["last_event_time"] = time.time()

    def _flush_structural_events(self) -> None:
        """Flush structural events to storage."""
        if not self._structural_buffer:
            return

        events = list(self._structural_buffer)
        self._structural_buffer.clear()

        filename = (
            self.config.output_directory
            / f"structural_telemetry_{int(time.time())}.{self.config.file_format}"
        )
        self._write_events_to_file(events, filename)

    def _flush_performance_events(self) -> None:
        """Flush performance events to storage."""
        if not self._performance_buffer:
            return

        events = list(self._performance_buffer)
        self._performance_buffer.clear()

        filename = (
            self.config.output_directory
            / f"performance_telemetry_{int(time.time())}.{self.config.file_format}"
        )
        self._write_events_to_file(events, filename)

    def _flush_failure_events(self) -> None:
        """Flush failure events to storage."""
        if not self._failure_buffer:
            return

        events = list(self._failure_buffer)
        self._failure_buffer.clear()

        filename = (
            self.config.output_directory
            / f"failure_telemetry_{int(time.time())}.{self.config.file_format}"
        )
        self._write_events_to_file(events, filename)

    def _write_events_to_file(self, events: list[Any], filename: Path) -> None:
        """Write events to file in specified format."""
        try:
            if self.config.file_format == "jsonl":
                with open(filename, "w") as f:
                    for event in events:
                        json.dump(asdict(event), f)
                        f.write("\n")

            elif self.config.file_format == "json":
                with open(filename, "w") as f:
                    json.dump([asdict(event) for event in events], f, indent=2)

            # Update statistics
            self._emission_stats["bytes_emitted"] += (
                filename.stat().st_size if filename.exists() else 0
            )

        except Exception as e:
            logger.error(f"Failed to write telemetry events to {filename}: {e}")

    def _schedule_flush(self) -> None:
        """Schedule periodic flushing of telemetry buffers."""
        if self._flush_timer:
            self._flush_timer.cancel()

        self._flush_timer = threading.Timer(
            self.config.flush_interval_seconds, self._periodic_flush
        )
        self._flush_timer.daemon = True
        self._flush_timer.start()

    def _periodic_flush(self) -> None:
        """Periodic flush callback."""
        try:
            self.flush_all()
        except Exception as e:
            logger.error(f"Periodic flush failed: {e}")
        finally:
            # Schedule next flush
            self._schedule_flush()

    def get_statistics(self) -> dict[str, Any]:
        """Get telemetry system statistics."""
        stats = self._emission_stats.copy()

        stats.update(
            {
                "buffer_sizes": {
                    "structural": len(self._structural_buffer),
                    "performance": len(self._performance_buffer),
                    "failure": len(self._failure_buffer),
                },
                "active_correlations": len(self._active_correlations),
                "correlation_types": len(self._correlation_counters),
                "config": asdict(self.config),
            }
        )

        return stats

    def cleanup(self) -> None:
        """Clean up telemetry system resources."""
        # Cancel flush timer
        if self._flush_timer:
            self._flush_timer.cancel()

        # Final flush
        self.flush_all()

        # Clear buffers
        self._structural_buffer.clear()
        self._performance_buffer.clear()
        self._failure_buffer.clear()

        logger.info("Unified telemetry system cleanup completed")


# ============================================================================
# PUBLIC API - Unified Telemetry Interface
# ============================================================================

# Global unified telemetry system instance
_unified_telemetry_system: TNFRUnifiedTelemetrySystem | None = None


def get_unified_telemetry_system(
    config: TelemetryConfiguration | None = None,
) -> TNFRUnifiedTelemetrySystem:
    """Get or create global unified telemetry system.

    This provides a singleton interface for all TNFR telemetry operations
    to eliminate redundant system creation across modules.

    Parameters
    ----------
    config : TelemetryConfiguration, optional
        Configuration for system (only used on first call)

    Returns
    -------
    TNFRUnifiedTelemetrySystem
        Global unified telemetry system instance
    """
    global _unified_telemetry_system

    if _unified_telemetry_system is None:
        _unified_telemetry_system = TNFRUnifiedTelemetrySystem(config)
        logger.info("Created global unified telemetry system")

    return _unified_telemetry_system


# Convenience functions for direct telemetry operations
def emit_structural_telemetry(**kwargs: Any) -> str:
    """Emit structural telemetry - convenience function."""
    return get_unified_telemetry_system().emit_structural_event(**kwargs)


def emit_performance_telemetry(
    operation_name: str, duration_ms: float, **kwargs: Any
) -> str:
    """Emit performance telemetry - convenience function."""
    return get_unified_telemetry_system().emit_performance_event(
        operation_name, duration_ms, **kwargs
    )


def emit_failure_telemetry(
    failure_type: str, error_message: str = "", **kwargs: Any
) -> str:
    """Emit failure telemetry - convenience function."""
    return get_unified_telemetry_system().emit_failure_event(
        failure_type, error_message, **kwargs
    )


def flush_unified_telemetry() -> None:
    """Flush unified telemetry buffers - convenience function."""
    if _unified_telemetry_system is not None:
        _unified_telemetry_system.flush_all()


def get_unified_telemetry_stats() -> dict[str, Any]:
    """Get unified telemetry statistics - convenience function."""
    if _unified_telemetry_system is not None:
        return _unified_telemetry_system.get_statistics()
    return {"status": "system_not_initialized"}