"""
Alerting System - Monitors and alerts on agent issues.

Implements:
- Stuck agent detection alerts
- Cost threshold alerts
- Error spike alerts
- Budget exhaustion alerts
- Health degradation alerts
"""

from dataclasses import dataclass, field
from datetime import datetime, timedelta
from enum import Enum
from typing import Optional, Dict, List, Any, Callable, Set
import asyncio
import logging
import json

try:
    import aiohttp
    AIOHTTP_AVAILABLE = True
except ImportError:
    AIOHTTP_AVAILABLE = False

logger = logging.getLogger(__name__)


class AlertSeverity(Enum):
    """Severity level of an alert."""
    INFO = "info"
    WARNING = "warning"
    ERROR = "error"
    CRITICAL = "critical"


class AlertType(Enum):
    """Type of alert."""
    STUCK_AGENT = "stuck_agent"
    COST_THRESHOLD = "cost_threshold"
    ERROR_SPIKE = "error_spike"
    BUDGET_EXHAUSTED = "budget_exhausted"
    HEALTH_DEGRADED = "health_degraded"
    MERGE_BLOCKED = "merge_blocked"
    APPROVAL_TIMEOUT = "approval_timeout"
    RATE_LIMIT = "rate_limit"


class AlertState(Enum):
    """Current state of an alert."""
    FIRING = "firing"
    RESOLVED = "resolved"
    ACKNOWLEDGED = "acknowledged"
    SILENCED = "silenced"


@dataclass
class AlertRule:
    """Configuration for an alert rule."""

    name: str
    alert_type: AlertType
    severity: AlertSeverity = AlertSeverity.WARNING

    # Thresholds
    threshold: float = 0.0
    duration_seconds: int = 0  # How long condition must persist

    # Targeting
    target_agents: Optional[Set[str]] = None  # None = all agents

    # Notification
    notify_channels: List[str] = field(default_factory=lambda: ["console"])
    cooldown_seconds: int = 300  # Don't re-alert for 5 min

    # State
    enabled: bool = True
    last_fired_at: Optional[datetime] = None


@dataclass
class Alert:
    """An active or resolved alert."""

    alert_id: str
    rule_name: str
    alert_type: AlertType
    severity: AlertSeverity

    # Content
    title: str
    message: str
    details: Dict[str, Any] = field(default_factory=dict)

    # Target
    agent_id: Optional[str] = None
    resource_id: Optional[str] = None

    # State
    state: AlertState = AlertState.FIRING
    firing_since: datetime = field(default_factory=datetime.now)
    resolved_at: Optional[datetime] = None

    # Response
    acknowledged_by: Optional[str] = None
    acknowledged_at: Optional[datetime] = None
    notes: str = ""


@dataclass
class NotificationChannel:
    """Configuration for a notification channel."""

    name: str
    channel_type: str  # console, slack, webhook, email

    # Channel-specific config
    webhook_url: Optional[str] = None
    email_addresses: List[str] = field(default_factory=list)
    slack_channel: Optional[str] = None
    slack_token: Optional[str] = None  # Bot token for Slack API

    # Webhook settings
    webhook_headers: Dict[str, str] = field(default_factory=dict)
    webhook_timeout_seconds: int = 10

    # Settings
    enabled: bool = True
    min_severity: AlertSeverity = AlertSeverity.WARNING

    # Retry settings
    retry_count: int = 3
    retry_delay_seconds: float = 1.0


class AlertManager:
    """
    Manages alerts and notifications.

    Monitors for various conditions and fires alerts when
    thresholds are exceeded.
    """

    def __init__(
        self,
        db: Any = None,
        config: Optional[Dict[str, Any]] = None,
    ):
        """
        Initialize alert manager.

        Args:
            db: Database for persistence
            config: Configuration options
        """
        self.db = db
        self.config = config or {}

        # Alert rules
        self._rules: Dict[str, AlertRule] = {}

        # Active alerts
        self._active_alerts: Dict[str, Alert] = {}
        self._alert_history: List[Alert] = []

        # Notification channels
        self._channels: Dict[str, NotificationChannel] = {}

        # Callbacks
        self._alert_callbacks: List[Callable] = []

        # Tracking state for condition detection
        self._condition_start_times: Dict[str, datetime] = {}
        self._error_counts: Dict[str, int] = {}
        self._last_error_window: datetime = datetime.now()

        # Alert ID generation
        self._next_alert_id = 1

        # Initialize default rules
        self._init_default_rules()
        self._init_default_channels()

    def _init_default_rules(self) -> None:
        """Initialize default alert rules."""
        default_rules = [
            AlertRule(
                name="stuck_agent_warning",
                alert_type=AlertType.STUCK_AGENT,
                severity=AlertSeverity.WARNING,
                duration_seconds=300,  # 5 minutes
            ),
            AlertRule(
                name="stuck_agent_critical",
                alert_type=AlertType.STUCK_AGENT,
                severity=AlertSeverity.CRITICAL,
                duration_seconds=900,  # 15 minutes
            ),
            AlertRule(
                name="cost_threshold_warning",
                alert_type=AlertType.COST_THRESHOLD,
                severity=AlertSeverity.WARNING,
                threshold=50.0,  # $50/day
            ),
            AlertRule(
                name="cost_threshold_critical",
                alert_type=AlertType.COST_THRESHOLD,
                severity=AlertSeverity.CRITICAL,
                threshold=100.0,  # $100/day
            ),
            AlertRule(
                name="error_spike",
                alert_type=AlertType.ERROR_SPIKE,
                severity=AlertSeverity.WARNING,
                threshold=10.0,  # 10 errors per minute
            ),
            AlertRule(
                name="budget_exhausted",
                alert_type=AlertType.BUDGET_EXHAUSTED,
                severity=AlertSeverity.ERROR,
                threshold=1.0,  # 100% of budget
            ),
            AlertRule(
                name="health_degraded",
                alert_type=AlertType.HEALTH_DEGRADED,
                severity=AlertSeverity.WARNING,
                threshold=0.5,  # Health score below 50%
            ),
        ]

        for rule in default_rules:
            self._rules[rule.name] = rule

    def _init_default_channels(self) -> None:
        """Initialize default notification channels."""
        self._channels["console"] = NotificationChannel(
            name="console",
            channel_type="console",
            min_severity=AlertSeverity.INFO,
        )

    def add_rule(self, rule: AlertRule) -> None:
        """Add or update an alert rule."""
        self._rules[rule.name] = rule
        logger.info(f"Added alert rule: {rule.name}")

    def remove_rule(self, rule_name: str) -> bool:
        """Remove an alert rule."""
        if rule_name in self._rules:
            del self._rules[rule_name]
            return True
        return False

    def add_channel(self, channel: NotificationChannel) -> None:
        """Add a notification channel."""
        self._channels[channel.name] = channel
        logger.info(f"Added notification channel: {channel.name}")

    def on_alert(self, callback: Callable) -> None:
        """Register a callback for alerts."""
        self._alert_callbacks.append(callback)

    def _generate_alert_id(self) -> str:
        """Generate unique alert ID."""
        alert_id = f"alert-{self._next_alert_id:06d}"
        self._next_alert_id += 1
        return alert_id

    async def check_stuck_agent(
        self,
        agent_id: str,
        is_stuck: bool,
        stuck_duration_seconds: int = 0,
        reason: str = "",
    ) -> Optional[Alert]:
        """
        Check for stuck agent condition.

        Args:
            agent_id: Agent to check
            is_stuck: Whether agent is currently stuck
            stuck_duration_seconds: How long agent has been stuck
            reason: Reason agent is stuck

        Returns:
            Alert if triggered, None otherwise
        """
        for rule_name, rule in self._rules.items():
            if rule.alert_type != AlertType.STUCK_AGENT:
                continue
            if not rule.enabled:
                continue

            # Check if targeting this agent
            if rule.target_agents and agent_id not in rule.target_agents:
                continue

            # Check duration threshold
            if is_stuck and stuck_duration_seconds >= rule.duration_seconds:
                # Check cooldown
                if rule.last_fired_at:
                    cooldown_elapsed = (datetime.now() - rule.last_fired_at).total_seconds()
                    if cooldown_elapsed < rule.cooldown_seconds:
                        continue

                # Fire alert
                alert = await self._fire_alert(
                    rule=rule,
                    title=f"Agent {agent_id} is stuck",
                    message=f"Agent has been stuck for {stuck_duration_seconds}s. Reason: {reason}",
                    agent_id=agent_id,
                    details={
                        "stuck_duration": stuck_duration_seconds,
                        "reason": reason,
                    },
                )
                return alert

            elif not is_stuck:
                # Resolve any existing alerts
                await self._resolve_alerts_for(
                    alert_type=AlertType.STUCK_AGENT,
                    agent_id=agent_id,
                )

        return None

    async def check_cost_threshold(
        self,
        current_cost: float,
        agent_id: Optional[str] = None,
    ) -> Optional[Alert]:
        """
        Check for cost threshold breach.

        Args:
            current_cost: Current cost (daily)
            agent_id: Optional agent to attribute

        Returns:
            Alert if triggered, None otherwise
        """
        for rule_name, rule in self._rules.items():
            if rule.alert_type != AlertType.COST_THRESHOLD:
                continue
            if not rule.enabled:
                continue

            if current_cost >= rule.threshold:
                # Check cooldown
                if rule.last_fired_at:
                    cooldown_elapsed = (datetime.now() - rule.last_fired_at).total_seconds()
                    if cooldown_elapsed < rule.cooldown_seconds:
                        continue

                alert = await self._fire_alert(
                    rule=rule,
                    title=f"Cost threshold exceeded (${rule.threshold:.2f})",
                    message=f"Daily cost has reached ${current_cost:.2f}",
                    agent_id=agent_id,
                    details={
                        "current_cost": current_cost,
                        "threshold": rule.threshold,
                    },
                )
                return alert

        return None

    async def check_error_spike(
        self,
        error_count: int,
        time_window_seconds: int = 60,
        agent_id: Optional[str] = None,
    ) -> Optional[Alert]:
        """
        Check for error spike.

        Args:
            error_count: Number of errors in time window
            time_window_seconds: Time window for counting
            agent_id: Optional agent to attribute

        Returns:
            Alert if triggered, None otherwise
        """
        errors_per_minute = error_count * (60 / time_window_seconds)

        for rule_name, rule in self._rules.items():
            if rule.alert_type != AlertType.ERROR_SPIKE:
                continue
            if not rule.enabled:
                continue

            if errors_per_minute >= rule.threshold:
                # Check cooldown
                if rule.last_fired_at:
                    cooldown_elapsed = (datetime.now() - rule.last_fired_at).total_seconds()
                    if cooldown_elapsed < rule.cooldown_seconds:
                        continue

                alert = await self._fire_alert(
                    rule=rule,
                    title="Error spike detected",
                    message=f"{errors_per_minute:.1f} errors/minute exceeds threshold of {rule.threshold}",
                    agent_id=agent_id,
                    details={
                        "errors_per_minute": errors_per_minute,
                        "threshold": rule.threshold,
                        "error_count": error_count,
                    },
                )
                return alert

        return None

    async def check_budget_exhausted(
        self,
        agent_id: str,
        budget_percentage: float,
    ) -> Optional[Alert]:
        """
        Check for budget exhaustion.

        Args:
            agent_id: Agent to check
            budget_percentage: Percentage of budget used (0-1)

        Returns:
            Alert if triggered, None otherwise
        """
        for rule_name, rule in self._rules.items():
            if rule.alert_type != AlertType.BUDGET_EXHAUSTED:
                continue
            if not rule.enabled:
                continue

            if budget_percentage >= rule.threshold:
                # Check cooldown
                if rule.last_fired_at:
                    cooldown_elapsed = (datetime.now() - rule.last_fired_at).total_seconds()
                    if cooldown_elapsed < rule.cooldown_seconds:
                        continue

                alert = await self._fire_alert(
                    rule=rule,
                    title=f"Budget exhausted for {agent_id}",
                    message=f"Agent has used {budget_percentage*100:.1f}% of daily budget",
                    agent_id=agent_id,
                    details={
                        "budget_percentage": budget_percentage,
                    },
                )
                return alert

        return None

    async def check_health_degraded(
        self,
        agent_id: str,
        health_score: float,
    ) -> Optional[Alert]:
        """
        Check for health degradation.

        Args:
            agent_id: Agent to check
            health_score: Current health score (0-1)

        Returns:
            Alert if triggered, None otherwise
        """
        for rule_name, rule in self._rules.items():
            if rule.alert_type != AlertType.HEALTH_DEGRADED:
                continue
            if not rule.enabled:
                continue

            if health_score < rule.threshold:
                # Check cooldown
                if rule.last_fired_at:
                    cooldown_elapsed = (datetime.now() - rule.last_fired_at).total_seconds()
                    if cooldown_elapsed < rule.cooldown_seconds:
                        continue

                alert = await self._fire_alert(
                    rule=rule,
                    title=f"Health degraded for {agent_id}",
                    message=f"Health score {health_score:.1%} below threshold {rule.threshold:.1%}",
                    agent_id=agent_id,
                    details={
                        "health_score": health_score,
                        "threshold": rule.threshold,
                    },
                )
                return alert
            else:
                # Resolve if health recovered
                await self._resolve_alerts_for(
                    alert_type=AlertType.HEALTH_DEGRADED,
                    agent_id=agent_id,
                )

        return None

    async def _fire_alert(
        self,
        rule: AlertRule,
        title: str,
        message: str,
        agent_id: Optional[str] = None,
        resource_id: Optional[str] = None,
        details: Optional[Dict[str, Any]] = None,
    ) -> Alert:
        """Fire an alert."""
        alert = Alert(
            alert_id=self._generate_alert_id(),
            rule_name=rule.name,
            alert_type=rule.alert_type,
            severity=rule.severity,
            title=title,
            message=message,
            agent_id=agent_id,
            resource_id=resource_id,
            details=details or {},
        )

        # Track alert
        self._active_alerts[alert.alert_id] = alert
        rule.last_fired_at = datetime.now()

        # Log
        log_level = {
            AlertSeverity.INFO: logging.INFO,
            AlertSeverity.WARNING: logging.WARNING,
            AlertSeverity.ERROR: logging.ERROR,
            AlertSeverity.CRITICAL: logging.CRITICAL,
        }.get(rule.severity, logging.WARNING)

        logger.log(log_level, f"[ALERT] {title}: {message}")

        # Send notifications
        await self._send_notifications(alert, rule.notify_channels)

        # Fire callbacks
        for callback in self._alert_callbacks:
            try:
                if asyncio.iscoroutinefunction(callback):
                    await callback(alert)
                else:
                    callback(alert)
            except Exception as e:
                logger.error(f"Alert callback failed: {e}")

        return alert

    async def _resolve_alerts_for(
        self,
        alert_type: AlertType,
        agent_id: Optional[str] = None,
    ) -> int:
        """Resolve alerts matching criteria."""
        resolved_count = 0

        for alert_id, alert in list(self._active_alerts.items()):
            if alert.alert_type != alert_type:
                continue
            if agent_id and alert.agent_id != agent_id:
                continue
            if alert.state != AlertState.FIRING:
                continue

            alert.state = AlertState.RESOLVED
            alert.resolved_at = datetime.now()

            self._alert_history.append(alert)
            del self._active_alerts[alert_id]

            resolved_count += 1
            logger.info(f"Resolved alert: {alert.title}")

        return resolved_count

    async def _send_notifications(
        self,
        alert: Alert,
        channel_names: List[str],
    ) -> None:
        """Send alert notifications."""
        for channel_name in channel_names:
            channel = self._channels.get(channel_name)
            if not channel or not channel.enabled:
                continue

            # Check minimum severity
            severity_order = [
                AlertSeverity.INFO,
                AlertSeverity.WARNING,
                AlertSeverity.ERROR,
                AlertSeverity.CRITICAL,
            ]
            if severity_order.index(alert.severity) < severity_order.index(channel.min_severity):
                continue

            try:
                if channel.channel_type == "console":
                    await self._notify_console(alert)
                elif channel.channel_type == "slack":
                    await self._notify_slack(alert, channel)
                elif channel.channel_type == "webhook":
                    await self._notify_webhook(alert, channel)
                elif channel.channel_type == "email":
                    await self._notify_email(alert, channel)

            except Exception as e:
                logger.error(f"Failed to send notification to {channel_name}: {e}")

    async def _notify_console(self, alert: Alert) -> None:
        """Print alert to console."""
        severity_prefix = {
            AlertSeverity.INFO: "[INFO]",
            AlertSeverity.WARNING: "[WARN]",
            AlertSeverity.ERROR: "[ERROR]",
            AlertSeverity.CRITICAL: "[CRITICAL]",
        }.get(alert.severity, "[ALERT]")

        print(f"\n{severity_prefix} {alert.title}")
        print(f"  {alert.message}")
        if alert.agent_id:
            print(f"  Agent: {alert.agent_id}")
        print()

    async def _notify_slack(
        self,
        alert: Alert,
        channel: NotificationChannel,
    ) -> None:
        """
        Send alert to Slack.

        Supports two modes:
        1. Incoming Webhook (webhook_url) - Simple, no token needed
        2. Slack API (slack_token + slack_channel) - Full API access

        Args:
            alert: The alert to send
            channel: Notification channel configuration
        """
        if not AIOHTTP_AVAILABLE:
            logger.warning("aiohttp not available, cannot send Slack notification")
            return

        # Build Slack message with Block Kit
        severity_emoji = {
            AlertSeverity.INFO: ":information_source:",
            AlertSeverity.WARNING: ":warning:",
            AlertSeverity.ERROR: ":x:",
            AlertSeverity.CRITICAL: ":rotating_light:",
        }.get(alert.severity, ":bell:")

        severity_color = {
            AlertSeverity.INFO: "#36a64f",
            AlertSeverity.WARNING: "#ffcc00",
            AlertSeverity.ERROR: "#ff6600",
            AlertSeverity.CRITICAL: "#ff0000",
        }.get(alert.severity, "#808080")

        # Build blocks for rich formatting
        blocks = [
            {
                "type": "header",
                "text": {
                    "type": "plain_text",
                    "text": f"{severity_emoji} {alert.title}",
                    "emoji": True,
                }
            },
            {
                "type": "section",
                "text": {
                    "type": "mrkdwn",
                    "text": alert.message,
                }
            },
        ]

        # Add context fields
        context_elements = [
            {
                "type": "mrkdwn",
                "text": f"*Severity:* {alert.severity.value.upper()}",
            },
            {
                "type": "mrkdwn",
                "text": f"*Type:* {alert.alert_type.value}",
            },
        ]

        if alert.agent_id:
            context_elements.append({
                "type": "mrkdwn",
                "text": f"*Agent:* {alert.agent_id}",
            })

        blocks.append({
            "type": "context",
            "elements": context_elements,
        })

        # Add details if present
        if alert.details:
            detail_text = "\n".join(f"• *{k}:* {v}" for k, v in alert.details.items())
            blocks.append({
                "type": "section",
                "text": {
                    "type": "mrkdwn",
                    "text": f"*Details:*\n{detail_text}",
                }
            })

        # Add timestamp
        blocks.append({
            "type": "context",
            "elements": [{
                "type": "mrkdwn",
                "text": f"Alert ID: `{alert.alert_id}` | Fired at: {alert.firing_since.isoformat()}",
            }]
        })

        # Build payload
        payload = {
            "blocks": blocks,
            "attachments": [{
                "color": severity_color,
                "fallback": f"{alert.title}: {alert.message}",
            }],
        }

        # Determine URL and headers
        if channel.webhook_url:
            # Incoming webhook mode
            url = channel.webhook_url
            headers = {"Content-Type": "application/json"}
        elif channel.slack_token and channel.slack_channel:
            # Slack API mode
            url = "https://slack.com/api/chat.postMessage"
            headers = {
                "Content-Type": "application/json",
                "Authorization": f"Bearer {channel.slack_token}",
            }
            payload["channel"] = channel.slack_channel
        else:
            logger.error("Slack channel requires webhook_url or (slack_token + slack_channel)")
            return

        # Send with retry
        last_error = None
        for attempt in range(channel.retry_count):
            try:
                async with aiohttp.ClientSession() as session:
                    async with session.post(
                        url,
                        json=payload,
                        headers=headers,
                        timeout=aiohttp.ClientTimeout(total=channel.webhook_timeout_seconds),
                    ) as response:
                        if response.status == 200:
                            response_data = await response.json()
                            # Check Slack API response
                            if url.startswith("https://slack.com/api/") and not response_data.get("ok"):
                                raise Exception(f"Slack API error: {response_data.get('error', 'unknown')}")
                            logger.info(f"Slack notification sent: {alert.title}")
                            return
                        else:
                            last_error = f"HTTP {response.status}: {await response.text()}"
            except asyncio.TimeoutError:
                last_error = "Request timed out"
            except Exception as e:
                last_error = str(e)

            if attempt < channel.retry_count - 1:
                await asyncio.sleep(channel.retry_delay_seconds * (attempt + 1))

        logger.error(f"Failed to send Slack notification after {channel.retry_count} attempts: {last_error}")

    async def _notify_webhook(
        self,
        alert: Alert,
        channel: NotificationChannel,
    ) -> None:
        """
        Send alert to a generic webhook.

        Sends a JSON payload with alert details to the configured webhook URL.
        Supports custom headers for authentication.

        Args:
            alert: The alert to send
            channel: Notification channel configuration
        """
        if not AIOHTTP_AVAILABLE:
            logger.warning("aiohttp not available, cannot send webhook notification")
            return

        if not channel.webhook_url:
            logger.error("Webhook channel requires webhook_url")
            return

        # Build payload
        payload = {
            "alert_id": alert.alert_id,
            "rule_name": alert.rule_name,
            "alert_type": alert.alert_type.value,
            "severity": alert.severity.value,
            "title": alert.title,
            "message": alert.message,
            "agent_id": alert.agent_id,
            "resource_id": alert.resource_id,
            "state": alert.state.value,
            "firing_since": alert.firing_since.isoformat(),
            "details": alert.details,
            "timestamp": datetime.now().isoformat(),
        }

        # Build headers
        headers = {
            "Content-Type": "application/json",
            "User-Agent": "AgentOrchestrator/1.0",
        }
        headers.update(channel.webhook_headers)

        # Send with retry
        last_error = None
        for attempt in range(channel.retry_count):
            try:
                async with aiohttp.ClientSession() as session:
                    async with session.post(
                        channel.webhook_url,
                        json=payload,
                        headers=headers,
                        timeout=aiohttp.ClientTimeout(total=channel.webhook_timeout_seconds),
                    ) as response:
                        if 200 <= response.status < 300:
                            logger.info(f"Webhook notification sent: {alert.title}")
                            return
                        else:
                            last_error = f"HTTP {response.status}: {await response.text()}"
            except asyncio.TimeoutError:
                last_error = "Request timed out"
            except Exception as e:
                last_error = str(e)

            if attempt < channel.retry_count - 1:
                await asyncio.sleep(channel.retry_delay_seconds * (attempt + 1))

        logger.error(f"Failed to send webhook notification after {channel.retry_count} attempts: {last_error}")

    async def _notify_email(
        self,
        alert: Alert,
        channel: NotificationChannel,
    ) -> None:
        """
        Send alert via email using SMTP.

        Note: This is a basic implementation. For production use,
        consider using a dedicated email service like SendGrid or AWS SES.

        Args:
            alert: The alert to send
            channel: Notification channel configuration
        """
        if not channel.email_addresses:
            logger.error("Email channel requires email_addresses")
            return

        # Get SMTP settings from environment or config
        import os
        smtp_host = os.environ.get("SMTP_HOST", "localhost")
        smtp_port = int(os.environ.get("SMTP_PORT", "587"))
        smtp_user = os.environ.get("SMTP_USER", "")
        smtp_pass = os.environ.get("SMTP_PASS", "")
        smtp_from = os.environ.get("SMTP_FROM", "alerts@agent-orchestrator.local")

        severity_prefix = {
            AlertSeverity.INFO: "[INFO]",
            AlertSeverity.WARNING: "[WARNING]",
            AlertSeverity.ERROR: "[ERROR]",
            AlertSeverity.CRITICAL: "[CRITICAL]",
        }.get(alert.severity, "[ALERT]")

        subject = f"{severity_prefix} {alert.title}"

        # Build email body
        body_lines = [
            f"Alert: {alert.title}",
            f"Severity: {alert.severity.value.upper()}",
            f"Type: {alert.alert_type.value}",
            "",
            f"Message: {alert.message}",
            "",
        ]

        if alert.agent_id:
            body_lines.append(f"Agent: {alert.agent_id}")

        if alert.details:
            body_lines.append("")
            body_lines.append("Details:")
            for k, v in alert.details.items():
                body_lines.append(f"  - {k}: {v}")

        body_lines.extend([
            "",
            f"Alert ID: {alert.alert_id}",
            f"Fired at: {alert.firing_since.isoformat()}",
            "",
            "---",
            "Agent Orchestrator Alert System",
        ])

        body = "\n".join(body_lines)

        # Send email asynchronously
        try:
            import smtplib
            from email.mime.text import MIMEText
            from email.mime.multipart import MIMEMultipart

            # Run SMTP in executor to avoid blocking
            loop = asyncio.get_event_loop()
            await loop.run_in_executor(
                None,
                self._send_email_sync,
                smtp_host,
                smtp_port,
                smtp_user,
                smtp_pass,
                smtp_from,
                channel.email_addresses,
                subject,
                body,
            )
            logger.info(f"Email notification sent: {alert.title}")

        except Exception as e:
            logger.error(f"Failed to send email notification: {e}")

    def _send_email_sync(
        self,
        smtp_host: str,
        smtp_port: int,
        smtp_user: str,
        smtp_pass: str,
        from_addr: str,
        to_addrs: List[str],
        subject: str,
        body: str,
    ) -> None:
        """Synchronous email sending helper."""
        import smtplib
        from email.mime.text import MIMEText
        from email.mime.multipart import MIMEMultipart

        msg = MIMEMultipart()
        msg["From"] = from_addr
        msg["To"] = ", ".join(to_addrs)
        msg["Subject"] = subject
        msg.attach(MIMEText(body, "plain"))

        with smtplib.SMTP(smtp_host, smtp_port) as server:
            if smtp_user and smtp_pass:
                server.starttls()
                server.login(smtp_user, smtp_pass)
            server.sendmail(from_addr, to_addrs, msg.as_string())

    def acknowledge_alert(
        self,
        alert_id: str,
        acknowledged_by: str,
        notes: str = "",
    ) -> bool:
        """
        Acknowledge an alert.

        Args:
            alert_id: ID of alert to acknowledge
            acknowledged_by: Who is acknowledging
            notes: Optional notes

        Returns:
            True if acknowledged successfully
        """
        if alert_id not in self._active_alerts:
            return False

        alert = self._active_alerts[alert_id]
        alert.state = AlertState.ACKNOWLEDGED
        alert.acknowledged_by = acknowledged_by
        alert.acknowledged_at = datetime.now()
        alert.notes = notes

        logger.info(f"Alert acknowledged: {alert.title} by {acknowledged_by}")
        return True

    def silence_alert(self, alert_id: str) -> bool:
        """Silence an alert."""
        if alert_id not in self._active_alerts:
            return False

        alert = self._active_alerts[alert_id]
        alert.state = AlertState.SILENCED
        return True

    def get_active_alerts(
        self,
        severity: Optional[AlertSeverity] = None,
        alert_type: Optional[AlertType] = None,
        agent_id: Optional[str] = None,
    ) -> List[Alert]:
        """Get active alerts with optional filtering."""
        alerts = list(self._active_alerts.values())

        if severity:
            alerts = [a for a in alerts if a.severity == severity]
        if alert_type:
            alerts = [a for a in alerts if a.alert_type == alert_type]
        if agent_id:
            alerts = [a for a in alerts if a.agent_id == agent_id]

        return alerts

    def get_alert_history(
        self,
        limit: int = 100,
        alert_type: Optional[AlertType] = None,
    ) -> List[Alert]:
        """Get alert history."""
        alerts = self._alert_history

        if alert_type:
            alerts = [a for a in alerts if a.alert_type == alert_type]

        return alerts[-limit:]

    def get_alert_summary(self) -> Dict[str, Any]:
        """Get summary of alert status."""
        active_by_severity = {}
        for severity in AlertSeverity:
            count = len([a for a in self._active_alerts.values() if a.severity == severity])
            if count > 0:
                active_by_severity[severity.value] = count

        return {
            "active_count": len(self._active_alerts),
            "active_by_severity": active_by_severity,
            "acknowledged_count": len([
                a for a in self._active_alerts.values()
                if a.state == AlertState.ACKNOWLEDGED
            ]),
            "rules_enabled": len([r for r in self._rules.values() if r.enabled]),
            "channels_enabled": len([c for c in self._channels.values() if c.enabled]),
        }


# Singleton instance
_alert_manager: Optional[AlertManager] = None


def get_alert_manager() -> Optional[AlertManager]:
    """Get the global alert manager instance."""
    return _alert_manager


def set_alert_manager(manager: AlertManager) -> None:
    """Set the global alert manager instance."""
    global _alert_manager
    _alert_manager = manager


# =============================================================================
# Convenience Functions for Channel Configuration
# =============================================================================

def create_slack_webhook_channel(
    name: str,
    webhook_url: str,
    min_severity: AlertSeverity = AlertSeverity.WARNING,
) -> NotificationChannel:
    """
    Create a Slack notification channel using an incoming webhook.

    To set up a Slack incoming webhook:
    1. Go to https://api.slack.com/apps
    2. Create a new app or select existing one
    3. Enable "Incoming Webhooks"
    4. Create a webhook for your desired channel
    5. Copy the webhook URL

    Args:
        name: Unique name for this channel
        webhook_url: Slack incoming webhook URL
        min_severity: Minimum severity to send (default: WARNING)

    Returns:
        Configured NotificationChannel

    Example:
        channel = create_slack_webhook_channel(
            name="slack-alerts",
            webhook_url="https://hooks.slack.com/services/T.../B.../xxx",
        )
        alert_manager.add_channel(channel)
    """
    return NotificationChannel(
        name=name,
        channel_type="slack",
        webhook_url=webhook_url,
        min_severity=min_severity,
    )


def create_slack_api_channel(
    name: str,
    token: str,
    channel: str,
    min_severity: AlertSeverity = AlertSeverity.WARNING,
) -> NotificationChannel:
    """
    Create a Slack notification channel using the Slack API.

    Requires a bot token with chat:write permission.

    To set up a Slack bot:
    1. Go to https://api.slack.com/apps
    2. Create a new app
    3. Under "OAuth & Permissions", add "chat:write" scope
    4. Install the app to your workspace
    5. Copy the "Bot User OAuth Token"
    6. Invite the bot to the desired channel

    Args:
        name: Unique name for this channel
        token: Slack bot token (xoxb-...)
        channel: Channel name (#channel) or ID (C1234567890)
        min_severity: Minimum severity to send (default: WARNING)

    Returns:
        Configured NotificationChannel

    Example:
        channel = create_slack_api_channel(
            name="slack-critical",
            token="xoxb-1234567890-abcdefg",
            channel="#production-alerts",
            min_severity=AlertSeverity.CRITICAL,
        )
        alert_manager.add_channel(channel)
    """
    return NotificationChannel(
        name=name,
        channel_type="slack",
        slack_token=token,
        slack_channel=channel,
        min_severity=min_severity,
    )


def create_webhook_channel(
    name: str,
    url: str,
    headers: Optional[Dict[str, str]] = None,
    min_severity: AlertSeverity = AlertSeverity.WARNING,
    timeout_seconds: int = 10,
) -> NotificationChannel:
    """
    Create a generic webhook notification channel.

    The webhook will receive POST requests with JSON payloads containing:
    - alert_id, rule_name, alert_type, severity
    - title, message, details
    - agent_id, resource_id
    - state, firing_since, timestamp

    Args:
        name: Unique name for this channel
        url: Webhook URL to POST alerts to
        headers: Optional headers (e.g., for authentication)
        min_severity: Minimum severity to send (default: WARNING)
        timeout_seconds: Request timeout (default: 10)

    Returns:
        Configured NotificationChannel

    Example:
        # Basic webhook
        channel = create_webhook_channel(
            name="monitoring",
            url="https://monitoring.example.com/webhook",
        )

        # With authentication
        channel = create_webhook_channel(
            name="pagerduty",
            url="https://events.pagerduty.com/v2/enqueue",
            headers={"Authorization": "Token token=xxx"},
            min_severity=AlertSeverity.ERROR,
        )
        alert_manager.add_channel(channel)
    """
    return NotificationChannel(
        name=name,
        channel_type="webhook",
        webhook_url=url,
        webhook_headers=headers or {},
        webhook_timeout_seconds=timeout_seconds,
        min_severity=min_severity,
    )


def create_email_channel(
    name: str,
    recipients: List[str],
    min_severity: AlertSeverity = AlertSeverity.ERROR,
) -> NotificationChannel:
    """
    Create an email notification channel.

    SMTP settings are read from environment variables:
    - SMTP_HOST: SMTP server hostname (default: localhost)
    - SMTP_PORT: SMTP server port (default: 587)
    - SMTP_USER: SMTP username (optional)
    - SMTP_PASS: SMTP password (optional)
    - SMTP_FROM: From address (default: alerts@agent-orchestrator.local)

    Args:
        name: Unique name for this channel
        recipients: List of email addresses
        min_severity: Minimum severity to send (default: ERROR)

    Returns:
        Configured NotificationChannel

    Example:
        channel = create_email_channel(
            name="email-alerts",
            recipients=["oncall@example.com", "team@example.com"],
            min_severity=AlertSeverity.CRITICAL,
        )
        alert_manager.add_channel(channel)
    """
    return NotificationChannel(
        name=name,
        channel_type="email",
        email_addresses=recipients,
        min_severity=min_severity,
    )


def setup_default_alerting(
    slack_webhook_url: Optional[str] = None,
    webhook_url: Optional[str] = None,
    email_recipients: Optional[List[str]] = None,
) -> AlertManager:
    """
    Set up alerting with common defaults.

    Creates an AlertManager with default rules and optionally configures
    notification channels based on provided arguments.

    Args:
        slack_webhook_url: Optional Slack incoming webhook URL
        webhook_url: Optional generic webhook URL
        email_recipients: Optional list of email addresses

    Returns:
        Configured AlertManager (also sets as global instance)

    Example:
        manager = setup_default_alerting(
            slack_webhook_url="https://hooks.slack.com/services/...",
        )
    """
    manager = AlertManager()

    if slack_webhook_url:
        manager.add_channel(create_slack_webhook_channel(
            name="slack",
            webhook_url=slack_webhook_url,
        ))

    if webhook_url:
        manager.add_channel(create_webhook_channel(
            name="webhook",
            url=webhook_url,
        ))

    if email_recipients:
        manager.add_channel(create_email_channel(
            name="email",
            recipients=email_recipients,
        ))

    set_alert_manager(manager)
    return manager
