Incident Schema Reference

Incident Record

pub struct Incident {
    pub id: Uuid,
    pub tenant_id: String,
    pub cluster_id: String,
    pub state: IncidentState,          // Open, Approved, Dismissed, Remediating, Resolved, Failed
    pub severity: Severity,            // Info, Warning, Error, Critical
    pub host: String,
    pub rank: Option<u32>,
    pub job_id: Option<String>,
    pub root_cause: String,
    pub confidence: f64,               // 0.0..1.0
    pub confidence_breakdown: BTreeMap<String, f64>,
    pub rules: Vec<RuleHit>,
    pub evidence: Vec<EvidenceRef>,
    pub most_likely_root_cause_node_id: String,
    pub remediation: Option<Remediation>,
    pub created_at_unix_nano: u128,
    pub updated_at_unix_nano: u128,
    pub labels: BTreeMap<String, String>,
    pub annotations: BTreeMap<String, String>,
}

pub enum IncidentState {
    Open,
    Approved,
    Dismissed,
    Remediating,
    Resolved,
    Failed,
}

pub struct RuleHit {
    pub rule_id: String,
    pub severity: Severity,
    pub evidence_ids: Vec<String>,
    pub description: String,
    pub fired_at_unix_nano: u128,
    pub confidence: f64,
}

pub struct EvidenceRef {
    pub id: String,
    pub source: EvidenceSource,
    pub ts_unix_nano: u128,
    pub uri: String,    // data-platform location
    pub sha256: String,
}

pub struct Remediation {
    pub id: Uuid,
    pub final_state: RemediationState,
    pub steps: Vec<RemediationStep>,
    pub started_at_unix_nano: u128,
    pub completed_at_unix_nano: Option<u128>,
    pub rma_ticket: Option<String>,
    pub warm_spare: Option<String>,
}

pub enum RemediationState {
    Detected,
    NodeCordoned,
    CheckpointTriggered,
    PodEvicted,
    JobRerouted,
    DiagnosticsCompleted,
    RmaIssued,
    Completed,
    Failed,
}

JSON Shape (External API)

{
  "id": "uuid",
  "tenant_id": "acme",
  "cluster_id": "prod",
  "state": "OPEN",
  "severity": "CRITICAL",
  "host": "node-3",
  "rank": 7,
  "job_id": "j-1234",
  "root_cause": "Xid 79",
  "confidence": 0.97,
  "confidence_breakdown": {
    "deterministic_rule_weight": 0.6,
    "source_reliability": 0.18,
    "timestamp_proximity": 0.13,
    "recurrence_count": 0.05,
    "conflicting_evidence_penalty": 0.0,
    "missing_data_penalty": 0.0
  },
  "rules": [
    { "rule_id": "R1", "severity": "CRITICAL", "confidence": 0.97, "evidence_ids": ["ev-1"] }
  ],
  "evidence": [
    { "id": "ev-1", "source": "DCGM", "ts": "2024-01-01T00:00:00Z", "uri": "denpex-platform://.../ev-1.json", "sha256": "..." }
  ],
  "most_likely_root_cause_node_id": "GPU-abc",
  "remediation": null,
  "created_at": "2024-01-01T00:00:00Z",
  "updated_at": "2024-01-01T00:00:00Z",
  "labels": { "owner": "ml-platform" }
}

State Transitions

Open -> Approved       (operator approved)
Open -> Dismissed      (operator dismissed as FP)
Open -> Remediating    (auto tier allowed)
Approved -> Remediating
Remediating -> Resolved
Remediating -> Failed
Resolved -> Closed     (after 7d)

Every transition emits an AuditRecord and is reflected on the REST/GraphQL API within 1 second.