LLM-powered incident analysis dashboard on Civo

1incident-analysis-dashboard/
2├── backend/
3│   ├── main.py
4│   ├── requirements.txt
5│   └── Dockerfile
6├── frontend/
7│   └── index.html
8└── kubernetes/
9    ├── backend-deployment.yaml
10    └── frontend-deployment.yaml

1kubectl get nodes
2kubectl get pods -A

1from fastapi import FastAPI, HTTPException
2from fastapi.middleware.cors import CORSMiddleware
3from pydantic import BaseModel, Field
4from typing import List, Dict, Any, Optional
5import os
6import json
7import logging
8import re
9import httpx
10
11
12
13
14# ---------------------------------------------------------
15# Logging setup
16# ---------------------------------------------------------
17logger = logging.getLogger("incident-analysis-api")
18logging.basicConfig(
19    level=logging.INFO,
20    format="%(asctime)s [%(levelname)s] %(name)s - %(message)s",
21)
22
23
24
25
26# ---------------------------------------------------------
27# RelaxAI client initialization
28# ---------------------------------------------------------
29RELAX_API_KEY = os.getenv("RELAX_API_KEY")
30RELAX_MODEL = "Llama-4-Maverick-17B-128E"
31RELAX_URL = "https://api.relax.ai/v1/chat/completions"
32
33
34# ---------------------------------------------------------
35# FastAPI app + CORS
36# ---------------------------------------------------------
37app = FastAPI(title="Incident Analysis API")
38
39
40
41
42# CORS middleware so browser frontend can talk to this API
43app.add_middleware(
44    CORSMiddleware,
45    allow_origins=["*"],          # for demo only; lock this down in real life
46    allow_credentials=True,
47    allow_methods=["*"],          # includes GET, POST, OPTIONS, etc.
48    allow_headers=["*"],          # includes Content-Type
49)
50
51
52
53
54# ---------------------------------------------------------
55# Pydantic models
56# ---------------------------------------------------------
57class LogEntry(BaseModel):
58    timestamp: str = Field(..., description="ISO8601 timestamp of the log entry")
59    severity: str = Field(..., description="Log level, e.g. ERROR, WARNING, CRITICAL")
60    service: str = Field(..., description="Service or component name")
61    message: str = Field(..., description="Raw log message")
62    metadata: Dict[str, Any] = Field(default_factory=dict)
63
64
65
66
67
68
69
70
71class AnalysisResult(BaseModel):
72    root_cause: str
73    category: str
74    summary: str
75    severity: str
76    recommendations: List[str]
77
78
79
80
81
82
83
84
85# ---------------------------------------------------------
86# Helper: safely extract JSON from model output
87# ---------------------------------------------------------
88def parse_relax_json(raw: str) -> Dict[str, Any]:
89    """
90    Best-effort JSON extraction from model output.
91
92
93
94
95    Handles:
96    - Pure JSON
97    - JSON wrapped in prose
98    - JSON inside ```json ... ``` code blocks
99
100
101
102
103    Raises ValueError if nothing JSON-like can be parsed.
104    """
105    if not raw:
106        raise ValueError("Empty response from model")
107
108
109
110
111    raw = raw.strip()
112
113
114
115
116    # Attempt direct JSON
117    try:
118        return json.loads(raw)
119    except json.JSONDecodeError:
120        pass
121
122
123
124
125    # Handle code-fenced JSON
126    if raw.startswith("```"):
127        raw_stripped = re.sub(r"^```[a-zA-Z]*", "", raw)
128        raw_stripped = raw_stripped.strip("` \n")
129        try:
130            return json.loads(raw_stripped)
131        except json.JSONDecodeError:
132            raw = raw_stripped  # fallback to generic extraction
133
134
135
136
137    # Extract first JSON object found in mixed text
138    start = raw.find("{")
139    end = raw.rfind("}")
140    if start != -1 and end != -1 and end > start:
141        candidate = raw[start : end + 1]
142        try:
143            return json.loads(candidate)
144        except json.JSONDecodeError:
145            logger.warning("Failed to parse JSON candidate from model output")
146
147
148
149
150    # Nothing worked
151    raise ValueError(f"Could not parse JSON from model output: {raw[:200]}...")
152
153
154# ---------------------------------------------------------
155# Manual RelaxAI call
156# ---------------------------------------------------------
157async def call_relax_ai(prompt: str) -> Dict[str, Any]:
158    if not RELAX_API_KEY:
159        raise HTTPException(status_code=500, detail="RELAX_API_KEY not configured.")
160
161
162    headers = {
163        "Authorization": f"Bearer {RELAX_API_KEY}",
164        "Content-Type": "application/json",
165    }
166
167
168    payload = {
169        "model": RELAX_MODEL,
170        "messages": [{"role": "user", "content": prompt}],
171        "temperature": 0.3,
172        "max_tokens": 500,
173    }
174
175
176    logger.debug("Sending request to RelaxAI...")
177    try:
178        async with httpx.AsyncClient(timeout=10.0) as client:
179            response = await client.post(RELAX_URL, json=payload, headers=headers)
180            response.raise_for_status()
181            data = response.json()
182            logger.debug("Received RelaxAI response: %s", data)
183            return data
184    except Exception as e:
185        logger.error("RelaxAI request failed: %s", e)
186        raise HTTPException(status_code=500, detail="RelaxAI call failed.")
187
188
189# ---------------------------------------------------------
190# Main endpoint
191# ---------------------------------------------------------
192@app.post("/analyze/logs", response_model=List[AnalysisResult])
193async def analyze_logs(logs: List[LogEntry]):
194    if not logs:
195        raise HTTPException(status_code=400, detail="No logs provided")
196
197
198    logger.info("Received %d log entries for analysis", len(logs))
199    results: List[AnalysisResult] = []
200
201
202    for idx, log in enumerate(logs):
203        logger.info("Analyzing log %d: %s", idx, log.service)
204
205
206        prompt = f"""
207        Analyze this system log entry:
208
209
210        Service: {log.service}
211        Severity: {log.severity}
212        Message: {log.message}
213        Metadata: {log.metadata}
214
215
216        Respond ONLY with valid JSON and follow this exact structure:
217
218
219        {{
220        "root_cause": "...",
221        "category": "...",
222        "summary": "...",
223        "recommendations": ["...", "...", "..."]
224        }}
225
226
227        Do NOT include any text before or after the JSON object.
228        """
229
230
231        try:
232            resp = await call_relax_ai(prompt)
233            content = resp["choices"][0]["message"]["content"]
234            parsed = parse_relax_json(content)
235        except Exception as e:
236            logger.error("Error analyzing: %s", e)
237            parsed = {
238                "root_cause": "Analysis failed",
239                "category": "error",
240                "summary": log.message[:200],
241                "recommendations": ["Check logs manually"],
242            }
243
244
245        results.append(
246            AnalysisResult(
247                root_cause=parsed["root_cause"],
248                category=parsed["category"],
249                summary=parsed["summary"],
250                severity=log.severity,
251                recommendations=parsed["recommendations"],
252            )
253        )
254
255
256    logger.info("Returning %d results", len(results))
257    return results
258
259
260# ---------------------------------------------------------
261# Health check
262# ---------------------------------------------------------
263@app.get("/health")
264async def health_check():
265    """
266    Simple health endpoint for Kubernetes liveness / readiness probes.
267    """
268    return {"status": "healthy", "service": "incident-analysis-api"}

1fastapi==0.104.1
2uvicorn[standard]==0.24.0
3pydantic==2.5.0
4relaxai==0.1.0
5httpx==0.27.0

1<!DOCTYPE html>
2<html lang="en">
3<head>
4<meta charset="UTF-8">
5<meta name="viewport" content="width=device-width, initial-scale=1.0">
6<title>Incident Analysis Dashboard</title>
7
8<style>
9body { font-family: Arial, sans-serif; background: #f7f7f7; padding: 20px; }
10.container { max-width: 900px; margin: 0 auto; background: white; padding: 25px; border-radius: 10px; box-shadow: 0 2px 8px rgba(0,0,0,0.1); }
11h1 { margin-bottom: 10px; font-size: 32px; }
12button { padding: 8px 15px; border: 1px solid #333; background: #fff; cursor: pointer; border-radius: 4px; }
13#status { margin-left: 15px; font-size: 14px; }
14.stats { margin-top: 20px; display: flex; gap: 20px; }
15.stat-card { padding: 15px; background: #eef2ff; border-radius: 8px; width: 180px; }
16.stat-label { font-size: 14px; color: #555; }
17.stat-value { font-size: 24px; font-weight: bold; margin-top: 5px; }
18.incidents { margin-top: 30px; }
19.incident { background: #f9fafb; border-left: 4px solid #ccc; padding: 15px; margin-bottom: 20px; border-radius: 6px; }
20.incident-header { display: flex; justify-content: space-between; font-size: 16px; margin-bottom: 8px; }
21.severity-badge { padding: 3px 6px; border-radius: 4px; font-size: 12px; font-weight: bold; color: white; }
22.severity-error { background: #dc2626; }
23.severity-critical { background: #7f1d1d; }
24.severity-warning { background: #d97706; }
25.rec-item { margin-left: 10px; }
26</style>
27
28</head>
29<body>
30
31<div class="container">
32
33<h1>🔍 Incident Analysis Dashboard</h1>
34
35<div class="controls">
36  <div style="margin-top:20px;">
37    <label><strong>Paste Logs (JSON Array):</strong></label><br>
38    <textarea id="logInput" style="width:100%; height:150px; margin-top:5px; font-family:monospace;">
39[
40  {
41    "timestamp": "2025-01-15T14:23:45Z",
42    "severity": "ERROR",
43    "service": "api-gateway",
44    "message": "Connection timeout to database service after 30000ms",
45    "metadata": { "request_id": "req-789012" }
46  }
47]
48    </textarea>
49  </div>
50
51  <button onclick="analyzeIncidents()">Analyze New Incidents</button>
52  <span id="status"></span>
53</div>
54
55<div class="stats" id="stats"></div>
56<div class="incidents" id="incidents"></div>
57
58</div>
59
60<script>
61// IMPORTANT: Change this before building the Docker image
62const API_URL = "http://<BACKEND-EXTERNAL-IP>:8000";
63
64async function analyzeIncidents() {
65  const statusEl = document.getElementById("status");
66  statusEl.textContent = "Analyzing...";
67
68  let logs;
69  try {
70    logs = JSON.parse(document.getElementById("logInput").value);
71  } catch (e) {
72    statusEl.textContent = "Invalid JSON: " + e.message;
73    return;
74  }
75
76  try {
77    const response = await fetch(`${API_URL}/analyze/logs`, {
78      method: "POST",
79      headers: { "Content-Type": "application/json" },
80      body: JSON.stringify(logs)
81    });
82
83    const results = await response.json();
84    displayResults(results);
85    statusEl.textContent = `Analyzed ${results.length} incidents`;
86
87  } catch (error) {
88    statusEl.textContent = "Analysis failed: " + error.message;
89    console.error(error);
90  }
91}
92
93function displayResults(results) {
94  const categories = {};
95  results.forEach(r => categories[r.category] = (categories[r.category] || 0) + 1);
96
97  // Stats section
98  document.getElementById("stats").innerHTML = `
99<div class="stat-card">
100  <div class="stat-label">Total Incidents</div>
101  <div class="stat-value">${results.length}</div>
102</div>
103
104<div class="stat-card">
105  <div class="stat-label">Critical</div>
106  <div class="stat-value" style="color:#dc2626">
107    ${results.filter(r => r.severity === "CRITICAL").length}
108  </div>
109</div>
110
111<div class="stat-card">
112  <div class="stat-label">Top Category</div>
113  <div class="stat-value" style="font-size:20px; color:#3b82f6">
114    ${Object.keys(categories)[0] || "None"}
115  </div>
116</div>
117`;
118
119  // Incident cards
120  document.getElementById("incidents").innerHTML = results.map(incident => `
121<div class="incident">
122
123  <div class="incident-header">
124    <strong>${incident.category.toUpperCase()}</strong>
125    <span class="severity-badge severity-${incident.severity.toLowerCase()}">
126      ${incident.severity}
127    </span>
128  </div>
129
130  <div><strong>Root Cause:</strong> ${incident.root_cause}</div>
131
132  <div style="margin:10px 0; color:#6b7280;">
133    ${incident.summary}
134  </div>
135
136  <div class="recommendations">
137    <strong>Recommendations:</strong>
138    ${incident.recommendations.map(r => `<div class="rec-item">• ${r}</div>`).join("")}
139  </div>
140
141</div>
142`).join("");
143}
144
145window.onload = () => analyzeIncidents();
146</script>
147
148</body>
149</html>

1FROM python:3.11-slim
2
3RUN apt-get update && apt-get install -y --no-install-recommends \
4    ca-certificates \
5 && rm -rf /var/lib/apt/lists/*
6
7WORKDIR /app
8
9COPY requirements.txt .
10RUN pip install --no-cache-dir -r requirements.txt
11
12COPY main.py .
13
14CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

1# Build backend
2cd backend
3docker build -t your-registry/incident-backend:latest .
4docker push your-registry/incident-backend:latest
5
6
7# For frontend, use nginx
8cd ../frontend
9docker build -t your-registry/incident-frontend:latest -f - . <<EOF
10FROM nginx:alpine
11COPY index.html /usr/share/nginx/html/
12EOF
13docker push your-registry/incident-frontend:latest

1kubectl create secret generic relaxai-secret \
2  --from-literal=RELAX_API_KEY='your_relaxai_api_key_here' \

1apiVersion: apps/v1
2kind: Deployment
3metadata:
4  name: incident-backend
5spec:
6  replicas: 1
7  selector:
8    matchLabels:
9      app: incident-backend
10  template:
11    metadata:
12      labels:
13        app: incident-backend
14    spec:
15      containers:
16      - name: backend
17        image: alimohamed782/incident-backend:latest
18        ports:
19        - containerPort: 8000
20        envFrom:
21        - secretRef:
22            name: relaxai-secret  
23        resources:                  
24          requests:
25            cpu: "250m"
26            memory: "256Mi"
27          limits:
28            cpu: "500m"
29            memory: "512Mi"
30
31
32---
33apiVersion: v1
34kind: Service
35metadata:
36  name: incident-backend-service
37spec:
38  selector:
39    app: incident-backend
40  ports:
41  - port: 8000
42    targetPort: 8000
43  type: LoadBalancer

1apiVersion: apps/v1
2kind: Deployment
3metadata:
4  name: incident-frontend
5spec:
6  replicas: 1
7  selector:
8    matchLabels:
9      app: incident-frontend
10  template:
11    metadata:
12      labels:
13        app: incident-frontend
14    spec:
15      containers:
16      - name: frontend
17        image: alimohamed782/incident-frontend:latest
18        ports:
19        - containerPort: 80
20---
21apiVersion: v1
22kind: Service
23metadata:
24  name: incident-frontend-service
25spec:
26  selector:
27    app: incident-frontend
28  ports:
29  - port: 80
30    targetPort: 80
31  type: LoadBalancer

1kubectl apply -f backend-deployment.yaml
2kubectl apply -f frontend-deployment.yaml
3
4
5# Get service URLs
6kubectl get services

1[
2  {
3    "timestamp": "2025-01-15T14:23:45Z",
4    "severity": "ERROR",
5    "service": "api-gateway",
6    "message": "Connection timeout to database service after 30000ms. Retry attempts exhausted. Last error: Connection refused on port 5432.",
7    "metadata": {
8      "request_id": "req-789012",
9      "user_id": "user-45678"
10    }
11  },
12  {
13    "timestamp": "2025-01-15T14:24:12Z",
14    "severity": "CRITICAL",
15    "service": "payment-processor",
16    "message": "Transaction processing failed with exception: java.lang.OutOfMemoryError: Java heap space. Current heap usage: 95%. GC overhead limit exceeded.",
17    "metadata": {
18      "transaction_id": "txn-234567",
19      "amount": 1250.00
20    }
21  },
22  {
23    "timestamp": "2025-01-15T14:25:33Z",
24    "severity": "WARNING",
25    "service": "auth-service",
26    "message": "High latency detected for authentication requests. Average response time: 2.5s (threshold: 500ms). Possible causes: database connection pool exhaustion.",
27    "metadata": {
28      "endpoint": "/api/v1/auth/login",
29      "requests_per_minute": 450
30    }
31  }
32]