TERMINAL# Docker (quickest) docker run -d --name prometheus -p 9090:9090 \ -v /opt/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml \ prom/prometheus # Access: http://your-server:9090 # Key exporters you need: # Node Exporter → CPU, RAM, disk metrics (install on every server) # kube-state-metrics → Kubernetes pod/node/deploy metrics # cAdvisor → Container-level metrics

YAML# /etc/prometheus/prometheus.yml global: scrape_interval: 15s # How often to collect metrics evaluation_interval: 15s # How often to evaluate alert rules alerting: alertmanagers: - static_configs: - targets: ["alertmanager:9093"] rule_files: - "alert_rules.yml" # Alert rules file scrape_configs: # Prometheus monitors itself - job_name: "prometheus" static_configs: - targets: ["localhost:9090"] # Monitor Linux servers via Node Exporter - job_name: "node-exporter" static_configs: - targets: - "web1:9100" - "web2:9100" - "db1:9100" # Prometheus visits each target every 15 seconds # and collects CPU, RAM, disk, network metrics # Monitor Kubernetes - job_name: "kube-state-metrics" static_configs: - targets: ["kube-state-metrics:8080"]

PROMQL# Total HTTP requests per second rate(http_requests_total[5m]) # HTTP error rate (5xx errors) sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m])) * 100 # Average response latency (95th percentile) histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) # Number of running pods count(kube_pod_status_phase{phase="Running"})

PROMQLcount(node_cpu_seconds_total{mode="idle", instance=~".*:9100"}) by (instance) # What this does: # node_cpu_seconds_total → metric from Node Exporter # {mode="idle"} → filter: only idle CPU (one entry per core when idle) # {instance=~".*:9100"} → regex match: all instances on port 9100 # count(...) → count the number of matching series = number of cores # by (instance) → group by server # # Example output: # {instance="web1:9100"} → 4 (web1 has 4 CPU cores) # {instance="web2:9100"} → 8 (web2 has 8 CPU cores)

PROMQLsum(rate(node_cpu_seconds_total{mode!="idle", instance=~".*:9100"}[5m])) by (instance) # What this does: # mode!="idle" → all CPU modes EXCEPT idle (user, system, iowait, etc.) # rate(...[5m]) → per-second rate over last 5 minutes # sum(...) → add all non-idle CPU modes together # by (instance) → per server # # Example: {instance="web1:9100"} → 2.3 (using 2.3 out of 4 cores)

PROMQLsum(rate(node_cpu_seconds_total{mode="idle", instance=~".*:9100"}[5m])) by (instance) # Same as above but mode="idle" → shows how much CPU is FREE # Example: {instance="web1:9100"} → 1.7 (1.7 cores available)

PROMQL100 - ( sum(rate(node_cpu_seconds_total{mode="idle", instance=~".*:9100"}[5m])) by (instance) / count(node_cpu_seconds_total{mode="idle", instance=~".*:9100"}) by (instance) * 100 ) # How it works step by step: # 1. sum(rate(idle[5m])) → idle cores (e.g., 1.7) # 2. count(idle) → total cores (e.g., 4) # 3. idle / total * 100 → idle percentage (42.5%) # 4. 100 - idle% → USED percentage (57.5%) # # Example: {instance="web1:9100"} → 57.5 (57.5% CPU used)

PROMQL( sum(rate(node_cpu_seconds_total{mode="idle", instance=~".*:9100"}[5m])) by (instance) / count(node_cpu_seconds_total{mode="idle", instance=~".*:9100"}) by (instance) * 100 ) # Same calculation without the "100 -" → gives available percentage # Example: {instance="web1:9100"} → 42.5 (42.5% CPU available)

PROMQLsum(node_memory_MemTotal_bytes{instance!~"default/node-exporter-.*"}) by (instance) / 1024 / 1024 / 1024 # node_memory_MemTotal_bytes → total RAM in bytes # Divide by 1024 three times → bytes → KB → MB → GB # Example: {instance="web1:9100"} → 15.6 (15.6 GB total RAM)

PROMQLsum(node_memory_MemAvailable_bytes{instance!~"default/node-exporter-.*"}) by (instance) / 1024 / 1024 / 1024 # MemAvailable = memory that can be used without swapping # Example: {instance="web1:9100"} → 8.2 (8.2 GB available)

PROMQL( sum(node_memory_MemTotal_bytes{instance!~"default/node-exporter-.*"}) by (instance) - sum(node_memory_MemAvailable_bytes{instance!~"default/node-exporter-.*"}) by (instance) ) / 1024 / 1024 / 1024 # Total minus Available = Used # Example: 15.6 - 8.2 = 7.4 GB used

PROMQL( sum(node_memory_MemAvailable_bytes{instance!~"default/node-exporter-.*"}) by (instance) / sum(node_memory_MemTotal_bytes{instance!~"default/node-exporter-.*"}) by (instance) ) * 100 # Available / Total * 100 = Available % # Example: 8.2 / 15.6 * 100 = 52.6% available

PROMQL( 1 - ( sum(node_memory_MemAvailable_bytes{instance!~"default/node-exporter-.*"}) by (instance) / sum(node_memory_MemTotal_bytes{instance!~"default/node-exporter-.*"}) by (instance) ) ) * 100 # 1 minus (Available/Total) = Used fraction → times 100 = Used % # Example: (1 - 0.526) * 100 = 47.4% RAM used

PROMQLsum by (phase) (kube_pod_status_phase) # Shows count of pods in each phase: # Running: 45 # Succeeded: 12 # Pending: 2 # Failed: 1 # Unknown: 0

PROMQLkube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"} > 0 # Shows pods stuck in CrashLoopBackOff # This means the container keeps crashing and K8s keeps restarting it # ALWAYS set an alert on this — it means something is seriously wrong # Common causes: wrong config, missing secrets, OOM killed, bad image

PROMQLkube_pod_status_phase{phase="Pending"} == 1 # Pods stuck in Pending = K8s cannot schedule them # Common causes: # - Not enough CPU/RAM on any node # - Node selector/affinity mismatch # - PVC not bound (storage not available) # - Taints preventing scheduling

PROMQLkube_node_status_condition{condition="Ready", status="true"} # Shows which nodes are Ready (healthy) # Value 1 = node is Ready # Value 0 = node is NOT Ready (serious problem!) # Alert if any node shows 0 — means workloads may be evicted

PROMQL# Pods NOT running (failed, pending, unknown) kube_pod_status_phase{phase!="Running", phase!="Succeeded"} == 1 # Container restart count (high restarts = unstable app) sum(kube_pod_container_status_restarts_total) by (pod, namespace) > 5 # Deployment replicas not matching desired kube_deployment_status_replicas_available != kube_deployment_spec_replicas # OOMKilled containers (ran out of memory) kube_pod_container_status_last_terminated_reason{reason="OOMKilled"} > 0

TERMINAL# Install Grafana with Docker docker run -d --name grafana -p 3000:3000 grafana/grafana # Access: http://your-server:3000 # Default login: admin / admin (change immediately!) # Add Prometheus as Data Source: # 1. Settings → Data Sources → Add → Prometheus # 2. URL: http://prometheus:9090 # 3. Click Save & Test

YAML# /etc/prometheus/alert_rules.yml groups: - name: critical_alerts rules: - alert: HighCPUUsage expr: 100 - (sum(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance) / count(node_cpu_seconds_total{mode="idle"}) by (instance) * 100) > 85 for: 5m labels: severity: warning annotations: summary: "High CPU on {{ $labels.instance }}" description: "CPU usage is above 85% for 5 minutes" - alert: PodCrashLooping expr: kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"} > 0 for: 2m labels: severity: critical annotations: summary: "Pod {{ $labels.pod }} is CrashLoopBackOff"

YAML# /etc/alertmanager/alertmanager.yml route: receiver: slack-critical routes: - match: severity: critical receiver: slack-critical - match: severity: warning receiver: slack-warnings receivers: - name: slack-critical slack_configs: - channel: "#alerts-critical" api_url: "https://hooks.slack.com/services/YOUR/WEBHOOK/URL" title: "CRITICAL: {{ .GroupLabels.alertname }}" text: "{{ .CommonAnnotations.description }}"

TERMINAL# Install Loki with Docker docker run -d --name loki -p 3100:3100 grafana/loki # Add Loki as Grafana Data Source: # Settings → Data Sources → Add → Loki # URL: http://loki:3100

LOGQL# Show all logs from nginx {job="nginx"} # Filter logs containing "error" {job="nginx"} |= "error" # Filter logs NOT containing "health" {job="nginx"} != "health" # Regex filter {job="nginx"} |~ "status=(500|502|503)" # Count errors per minute count_over_time({job="nginx"} |= "error" [1m])

YAML# /etc/promtail/config.yml server: http_listen_port: 9080 positions: filename: /tmp/positions.yaml # Remember where we stopped reading clients: - url: http://loki:3100/loki/api/v1/push scrape_configs: - job_name: system static_configs: - targets: [localhost] labels: job: system __path__: /var/log/syslog - job_name: nginx static_configs: - targets: [localhost] labels: job: nginx __path__: /var/log/nginx/*.log - job_name: myapp static_configs: - targets: [localhost] labels: job: myapp __path__: /opt/myapp/logs/*.log

DOCKER-COMPOSE# docker-compose.yml — Full monitoring stack version: "3" services: prometheus: image: prom/prometheus ports: ["9090:9090"] volumes: - ./prometheus.yml:/etc/prometheus/prometheus.yml - ./alert_rules.yml:/etc/prometheus/alert_rules.yml grafana: image: grafana/grafana ports: ["3000:3000"] environment: GF_SECURITY_ADMIN_PASSWORD: admin123 alertmanager: image: prom/alertmanager ports: ["9093:9093"] volumes: - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml loki: image: grafana/loki ports: ["3100:3100"] promtail: image: grafana/promtail volumes: - /var/log:/var/log - ./promtail.yml:/etc/promtail/config.yml node-exporter: image: prom/node-exporter ports: ["9100:9100"] # Start everything: docker-compose up -d # Grafana: http://localhost:3000 (admin/admin123) # Prometheus: http://localhost:9090