created monitoring stack

Monitor-Stack/.env.example

@@ -0,0 +1,71 @@
+# ============================================================
+# Monitor Stack - Environment Variables
+# Copy this file to .env and fill in your values.
+# ============================================================
+
+
+# ------------------------------------------------------------
+# Image Versions
+# Pin to specific versions to avoid unexpected upgrades.
+# ------------------------------------------------------------
+
+# Grafana Loki version (https://github.com/grafana/loki/releases)
+LOKI_VERSION=3.7.2
+
+# Prometheus version (https://github.com/prometheus/prometheus/releases)
+PROMETHEUS_VERSION=3.12.0
+
+# Alertmanager version (https://github.com/prometheus/alertmanager/releases)
+ALERTMANAGER_VERSION=0.32.1
+
+# Grafana version (https://github.com/grafana/grafana/releases)
+GRAFANA_VERSION=13.0.1
+
+
+# ------------------------------------------------------------
+# Paths
+# ------------------------------------------------------------
+
+# Absolute path to the directory containing your config files:
+#   loki-config.yml, prometheus.yml, alertmanager.yml
+MONITOR_STACK_CONFIG_PATH=/path/to/monitor-stack/config
+
+
+# ------------------------------------------------------------
+# Ports
+# Host port mappings for services that expose a UI or API.
+# ------------------------------------------------------------
+
+# Prometheus web UI and API
+PROMETHEUS_PORT=9090
+
+# Alertmanager web UI and API
+ALERTMANAGER_PORT=9093
+
+# Grafana web UI
+GRAFANA_PORT=3000
+
+
+# ------------------------------------------------------------
+# Grafana
+# ------------------------------------------------------------
+
+# Initial admin username (set on first boot only)
+GRAFANA_ADMIN_USER=admin
+
+# Initial admin password — change this before first boot
+GRAFANA_ADMIN_PASSWORD=changeme
+
+# Secret key used to sign sessions and cookies.
+# Changing this after first boot will invalidate all active sessions.
+# Generate with: openssl rand -hex 32
+GRAFANA_SECRET_KEY=
+
+
+# ------------------------------------------------------------
+# Timezone
+# Applied to all containers for consistent log timestamps.
+# Use a valid tz database name: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
+# ------------------------------------------------------------
+
+TZ=America/Los_Angeles

Monitor-Stack/alertmanager.yml

@@ -0,0 +1,19 @@
+global:
+  resolve_timeout: 5m
+
+route:
+  group_by: ['alertname', 'job']
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 1h
+  receiver: 'default'
+
+receivers:
+  - name: 'default'
+
+inhibit_rules:
+  - source_matchers:
+      - severity="critical"
+    target_matchers:
+      - severity="warning"
+    equal: ['alertname', 'job']

Monitor-Stack/docker-compose-volume.yaml

@@ -0,0 +1,127 @@
+services:
+  loki:
+    image: grafana/loki:${LOKI_VERSION:-3.7.2}
+    container_name: monitor-loki
+    volumes:
+      - ${MONITOR_STACK_CONFIG_PATH}/loki-config.yml:/etc/loki/local-config.yaml
+      - loki_data:/loki
+    command: -config.file=/etc/loki/local-config.yaml
+    environment:
+      - TZ=${TZ:-America/Los_Angeles}
+    networks:
+      - monitor_network
+    healthcheck:
+      test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3100/ready || exit 1"]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+      start_period: 30s
+    deploy:
+      resources:
+        limits:
+          memory: 512m
+    restart: unless-stopped
+
+  prometheus:
+    image: prom/prometheus:${PROMETHEUS_VERSION:-3.12.0}
+    container_name: prometheus
+    ports:
+      - "${PROMETHEUS_PORT:-9090}:9090"
+    volumes:
+      - ${MONITOR_STACK_CONFIG_PATH}/prometheus.yml:/etc/prometheus/prometheus.yml
+      - prometheus_data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--storage.tsdb.retention.time=30d'
+      - '--web.enable-lifecycle'
+    environment:
+      - TZ=${TZ:-America/Los_Angeles}
+    networks:
+      - monitor_network
+    healthcheck:
+      test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:9090/-/healthy || exit 1"]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+      start_period: 30s
+    deploy:
+      resources:
+        limits:
+          memory: 512m
+    restart: unless-stopped
+
+  alertmanager:
+    image: prom/alertmanager:${ALERTMANAGER_VERSION:-0.32.1}
+    container_name: alertmanager
+    ports:
+      - "${ALERTMANAGER_PORT:-9093}:9093"
+    volumes:
+      - ${MONITOR_STACK_CONFIG_PATH}/alertmanager.yml:/etc/alertmanager/alertmanager.yml
+      - alertmanager_data:/alertmanager
+    command:
+      - '--config.file=/etc/alertmanager/alertmanager.yml'
+      - '--storage.path=/alertmanager'
+    environment:
+      - TZ=${TZ:-America/Los_Angeles}
+    networks:
+      - monitor_network
+    healthcheck:
+      test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:9093/-/healthy || exit 1"]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+      start_period: 30s
+    deploy:
+      resources:
+        limits:
+          memory: 512m
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana:${GRAFANA_VERSION:-13.0.1}
+    container_name: grafana
+    ports:
+      - "${GRAFANA_PORT:-3000}:3000"
+    volumes:
+      - grafana_data:/var/lib/grafana
+    environment:
+      - TZ=${TZ:-America/Los_Angeles}
+      - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin}
+      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-changeme}
+      - GF_SECURITY_SECRET_KEY=${GRAFANA_SECRET_KEY}
+      - GF_LOG_MODE=console
+    depends_on:
+      loki:
+        condition: service_healthy
+      prometheus:
+        condition: service_healthy
+      alertmanager:
+        condition: service_healthy
+    networks:
+      - monitor_network
+    healthcheck:
+      test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3000/api/health || exit 1"]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+      start_period: 30s
+    deploy:
+      resources:
+        limits:
+          memory: 512m
+    restart: unless-stopped
+
+volumes:
+  loki_data:
+    name: loki_data
+  prometheus_data:
+    name: prometheus_data
+  alertmanager_data:
+    name: alertmanager_data
+  grafana_data:
+    name: grafana_data
+
+networks:
+  monitor_network:
+    external: true

Monitor-Stack/loki-config.yml

@@ -0,0 +1,37 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /loki
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+limits_config:
+  retention_period: 30d
+
+compactor:
+  working_directory: /loki/compactor
+  compaction_interval: 10m
+  retention_enabled: true
+  retention_delete_delay: 2h
+  retention_delete_worker_count: 150
+  delete_request_store: filesystem

Monitor-Stack/prometheus.yml

@@ -0,0 +1,17 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+            - alertmanager:9093
+
+rule_files:
+  # - alert-rules.yml
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: ['localhost:9090']