Large Software Projects: Collecting Logs

// Based of https://github.com/adityasinghcodes/nextjs-monitoring/blob/main/instrumentation.ts
// Node.js-specific imports are moved into dynamic imports within runtime checks
// Prevent Edge runtime from trying to import Node.js-specific modules
declare global {
    var metrics:
        | {
        registry: any;
    }
        | undefined;
    var logger: any | undefined;
}

export async function register() {
    if (process.env.NEXT_RUNTIME === "nodejs") {
        const { Registry, collectDefaultMetrics } = await import("prom-client");
        const pino = (await import("pino")).default;
        const pinoLoki = (await import("pino-loki")).default;

        //Prom-client initialization
        const prometheusRegistry = new Registry();
        collectDefaultMetrics({
            register: prometheusRegistry,
        });
        globalThis.metrics = {
            registry: prometheusRegistry,
        };

        //logger initialization
        globalThis.logger = pino(
            pinoLoki({
                host: "http://localhost:3100", // Connects to the loki container via localhost:3100
                batching: true,
                interval: 5,
                labels: { app: "next-app" }, // Crucial label for querying in Grafana
            })
        );
    }
}

// Based of https://github.com/adityasinghcodes/nextjs-monitoring/blob/main/app/api/examples/logging/route.ts
export const runtime = "nodejs";

export async function GET() {
    try {
        const { randomUUID } = await import("crypto");

        globalThis?.logger?.info({
            meta: {
                requestId: randomUUID(),
                extra: "This is some extra information that you can add to the meta",
                anything: "anything",
            },
            message: "Successful request handled",
        });
        return Response.json({
            message: "Hello world",
        });
    } catch (error) {
        globalThis?.logger?.error({
            err: error,
            message: "Something went wrong during success logging",
        });
    }
}

// Based of https://github.com/adityasinghcodes/nextjs-monitoring/blob/main/app/api/examples/logging/route.ts
export const runtime = "nodejs";

export async function GET() {
    try {
        throw new Error("Something is fundamentally wrong with this API endpoint");
    } catch (error) {
        globalThis?.logger?.error({
            err: error,
            message: "An error message here",
        });
        return new Response(JSON.stringify({ error: 'Internal Server Error' }), { status: 500 });
    }
}

# Based of https://github.com/adityasinghcodes/nextjs-monitoring/blob/main/loki-config.yml
# Disable authentication (NOT recommended for production)
auth_enabled: false

# Server configuration
server:
  # Port where Loki will listen for incoming connections
  http_listen_port: 3100

# Common configuration settings
common:
  # Base directory for Loki's data storage
  path_prefix: /tmp/loki
  # Storage configuration - using local filesystem
  storage:
    filesystem:
      # Directory where Loki stores data chunks
      chunks_directory: /tmp/loki/chunks
  # Number of copies of each stream to maintain (1 for single instance)
  replication_factor: 1
  # Ring is Loki's internal coordination system
  ring:
    # Key-Value store configuration for ring
    kvstore:
      # Using in-memory storage (good for testing, NOT for production)
      store: inmemory

# Schema configuration defines how Loki organizes and stores log data
schema_config:
  configs:
    - from: 2020-10-24
      # Using TSDB (Time Series Database) storage format
      store: tsdb
      # Using local filesystem for object storage
      object_store: filesystem
      # Schema version
      schema: v13
      # Index configuration
      index:
        # Prefix for index files
        prefix: index_
        # Create new index files every 24 hours
        period: 24h

limits_config:
  # Keep logs for 28 days before deletion
  retention_period: 672h

ruler:
  storage:
    type: local
    local:
      directory: /loki/rules
# NOTE: This configuration is suitable for development/testing only.
# For production, you should:
# 1. Enable authentication
# 2. Use persistent storage instead of filesystem
# 3. Use external kvstore (like etcd or consul) instead of inmemory
# 4. Use proper persistent directory instead of /tmp

# Based of https://github.com/adityasinghcodes/nextjs-monitoring/blob/main/docker-compose.yml
services:
  grafana:
    container_name: grafana
    image: grafana/grafana:11.4.0
    ports:
      - "3001:3000"
    environment:
      - GF_SECURITY_ADMIN_USER=admin_user
      - GF_SECURITY_ADMIN_PASSWORD=admin_password
    volumes:
      - grafana-storage:/var/lib/grafana
    networks:
      - monitoring

  prometheus:
    container_name: prometheus
    image: prom/prometheus:v3.0.1
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus-storage:/prometheus
    networks:
      - monitoring

  loki:
    container_name: loki
    image: grafana/loki:2.9.2
    ports:
      - "3100:3100"
    volumes:
      - ./loki-config.yml:/etc/loki/local-config.yml
    command: -config.file=/etc/loki/local-config.yml
    networks:
      - monitoring

networks:
  monitoring:
    name: monitoring
    driver: bridge

volumes:
  grafana-storage:
  prometheus-storage: