Centralizing logs across many services/containers (ELK/EFK, Loki) and tracing a single request end-to-end (correlation ID, OpenTelemetry, Jaeger/Zipkin)

# Without centralized logging:
kubectl logs transfer-service-pod-abc     # check one pod
kubectl logs fraud-service-pod-def        # check another pod
kubectl logs account-service-pod-ghi      # check yet another pod
# Slow, manual, and containers may be gone already

# With centralized logging:
# Search one store for all services at once:
{service="transfer"} |= "SAV2001" | json | traceId = "abc123"

// Spring Boot: configure structured JSON logging (logback-spring.xml)
// <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
//   <encoder class="net.logstash.logback.encoder.LogstashEncoder" />
// </appender>

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;

public class TransferService {
    private static final Logger log = LoggerFactory.getLogger(TransferService.class);

    public void processTransfer(String fromAccount, String toAccount,
                                java.math.BigDecimal amount) {
        MDC.put("fromAccount", fromAccount);
        MDC.put("toAccount", toAccount);
        MDC.put("amount", amount.toPlainString());
        log.info("Processing transfer");
        // Output: {"timestamp":"2024-03-15T10:05:03Z","level":"INFO",
        //          "service":"transfer-service","traceId":"abc123",
        //          "fromAccount":"SAV2001","toAccount":"CUR3001",
        //          "amount":"50000","message":"Processing transfer"}
    }
}

# Loki query (LogQL) -- filter by label, then grep
{service="transfer-service", namespace="production"}
  |= "SAV2001"
  | json
  | amount > 10000

# Elasticsearch query (KQL) -- arbitrary text search
fromAccount: "SAV2001" AND level: "ERROR" AND amount: [10000 TO *]

# Fluent Bit DaemonSet (simplified)
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluent-bit
  namespace: logging
spec:
  template:
    spec:
      containers:
        - name: fluent-bit
          image: fluent/fluent-bit:2.2
          volumeMounts:
            - name: varlog
              mountPath: /var/log
            - name: config
              mountPath: /fluent-bit/etc/
      volumes:
        - name: varlog
          hostPath:
            path: /var/log

# fluent-bit.conf
[INPUT]
    Name              tail
    Path              /var/log/containers/*.log
    Parser            docker
    Tag               kube.*

[FILTER]
    Name              kubernetes
    Match             kube.*
    # Enriches with pod name, namespace, labels

[OUTPUT]
    Name              loki
    Match             *
    Host              loki.logging.svc
    Port              3100
    Labels            job=fluent-bit

# W3C traceparent header format:
traceparent: 00-abc123def456789012345678abcdef01-1234567890abcdef-01
             ^^  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^  ^^^^^^^^^^^^^^^^  ^^
             ver          trace-id                    span-id      flags

// OpenTelemetry auto-instrumentation handles this automatically
// In Spring Boot, add the dependency and the agent:
// java -javaagent:opentelemetry-javaagent.jar \
//      -Dotel.service.name=transfer-service \
//      -Dotel.exporter.otlp.endpoint=http://otel-collector:4317 \
//      -jar transfer-service.jar

// The agent automatically:
// 1. Creates spans for incoming HTTP requests
// 2. Propagates traceparent header on outgoing HTTP calls
// 3. Creates spans for database queries
// 4. Exports spans to the OTLP collector

import io.opentelemetry.api.trace.Span;
import org.slf4j.MDC;

public class TraceLoggingFilter implements javax.servlet.Filter {
    public void doFilter(ServletRequest req, ServletResponse res,
                         FilterChain chain) throws Exception {
        String traceId = Span.current().getSpanContext().getTraceId();
        MDC.put("traceId", traceId);
        try {
            chain.doFilter(req, res);
        } finally {
            MDC.remove("traceId");
        }
    }
}

# Now you can pivot:
# 1. In Jaeger: find trace abc123, see fraud-svc span is slow
# 2. In Loki:  {service="fraud-service"} |= "abc123"
#    -> see the exact exception: "Timeout calling AML API after 3 retries"

# OpenTelemetry Collector with tail-based sampling
processors:
  tail_sampling:
    decision_wait: 10s
    policies:
      - name: errors-always
        type: status_code
        status_code:
          status_codes: [ERROR]
      - name: slow-always
        type: latency
        latency:
          threshold_ms: 500
      - name: sample-rest
        type: probabilistic
        probabilistic:
          sampling_percentage: 10    # Keep 10% of normal traces

// WRONG: Different field names across services
// transfer-service: {"acct_from": "SAV2001"}
// account-service:  {"sourceAccount": "SAV2001"}
// fraud-service:    {"from_account": "SAV2001"}

// CORRECT: Standardized field names
// All services: {"fromAccount": "SAV2001", "toAccount": "CUR3001"}

// WRONG: Logging sensitive data
log.info("Card number: {}", cardNumber);  // Never!

// CORRECT: Redact sensitive fields
log.info("Card ending: {}", cardNumber.substring(cardNumber.length() - 4));