Log analizcisi: parse, endpoint başına p50/p95, pencereli hata oranı ve akan veri

2026-10-22T10:15:30.123Z GET /api/orders/42?expand=items 200 123ms
<zaman (ISO-8601, UTC)> <method> <path> <status> <latency>ms

LogEntry.parse("2026-10-22T10:15:30.123Z GET /api/orders 200 123ms")
    -> Optional[LogEntry[timestamp=2026-10-22T10:15:30.123Z, method=GET, path=/api/orders, status=200, latencyMillis=123]]
LogEntry.parse("garbage")                                   -> Optional.empty
LogEntry.parse("2026-10-22T10:15:30Z GET /a 999 5ms")       -> Optional.empty   (geçersiz status)

GET /orders/1 ... GET /orders/20 (latency 1..20 ms), POST /orders (7 ms)
latencySummary() -> {GET /orders/{id}=EndpointSummary[count=20, p50=10, p95=19],
                     POST /orders=EndpointSummary[count=1, p50=7, p95=7]}

10:00:10 200, 10:01:10 503, 10:00:20 500 (geç geldi), 10:00:30 404
pencereler -> [10:00 total=3 errors=1], [10:01 total=1 errors=1]

import java.time.Instant;
import java.time.format.DateTimeParseException;
import java.util.Optional;
import java.util.regex.Pattern;

/**
 * Level 1: one parsed access-log line, e.g.
 * {@code 2026-10-22T10:15:30.123Z GET /api/orders/42?expand=items 200 123ms}
 */
public record LogEntry(Instant timestamp, String method, String path, int status, long latencyMillis) {

    private static final Pattern METHOD = Pattern.compile("[A-Z]{3,7}");
    private static final Pattern NUMERIC = Pattern.compile("\\d+");
    private static final Pattern UUID = Pattern.compile("[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}");

    /** Returns empty for malformed lines instead of throwing: one bad line must not stop a whole run. */
    public static Optional<LogEntry> parse(String line) {
        if (line == null) {
            return Optional.empty();
        }
        String[] parts = line.trim().split("\\s+");
        if (parts.length != 5 || !METHOD.matcher(parts[1]).matches()
                || !parts[2].startsWith("/") || !parts[4].endsWith("ms")) {
            return Optional.empty();
        }
        try {
            Instant timestamp = Instant.parse(parts[0]);
            int status = Integer.parseInt(parts[3]);
            long latency = Long.parseLong(parts[4].substring(0, parts[4].length() - 2));
            if (status < 100 || status > 599 || latency < 0) {
                return Optional.empty();
            }
            return Optional.of(new LogEntry(timestamp, parts[1], parts[2], status, latency));
        } catch (DateTimeParseException | NumberFormatException e) {
            return Optional.empty();
        }
    }

    public boolean isServerError() {
        return status >= 500;          // 4xx is the client's fault and does not page anyone
    }

    /** Level 2: "GET /api/orders/42?x=1" and "GET /api/orders/43" are the same endpoint. */
    public String endpoint() {
        String withoutQuery = path.contains("?") ? path.substring(0, path.indexOf('?')) : path;
        StringBuilder normalized = new StringBuilder(method).append(' ');
        String[] segments = withoutQuery.split("/", -1);
        for (int i = 1; i < segments.length; i++) {
            String segment = segments[i];
            boolean isId = NUMERIC.matcher(segment).matches() || UUID.matcher(segment).matches();
            normalized.append('/').append(isId ? "{id}" : segment);
        }
        return normalized.toString();
    }
}

/** Level 2 introduces it, Level 4 swaps the implementation without touching the analyzer. */
public interface LatencyRecorder {

    void record(long latencyMillis);

    long count();

    /** Nearest-rank percentile: the smallest recorded value with at least p% of values at or below it. */
    long percentile(double p);

    static long nearestRank(double p, long count) {
        if (!(p > 0 && p <= 100)) {
            throw new IllegalArgumentException("p must be in (0, 100]: " + p);
        }
        if (count == 0) {
            throw new IllegalStateException("no values recorded");
        }
        return (long) Math.ceil(p / 100.0 * count);   // 1-based
    }
}

import java.util.Arrays;

/** Level 2: keeps every value; exact, but memory grows with the number of requests. */
public final class ExactLatencyRecorder implements LatencyRecorder {

    private long[] values = new long[16];
    private int size;
    private boolean sorted = true;

    @Override
    public void record(long latencyMillis) {
        if (size == values.length) {
            values = Arrays.copyOf(values, size * 2);
        }
        values[size++] = latencyMillis;
        sorted = false;
    }

    @Override
    public long count() {
        return size;
    }

    @Override
    public long percentile(double p) {
        long rank = LatencyRecorder.nearestRank(p, size);
        if (!sorted) {
            Arrays.sort(values, 0, size);   // sort once, answer many percentile queries
            sorted = true;
        }
        return values[(int) rank - 1];
    }
}

/**
 * Level 4: fixed-size log-linear histogram (the idea behind HdrHistogram).
 * Values below 32 get their own bucket; above that each power of two is split into 32 buckets,
 * so a reported percentile is at most ~3% above the true value. Memory does not depend on traffic.
 */
public final class LatencyHistogram implements LatencyRecorder {

    private static final int SUB_BUCKET_BITS = 5;
    private static final int SUB_BUCKETS = 1 << SUB_BUCKET_BITS;          // 32

    private final long maxTrackable;
    private final long[] counts;
    private long total;
    private long maxRecorded;

    public LatencyHistogram(long maxTrackableMillis) {
        if (maxTrackableMillis < SUB_BUCKETS) {
            throw new IllegalArgumentException("maxTrackableMillis must be >= " + SUB_BUCKETS);
        }
        this.maxTrackable = maxTrackableMillis;
        this.counts = new long[indexOf(maxTrackableMillis) + 1];
    }

    @Override
    public void record(long latencyMillis) {
        if (latencyMillis < 0) {
            throw new IllegalArgumentException("negative latency");
        }
        long clamped = Math.min(latencyMillis, maxTrackable);  // e.g. a 2-hour outlier counts as "max"
        counts[indexOf(clamped)]++;
        total++;
        maxRecorded = Math.max(maxRecorded, clamped);
    }

    @Override
    public long count() {
        return total;
    }

    @Override
    public long percentile(double p) {
        long rank = LatencyRecorder.nearestRank(p, total);
        long seen = 0;
        for (int i = 0; i < counts.length; i++) {
            seen += counts[i];
            if (seen >= rank) {
                return Math.min(upperBoundOf(i), maxRecorded);
            }
        }
        throw new IllegalStateException("unreachable");
    }

    public int bucketCount() {
        return counts.length;
    }

    static int indexOf(long value) {
        if (value < SUB_BUCKETS) {
            return (int) value;
        }
        int highestBit = 63 - Long.numberOfLeadingZeros(value);         // >= 5
        int shift = highestBit - SUB_BUCKET_BITS;                         // keep the top 6 bits
        long subBucket = (value >>> shift) - SUB_BUCKETS;                 // 0..31
        return SUB_BUCKETS + shift * SUB_BUCKETS + (int) subBucket;
    }

    static long upperBoundOf(int index) {
        if (index < SUB_BUCKETS) {
            return index;
        }
        int shift = (index - SUB_BUCKETS) / SUB_BUCKETS;
        long subBucket = (index - SUB_BUCKETS) % SUB_BUCKETS;
        return ((SUB_BUCKETS + subBucket + 1) << shift) - 1;
    }
}

import java.time.Duration;
import java.time.Instant;
import java.util.Map;
import java.util.TreeMap;
import java.util.function.Consumer;

/**
 * Level 3: server-error rate per tumbling window, e.g. per minute.
 * Level 4: only windows that can still receive events stay in memory; a window is closed
 * (emitted and forgotten) once the newest event is {@code allowedLateness} past its end.
 */
public final class ErrorRateWindows {

    public record WindowStats(Instant start, long total, long errors) {
        public double errorRate() {
            return total == 0 ? 0.0 : (double) errors / total;
        }
    }

    private static final class Counts {
        long total;
        long errors;
    }

    private final long windowMillis;
    private final long allowedLatenessMillis;
    private final Consumer<WindowStats> onClose;
    private final TreeMap<Long, Counts> open = new TreeMap<>();   // window start -> counts
    private long maxSeenMillis = Long.MIN_VALUE;
    private long closedBeforeMillis = Long.MIN_VALUE;              // every window starting earlier is closed
    private long droppedLate;

    public ErrorRateWindows(Duration window, Duration allowedLateness, Consumer<WindowStats> onClose) {
        this.windowMillis = window.toMillis();
        this.allowedLatenessMillis = allowedLateness.toMillis();
        if (windowMillis <= 0 || allowedLatenessMillis < 0) {
            throw new IllegalArgumentException("window must be positive, lateness non-negative");
        }
        this.onClose = onClose;
    }

    public void record(Instant timestamp, boolean isError) {
        long millis = timestamp.toEpochMilli();
        long start = Math.floorDiv(millis, windowMillis) * windowMillis;
        if (start < closedBeforeMillis) {
            droppedLate++;                     // its window was already reported; count, do not reopen
            return;
        }
        Counts counts = open.computeIfAbsent(start, s -> new Counts());
        counts.total++;
        if (isError) {
            counts.errors++;
        }
        if (millis > maxSeenMillis) {
            maxSeenMillis = millis;
            closeWindowsEndingBefore(maxSeenMillis - allowedLatenessMillis);
        }
    }

    /** End of input: report everything that is still open. */
    public void flush() {
        closeWindowsEndingBefore(Long.MAX_VALUE);
    }

    public int openWindows() {
        return open.size();
    }

    public long droppedLate() {
        return droppedLate;
    }

    private void closeWindowsEndingBefore(long watermarkMillis) {
        while (!open.isEmpty()) {
            Map.Entry<Long, Counts> oldest = open.firstEntry();
            long end = oldest.getKey() + windowMillis;
            if (end > watermarkMillis) {
                return;
            }
            open.pollFirstEntry();
            closedBeforeMillis = Math.max(closedBeforeMillis, end);
            Counts c = oldest.getValue();
            onClose.accept(new WindowStats(Instant.ofEpochMilli(oldest.getKey()), c.total, c.errors));
        }
    }
}

import java.time.Duration;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.TreeMap;
import java.util.function.Consumer;
import java.util.function.Supplier;

/**
 * Levels 1-4 together: parses lines one at a time (works on an endless stream),
 * keeps per-endpoint latency percentiles and per-window error rates with bounded memory.
 */
public final class LogAnalyzer {

    public record EndpointSummary(long count, long p50, long p95) {}

    static final String OTHER_ENDPOINT = "OTHER";

    private final Supplier<LatencyRecorder> recorderFactory;
    private final int maxEndpoints;
    private final Map<String, LatencyRecorder> latencies = new HashMap<>();
    private final ErrorRateWindows errorWindows;
    private long validLines;
    private long invalidLines;

    /**
     * @param recorderFactory exact recorder (level 2) or histogram (level 4)
     * @param maxEndpoints    cardinality guard: endpoints beyond this share the "OTHER" bucket
     */
    public LogAnalyzer(Supplier<LatencyRecorder> recorderFactory, int maxEndpoints,
                       Duration window, Duration allowedLateness, Consumer<ErrorRateWindows.WindowStats> onWindowClosed) {
        if (maxEndpoints < 1) {
            throw new IllegalArgumentException("maxEndpoints must be positive");
        }
        this.recorderFactory = Objects.requireNonNull(recorderFactory);
        this.maxEndpoints = maxEndpoints;
        this.errorWindows = new ErrorRateWindows(window, allowedLateness, onWindowClosed);
    }

    public void accept(String line) {
        LogEntry.parse(line).ifPresentOrElse(this::accept, () -> invalidLines++);
    }

    private void accept(LogEntry entry) {
        validLines++;
        recorderFor(entry.endpoint()).record(entry.latencyMillis());
        errorWindows.record(entry.timestamp(), entry.isServerError());
    }

    /** Call at end of input so the last windows are reported. */
    public void finish() {
        errorWindows.flush();
    }

    /** Sorted by endpoint for stable output. */
    public Map<String, EndpointSummary> latencySummary() {
        Map<String, EndpointSummary> summary = new TreeMap<>();
        latencies.forEach((endpoint, recorder) -> summary.put(endpoint,
                new EndpointSummary(recorder.count(), recorder.percentile(50), recorder.percentile(95))));
        return summary;
    }

    public long validLines() {
        return validLines;
    }

    public long invalidLines() {
        return invalidLines;
    }

    public long droppedLateEvents() {
        return errorWindows.droppedLate();
    }

    public int trackedEndpoints() {
        return latencies.size();
    }

    public int openWindows() {
        return errorWindows.openWindows();
    }

    private LatencyRecorder recorderFor(String endpoint) {
        LatencyRecorder existing = latencies.get(endpoint);
        if (existing != null) {
            return existing;
        }
        // Keep the last slot for OTHER, so at most maxEndpoints recorders ever exist.
        String key = latencies.size() < maxEndpoints - 1 ? endpoint : OTHER_ENDPOINT;
        return latencies.computeIfAbsent(key, k -> recorderFactory.get());
    }
}

import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.IntStream;

public final class LogAnalyzerTest {

    public static void main(String[] args) {
        Map<String, Runnable> tests = new LinkedHashMap<>();
        tests.put("level1_parsesValidAndRejectsMalformed", LogAnalyzerTest::level1_parsesValidAndRejectsMalformed);
        tests.put("level2_endpointNormalization", LogAnalyzerTest::level2_endpointNormalization);
        tests.put("level2_exactNearestRankPercentiles", LogAnalyzerTest::level2_exactNearestRankPercentiles);
        tests.put("level3_errorRatePerWindowIncludingOutOfOrder", LogAnalyzerTest::level3_errorRatePerWindowIncludingOutOfOrder);
        tests.put("level4_histogramErrorIsBounded", LogAnalyzerTest::level4_histogramErrorIsBounded);
        tests.put("level4_lateEventsAreDroppedAfterWindowCloses", LogAnalyzerTest::level4_lateEventsAreDroppedAfterWindowCloses);
        tests.put("level4_endpointCardinalityIsCapped", LogAnalyzerTest::level4_endpointCardinalityIsCapped);
        tests.put("level4_millionLinesWithBoundedState", LogAnalyzerTest::level4_millionLinesWithBoundedState);

        int passed = 0;
        for (var test : tests.entrySet()) {
            test.getValue().run();
            System.out.println("PASS " + test.getKey());
            passed++;
        }
        System.out.println(passed + "/" + tests.size() + " tests passed");
    }

    static final Instant T0 = Instant.parse("2026-10-22T10:00:00Z");

    static String line(long secondsFromT0, String method, String path, int status, long latency) {
        return T0.plusSeconds(secondsFromT0) + " " + method + " " + path + " " + status + " " + latency + "ms";
    }

    static void level1_parsesValidAndRejectsMalformed() {
        var entry = LogEntry.parse("2026-10-22T10:15:30.123Z GET /api/orders 200 123ms").orElseThrow();
        check(entry.status() == 200 && entry.latencyMillis() == 123 && entry.method().equals("GET"), "fields");
        for (String bad : List.of("", "garbage", "2026-10-22T10:15:30Z GET /a 200",
                "not-a-time GET /a 200 5ms", "2026-10-22T10:15:30Z GET /a 999 5ms",
                "2026-10-22T10:15:30Z GET /a 200 -5ms", "2026-10-22T10:15:30Z get /a 200 5ms")) {
            check(LogEntry.parse(bad).isEmpty(), "should reject: " + bad);
        }
        var analyzer = exactAnalyzer(stats -> {});
        analyzer.accept("garbage");
        analyzer.accept(line(0, "GET", "/a", 200, 5));
        check(analyzer.invalidLines() == 1 && analyzer.validLines() == 1, "counted, not thrown");
    }

    static void level2_endpointNormalization() {
        check(LogEntry.parse(line(0, "GET", "/api/orders/42?expand=items", 200, 1)).orElseThrow()
                .endpoint().equals("GET /api/orders/{id}"), "numeric id + query");
        check(LogEntry.parse(line(0, "DELETE", "/carts/3f2b8c1e-1a2b-4c3d-9e8f-001122334455/items/7", 200, 1)).orElseThrow()
                .endpoint().equals("DELETE /carts/{id}/items/{id}"), "uuid");
        check(LogEntry.parse(line(0, "GET", "/v2/health", 200, 1)).orElseThrow()
                .endpoint().equals("GET /v2/health"), "v2 is not an id");
    }

    static void level2_exactNearestRankPercentiles() {
        var analyzer = exactAnalyzer(stats -> {});
        for (int latency = 1; latency <= 20; latency++) {
            analyzer.accept(line(latency, "GET", "/orders/" + latency, 200, latency));
        }
        analyzer.accept(line(0, "POST", "/orders", 201, 7));
        var summary = analyzer.latencySummary();
        check(summary.keySet().equals(java.util.Set.of("GET /orders/{id}", "POST /orders")), "keys: " + summary.keySet());
        var get = summary.get("GET /orders/{id}");
        // n = 20: p50 rank = 10 -> 10, p95 rank = 19 -> 19
        check(get.count() == 20 && get.p50() == 10 && get.p95() == 19, "nearest rank: " + get);
        check(summary.get("POST /orders").p95() == 7, "single value");
    }

    static void level3_errorRatePerWindowIncludingOutOfOrder() {
        List<ErrorRateWindows.WindowStats> closed = new ArrayList<>();
        var analyzer = exactAnalyzer(closed::add);           // unlimited lateness: everything waits for finish()
        analyzer.accept(line(10, "GET", "/a", 200, 1));
        analyzer.accept(line(70, "GET", "/a", 503, 1));
        analyzer.accept(line(20, "GET", "/a", 500, 1));      // out of order, still the first minute
        analyzer.accept(line(30, "GET", "/a", 404, 1));      // 4xx is not a server error
        analyzer.finish();
        check(closed.size() == 2, "two windows");
        check(closed.get(0).start().equals(T0) && closed.get(0).total() == 3 && closed.get(0).errors() == 1, "first: " + closed.get(0));
        check(closed.get(1).errorRate() == 1.0, "second minute is all errors");
    }

    static void level4_histogramErrorIsBounded() {
        var histogram = new LatencyHistogram(3_600_000);
        var exact = new ExactLatencyRecorder();
        for (int v = 1; v <= 100_000; v++) {
            long value = (v * 7919L) % 50_000;               // spread values without randomness
            histogram.record(value);
            exact.record(value);
        }
        for (double p : new double[] {50, 90, 95, 99, 99.9, 100}) {
            long truth = exact.percentile(p);
            long approx = histogram.percentile(p);
            check(approx >= truth && approx <= truth + Math.max(1, truth / 32), "p" + p + " " + truth + " vs " + approx);
        }
        check(histogram.bucketCount() < 600, "fixed size: " + histogram.bucketCount());
    }

    static void level4_lateEventsAreDroppedAfterWindowCloses() {
        List<ErrorRateWindows.WindowStats> closed = new ArrayList<>();
        var analyzer = new LogAnalyzer(() -> new LatencyHistogram(60_000), 100,
                Duration.ofMinutes(1), Duration.ofSeconds(30), closed::add);
        analyzer.accept(line(10, "GET", "/a", 500, 1));
        analyzer.accept(line(85, "GET", "/a", 200, 1));      // watermark 55s: first minute still open
        check(closed.isEmpty(), "within lateness");
        analyzer.accept(line(50, "GET", "/a", 200, 1));      // late but accepted
        analyzer.accept(line(95, "GET", "/a", 200, 1));      // watermark 65s: first minute closes
        check(closed.size() == 1 && closed.get(0).total() == 2, "closed with late event: " + closed);
        analyzer.accept(line(40, "GET", "/a", 500, 1));      // too late
        check(analyzer.droppedLateEvents() == 1, "dropped, not reopened");
    }

    static void level4_endpointCardinalityIsCapped() {
        var analyzer = new LogAnalyzer(() -> new LatencyHistogram(60_000), 3,
                Duration.ofMinutes(1), Duration.ofMinutes(1), stats -> {});
        for (String path : List.of("/a", "/b", "/c", "/d", "/a")) {
            analyzer.accept(line(0, "GET", path, 200, 1));
        }
        var summary = analyzer.latencySummary();
        check(analyzer.trackedEndpoints() == 3, "capped");
        check(summary.get("GET /a").count() == 2 && summary.get("OTHER").count() == 2, "overflow: " + summary);
    }

    static void level4_millionLinesWithBoundedState() {
        List<ErrorRateWindows.WindowStats> alerts = new ArrayList<>();
        var analyzer = new LogAnalyzer(() -> new LatencyHistogram(3_600_000), 50,
                Duration.ofMinutes(1), Duration.ofMinutes(2),
                window -> { if (window.total() >= 100 && window.errorRate() > 0.05) alerts.add(window); });
        int maxOpenWindows = 0;
        // Lines are generated on the fly and never stored: 1,000,000 requests over ~2.8 hours.
        var lines = IntStream.range(0, 1_000_000).mapToObj(i -> line(i / 100, "GET",
                "/users/" + i + "/orders",                                // high-cardinality path, normalized
                (i / 100) % 600 < 60 && i % 10 == 0 ? 500 : 200,          // 10% errors in the first minute of every 10
                1 + (i % 1_000))).iterator();
        while (lines.hasNext()) {
            analyzer.accept(lines.next());
            maxOpenWindows = Math.max(maxOpenWindows, analyzer.openWindows());
        }
        analyzer.finish();
        var summary = analyzer.latencySummary().get("GET /users/{id}/orders");
        check(summary.count() == 1_000_000, "count");
        check(summary.p95() >= 950 && summary.p95() <= 950 + 950 / 32, "p95 ~ 950: " + summary.p95());
        check(analyzer.trackedEndpoints() == 1, "ids normalized away");
        check(maxOpenWindows <= 4, "open windows bounded by lateness: " + maxOpenWindows);
        check(alerts.size() == 17, "one alert per error minute: " + alerts.size());
    }

    static LogAnalyzer exactAnalyzer(java.util.function.Consumer<ErrorRateWindows.WindowStats> onClose) {
        return new LogAnalyzer(ExactLatencyRecorder::new, 1_000, Duration.ofMinutes(1),
                Duration.ofDays(365), onClose);
    }

    static void check(boolean condition, String message) {
        if (!condition) {
            throw new AssertionError(message);
        }
    }
}