From 5334b15aa453358d1a67e0b0c5a56a4795cbd18f Mon Sep 17 00:00:00 2001 From: Michael Gross Date: Mon, 6 Apr 2026 17:46:04 +0200 Subject: [PATCH] Fix markdown encoding --- benchmark_v2_parameter.md | 72 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/benchmark_v2_parameter.md b/benchmark_v2_parameter.md index 80401f2..c7e2593 100644 --- a/benchmark_v2_parameter.md +++ b/benchmark_v2_parameter.md @@ -1 +1,71 @@ -IyBMTE0gQmVuY2htYXJrIHYyIOKAlCBQYXJhbWV0ZXItw5xiZXJzaWNodAoKIyMgQ0xJLVBhcmFtZXRlcgoKfCBQYXJhbWV0ZXIgfCBUeXAgfCBTdGFuZGFyZCB8IEJlc2NocmVpYnVuZyB8CnwtLS0tLS0tLS0tLXwtLS0tLXwtLS0tLS0tLS0tfC0tLS0tLS0tLS0tLS0tfAp8IGBBTlpBSExgIHwgYGludGAgKHBvc2l0aW9uYWwpIHwg4oCUIHwgQW56YWhsIGRlciB6dSB0ZXN0ZW5kZW4gTW9kZWxsZSAoei5CLiBgNGApIHwKfCBgLS1iYWNrZW5kYCB8IGNob2ljZSB8IGB2bGxtYCB8IEJhY2tlbmQtUHJlc2V0OiBgdmxsbWAsIGBvbGxhbWFgLCBgbG1zdHVkaW9gIHwKfCBgLS11cmxgIHwgc3RyaW5nIHwgYE5vbmVgIHwgRWlnZW5lIEJhc2UtVVJMLCDDvGJlcnNjaHJlaWJ0IGAtLWJhY2tlbmRgICh6LkIuIGBodHRwOi8vbG9jYWxob3N0OjkwMDAvdjFgKSB8CnwgYC0tbW9kZWxgIHwgc3RyaW5nIHwgYE5vbmVgIHwgTW9kZWxsbmFtZSBleHBsaXppdCBhbmdlYmVuLCDDvGJlcnNwcmluZ3QgQXV0by1EZXRlY3QgKHouQi4gYGdlbW1hNDozMWJgKSB8CnwgYC0tcmVzdWx0cy1kaXJgIHwgc3RyaW5nIHwgYHJlc3VsdHMvYCB8IEF1c2dhYmV2ZXJ6ZWljaG5pcyB8CgojIyBCYWNrZW5kLVByZXNldHMKCnwgTmFtZSB8IFVSTCB8CnwtLS0tLS18LS0tLS18CnwgYHZsbG1gIHwgYGh0dHA6Ly9sb2NhbGhvc3Q6ODAwMC92MWAgfAp8IGBvbGxhbWFgIHwgYGh0dHA6Ly9sb2NhbGhvc3Q6MTE0MzQvdjFgIHwKfCBgbG1zdHVkaW9gIHwgYGh0dHA6Ly9sb2NhbGhvc3Q6MTIzNC92MWAgfAoKIyMgSW50ZXJuZSBLb25zdGFudGVuCgp8IEtvbnN0YW50ZSB8IFdlcnQgfCBCZXNjaHJlaWJ1bmcgfAp8LS0tLS0tLS0tLS18LS0tLS0tfC0tLS0tLS0tLS0tLS0tfAp8IGBERUZBVUxUX1RJTUVPVVRgIHwgYDMwMC4wIHNgIHwgSFRUUC1UaW1lb3V0IHBybyBSZXF1ZXN0IHwKfCBgTUFYX1JFVFJJRVNgIHwgYDNgIHwgV2llZGVyaG9sdW5nZW4gYmVpIEZlaGxlciAoNDI5LCA1eHgsIFRpbWVvdXQpIHwKCiMjIFByb21wdC1CbMO2Y2tlCgp8IElEIHwgQmxvY2sgfCBCZXplaWNobnVuZyB8CnwtLS0tfC0tLS0tLS18LS0tLS0tLS0tLS0tLXwKfCBBMSB8IENvZGUgfCBTb3J0aWVyZnVua3Rpb24gbWl0IGZlaGxlbmRlbiBTY2hsw7xzc2VsbiB8CnwgQTIgfCBDb2RlIHwgQ1NWLURlYnVnZ2luZyB8CnwgQTMgfCBDb2RlIHwgSFRUUC1BUEktQ2xpZW50IHwKfCBCMSB8IEJ1c2luZXNzIHwgTW9FLUVya2zDpHJ1bmcgZsO8ciBHZXNjaMOkZnRza3VuZGVuIHwKfCBCMiB8IEJ1c2luZXNzIHwgRS1NYWlsLUFic2FnZSB8CnwgQjMgfCBCdXNpbmVzcyB8IHJldkRTRy1Bcmd1bWVudGUgfAoKIyMgR2VtZXNzZW5lIE1ldHJpa2VuIChwcm8gUnVuKQoKfCBNZXRyaWsgfCBCZXNjaHJlaWJ1bmcgfAp8LS0tLS0tLS18LS0tLS0tLS0tLS0tLXwKfCBgdHRmdF9zYCB8IFRpbWUgdG8gRmlyc3QgVG9rZW4gKFNla3VuZGVuKSB8CnwgYHRoaW5raW5nX3RpbWVfc2AgfCBEYXVlciBkZXMgYDx0aGluaz5gLUJsb2NrcyAoMCB3ZW5uIGtlaW4gVGhpbmtpbmcpIHwKfCBgdG90YWxfdGltZV9zYCB8IEdlc2FtdGxhdWZ6ZWl0IHwKfCBgdG90YWxfdG9rZW5zYCB8IEFuemFobCBnZW5lcmllcnRlciBUb2tlbnMgfAp8IGB0b2tlbnNfcGVyX3NlY2AgfCBUaHJvdWdocHV0ICh0b2svcykgfAoKIyMgQmVpc3BpZWxhdWZydWZlCgpgYGBiYXNoCiMgNCBNb2RlbGxlIG1pdCB2bGxtIChTdGFuZGFyZCkKcHl0aG9uIGJlbmNobWFya192Mi5weSA0CgojIDIgTW9kZWxsZSBtaXQgb2xsYW1hCnB5dGhvbiBiZW5jaG1hcmtfdjIucHkgMiAtLWJhY2tlbmQgb2xsYW1hCgojIDEgTW9kZWxsIG1pdCBjdXN0b20gVVJMIHVuZCBmZXN0ZW0gTW9kZWxsbmFtZW4KcHl0aG9uIGJlbmNobWFya192Mi5weSAxIC0tdXJsIGh0dHA6Ly9sb2NhbGhvc3Q6OTAwMC92MSAtLW1vZGVsIGdlbW1hNDozMWIKCiMgRWlnZW5lcyBBdXNnYWJldmVyemVpY2huaXMKcHl0aG9uIGJlbmNobWFya192Mi5weSAyIC0tcmVzdWx0cy1kaXIgL3RtcC9iZW5jaApgYGAKCiMjIEF1c2dhYmUtRGF0ZWllbgoKfCBEYXRlaS9QZmFkIHwgSW5oYWx0IHwKfC0tLS0tLS0tLS0tLXwtLS0tLS0tLXwKfCBgcmVzdWx0cy88bW9kZWxsPi5qc29uYCB8IE1ldHJpa2VuIGFsbGVyIFJ1bnMgKG9obmUgUm9oYW50d29ydGVuKSB8CnwgYHJlc3VsdHMvPG1vZGVsbD4vPHByb21wdF9pZD4udHh0YCB8IFJvaGFudHdvcnQgcHJvIFByb21wdCB8CnwgYHJlc3VsdHMvYmVuY2htYXJrX3YyXzx0aW1lc3RhbXA+Lm1kYCB8IE1hcmtkb3duLVJlcG9ydCBtaXQgWnVzYW1tZW5mYXNzdW5nICsgRGV0YWlscyB8Cg== \ No newline at end of file +# LLM Benchmark v2 — Parameter-Übersicht + +## CLI-Parameter + +| Parameter | Typ | Standard | Beschreibung | +|-----------|-----|----------|--------------| +| `ANZAHL` | `int` (positional) | — | Anzahl der zu testenden Modelle (z.B. `4`) | +| `--backend` | choice | `vllm` | Backend-Preset: `vllm`, `ollama`, `lmstudio` | +| `--url` | string | `None` | Eigene Base-URL, überschreibt `--backend` (z.B. `http://localhost:9000/v1`) | +| `--model` | string | `None` | Modellname explizit angeben, überspringt Auto-Detect (z.B. `gemma4:31b`) | +| `--results-dir` | string | `results/` | Ausgabeverzeichnis | + +## Backend-Presets + +| Name | URL | +|------|-----| +| `vllm` | `http://localhost:8000/v1` | +| `ollama` | `http://localhost:11434/v1` | +| `lmstudio` | `http://localhost:1234/v1` | + +## Interne Konstanten + +| Konstante | Wert | Beschreibung | +|-----------|------|--------------| +| `DEFAULT_TIMEOUT` | `300.0 s` | HTTP-Timeout pro Request | +| `MAX_RETRIES` | `3` | Wiederholungen bei Fehler (429, 5xx, Timeout) | + +## Prompt-Blöcke + +| ID | Block | Bezeichnung | +|----|-------|-------------| +| A1 | Code | Sortierfunktion mit fehlenden Schlüsseln | +| A2 | Code | CSV-Debugging | +| A3 | Code | HTTP-API-Client | +| B1 | Business | MoE-Erklärung für Geschäftskunden | +| B2 | Business | E-Mail-Absage | +| B3 | Business | revDSG-Argumente | + +## Gemessene Metriken (pro Run) + +| Metrik | Beschreibung | +|--------|-------------| +| `ttft_s` | Time to First Token (Sekunden) | +| `thinking_time_s` | Dauer des ``-Blocks (0 wenn kein Thinking) | +| `total_time_s` | Gesamtlaufzeit | +| `total_tokens` | Anzahl generierter Tokens | +| `tokens_per_sec` | Throughput (tok/s) | + +## Beispielaufrufe + +```bash +# 4 Modelle mit vllm (Standard) +python benchmark_v2.py 4 + +# 2 Modelle mit ollama +python benchmark_v2.py 2 --backend ollama + +# 1 Modell mit custom URL und festem Modellnamen +python benchmark_v2.py 1 --url http://localhost:9000/v1 --model gemma4:31b + +# Eigenes Ausgabeverzeichnis +python benchmark_v2.py 2 --results-dir /tmp/bench +``` + +## Ausgabe-Dateien + +| Datei/Pfad | Inhalt | +|------------|--------| +| `results/.json` | Metriken aller Runs (ohne Rohantworten) | +| `results//.txt` | Rohantwort pro Prompt | +| `results/benchmark_v2_.md` | Markdown-Report mit Zusammenfassung + Details |