Elasticsearch Rally ベンチマーク入門: バージョン 8.x と 9.x の性能比較

{
  "challenges": [
    {
      "name": "default",
      "description": "標準的な4クライアントベンチマーク",
      "default": true
    },
    {
      "name": "high-concurrency",
      "description": "8クライアントでのストレステスト"
    },
    {
      "name": "ramp-up",
      "description": "1から8クライアントへの段階的負荷増加"
    }
  ]
}

{
  "operation": "bulk-index",
  "clients": 4  // 4つのスレッドが同時に実行
}

クライアント 1: [リクエスト] → [待機] → [リクエスト] → [待機] → ...

クライアント 2: [リクエスト] → [待機] → [リクエスト] → [待機] → ...

クライアント 3: [リクエスト] → [待機] → [リクエスト] → [待機] → ...

クライアント 4: [リクエスト] → [待機] → [リクエスト] → [待機] → ...

ssh -i /path/to/your-key.pem ubuntu@<ec2-public-ip>

# システムパッケージの更新
sudo apt update && sudo apt upgrade -y

# Pythonと開発ツールのインストール
sudo apt install -y python3-pip python3-venv python3-dev build-essential git curl

# Pythonバージョンの確認（3.9以上である必要があります）
python3 --version

sudo apt install -y openjdk-17-jdk

# JAVA_HOMEの設定
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64
echo 'export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64' >> ~/.bashrc

# 確認
java -version

sudo apt install -y openjdk-21-jdk

# JAVA21_HOMEの設定
export JAVA21_HOME=/usr/lib/jvm/java-21-openjdk-amd64
echo 'export JAVA21_HOME=/usr/lib/jvm/java-21-openjdk-amd64' >> ~/.bashrc

# 変更を適用
source ~/.bashrc

# 確認
ls -la /usr/lib/jvm/java-21-openjdk-amd64

# ワークスペースディレクトリの作成
mkdir -p ~/rally-workspace
cd ~/rally-workspace

# Python仮想環境の作成
python3 -m venv rally-venv

# 仮想環境の有効化
source rally-venv/bin/activate

# Rallyのインストール
pip install --upgrade pip
pip install esrally

# インストールの確認
esrally --version

esrally 2.x.x (Python 3.12.x)

# 組み込みトラックで簡単なテストを実行
esrally race --distribution-version=8.11.0 --track=geonames --test-mode

cd ~/rally-workspace
mkdir -p log_ingest/corpora
cd log_ingest

log_ingest/
├── track.json                 # メインのトラック定義
├── index-settings-mapping.json # インデックス設定
└── corpora/
    ├── logs.jsonl            # フルデータセット（100K docs）
    └── logs-1k.jsonl         # テストサブセット（1K docs）

{
  "settings": {
    "index": {
      "number_of_shards": 1,
      "number_of_replicas": 0
    }
  },
  "mappings": {
    "properties": {
      "@timestamp": {
        "type": "date"
      },
      "clientip": {
        "type": "ip"
      },
      "response": {
        "type": "integer"
      },
      "request": {
        "type": "text"
      },
      "machine": {
        "properties": {
          "os": {
            "type": "keyword"
          },
          "ram": {
            "type": "long"
          }
        }
      },
      "ingest_time": {
        "type": "date"
      }
    }
  }
}

{"@timestamp":"2025-10-16T10:00:00Z","clientip":"192.168.1.100","response":200,"request":"GET /api/users HTTP/1.1","machine":{"os":"linux","ram":8589934592},"referer":"https://example.com/login"}

{
  "version": 2,
  "description": "ES 8.x vs 9.x のログインジェストパイプラインベンチマーク",
  "indices": [
    {
      "name": "logs",
      "body": "index-settings-mapping.json"
    }
  ],
  "corpora": [
    {
      "name": "logcorpus",
      "documents": [
        {
          "source-file": "corpora/logs.jsonl",
          "document-count": 100000
        }
      ]
    }
  ],
  "operations": [
    {
      "name": "delete-index",
      "operation-type": "delete-index"
    },
    {
      "name": "create-index",
      "operation-type": "create-index"
    },
    {
      "name": "create-log-pipeline",
      "operation-type": "put-pipeline",
      "id": "log-pipe",
      "body": {
        "description": "ログ処理パイプライン",
        "processors": [
          {
            "set": {
              "field": "ingest_time",
              "value": "{{'{{'}}_ingest.timestamp{{'}}'}}"
            }
          },
          {
            "uppercase": {
              "field": "machine.os"
            }
          },
          {
            "convert": {
              "field": "response",
              "type": "integer"
            }
          },
          {
            "remove": {
              "field": "referer"
            }
          }
        ]
      }
    },
    {
      "name": "bulk-index-with-pipeline",
      "operation-type": "bulk",
      "bulk-size": 1000,
      "pipeline": "log-pipe"
    }
  ],
  "challenges": [
    {
      "name": "default",
      "description": "4クライアントでの標準ベンチマーク",
      "default": true,
      "schedule": [
        {
          "operation": "delete-index"
        },
        {
          "operation": "create-index"
        },
        {
          "operation": "create-log-pipeline"
        },
        {
          "operation": "bulk-index-with-pipeline",
          "warmup-iterations": 10,
          "iterations": 100,
          "clients": 4
        }
      ]
    },
    {
      "name": "high-concurrency",
      "description": "8クライアントでの高負荷テスト",
      "schedule": [
        {
          "operation": "delete-index"
        },
        {
          "operation": "create-index"
        },
        {
          "operation": "create-log-pipeline"
        },
        {
          "operation": "bulk-index-with-pipeline",
          "warmup-iterations": 10,
          "iterations": 100,
          "clients": 8
        }
      ]
    },
    {
      "name": "ramp-up",
      "description": "1から8クライアントへの段階的負荷増加",
      "schedule": [
        {
          "operation": "delete-index"
        },
        {
          "operation": "create-index"
        },
        {
          "operation": "create-log-pipeline"
        },
        {
          "operation": "bulk-index-with-pipeline",
          "warmup-iterations": 5,
          "iterations": 50,
          "clients": 1,
          "target-throughput": 500
        },
        {
          "operation": "bulk-index-with-pipeline",
          "warmup-iterations": 5,
          "iterations": 50,
          "clients": 2,
          "target-throughput": 1000
        },
        {
          "operation": "bulk-index-with-pipeline",
          "warmup-iterations": 5,
          "iterations": 50,
          "clients": 4,
          "target-throughput": 2000
        },
        {
          "operation": "bulk-index-with-pipeline",
          "warmup-iterations": 5,
          "iterations": 50,
          "clients": 8
        }
      ]
    }
  ]
}

// ❌ 間違い - Rallyがこれを解釈しようとします
"value": "{{_ingest.timestamp}}"

// ✅ 正しい - Jinja2のためにエスケープ
"value": "{{'{{'}}_ingest.timestamp{{'}}'}}"

{
  "set": {
    "field": "ingest_time",
    "value": "{{'{{'}}_ingest.timestamp{{'}}'}}"
  }
}

{
  "uppercase": {
    "field": "machine.os"
  }
}

{
  "convert": {
    "field": "response",
    "type": "integer"
  }
}

{
  "remove": {
    "field": "referer"
  }
}

"ingest_pipelines": [
  {
    "name": "log-pipe",
    "body": "ingest-pipeline.json"
  }
]

"operations": [
  {
    "name": "create-log-pipeline",
    "operation-type": "put-pipeline",
    "id": "log-pipe",
    "body": {
      "processors": [...]
    }
  }
]

"schedule": [
  {"operation": "create-index"},
  {"operation": "create-log-pipeline"},  // 明示的な作成
  {"operation": "bulk-index-with-pipeline"}
]

# Rally出力でパイプラインメトリクスを確認
grep "Ingest Pipeline" ~/.rally/logs/rally.log

# --preserve-installを使用した場合、Elasticsearchに直接クエリ
curl -X GET "localhost:PORT/_ingest/pipeline/log-pipe?pretty"
curl -X GET "localhost:PORT/logs/_search?size=1&pretty"

# Rally環境を有効化
source ~/rally-workspace/rally-venv/bin/activate

# 1Kサブセットでテスト
esrally race \
  --track-path=~/rally-workspace/log_ingest \
  --distribution-version=8.11.0 \
  --test-mode

ES 8.11.0 (Java 17) vs ES 9.0.0 (Java 21)
→ 2つの変数が変化: ESバージョンとJavaバージョン

ES 8.11.0 (Java 21) vs ES 9.0.0 (Java 21)
→ ESバージョンのみが変化

# Java 17が設定されていることを確認
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64
source ~/.bashrc

# 確認
java -version  # Java 17であることを確認

# デフォルトチャレンジを実行（4クライアント）
esrally race \
  --track-path=~/rally-workspace/log_ingest \
  --distribution-version=8.11.0 \
  --user-tag="version:8.11.0,java:17,challenge:default"

# 最初の試み（失敗）
esrally race \
  --track-path=~/rally-workspace/log_ingest \
  --distribution-version=9.0.0

# エラー:
# [ERROR] Cannot race. Daemon startup failed with exit code [1]

# ログを確認
tail -100 ~/.rally/logs/rally.log

# "Java 21 or higher required" のようなメッセージを発見

# Java 21をインストール
sudo apt install -y openjdk-21-jdk

# JAVA21_HOMEを設定
export JAVA21_HOME=/usr/lib/jvm/java-21-openjdk-amd64
echo 'export JAVA21_HOME=/usr/lib/jvm/java-21-openjdk-amd64' >> ~/.bashrc
source ~/.bashrc

# 確認
echo $JAVA21_HOME
ls -la $JAVA21_HOME

# 再試行
esrally race \
  --track-path=~/rally-workspace/log_ingest \
  --distribution-version=9.0.0 \
  --user-tag="version:9.0.0,java:21,challenge:default"

# デフォルトチャレンジの実行
esrally race \
  --track-path=~/rally-workspace/log_ingest \
  --distribution-version=8.11.0 \
  --user-tag="version:8.11.0,java:17"

# 高並行チャレンジ（8クライアント）
esrally race \
  --track-path=~/rally-workspace/log_ingest \
  --distribution-version=8.11.0 \
  --challenge=high-concurrency \
  --user-tag="version:8.11.0,java:21,challenge:highcon"

# ランプアップチャレンジ（段階的負荷増加）
esrally race \
  --track-path=~/rally-workspace/log_ingest \
  --distribution-version=8.11.0 \
  --challenge=ramp-up \
  --user-tag="version:8.11.0,java:21,challenge:rampup"

[INFO] Racing on track [log_ingest], challenge [default]
[INFO] Preparing track [log_ingest] for race
[INFO] Downloading Elasticsearch 8.11.0...
[INFO] Starting Elasticsearch cluster
[INFO] Racing (please be patient)...

Running delete-index                    [100% done]
Running create-index                    [100% done]
Running create-log-pipeline             [100% done]
Running bulk-index-with-pipeline        [100% done]

------------------------------------------------------
    Final Results
------------------------------------------------------

|                   Metric |         Task |     Value |   Unit |
|--------------------------|--------------|-----------|--------|
|           Total indexing | bulk-index   |      2.50 |    min |
|            Total merge   | bulk-index   |      0.12 |    min |
|           Throughput     | bulk-index   |   1,650.5 | docs/s |
|       Service time (p50) | bulk-index   |     15.23 |     ms |
|       Service time (p99) | bulk-index   |     42.18 |     ms |
| Total Ingest Pipeline    | bulk-index   |   100,000 |  count |
| Total Ingest Pipeline    | bulk-index   |      1.85 |      s |

# すべてのレースをリスト表示
esrally list races

# 2つの特定のレースを比較
esrally compare \
  --baseline=<race-id-es8> \
  --contender=<race-id-es9>

Total Ingest Pipeline count: 0
Total Ingest Pipeline time: 0 s

{
  "operation": "bulk-index-with-pipeline",
  "operation-type": "bulk",
  "bulk-size": 1000,
  "pipeline": "log-pipe"  // パイプラインIDと一致する必要があります
}

jinja2.exceptions.UndefinedError: '_ingest' is undefined

"value": "{{'{{'}}_ingest.timestamp{{'}}'}}"

[WARNING] No throughput metrics available for [bulk-index-with-pipeline].
Likely cause: The benchmark ended already during warmup.

// ❌ 小さなデータセットでの問題
{
  "warmup-time-period": 60,
  "clients": 4
}

// ✅ 解決策
{
  "warmup-iterations": 10,
  "iterations": 100,
  "clients": 4
}

jsonschema.exceptions.ValidationError: 0 is less than the minimum of 1
Path: ['corpora', 0, 'documents', 0, 'uncompressed-bytes']

// ❌ 間違い
{
  "source-file": "corpora/logs.jsonl",
  "document-count": 100000,
  "uncompressed-bytes": 0
}

// ✅ 正しい
{
  "source-file": "corpora/logs.jsonl",
  "document-count": 100000
}

[ERROR] Cannot race. Daemon startup failed with exit code [1]

# Javaバージョンを確認
java -version

# Rallyログを確認
tail -100 ~/.rally/logs/rally.log

# Java 21をインストール
sudo apt install -y openjdk-21-jdk

# 環境変数を設定
export JAVA21_HOME=/usr/lib/jvm/java-21-openjdk-amd64
echo 'export JAVA21_HOME=/usr/lib/jvm/java-21-openjdk-amd64' >> ~/.bashrc
source ~/.bashrc

# 再試行
esrally race --track-path=~/log_ingest --distribution-version=9.0.0

Parameter source for operation 'put-pipeline' did not provide 
the mandatory parameter 'id' or 'body'

{
  "name": "create-log-pipeline",
  "operation-type": "put-pipeline",
  "id": "log-pipe",          // 必須
  "body": {                   // 必須
    "processors": [...]
  }
}

tail -f ~/.rally/logs/rally.log

# レースディレクトリを検索
ls -lt ~/.rally/benchmarks/races/

# ESログを表示
cat ~/.rally/benchmarks/races/<race-id>/rally-node-0/logs/server/rally-benchmark.log

esrally race \
  --track-path=~/log_ingest \
  --distribution-version=8.11.0 \
  --preserve-install

# クラスタは手動検査のために実行され続けます

# 両方のバージョンをインストール
sudo apt install -y openjdk-17-jdk openjdk-21-jdk

# 環境変数を設定
export JAVA_HOME=/usr/lib/jvm/java-21-openjdk-amd64
export JAVA17_HOME=/usr/lib/jvm/java-17-openjdk-amd64
export JAVA21_HOME=/usr/lib/jvm/java-21-openjdk-amd64

# 永続化のために.bashrcに追加
echo 'export JAVA_HOME=/usr/lib/jvm/java-21-openjdk-amd64' >> ~/.bashrc
echo 'export JAVA17_HOME=/usr/lib/jvm/java-17-openjdk-amd64' >> ~/.bashrc
echo 'export JAVA21_HOME=/usr/lib/jvm/java-21-openjdk-amd64' >> ~/.bashrc

{
  "warmup-iterations": 10,
  "iterations": 100,
  "clients": 4
}

rally-workspace/
├── rally-venv/              # Python仮想環境
├── tracks/
│   ├── log_ingest/
│   │   ├── track.json
│   │   ├── index-settings-mapping.json
│   │   └── corpora/
│   │       ├── logs.jsonl
│   │       └── logs-1k.jsonl
│   ├── search_performance/
│   └── bulk_indexing/
├── scripts/
│   ├── synthetic-log-generator.py
│   └── verify_pipeline.sh
└── results/
    └── comparison_reports/

# CPUとメモリ
htop

# ディスク容量
df -h

# ネットワーク
iftop

# Rallyプロセス
ps aux | grep esrally

# Elasticsearchプロセス
ps aux | grep elasticsearch

# 最初の1000行
head -n 1000 logs.jsonl > logs-1k.jsonl

# ランダムな1000行
shuf -n 1000 logs.jsonl > logs-1k-random.jsonl

# ドキュメント数を確認
wc -l logs.jsonl
wc -l logs-1k.jsonl

# resultsディレクトリを作成
mkdir -p ~/rally-workspace/results

# すべてのレースをリスト表示
esrally list races > results/all_races.txt

# 特定のレース詳細をエクスポート
esrally list races --race-id=<race-id> > results/race_<version>.txt

# 比較して保存
esrally compare \
  --baseline=<race-id-8> \
  --contender=<race-id-9> \
  > results/comparison_8_vs_9.txt

esrally race \
  --track-path=~/log_ingest \
  --distribution-version=8.11.0 \
  --user-tag="env:test,purpose:learning,java:17,challenge:default"

Service time p50: 15 ms   (典型的なリクエスト)
Service time p95: 35 ms   (まだ良好)
Service time p99: 120 ms  (スパイクに注意)
Service time p100: 500 ms (外れ値が存在)

Total Ingest Pipeline count: 100,000  # ドキュメント数と一致する必要があります
Total Ingest Pipeline time: 1.85 s    # パイプラインで費やした時間

Total merge time: 0.12 min

Error rate: 0.00%

ベースライン（ES 8.11.0）とコンテンダー（ES 9.0.0）の比較

|                     Metric |   Baseline |  Contender |    Diff |   Unit |
|----------------------------|------------|------------|---------|--------|
|             Total indexing |       2.50 |       2.35 |  -0.15  |    min |
|                 Throughput |    1650.50 |    1750.25 | +99.75  | docs/s |
|          Service time (p50)|      15.23 |      14.10 |  -1.13  |     ms |
|          Service time (p99)|      42.18 |      38.50 |  -3.68  |     ms |
|    Ingest Pipeline time    |       1.85 |       1.72 |  -0.13  |      s |