日本語アナライザーの比較：Kuromoji・Sudachi・MeCab・Janome・LLMの性能検証

curl -LsSf https://astral.sh/uv/install.sh | sh
mkdir analyzer-project && cd analyzer-project
uv venv
source .venv/bin/activate

uv pip install elasticsearch janome openai sudachipy sudachidict_core mecab-python3

bin/elasticsearch-plugin install analysis-kuromoji

brew install mecab mecab-ipadic

export OPENAI_API_KEY="sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

import os
import sys
import platform
import re
from dotenv import load_dotenv

# Import the working elasticsearch connection utility
from elastic_conection import es, test_connection

# Japanese text analysis libraries
import MeCab
from janome.tokenizer import Tokenizer
from sudachipy import tokenizer as sudachi_tokenizer
from sudachipy import dictionary as sudachi_dictionary

# OpenAI for LLM comparison
from openai import OpenAI

# Environment and target text setup
print("Python環境情報")
print("="*30)
print(f"Python Version: {sys.version.split()[0]}")
print(f"Platform: {platform.platform()}")
print(f"Architecture: {platform.machine()}")

# Check if in virtual environment
if hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix):
    print("仮想環境で実行中")
else:
    print("仮想環境ではありません")

print(f"実行環境: {sys.executable}")

# Define the target text for analysis
TARGET_TEXT = """ロキソニン錠の説明書ですね。ロキソニンは、解熱鎮痛作用のある非ステロイド性抗炎症薬（NSAIDs）で、
痛みや発熱、炎症を抑える効果があります。具体的には、関節リウマチ、変形性関節症、腰痛症、肩こり、
歯痛、手術後や外傷後の炎症や痛み、風邪による熱や痛みなどに用いられます。"""

print(f"\n分析対象テキスト: {TARGET_TEXT}")
print(f"文字数: {len(TARGET_TEXT)}")
print()

def clean_tokens_like_kuromoji(tokens):
    """
    Clean tokens to match Kuromoji's behavior by removing stop words and punctuation.
    
    Args:
        tokens (list): List of token strings
        
    Returns:
        list: Cleaned tokens with stop words and punctuation removed
    """
    
    # Japanese stop words and particles commonly filtered out
    stop_words = {
        'は', 'の', 'を', 'に', 'が', 'で', 'と', 'から', 'まで', 'より', 'へ',
        'や', 'か', 'も', 'て', 'た', 'だ', 'である', 'です', 'ます', 'した',
        'する', 'ある', 'いる', 'なる', 'この', 'その', 'あの', 'どの',
        'これ', 'それ', 'あれ', 'どれ', 'ここ', 'そこ', 'あそこ', 'どこ',
        'こう', 'そう', 'ああ', 'どう', 'という', 'といった', 'による',
        'において', 'について', 'に関して', 'に対して', 'に関する', 'について',
        'さん', 'ちゃん', 'くん', 'さま', 'さあ', 'まあ', 'ああ', 'いや',
        'はい', 'いいえ', 'うん', 'ううん', 'えー', 'あー', 'うー', 'おー'
    }
    
    # Punctuation and symbols to remove
    punctuation_patterns = [
        r'^[、。！？.,!?;:()（）\[\]「」『』【】〈〉《》〔〕…‥・]+$',  # Pure punctuation
        r'^[ー\-~〜]+$',  # Long vowel marks and dashes
        r'^[　\s]+$',     # Whitespace (including full-width)
        r'^\d+$',         # Pure numbers
        r'^[a-zA-Z]+$',   # Pure alphabet
        r'^[０-９]+$',    # Full-width numbers
        r'^[ａ-ｚＡ-Ｚ]+$' # Full-width alphabet
    ]
    
    cleaned = []
    for token in tokens:
        if not token or not token.strip():
            continue
            
        # Remove stop words
        if token in stop_words:
            continue
            
        # Remove punctuation and unwanted patterns
        is_punctuation = False
        for pattern in punctuation_patterns:
            if re.match(pattern, token):
                is_punctuation = True
                break
                
        if not is_punctuation:
            cleaned.append(token)
    
    return cleaned

print("Token cleaning function defined")

# === システム接続確認とトークナイザー初期化 ===
print("=== システム接続確認 ===")

# Elasticsearch接続と初期化
print("Elasticsearchに接続中...")
try:
    # elastic_conection.pyからESクライアントをインポート
    from elastic_conection import es, test_connection
    
    # 接続テスト
    if test_connection():
        es_available = True
        print("Elasticsearch接続成功")
    else:
        es_available = False
        print("Elasticsearch接続失敗")
        print("解決方法: Elasticsearchサーバーを起動してください")
        print("   brew services start elasticsearch")
        
except Exception as e:
    es_available = False
    print(f"Elasticsearch接続失敗: {e}")
    print("解決方法: Elasticsearchサーバーを起動してください")
    print("   brew services start elasticsearch")

# 各トークナイザーの初期化
print("\nトークナイザーを初期化中...")

# MeCab初期化 (Homebrew環境対応)
tagger = None
try:
    import MeCab
    
    # Homebrew環境用の設定リスト（正しいmecabrcパスを含む）
    configs_to_try = [
        "-r /opt/homebrew/etc/mecabrc -Owakati",  # Homebrew mecabrc + wakati mode
        "-r /opt/homebrew/etc/mecabrc",            # Homebrew mecabrc only  
        "-Owakati",                                # wakati mode only
        "",                                        # デフォルト設定
        "-d /opt/homebrew/lib/mecab/dic/ipadic",   # ipadic辞書パス (Homebrew)
        "-d /usr/local/lib/mecab/dic/ipadic",      # ipadic辞書パス (従来のパス)
    ]
    
    for config in configs_to_try:
        try:
            print(f"   MeCab設定を試行中: '{config if config else 'デフォルト'}'")
            tagger = MeCab.Tagger(config)
            # テスト実行して動作確認
            test_result = tagger.parse("テスト")
            if test_result and len(test_result.strip()) > 0:
                mecab_config = config if config else "デフォルト設定"
                print(f"MeCab初期化成功 (設定: {mecab_config})")
                print(f"   テスト結果: {test_result.strip()}")
                break
        except Exception as e:
            print(f"   設定失敗: {e}")
            continue
    
    if not tagger:
        print("MeCab初期化失敗: MeCab could not be initialized with any configuration")
        print("解決方法: brew install mecab mecab-ipadic")
        
except ImportError:
    print("MeCab初期化失敗: MeCabがインストールされていません")
    print("解決方法: brew install mecab mecab-ipadic")

# SudachiPy初期化
tokenizer_obj = None
try:
    from sudachipy import tokenizer
    from sudachipy import dictionary
    
    tokenizer_obj = dictionary.Dictionary().create()
    print("SudachiPy初期化成功")
except Exception as e:
    print(f"SudachiPy初期化失敗: {e}")

# Janome初期化
janome_tokenizer = None
try:
    from janome.tokenizer import Tokenizer
    janome_tokenizer = Tokenizer()
    print("Janome初期化成功")
except Exception as e:
    print(f"Janome初期化失敗: {e}")

# OpenAI API初期化
openai_client = None
try:
    import openai
    from dotenv import load_dotenv
    import os
    
    load_dotenv()
    api_key = os.getenv("OPENAI_API_KEY")
    
    if api_key:
        openai_client = openai.OpenAI(api_key=api_key)
        print("OpenAI API初期化成功")
    else:
        print("OpenAI API初期化失敗: APIキーが設定されていません")
        print("解決方法: .envファイルにOPENAI_API_KEYを設定してください")
        
except Exception as e:
    print(f"OpenAI API初期化失敗: {e}")

# 初期化サマリー
print("\n初期化サマリー:")
print(f"   Elasticsearch: {'OK' if es_available else 'NG'}")
print(f"   MeCab: {'OK' if tagger else 'NG'}")
print(f"   SudachiPy: {'OK' if tokenizer_obj else 'NG'}")
print(f"   Janome: {'OK' if janome_tokenizer else 'NG'}")
print(f"   OpenAI: {'OK' if openai_client else 'NG'}")

def tokenize_with_kuromoji(text):
    """Tokenize text using Elasticsearch Kuromoji analyzer"""
    if not es_available:
        print("Kuromoji tokenization skipped: Elasticsearch not available")
        return []
        
    try:
        response = es.indices.analyze(
            body={
                "analyzer": "kuromoji",
                "text": text
            }
        )
        return [token['token'] for token in response['tokens']]
    except Exception as e:
        print(f"Kuromoji tokenization error: {e}")
        return []

def tokenize_with_mecab(text):
    """Tokenize text using MeCab"""
    if not tagger:
        print("MeCab tokenization skipped: MeCab not available")
        return []
        
    try:
        result = tagger.parse(text).strip().split()
        return [token for token in result if token]
    except Exception as e:
        print(f"MeCab tokenization error: {e}")
        return []

def tokenize_with_sudachi(text, mode='C'):
    """Tokenize text using SudachiPy with specified mode (A, B, or C)"""
    if not tokenizer_obj:
        print("SudachiPy tokenization skipped: SudachiPy not available")
        return []
        
    try:
        mode_map = {'A': sudachi_tokenizer.Tokenizer.SplitMode.A,
                   'B': sudachi_tokenizer.Tokenizer.SplitMode.B,
                   'C': sudachi_tokenizer.Tokenizer.SplitMode.C}
        
        tokens = tokenizer_obj.tokenize(text, mode_map[mode])
        return [token.surface() for token in tokens]
    except Exception as e:
        print(f"SudachiPy tokenization error: {e}")
        return []

def tokenize_with_janome(text):
    """Tokenize text using Janome"""
    if not janome_tokenizer:
        print("Janome tokenization skipped: Janome not available")
        return []
        
    try:
        tokens = janome_tokenizer.tokenize(text, wakati=True)
        return list(tokens)
    except Exception as e:
        print(f"Janome tokenization error: {e}")
        return []

def tokenize_with_openai(text):
    """Tokenize text using OpenAI GPT-4"""
    if not openai_client:
        print("OpenAI tokenization skipped: OpenAI client not available")
        return []
    
    try:
        prompt = f"""
Please tokenize the following Japanese text into meaningful segments. 
Return only a comma-separated list of tokens, no explanations.

Text: {text}

Tokens:"""

        response = openai_client.chat.completions.create(
            model="gpt-4",
            messages=[{"role": "user", "content": prompt}],
            max_tokens=200,
            temperature=0
        )
        
        result = response.choices[0].message.content.strip()
        tokens = [token.strip() for token in result.split(',')]
        return [token for token in tokens if token]
    
    except Exception as e:
        print(f"OpenAI tokenization error: {e}")
        return []

print("All tokenization functions defined (with availability checks)")

def compare_all_tokenizers(text):
    """Compare all tokenizers on the given text - using Kuromoji as baseline (no cleaning)"""
    
    print(f"分析対象テキスト: {text}")
    print("=" * 80)
    
    results = {}
    
    # 1. Kuromoji (Elasticsearch) - BASELINE (no cleaning applied)
    print("\n1. Kuromoji (Elasticsearch) - ベースライン")
    kuromoji_tokens = tokenize_with_kuromoji(text)
    results['kuromoji'] = {
        'raw': kuromoji_tokens,
        'cleaned': kuromoji_tokens  # No cleaning - use as baseline
    }
    print(f"Raw/Baseline ({len(kuromoji_tokens)}): {kuromoji_tokens}")
    print("Kuromojiはベースラインとして使用（クリーニングなし）")
    
    # 2. SudachiPy (all modes) - cleaned to match Kuromoji behavior
    for mode in ['A', 'B', 'C']:
        print(f"\n2. SudachiPy Mode {mode} - Kuromojiベース調整済み")
        sudachi_tokens = tokenize_with_sudachi(text, mode)
        sudachi_cleaned = clean_tokens_like_kuromoji(sudachi_tokens)
        results[f'sudachi_{mode}'] = {
            'raw': sudachi_tokens,
            'cleaned': sudachi_cleaned
        }
        print(f"Raw ({len(sudachi_tokens)}): {sudachi_tokens}")
        print(f"Cleaned ({len(sudachi_cleaned)}): {sudachi_cleaned}")
    
    # 3. MeCab - cleaned to match Kuromoji behavior
    print(f"\n3. MeCab - Kuromojiベース調整済み")
    mecab_tokens = tokenize_with_mecab(text)
    mecab_cleaned = clean_tokens_like_kuromoji(mecab_tokens)
    results['mecab'] = {
        'raw': mecab_tokens,
        'cleaned': mecab_cleaned
    }
    print(f"Raw ({len(mecab_tokens)}): {mecab_tokens}")
    print(f"Cleaned ({len(mecab_cleaned)}): {mecab_cleaned}")
    
    # 4. Janome - cleaned to match Kuromoji behavior
    print(f"\n4. Janome - Kuromojiベース調整済み")
    janome_tokens = tokenize_with_janome(text)
    janome_cleaned = clean_tokens_like_kuromoji(janome_tokens)
    results['janome'] = {
        'raw': janome_tokens,
        'cleaned': janome_cleaned
    }
    print(f"Raw ({len(janome_tokens)}): {janome_tokens}")
    print(f"Cleaned ({len(janome_cleaned)}): {janome_cleaned}")
    
    # 5. OpenAI GPT-4 - cleaned to match Kuromoji behavior
    if openai_client:
        print(f"\n5. OpenAI GPT-4 - Kuromojiベース調整済み")
        openai_tokens = tokenize_with_openai(text)
        openai_cleaned = clean_tokens_like_kuromoji(openai_tokens)
        results['openai'] = {
            'raw': openai_tokens,
            'cleaned': openai_cleaned
        }
        print(f"Raw ({len(openai_tokens)}): {openai_tokens}")
        print(f"Cleaned ({len(openai_cleaned)}): {openai_cleaned}")
    else:
        print(f"\n5. OpenAI GPT-4 (スキップ - API key not available)")
    
    # Comparison summary with Kuromoji as baseline
    print(f"\nKuromojiベースライン比較:")
    kuromoji_baseline = set(results['kuromoji']['cleaned'])
    for name, data in results.items():
        if name != 'kuromoji':
            cleaned_tokens = set(data['cleaned'])
            overlap = len(kuromoji_baseline & cleaned_tokens)
            total_unique = len(kuromoji_baseline | cleaned_tokens)
            similarity = (overlap / total_unique * 100) if total_unique > 0 else 0
            print(f"  {name:<12}: {similarity:.1f}% similarity to Kuromoji baseline")
    
    return results

# Run the comparison
analysis_results = compare_all_tokenizers(TARGET_TEXT)