RLHFの報酬モデル分析：OpenAssistantの報酬モデルが日本語応答で機能するか調査してみた

import pandas as pd

# Load the spreadsheet
xls = pd.ExcelFile('/mnt/data/reward_model_evaluations.xlsx')

# Get the names of all sheets in the Excel file
sheet_names = xls.sheet_names
# ['reward-model-deberta-v3-base',
# 'reward-model-deberta-v3-large',
# 'reward-model-deberta-v3-large-v2',
# 'reward-model-electra-large-discriminator']

# Load the first sheet into a DataFrame
df1 = xls.parse(sheet_names[0])

# Load the second sheet into a DataFrame
df2 = xls.parse(sheet_names[1])

# Load the third sheet into a DataFrame
df3 = xls.parse(sheet_names[2])

# Load the fourth sheet into a DataFrame
df4 = xls.parse(sheet_names[3])

# Concatenate the statistics of all sheets
all_stats = pd.concat([df1_stats, df2_stats, df3_stats, df4_stats], keys=[df1['reward_model_name'].iloc[0], df2['reward_model_name'].iloc[0], df3['reward_model_name'].iloc[0], df4['reward_model_name'].iloc[0]])

# Reset the index for a cleaner look
all_stats.reset_index(inplace=True)
all_stats.rename(columns={'level_0': 'Model', 'level_1': 'Statistic'}, inplace=True)

all_stats.pivot(index='Model', columns='Statistic')

# Function to plot box plots with fixed y-axis
def plot_boxplot_fixed_yaxis(df, title, y_min, y_max):
    # Prepare data
    data = [df['chosen_text_reward'], df['rejected_text_reward']]

    plt.figure(figsize=(8, 6))

    plt.boxplot(data)
    plt.title(title)
    plt.xticks([1, 2], ['Chosen Text Reward', 'Rejected Text Reward'])
    plt.ylabel('Reward')
    plt.ylim(y_min, y_max)

    plt.show()

# Determine the y-axis limits
y_min = min(df1['chosen_text_reward'].min(), df1['rejected_text_reward'].min(),
            df2['chosen_text_reward'].min(), df2['rejected_text_reward'].min(),
            df3['chosen_text_reward'].min(), df3['rejected_text_reward'].min(),
            df4['chosen_text_reward'].min(), df4['rejected_text_reward'].min()) - 0.5  # Adding some padding

y_max = max(df1['chosen_text_reward'].max(), df1['rejected_text_reward'].max(),
            df2['chosen_text_reward'].max(), df2['rejected_text_reward'].max(),
            df3['chosen_text_reward'].max(), df3['rejected_text_reward'].max(),
            df4['chosen_text_reward'].max(), df4['rejected_text_reward'].max()) + 0.5  # Adding some padding

# Plot box plots for each sheet with fixed y-axis
for df in [df1, df2, df3, df4]:
    model_name = df['reward_model_name'].iloc[0]
    plot_boxplot_fixed_yaxis(df, model_name, y_min, y_max)

import matplotlib.pyplot as plt

# Function to plot histograms
def plot_histograms(df, title):
    plt.figure(figsize=(14, 6))

    plt.hist(df['chosen_text_reward'], bins=100, alpha=0.5, label='Chosen Text Reward')
    plt.hist(df['rejected_text_reward'], bins=100, alpha=0.5, label='Rejected Text Reward')

    plt.title(title)
    plt.xlabel('Reward')
    plt.ylabel('Frequency')
    plt.legend(loc='upper right')

    plt.show()

# Plot histograms for each sheet
plot_histograms(df1, df1['reward_model_name'].iloc[0])
plot_histograms(df2, df2['reward_model_name'].iloc[0])
plot_histograms(df3, df3['reward_model_name'].iloc[0])
plot_histograms(df4, df4['reward_model_name'].iloc[0])

# Extract the model names from the 'reward_model_name' column
model_names = [df1['reward_model_name'][0], df2['reward_model_name'][0], 
               df3['reward_model_name'][0], df4['reward_model_name'][0]]

# Plot 2D histograms for all models with increased resolution
plot_2d_hist_fixed_color_high_res(df1, df1, model_names[0])
plot_2d_hist_fixed_color_high_res(df2, df2, model_names[1])
plot_2d_hist_fixed_color_high_res(df3, df3, model_names[2])
plot_2d_hist_fixed_color_high_res(df4, df4, model_names[3])