import pandas as pd
import os

## get all output files from simulation study, each file contains p-values for all bootstrap samples for 6 estimators under one combination of lambda1, lambda2, N, p_rw. 
## The combination each file refers to is represented by the filename. 
filenames = os.listdir('../ALLEXPR/')
filenames_expr = [f for f in filenames if f[:4] == 'expr']

colnames = ['p_naive_i', 'p_neighbor_i', 'p_lm_i', 'p_naive_c', 'p_neighbor_c', 'p_lm_c']
d = {'lambda1':[], 'lambda2':[], 'N':[], 'pr':[], 'epsilon':[], 'p_naive_i':[], 'p_neighbor_i':[], 'p_lm_i':[], 'p_naive_c':[], 'p_neighbor_c':[], 'p_lm_c':[], 'count':[]} 
for filename in filenames_expr:
    # for the file, obtain the combination of lambda1, lambda2, N, pr, epsilon the file is for. 
    lambda1, lambda2, N, pr, epsilon = filename.split('_')[1:6]
    d['lambda1'].append(lambda1)
    d['lambda2'].append(lambda2)
    d['N'].append(N)
    d['pr'].append(pr)
    d['epsilon'].append(epsilon)
    df = pd.read_csv('../ALLEXPR/' + filename, sep='\t', header=None, names=colnames)
    B = df.count()[0]
    d['count'].append(B)
    rejection_rate = []
    # compute rejection rate for each estimator
    for name in colnames: 
        l = df[name].values.tolist()
        reject_num = sum([p<0.05 for p in l]) + 0.0
        reject_rate = reject_num / B
        d[name].append(reject_rate)

df = pd.DataFrame(data=d)
df = df.sort_values(by=['lambda1','lambda2','N','pr','epsilon'])
## save rejection rates under all combinations of lambda1, lambda2, N, pr.
df.to_csv('./result_all', sep='\t', header=True)

