Benchmark NSGA-II

In the following, we benchmark the results of the NSGA-II implementation in pymoo and the original implementation in C which is available here.

In [11]:
%run custom.ipynb
In [12]:
%matplotlib inline
%reload_ext autoreload
%autoreload 2

problems = ['zdt1', 'zdt2', 'zdt3', 'zdt4','zdt6', 'kursawe', 'bnh', 'osy']


# read the setup file to get the experiment details
def get_setup(str_problem):
    with open("input_data/%s.in" % str_problem) as f:
        setup = f.readlines()
    setup = [x.strip() for x in setup]

    val = {
        'pop_size' : int(setup[0]),
        'n_gen' :  int(setup[1]),
        'n_obj' :int(setup[2]),
        'n_var' : int(setup[4]),
        'problem' : get_problem(str_problem)
    }
    
    return val


        

def load_results(algorithm, func_load=None, func_load_history=None):
    
    data = []
    
    # for each problem to compare
    for problem in problems:
        
        setup = get_setup(problem)
        p = setup['problem']
        
        pf = p.pareto_front()

        folder = os.path.join(".", algorithm, problem)
        
        
        if not os.path.exists(folder):
            continue
        
        files = [f for f in os.listdir(folder) if re.match(r'.*out', f)]
        
        print("%s: Parsing %s files." % (problem, len(files)))
            
        # for each run that was executed
        for run, f in enumerate(files):
            
            entry = {}
            
            path_to_file = os.path.join(folder, f)

            if func_load is None:
                F = np.loadtxt(path_to_file)[:, :setup['n_obj']]
            else:
                F = func_load(path_to_file, setup)
                
            I = NonDominatedSorting().do(F, only_non_dominated_front=True)
            F = F[I,:]
            
            igd = IGD(pf).calc(F)
            hv = Hypervolume(pf=pf).calc(F)

            data.append({'problem' : problem, 'algorithm' : algorithm, 
                         'run' : run, 'igd' : igd, 'hv' : hv, 'fname' : f})
            
            
    pickle.dump(data, open("%s.dat" % algorithm, "wb" ))
    return data

NSGA-II (C-Code)

Load the data executed from the C code and convert them into the right format.

In [4]:
def func_load(fname, setup):
    M = np.genfromtxt(fname,delimiter='\t', comments='#')
    F = M[:,0:setup['n_obj']]
    return F
    
def func_load_history(fname, problem):
    H = np.genfromtxt(fname,delimiter='\t', comments='#')[:, 0:n_obj]
    H = np.reshape(H, (n_gen, pop_size, problem.n_obj))
    return H
In [5]:
cnsga2 = load_results("cnsga2",func_load, func_load_history)

zdt1: Parsing 100 files.
zdt2: Parsing 100 files.
zdt3: Parsing 100 files.
zdt4: Parsing 100 files.
zdt6: Parsing 100 files.
kursawe: Parsing 100 files.
bnh: Parsing 100 files.
osy: Parsing 100 files.

NSGA-II (pymoo)

In [7]:
pynsga2 = load_results("pynsga2")

zdt1: Parsing 100 files.
zdt2: Parsing 100 files.
zdt3: Parsing 100 files.
zdt4: Parsing 100 files.
zdt6: Parsing 100 files.
kursawe: Parsing 100 files.
bnh: Parsing 100 files.
osy: Parsing 100 files.
In [8]:
df = pd.DataFrame(cnsga2 + pynsga2,columns=['problem', 'algorithm' , 'run', 'igd', 'hv', 'fname'])
In [9]:
metric = 'igd'

for name in problems:
    
    _df = df[df['problem'] == name]
    val = _df.groupby(['algorithm']).agg({metric: ['median', 'min','max','mean', 'std']})
    print(val)
    
    
    _df.boxplot(column=metric, by='algorithm')
    plt.title(name)
    plt.suptitle("")
    plt.show()
    print("----------------------------------------------")

                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga2     0.005345  0.004813  0.006178  0.005331  0.000252
pynsga2    0.005184  0.004503  0.006119  0.005201  0.000284

----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga2     0.005546  0.004920  0.006374  0.005550  0.000305
pynsga2    0.005260  0.004605  0.006107  0.005303  0.000308

----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga2     0.005418  0.004674  0.060628  0.006552  0.007770
pynsga2    0.005309  0.004230  0.006263  0.005344  0.000349

----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga2     0.010910  0.005117  0.171508  0.019516  0.028515
pynsga2    0.009697  0.004895  0.081879  0.011814  0.010016

----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga2     0.004353  0.003810  0.004806  0.004363  0.000214
pynsga2    0.003586  0.003116  0.004103  0.003620  0.000216

----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga2     0.042316  0.035835  0.048094  0.042363  0.002133
pynsga2    0.039819  0.034388  0.044825  0.039778  0.002097

----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga2     0.523537  0.463842  0.611596  0.526354  0.028452
pynsga2    0.519662  0.460080  0.595565  0.522908  0.032905

----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga2     0.373246  0.299593  5.085718  0.695528  1.106363
pynsga2    0.358534  0.262275  5.034631  0.760694  1.285847

----------------------------------------------