Benchmark pymoo vs. cnsga3

This purpose of the comparison is to benchmark the original NSGA-III code with pymoo.

In [4]:
%run custom.ipynb
In [7]:
import os
import pickle

from pymoo.operators.crossover.simulated_binary_crossover import SimulatedBinaryCrossover
from pymoo.operators.mutation.polynomial_mutation import PolynomialMutation
from pymoo.util.reference_direction import MultiLayerReferenceDirectionFactory, UniformReferenceDirectionFactory
from pymop.factory import get_problem


def get_setup(n_obj):
    if n_obj == 3:
        pop_size = 92
        ref_dirs = UniformReferenceDirectionFactory(n_obj, n_partitions=12).do()
    elif n_obj == 5:
        pop_size = 212
        ref_dirs = UniformReferenceDirectionFactory(n_obj, n_partitions=6).do()
    elif n_obj == 8:
        pop_size = 156
        ref_dirs = MultiLayerReferenceDirectionFactory([
            UniformReferenceDirectionFactory(n_obj, n_partitions=3, scaling=1.0),
            UniformReferenceDirectionFactory(n_obj, n_partitions=2, scaling=0.5)]).do()
    elif n_obj == 10:
        pop_size = 276
        ref_dirs = MultiLayerReferenceDirectionFactory([
            UniformReferenceDirectionFactory(n_obj, n_partitions=3, scaling=1.0),
            UniformReferenceDirectionFactory(n_obj, n_partitions=2, scaling=0.5)]).do()
    elif n_obj == 15:
        pop_size = 136
        ref_dirs = MultiLayerReferenceDirectionFactory([
            UniformReferenceDirectionFactory(n_obj, n_partitions=2, scaling=1.0),
            UniformReferenceDirectionFactory(n_obj, n_partitions=1, scaling=0.5)]).do()

    return {
        'ref_dirs': ref_dirs,
        'pop_size': pop_size,
        'crossover': SimulatedBinaryCrossover(1.0, 30),
        'mutation': PolynomialMutation(20)
    }


setup = {

    # ==========================================
    # DTLZ1
    # ==========================================

    'dtlz1_3obj': {
        'termination': ('n_gen', 400),
        'problem': get_problem("dtlz1", None, 3, k=5),
        **get_setup(3)

    },

    'dtlz1_5obj': {
        'termination': ('n_gen', 600),
        'problem': get_problem("dtlz1", None, 5, k=5),
        **get_setup(5)
    },

    'dtlz1_8obj': {
        'termination': ('n_gen', 750),
        'problem': get_problem("dtlz1", None, 8, k=5),
        **get_setup(8)
    },

    'dtlz1_10obj': {
        'termination': ('n_gen', 1000),
        'problem': get_problem("dtlz1", None, 10, k=5),
        **get_setup(10)
    },

    'dtlz1_15obj': {
        'termination': ('n_gen', 1500),
        'problem': get_problem("dtlz1", None, 15, k=5),
        **get_setup(15)
    },

    # ==========================================
    # DTLZ2
    # ==========================================

    'dtlz2_3obj': {
        'termination': ('n_gen', 250),
        'problem': get_problem("dtlz2", None, 3, k=10),
        **get_setup(3)

    },

    'dtlz2_5obj': {
        'termination': ('n_gen', 350),
        'problem': get_problem("dtlz2", None, 5, k=10),
        **get_setup(5)

    },

    'dtlz2_8obj': {
        'termination': ('n_gen', 500),
        'problem': get_problem("dtlz2", None, 8, k=10),
        **get_setup(8)
    },

    'dtlz2_10obj': {
        'termination': ('n_gen', 750),
        'problem': get_problem("dtlz2", None, 10, k=10),
        **get_setup(10)

    },

    'dtlz2_15obj': {
        'termination': ('n_gen', 1000),
        'problem': get_problem("dtlz2", None, 15, k=10),
        **get_setup(15)
    },

    # ==========================================
    # DTLZ3
    # ==========================================

    'dtlz3_3obj': {
        'termination': ('n_gen', 1000),
        'problem': get_problem("dtlz3", None, 3, k=10),
        **get_setup(3)

    },

    'dtlz3_5obj': {
        'termination': ('n_gen', 1000),
        'problem': get_problem("dtlz3", None, 5, k=10),
        **get_setup(5)

    },

    'dtlz3_8obj': {
        'termination': ('n_gen', 1000),
        'problem': get_problem("dtlz3", None, 8, k=10),
        **get_setup(8)
    },

    'dtlz3_10obj': {
        'termination': ('n_gen', 1500),
        'problem': get_problem("dtlz3", None, 10, k=10),
        **get_setup(10)

    },

    'dtlz3_15obj': {
        'termination': ('n_gen', 2000),
        'problem': get_problem("dtlz3", None, 15, k=10),
        **get_setup(15)
    },

    # ==========================================
    # DTLZ4
    # ==========================================

    'dtlz4_3obj': {
        'termination': ('n_gen', 600),
        'problem': get_problem("dtlz3", None, 3, k=10),
        **get_setup(3)

    },

    'dtlz4_5obj': {
        'termination': ('n_gen', 1000),
        'problem': get_problem("dtlz4", None, 5, k=10),
        **get_setup(5)

    },

    'dtlz4_8obj': {
        'termination': ('n_gen', 1250),
        'problem': get_problem("dtlz4", None, 8, k=10),
        **get_setup(8)
    },

    'dtlz4_10obj': {
        'termination': ('n_gen', 2000),
        'problem': get_problem("dtlz4", None, 10, k=10),
        **get_setup(10)

    },

    'dtlz4_15obj': {
        'termination': ('n_gen', 3000),
        'problem': get_problem("dtlz4", None, 15, k=10),
        **get_setup(15)
    },

}
In [8]:
%matplotlib inline
%reload_ext autoreload
%autoreload 2


problems = setup.keys()


def load_results(algorithm, func_load=None, func_load_history=None):
    
    data = []
    
    # for each problem to compare
    for problem in problems:
        
        _setup = setup[problem]
        p = _setup['problem']
        
        pf = p.pareto_front(_setup['ref_dirs'])

        
        folder = os.path.join(".", algorithm, problem.replace('_', '/'))
        
        if not os.path.exists(folder):
            continue
        
        files = [f for f in os.listdir(folder) if re.match(r'.*out', f)]
        
        print("%s: Parsing %s files." % (problem, len(files)))
            
        # for each run that was executed
        for run, f in enumerate(files):
            
            entry = {}
            
            path_to_file = os.path.join(folder, f)

            if func_load is None:
                F = np.loadtxt(path_to_file)[:,:p.n_obj]
            else:
                F = func_load(path_to_file, _setup)
                
            I = NonDominatedSorting().do(F, only_non_dominated_front=True)
            F = F[I,:]

            igd = IGD(pf).calc(F)
            gd = GD(pf).calc(F)
            hv = None
            data.append({'fname' : f, 'problem' : problem, 'algorithm' : algorithm, 
                         'run' : run, 'igd' : igd, 'gd' : gd, 'hv' : hv})
            
            
    pickle.dump(data, open("%s.dat" % algorithm, "wb" ))
    return data
    

NSGA-III (C-Code)

Load the data executed from the C code and convert them into the right format.

In [9]:
def func_load(fname, setup):
    M = np.genfromtxt(fname,delimiter='\t', comments='#')
    F = M[:,0:setup['problem'].n_obj]
    return F
    
def func_load_history(fname, problem):
    H = np.genfromtxt(fname,delimiter='\t', comments='#')[:, 0:n_obj]
    H = np.reshape(H, (n_gen, pop_size, problem.n_obj))
    return H
In [11]:
cnsga3 = load_results("cnsga3",func_load, func_load_history)
dtlz1_3obj: Parsing 100 files.
dtlz1_5obj: Parsing 100 files.
dtlz1_8obj: Parsing 100 files.
dtlz1_10obj: Parsing 100 files.
dtlz1_15obj: Parsing 100 files.
dtlz2_3obj: Parsing 100 files.
dtlz2_5obj: Parsing 100 files.
dtlz2_8obj: Parsing 100 files.
dtlz2_10obj: Parsing 100 files.
dtlz2_15obj: Parsing 100 files.
dtlz3_3obj: Parsing 100 files.
dtlz3_5obj: Parsing 100 files.
dtlz3_8obj: Parsing 100 files.
dtlz4_3obj: Parsing 100 files.
dtlz4_5obj: Parsing 100 files.
dtlz4_8obj: Parsing 100 files.

NSGA-III (Python-Code)

In [10]:
pynsga3 = load_results("pynsga3")
dtlz1_3obj: Parsing 50 files.
dtlz1_5obj: Parsing 50 files.
dtlz1_8obj: Parsing 50 files.
dtlz1_10obj: Parsing 50 files.
dtlz1_15obj: Parsing 50 files.
dtlz2_3obj: Parsing 50 files.
dtlz2_5obj: Parsing 50 files.
dtlz2_8obj: Parsing 50 files.
dtlz2_10obj: Parsing 50 files.
dtlz2_15obj: Parsing 50 files.
dtlz3_3obj: Parsing 50 files.
dtlz3_5obj: Parsing 50 files.
dtlz3_8obj: Parsing 50 files.
dtlz3_10obj: Parsing 50 files.
dtlz3_15obj: Parsing 50 files.
dtlz4_3obj: Parsing 50 files.
dtlz4_5obj: Parsing 50 files.
dtlz4_8obj: Parsing 50 files.
dtlz4_10obj: Parsing 50 files.
dtlz4_15obj: Parsing 50 files.
In [13]:
algorithms = cnsga3 + pynsga3

df = pd.DataFrame(algorithms, 
                  columns=['fname', 'problem', 'algorithm' , 'run', 'igd', 'hv', 'gd'])
In [14]:
metric = 'igd'

for name in problems:
    
    _df = df[df['problem'] == name]
    val = _df.groupby(['algorithm']).agg({metric: ['median', 'min','max','mean', 'std']})
    print(val)
    
    
    _df.boxplot(column=metric, by='algorithm')
    plt.title(name)
    plt.suptitle("")
    plt.show()
    print("----------------------------------------------")
                igd                                       
             median      min       max      mean       std
algorithm                                                 
cnsga3     0.001588  0.00040  0.007235  0.001878  0.001199
pynsga3    0.001332  0.00041  0.005309  0.001638  0.001100
----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga3     0.001179  0.000508  0.004287  0.001361  0.000690
pynsga3    0.001001  0.000522  0.002496  0.001171  0.000514
----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga3     0.004222  0.002220  0.009229  0.004467  0.001415
pynsga3    0.004111  0.002247  0.013440  0.004535  0.002069
----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga3     0.003573  0.002353  0.006712  0.003786  0.000981
pynsga3    0.003149  0.002363  0.005609  0.003371  0.000744
----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga3     0.005670  0.002474  0.023337  0.006479  0.003378
pynsga3    0.005724  0.003039  0.012860  0.006428  0.002341
----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga3     0.001402  0.000948  0.002627  0.001453  0.000288
pynsga3    0.001372  0.001039  0.002480  0.001409  0.000251
----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga3     0.005033  0.004112  0.006394  0.005053  0.000472
pynsga3    0.004918  0.004127  0.006234  0.004954  0.000474
----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga3     0.015633  0.012708  0.019918  0.015822  0.001341
pynsga3    0.015510  0.012314  0.019487  0.015521  0.001448
----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga3     0.015301  0.013140  0.018440  0.015489  0.000993
pynsga3    0.015598  0.013414  0.017909  0.015631  0.001056
----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga3     0.017208  0.014270  0.021679  0.017444  0.001676
pynsga3    0.017601  0.014955  0.023448  0.017892  0.001653
----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga3     0.003544  0.000644  0.015692  0.004419  0.003231
pynsga3    0.003849  0.001440  0.013401  0.004924  0.003051
----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga3     0.005555  0.001861  0.031697  0.006519  0.003890
pynsga3    0.005510  0.002096  0.014537  0.006148  0.002664
----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga3     0.029281  0.012009  0.900872  0.042745  0.089346
pynsga3    0.028957  0.014583  0.252772  0.036903  0.035020
----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
pynsga3    0.012717  0.008708  0.026254  0.014036  0.003859
----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
pynsga3    0.025578  0.013197  0.425843  0.033572  0.057097
----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga3     0.000690  0.000379  0.429141  0.106576  0.133910
pynsga3    0.014229  0.005449  0.100220  0.017849  0.015761
----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga3     0.001104  0.000816  0.407622  0.008047  0.045258
pynsga3    0.000568  0.000360  0.001053  0.000608  0.000138
----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
cnsga3     0.007374  0.005083  0.587927  0.086980  0.144686
pynsga3    0.003921  0.003027  0.005736  0.004045  0.000618
----------------------------------------------
                igd                                        
             median       min       max      mean       std
algorithm                                                  
pynsga3    0.004571  0.003579  0.005506  0.004627  0.000472
----------------------------------------------
                igd                                       
             median       min      max      mean       std
algorithm                                                 
pynsga3    0.007048  0.005298  0.00926  0.007187  0.001161
----------------------------------------------
In [15]:
_df = df[(df['algorithm']=='pynsga3-pbi') & (df['problem']=='dtlz3_15obj')]