boundml.evaluation

19def evaluate_solvers(solvers: [Solver], instances: Instances, n_instances, metrics, n_cpu=0):
20    if n_cpu == 0:
21        n_cpu = mp.cpu_count()
22
23    n_cpu = min(n_cpu, n_instances + 1)
24
25    data = np.zeros((n_instances, len(solvers), len(metrics)))
26
27    files = {}
28    async_results = {}
29
30    # Start the jobs
31    with mp.Pool(processes=n_cpu) as pool:
32        for i, instance in zip(range(n_instances), instances):
33            for j, solver in enumerate(solvers):
34                prob_file = tempfile.NamedTemporaryFile(suffix=".lp")
35                instance.writeProblem(prob_file.name, verbose=False)
36
37                files[i,j] = prob_file
38                async_results[i,j] = pool.apply_async(_solve, [solver, prob_file.name, metrics])
39
40        print(f"{'Instance':<15}" + "".join([f"{str(solver):<{15 * len(metrics)}}" for solver in solvers]))
41        print(f"{'':<15}" + len(solvers) * ("".join([f"{metric:<15}" for metric in metrics])))
42        for i, instance in zip(range(n_instances), instances):
43            print(f"{i:<15}", end="")
44            for j, solver in enumerate(solvers):
45                line = async_results[i,j].get()
46                files[i,j].close()
47                for k, d in enumerate(line):
48                    data[i, j, k] = d
49                print("".join([f"{d:{'<15.3f' if type(d) == float else '<15'}}" for d in line]), end="", flush=True)
50
51            print()
52
53    res = SolverEvaluationResults(data, [str(s) for s in solvers], metrics)
54
55    print("=" * (15 * (len(solvers) * len(metrics) + 1)))
56
57    ss = {
58        "nnodes": 10,
59        "time": 1,
60        "gap": 1,
61    }
62
63    means = {}
64    for k, metric in enumerate(metrics):
65        mean = res.aggregate(metrics[k], lambda values: shifted_geometric_mean(values, shift=ss[metric]))
66        means[metrics[k]] = mean
67
68    info = []
69    for j in range(len(solvers)):
70        for metric in metrics:
71            info.append(means[metric][j])
72    print(f"{'sg mean': <15}" + "".join([f"{val: <15.3f}" for val in info]))
73
74    return res

 11class SolverEvaluationResults:
 12    def __init__(self, raw_data: np.array, solvers: [str], metrics: [str]):
 13        self.data = raw_data
 14        self.solvers = solvers
 15        self.metrics = metrics
 16
 17    @property
 18    def metric_index(self) -> dict:
 19        """Returns a dictionary mapping metric names to their indices"""
 20        return {metric: idx for idx, metric in enumerate(self.metrics)}
 21
 22    def get_metric_data(self, metric: str) -> np.array:
 23        """Get all data for a specific metric"""
 24        return self.data[:, :, self.metric_index[metric]]
 25
 26    def aggregate(self, metric: str, aggregation_func: callable) -> np.array:
 27        """
 28        Apply aggregation function to a specific metric
 29        Args:
 30            metric: metric name to aggregate
 31            aggregation_func: function to apply (e.g., np.sum, np.mean)
 32        """
 33        return np.array([aggregation_func(self.get_metric_data(metric)[:, i]) for i in range(len(self.solvers))])
 34
 35    def split_instances_over(self, metric: str, condition):
 36        assert metric in self.metrics, "Cannot make a split on a non-existing metric"
 37
 38        index = self.metrics.index(metric)
 39        d = self.data[:, :, index]  # keep only the compared metrix
 40
 41        indexes = np.where(np.prod(np.apply_along_axis(condition, 1, d), axis=1))[0]
 42
 43        positives = self.data[indexes,]
 44        negatives = np.delete(self.data, indexes, axis=0)
 45        return SolverEvaluationResults(positives, self.solvers, self.metrics), SolverEvaluationResults(negatives,
 46                                                                                                       self.solvers,
 47                                                                                                       self.metrics)
 48
 49    def remove_solver(self, solver: str):
 50        index = self.solvers.index(solver)
 51        self.data = np.delete(self.data, index, axis=1)
 52        self.solvers.remove(solver)
 53
 54    def performance_profile(self, metric: str = "nnodes", ratios=np.arange(0, 1.00, .01), filename=None, plot=True, logx=True):
 55
 56        if filename:
 57            backend = matplotlib.get_backend()
 58            matplotlib.use('pgf')
 59
 60        metric_index = self.metrics.index(metric)
 61        n_instances = self.data.shape[0]
 62
 63        data = self.data[:, :, metric_index]
 64        min = np.min(data)
 65        max = np.max(data)
 66
 67        xs = ratios * (max - min) + min
 68
 69        res = []
 70        for s, solver in enumerate(self.solvers):
 71            ys = np.zeros(len(ratios))
 72            for i in range(n_instances):
 73                val = data[i, s]
 74                indexes = np.where(val <= xs)
 75                ys[indexes] += 1
 76
 77            ys /= n_instances
 78            label = solver
 79            if label == "relpscost":
 80                label = "RPB"
 81            elif "GNN" in label and "sb" in label:
 82                label = "$LSB$"
 83            elif "GNN" in label and "ub" in label:
 84                x = label.split("_")[4]
 85                label = f"$LLB^{x}$"
 86
 87            if logx:
 88                auc = np.trapezoid(ys, np.log(xs)) / np.log(max)
 89            else:
 90                auc = np.trapezoid(ys, xs) / max
 91
 92            res.append(auc)
 93            if plot:
 94                plt.plot(xs, ys, label=label)
 95
 96        if plot:
 97            plt.legend()
 98            plt.xlabel(metric)
 99            plt.ylabel("frequency")
100            plt.title(f"Performance profile w.r.t. {metric}")
101
102            if logx:
103                plt.xscale("log")
104
105            if filename:
106                plt.savefig(filename)
107                matplotlib.use(backend)
108
109            else:
110                plt.show()
111
112        return np.array(res)
113
114    def compute_report(self, *aggregations: tuple[str, callable], **kwargs):
115        data = {"solver": [s for s in self.solvers]}
116
117        for i, aggregation in enumerate(aggregations):
118            data[aggregation[0]] = list(aggregation[1](self))
119
120        return SolverEvaluationReport(data, **kwargs)
121
122    def __add__(self, other):
123        assert self.metrics == other.metrics, "Metrics must be the same when combining Results of different solvers"
124        assert self.data.shape
125        solvers = self.solvers + other.solvers
126        data = np.hstack((self.data, other.data))
127        return SolverEvaluationResults(data, solvers, self.metrics)
128
129    @staticmethod
130    def sg_metric(metric, s):
131        return (metric, lambda evaluationResults:
132        evaluationResults.aggregate(metric, lambda values: shifted_geometric_mean(values, shift=s))
133                )
134
135    @staticmethod
136    def nwins(metric, dir=1):
137        def get_wins(evaluationResults: SolverEvaluationResults):
138            data = evaluationResults.get_metric_data(metric)
139            res = []
140            for i in range(len(evaluationResults.solvers)):
141                c = 0
142                for j in range(len(data[:, i])):
143                    c += dir * data[j, i] <= dir * np.min(data[j, :])
144                res.append(c)
145            return np.array(res)
146
147        return f"wins ({metric})", get_wins
148
149    @staticmethod
150    def nsolved():
151        return ("nsolved", lambda evaluationResults: evaluationResults.aggregate("gap", lambda values: values.shape[
152                                                                                                           0] - np.count_nonzero(
153            values)))
154
155    @staticmethod
156    def auc_score(metric, **kwargs):
157        return ("AUC", lambda evaluationResults: evaluationResults.performance_profile(metric, plot=False, **kwargs))

159class SolverEvaluationReport:
160    def __init__(self, data=None, header=None, df_=None):
161        assert (data is None) != (df_ is None), "Only one of data and df_ must be given"
162
163        if df_ is not None:
164            self.df = df_
165            return
166
167        if header is not None:
168            data_ = {}
169            for key in data:
170                if key != "solver":
171                    data_[(header, key)] = data[key]
172                else:
173                    data_[("", key)] = data[key]
174
175        else:
176            data_ = data
177
178        self.df = pd.DataFrame(data_)
179        if header is not None:
180            self.df.set_index(("","solver"), inplace=True)
181
182    def __str__(self):
183        return tabulate(self.df, headers="keys", tablefmt='grid', showindex=False)
184
185    def to_latex(self, *args, **kwargs):
186        return self.df.to_latex(index=False, *args, **kwargs)
187
188    def __add__(self, other):
189        print(self.df.to_dict(orient='list'))
190        print(other.df.to_dict(orient='list'))
191        df2 = pd.concat(
192            [self.df, other.df],
193            axis=1
194        )
195
196        df2 = df2.reset_index().rename(columns={'index': ('', 'solver')})
197        return SolverEvaluationReport(df_ = df2)