boundml.evaluation

21def evaluate_solvers(solvers: [Solver], instances: Instances, n_instances, metrics, n_cpu=0):
22    if n_cpu == 0:
23        n_cpu = mp.cpu_count()
24
25    n_cpu = min(n_cpu, n_instances + 1)
26
27    data = np.zeros((n_instances, len(solvers), len(metrics)))
28
29    files = {}
30    async_results = {}
31
32    print(f"{'Instance':<15}" + "".join([f"{str(solver):<{15 * len(metrics)}}" for solver in solvers]))
33    print(f"{'':<15}" + len(solvers) * ("".join([f"{metric:<15}" for metric in metrics])))
34
35    # Start the jobs
36    if n_cpu > 1:
37        with mp.Pool(processes=n_cpu, maxtasksperchild=1) as pool:
38            for i, instance in zip(range(n_instances), instances):
39                for j, solver in enumerate(solvers):
40                    prob_file = tempfile.NamedTemporaryFile(suffix=".lp")
41                    instance.writeProblem(prob_file.name, verbose=False)
42
43                    files[i,j] = prob_file
44                    async_results[i,j] = pool.apply_async(_solve, [solver, prob_file.name, metrics])
45
46            for i, instance in zip(range(n_instances), instances):
47                print(f"{i:<15}", end="")
48                for j, solver in enumerate(solvers):
49                    line = async_results[i,j].get()
50                    files[i,j].close()
51                    for k, d in enumerate(line):
52                        data[i, j, k] = d
53                    _print_result(line)
54
55                print()
56    else:
57        for i, instance in zip(range(n_instances), instances):
58            print(f"{i:<15}", end="")
59            for j, solver in enumerate(solvers):
60                prob_file = tempfile.NamedTemporaryFile(suffix=".lp")
61                instance.writeProblem(prob_file.name, verbose=False)
62
63                files[i, j] = prob_file
64                line = _solve(solver, prob_file.name, metrics)
65                for k, d in enumerate(line):
66                    data[i, j, k] = d
67                _print_result(line)
68            print()
69
70    res = SolverEvaluationResults(data, [str(s) for s in solvers], metrics)
71
72    print("=" * (15 * (len(solvers) * len(metrics) + 1)))
73
74    ss = {
75        "nnodes": 10,
76        "time": 1,
77        "gap": 1,
78    }
79
80    means = {}
81    for k, metric in enumerate(metrics):
82        s = ss[metric] if metric in ss else 1
83        mean = res.aggregate(metrics[k], lambda values: shifted_geometric_mean(values, shift=s))
84        means[metrics[k]] = mean
85
86    info = []
87    for j in range(len(solvers)):
88        for metric in metrics:
89            info.append(means[metric][j])
90    print(f"{'sg mean': <15}" + "".join([f"{val: <15.3f}" for val in info]))
91
92    return res

 11class SolverEvaluationResults:
 12    def __init__(self, raw_data: np.array, solvers: [str], metrics: [str]):
 13        self.data = raw_data
 14        self.solvers = solvers
 15        self.metrics = metrics
 16
 17    @property
 18    def metric_index(self) -> dict:
 19        """Returns a dictionary mapping metric names to their indices"""
 20        return {metric: idx for idx, metric in enumerate(self.metrics)}
 21
 22    def get_metric_data(self, metric: str) -> np.array:
 23        """Get all data for a specific metric"""
 24        return self.data[:, :, self.metric_index[metric]]
 25
 26    def aggregate(self, metric: str, aggregation_func: callable) -> np.array:
 27        """
 28        Apply aggregation function to a specific metric
 29        Args:
 30            metric: metric name to aggregate
 31            aggregation_func: function to apply (e.g., np.sum, np.mean)
 32        """
 33        return np.array([aggregation_func(self.get_metric_data(metric)[:, i]) for i in range(len(self.solvers))])
 34
 35    def split_instances_over(self, metric: str, condition):
 36        assert metric in self.metrics, "Cannot make a split on a non-existing metric"
 37
 38        index = self.metrics.index(metric)
 39        d = self.data[:, :, index]  # keep only the compared metrix
 40
 41        indexes = np.where(np.prod(np.apply_along_axis(condition, 1, d), axis=1))[0]
 42
 43        positives = self.data[indexes,]
 44        negatives = np.delete(self.data, indexes, axis=0)
 45        return SolverEvaluationResults(positives, self.solvers, self.metrics), SolverEvaluationResults(negatives,
 46                                                                                                       self.solvers,
 47                                                                                                       self.metrics)
 48
 49    def remove_solver(self, solver: str):
 50        index = self.solvers.index(solver)
 51        self.data = np.delete(self.data, index, axis=1)
 52        self.solvers.remove(solver)
 53
 54    def performance_profile(self, metric: str = "nnodes", ratios=np.arange(0, 1.00, .01), filename=None, plot=True, logx=True):
 55
 56        if filename:
 57            backend = matplotlib.get_backend()
 58            matplotlib.use('pgf')
 59
 60        metric_index = self.metrics.index(metric)
 61        n_instances = self.data.shape[0]
 62
 63        data = self.data[:, :, metric_index]
 64        min = np.min(data)
 65        max = np.max(data)
 66
 67        xs = ratios * (max - min) + min
 68
 69        res = []
 70        for s, solver in enumerate(self.solvers):
 71            ys = np.zeros(len(ratios))
 72            for i in range(n_instances):
 73                val = data[i, s]
 74                indexes = np.where(val <= xs)
 75                ys[indexes] += 1
 76
 77            ys /= n_instances
 78            label = solver
 79
 80            if logx:
 81                auc = np.trapezoid(ys, np.log(xs)) / np.log(max)
 82            else:
 83                auc = np.trapezoid(ys, xs) / max
 84
 85            res.append(auc)
 86            if plot:
 87                plt.plot(xs, ys, label=label)
 88
 89        if plot:
 90            plt.legend()
 91            plt.xlabel(metric)
 92            plt.ylabel("frequency")
 93            plt.title(f"Performance profile w.r.t. {metric}")
 94
 95            if logx:
 96                plt.xscale("log")
 97
 98            if filename:
 99                plt.savefig(filename)
100                matplotlib.use(backend)
101
102            else:
103                plt.show()
104
105        return np.array(res)
106
107    def compute_report(self, *aggregations: tuple[str, callable], **kwargs):
108        data = {"solver": [s for s in self.solvers]}
109
110        for i, aggregation in enumerate(aggregations):
111            data[aggregation[0]] = list(aggregation[1](self))
112
113        return SolverEvaluationReport(data, **kwargs)
114
115    def __add__(self, other):
116        assert self.metrics == other.metrics, "Metrics must be the same when combining Results of different solvers"
117        assert self.data.shape
118        solvers = self.solvers + other.solvers
119        data = np.hstack((self.data, other.data))
120        return SolverEvaluationResults(data, solvers, self.metrics)
121
122    @staticmethod
123    def sg_metric(metric, s):
124        return (metric, lambda evaluationResults:
125        evaluationResults.aggregate(metric, lambda values: shifted_geometric_mean(values, shift=s))
126                )
127
128    @staticmethod
129    def nwins(metric, dir=1):
130        def get_wins(evaluationResults: SolverEvaluationResults):
131            data = evaluationResults.get_metric_data(metric)
132            gaps = evaluationResults.get_metric_data("gap")
133            res = []
134            for i in range(len(evaluationResults.solvers)):
135                c = 0
136                for j in range(len(data[:, i])):
137                    # Does not count as a win if the instance was not solved optimally.
138                    if gaps[j, i] == 0 or metric == "gap":
139                        c += dir * data[j, i] <= dir * np.min(data[j, :])
140                res.append(c)
141            return np.array(res)
142
143        return f"wins ({metric})", get_wins
144
145    @staticmethod
146    def nsolved():
147        return ("nsolved", lambda evaluationResults: evaluationResults.aggregate("gap", lambda values: values.shape[
148                                                                                                           0] - np.count_nonzero(
149            values)))
150
151    @staticmethod
152    def auc_score(metric, **kwargs):
153        return ("AUC", lambda evaluationResults: evaluationResults.performance_profile(metric, plot=False, **kwargs))

155class SolverEvaluationReport:
156    def __init__(self, data=None, header=None, df_=None):
157        assert (data is None) != (df_ is None), "Only one of data and df_ must be given"
158
159        if df_ is not None:
160            self.df = df_
161            return
162
163        if header is not None:
164            data_ = {}
165            for key in data:
166                if key != "solver":
167                    data_[(header, key)] = data[key]
168                else:
169                    data_[("", key)] = data[key]
170
171        else:
172            data_ = data
173
174        self.df = pd.DataFrame(data_)
175        if header is not None:
176            self.df.set_index(("","solver"), inplace=True)
177
178    def __str__(self):
179        return tabulate(self.df, headers="keys", tablefmt='grid', showindex=False)
180
181    def to_latex(self, *args, **kwargs):
182        return self.df.to_latex(index=False, *args, **kwargs)
183
184    def __add__(self, other):
185        print(self.df.to_dict(orient='list'))
186        print(other.df.to_dict(orient='list'))
187        df2 = pd.concat(
188            [self.df, other.df],
189            axis=1
190        )
191
192        df2 = df2.reset_index().rename(columns={'index': ('', 'solver')})
193        return SolverEvaluationReport(df_ = df2)