001    package net.sf.cpsolver.ifs.example.csp;
002    
003    import java.io.File;
004    import java.io.FileInputStream;
005    import java.io.FileOutputStream;
006    import java.io.FileWriter;
007    import java.io.PrintWriter;
008    import java.util.Date;
009    import java.util.Iterator;
010    import java.util.Locale;
011    import java.util.StringTokenizer;
012    
013    import net.sf.cpsolver.ifs.model.Constraint;
014    import net.sf.cpsolver.ifs.solution.Solution;
015    import net.sf.cpsolver.ifs.solver.Solver;
016    import net.sf.cpsolver.ifs.util.DataProperties;
017    import net.sf.cpsolver.ifs.util.Progress;
018    import net.sf.cpsolver.ifs.util.ProgressWriter;
019    import net.sf.cpsolver.ifs.util.ToolBox;
020    
021    /**
022     * Test of Structured CSP problems. It takes one argument -- property file with
023     * all the parameters. It allows to execute given number of tests. It also
024     * allows to define several configurations which will be executed. For instance
025     * CSP(20,15,5%..95%,5..95%), 10 runs of each configuration. All such
026     * configuration are processed in one run of Test class. <br>
027     * <br>
028     * In Structured CSP, variables are divided into several kernels (some variables
029     * may remain ouside kernels). Different constraints (in density and tightnes)
030     * are generated according to whether variables are from the same kernel or not. <br>
031     * <br>
032     * Test's parameters: <br>
033     * <table border='1'>
034     * <tr>
035     * <th>Parameter</th>
036     * <th>Type</th>
037     * <th>Comment</th>
038     * </tr>
039     * <tr>
040     * <td>General.MPP</td>
041     * <td>{@link String}</td>
042     * <td>Minimal perturbation problem (if true), this mj. means that initial
043     * assignment will be generated</td>
044     * </tr>
045     * <tr>
046     * <td>CSP.Seed</td>
047     * <td>{@link Long}</td>
048     * <td>Random number generator seed, {@link System#currentTimeMillis()} is taken
049     * if not present</td>
050     * </tr>
051     * <tr>
052     * <td>CSP.ForceSolutionExistance</td>
053     * <td>{@link Boolean}</td>
054     * <td>If true, generated problem will always have at least one feasible
055     * solution</td>
056     * </tr>
057     * <tr>
058     * <td>CPS.NrTests</td>
059     * <td>{@link Integer}</td>
060     * <td>Number of tests (for each input configuration)</td>
061     * </tr>
062     * <tr>
063     * <td>CSP.NrVariables</td>
064     * <td>{@link Integer}</td>
065     * <td>Number of variables</td>
066     * </tr>
067     * <tr>
068     * <td>CSP.NrVariablesMin<br>
069     * CSP.NrVariablesMax<br>
070     * CSP.NrVariablesStep</td>
071     * <td>{@link Integer}</td>
072     * <td>Range of the number variables (a set of different configurations will be
073     * generated)<br>
074     * Use either CSP.NrVariables or these CSP.NrVariablesMin, CSP.NrVariablesMax,
075     * CSP.NrVariablesStep</td>
076     * </tr>
077     * <tr>
078     * <td>CSP.DomainSize</td>
079     * <td>{@link Integer}</td>
080     * <td>Number of values of every variable</td>
081     * </tr>
082     * <tr>
083     * <td>CSP.DomainSizeRatio</td>
084     * <td>{@link Double}</td>
085     * <td>Number of values as a ration of the number of variables. This way we can
086     * model for instance CSP(N,2N,p1,p2) problems with one configuration.<br>
087     * Use either CSP.DomainSize or CSP.DomainSizeRatio</td>
088     * </tr>
089     * <tr>
090     * <td>CSP.Tightness</td>
091     * <td>{@link Double}</td>
092     * <td>Tightness of constraints outside kernels</td>
093     * </tr>
094     * <tr>
095     * <td>CSP.TightnessMin<br>
096     * CSP.TightnessMax<br>
097     * CSP.TightnessStep</td>
098     * <td>{@link Double}</td>
099     * <td>Tightness of constraints outside kernels given as a range -> respective
100     * configurations will be generated and tested</td>
101     * </tr>
102     * <tr>
103     * <td>CSP.Density</td>
104     * <td>{@link Double}</td>
105     * <td>Density of constraints outside kernels</td>
106     * </tr>
107     * <tr>
108     * <td>CSP.DensityMin<br>
109     * CSP.DensityMax<br>
110     * CSP.DensityStep</td>
111     * <td>{@link Double}</td>
112     * <td>Density of constraints outside kernels given as a range -> respective
113     * configurations will be generated and tested</td>
114     * </tr>
115     * <tr>
116     * <td>CSP.NrKernels</td>
117     * <td>{@link Integer}</td>
118     * <td>Number of kernels (Structured CSP, use 0 for "normal" CSP)</td>
119     * </tr>
120     * <tr>
121     * <td>CSP.KernelSize</td>
122     * <td>{@link Integer}</td>
123     * <td>Number of variables in each kernel</td>
124     * </tr>
125     * <tr>
126     * <td>CSP.KernelTightness</td>
127     * <td>{@link Double}</td>
128     * <td>Tightness of constraints inside a kernel</td>
129     * </tr>
130     * <tr>
131     * <td>CSP.KernelDensity</td>
132     * <td>{@link Double}</td>
133     * <td>Density of constraints inside a kernel</td>
134     * </tr>
135     * <tr>
136     * <td>CSP.SameProblemEachStep</td>
137     * <td>{@link Boolean}</td>
138     * <td>If true, each configuration will start with the same seed</td>
139     * </tr>
140     * <tr>
141     * <td>CSP.SameProblemEachTest</td>
142     * <td>{@link Boolean}</td>
143     * <td>If true, each test of the same configuration will start with the same
144     * seed</td>
145     * </tr>
146     * <tr>
147     * <td>General.Output</td>
148     * <td>{@link String}</td>
149     * <td>Output folder where a log file and tables with results. In order not to
150     * overwrite the results if executed more than once, a subfolder with the name
151     * taken from current date and time will be created in this folder and all
152     * results will go to this subfolder.</td>
153     * </tr>
154     * </table>
155     * <br>
156     * <br>
157     * Also, the configuration file can consist only from one parameter (named
158     * INCLUDE_REGEXP) which is processed as a regular expression of semicolon
159     * separated list of property files, for instance
160     * <ul>
161     * <code>INCLUDE_REGEXP=general.ini;{CSP(50,12,250,p2)|CSP(25,15,198,p2)}.ini;{std|opt}.ini;{10x1min}.ini;{cbs|rw1|tabu20}.ini</code>
162     * <br>
163     * </ul>
164     * where {a|b|c|...} means a selection of a, b, c, .. All possible combinations
165     * are taken and for each of them an input configuration is combined from the
166     * relevant files. So, for instance, the above example will result into the
167     * following configurations:
168     * <ul>
169     * <li>general.ini;CSP(50,12,250,p2).ini;std.ini;10x1min.ini;cbs.ini
170     * <li>general.ini;CSP(50,12,250,p2).ini;std.ini;10x1min.ini;rw1.ini
171     * <li>general.ini;CSP(50,12,250,p2).ini;std.ini;10x1min.ini;tabu20.ini
172     * <li>general.ini;CSP(50,12,250,p2).ini;opt.ini;10x1min.ini;cbs.ini
173     * <li>general.ini;CSP(50,12,250,p2).ini;opt.ini;10x1min.ini;rw1.ini
174     * <li>general.ini;CSP(50,12,250,p2).ini;opt.ini;10x1min.ini;tabu20.ini
175     * <li>general.ini;CSP(25,15,198,p2).ini;std.ini;10x1min.ini;cbs.ini
176     * <li>general.ini;CSP(25,15,198,p2).ini;std.ini;10x1min.ini;rw1.ini
177     * <li>general.ini;CSP(25,15,198,p2).ini;std.ini;10x1min.ini;tabu20.ini
178     * <li>general.ini;CSP(25,15,198,p2).ini;opt.ini;10x1min.ini;cbs.ini
179     * <li>general.ini;CSP(25,15,198,p2).ini;opt.ini;10x1min.ini;rw1.ini
180     * <li>general.ini;CSP(25,15,198,p2).ini;opt.ini;10x1min.ini;tabu20.ini
181     * </ul>
182     * To be able to distinguish such configuration a subfolder in General.Output
183     * folder is created, its name is combined from the names which are in
184     * parenthesis. So, for instance the first bunch of tests will output into the
185     * folder:
186     * <ul>
187     * ${General.Output}\CSP(50,12,250,p2)_std_10x1min_csb\25-Feb-05_191136
188     * </ul>
189     * If one parameter is defined in more than one configuration files (e.g. in
190     * general.ini as well as cbs.ini) the one from the file more on the right is
191     * taken. <br>
192     * <br>
193     * An example of the configurations:<br>
194     * File<b> general.ini</b>
195     * <ul>
196     * <code>
197     * #Default settings common for all configurations<br>
198     * General.MPP=false<br>
199     * General.InitialAssignment=false<br>
200     * General.Output=output\\RandomCSP\\IFS<br>
201     * <br>
202     * #Value selection heuristics<br>
203     * Value.Class=net.sf.cpsolver.ifs.heuristics.GeneralValueSelection<br>
204     * Value.WeightWeightedConflicts=0.0<br>
205     * Value.RandomWalkProb=0.0<br>
206     * Value.WeightConflicts=1.0<br>
207     * Value.WeightNrAssignments=0.0<br>
208     * Value.WeightValue=0.0<br>
209     * Value.Tabu=0<br>
210     * <br>
211     * #Variable selection heuristics<br>
212     * Variable.Class=net.sf.cpsolver.ifs.heuristics.GeneralVariableSelection<br>
213     * Variable.RandomSelection=true<br>
214     * <br>
215     * #Termination condition<br>
216     * Termination.Class=net.sf.cpsolver.ifs.termination.GeneralTerminationCondition<br>
217     * Termination.MaxIters=-1<br>
218     * Termination.TimeOut=-1<br>
219     * Termination.StopWhenComplete=true<br>
220     * <br>
221     * #Solution comparator<br>
222     * Comparator.Class=net.sf.cpsolver.ifs.solution.GeneralSolutionComparator<br>
223     * </code>
224     * </ul>
225     * <br>
226     * File<b> CSP(50,12,250,p2).ini</b>
227     * <ul>
228     * <code>
229     * #Sparse problem CSP(50,12,250/1225,p2)<br>
230     * CSP.NrVariables=50<br>
231     * CSP.DomainSize=12<br>
232     * CSP.Density=0.2<br>
233     * CSP.TightnessMin=0.10<br>
234     * CSP.TightnessMax=0.95<br>
235     * CSP.TightnessStep=0.02<br>
236     * <br> 
237     * CSP.Seed=780921<br>
238     * <br>
239     * CSP.ForceSolutionExistance=false<br>
240     * CSP.SameProblemEachStep=false<br>
241     * CSP.SameProblemEachTest=false<br>
242     * <br>
243     * CSP.NrKernels=0<br>
244     * </code>
245     * </ul>
246     * <br>
247     * File<b> std.ini</b>
248     * <ul>
249     * <code>
250     * #Standard problem<br>
251     * CSP.ForceSolutionExistance=false<br>
252     * </code>
253     * </ul>
254     * <br>
255     * File<b> opt.ini</b>
256     * <ul>
257     * <code>
258     * #Optimization problem (minCSP)<br>
259     * #Value selection: use weigh of a conflict, but when there are more than one value<br>
260     * #        with the same number of conflicts, use the one with lower value<br>
261     * Value.WeightValue=0.0001<br>
262     * Value.WeightConflicts=1.0<br>
263     * #Do not stop when a complete solution is found<br>
264     * Termination.StopWhenComplete=false<br>
265     * </code>
266     * </ul>
267     * <br>
268     * File<b> 10x1min.ini</b>
269     * <ul>
270     * <code>
271     * #For each configuration, execute 10 tests, each with 1 minute timeout<br>
272     * CPS.NrTests=10<br>
273     * Termination.TimeOut=60<br>
274     * </code>
275     * </ul>
276     * <br>
277     * File<b> cbs.ini</b>
278     * <ul>
279     * <code>
280     * #Use conflict-based statistics<br>
281     * Extensions.Classes=net.sf.cpsolver.ifs.extension.ConflictStatistics<br>
282     * Value.WeightWeightedConflicts=1.0<br>
283     * </code>
284     * </ul>
285     * <br>
286     * File<b> tabu20.ini</b>
287     * <ul>
288     * <code>
289     * #Use tabu-list of the length 20<br>
290     * Value.Tabu=20<br>
291     * </code>
292     * </ul>
293     * <br>
294     * File<b> rw1.ini</b>
295     * <ul>
296     * <code>
297     * #Use 1% random walk selection<br>
298     * Value.RandomWalkProb=0.01<br>
299     * </code>
300     * </ul>
301     * <br>
302     * 
303     * @see StructuredCSPModel
304     * @see net.sf.cpsolver.ifs.extension.ConflictStatistics
305     * @see net.sf.cpsolver.ifs.heuristics.GeneralValueSelection
306     * @see net.sf.cpsolver.ifs.heuristics.GeneralVariableSelection
307     * @see net.sf.cpsolver.ifs.termination.GeneralTerminationCondition
308     * @see net.sf.cpsolver.ifs.solution.GeneralSolutionComparator
309     * 
310     * @version IFS 1.2 (Iterative Forward Search)<br>
311     *          Copyright (C) 2006 - 2010 Tomas Muller<br>
312     *          <a href="mailto:muller@unitime.org">muller@unitime.org</a><br>
313     *          <a href="http://muller.unitime.org">http://muller.unitime.org</a><br>
314     * <br>
315     *          This library is free software; you can redistribute it and/or modify
316     *          it under the terms of the GNU Lesser General Public License as
317     *          published by the Free Software Foundation; either version 3 of the
318     *          License, or (at your option) any later version. <br>
319     * <br>
320     *          This library is distributed in the hope that it will be useful, but
321     *          WITHOUT ANY WARRANTY; without even the implied warranty of
322     *          MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
323     *          Lesser General Public License for more details. <br>
324     * <br>
325     *          You should have received a copy of the GNU Lesser General Public
326     *          License along with this library; if not see
327     *          <a href='http://www.gnu.org/licenses/'>http://www.gnu.org/licenses/</a>.
328     */
329    public class Test {
330        private static java.text.DecimalFormat sDoubleFormat = new java.text.DecimalFormat("0.000",
331                new java.text.DecimalFormatSymbols(Locale.US));
332        private static java.text.SimpleDateFormat sDateFormat = new java.text.SimpleDateFormat("dd-MMM-yy_HHmmss",
333                java.util.Locale.US);
334        private static org.apache.log4j.Logger sLogger = org.apache.log4j.Logger.getLogger(Test.class);
335    
336        private static void test(DataProperties properties) throws Exception {
337            boolean sameProblemStep = properties.getPropertyBoolean("CSP.SameProblemEachStep", false);
338            boolean sameProblemTest = properties.getPropertyBoolean("CSP.SameProblemEachTest", false);
339            int nrVars = properties.getPropertyInt("CSP.NrVariables", 20);
340            int nrKernels = properties.getPropertyInt("CSP.NrKernels", 2);
341            int nrKernelVariables = properties.getPropertyInt("CSP.KernelSize", 8);
342            int nrVariablesMin = properties.getPropertyInt("CSP.NrVariablesMin", nrVars);
343            int nrVariablesMax = properties.getPropertyInt("CSP.NrVariablesMax", nrVars);
344            int nrVariablesStep = properties.getPropertyInt("CSP.NrVariablesStep", 1);
345            int nrValues = properties.getPropertyInt("CSP.DomainSize", 10);
346            double nrValuesRatio = properties.getPropertyDouble("CSP.DomainSizeRatio", -1);
347            float kernelTightness = properties.getPropertyFloat("CSP.KernelTightness", 0.097f);
348            float kernelDensity = properties.getPropertyFloat("CSP.KernelDensity", 0.097f);
349            float tightnessInit = properties.getPropertyFloat("CSP.Tightness", 0.4f);
350            float tightnessMin = properties.getPropertyFloat("CSP.TightnessMin", tightnessInit);
351            float tightnessMax = properties.getPropertyFloat("CSP.TightnessMax", tightnessInit) + 1e-6f;
352            float tightnessStep = properties.getPropertyFloat("CSP.TightnessStep", 0.1f);
353            float densityInit = properties.getPropertyFloat("CSP.Density", 0.4f);
354            float densityMin = properties.getPropertyFloat("CSP.DensityMin", densityInit);
355            float densityMax = properties.getPropertyFloat("CSP.DensityMax", densityInit) + 1e-6f;
356            float densityStep = properties.getPropertyFloat("CSP.DensityStep", 0.1f);
357            long seed = properties.getPropertyLong("CSP.Seed", System.currentTimeMillis());
358            int nrTests = properties.getPropertyInt("CPS.NrTests", 10);
359            boolean mpp = properties.getPropertyBoolean("General.MPP", false);
360            PrintWriter logStat = new PrintWriter(new FileWriter(properties.getProperty("General.Output") + File.separator
361                    + "rcsp_" + nrVariablesMin + "_" + nrValues + ".csv"));
362            PrintWriter logAvgStat = new PrintWriter(new FileWriter(properties.getProperty("General.Output")
363                    + File.separator + "avg_stat.csv"));
364            PrintWriter log = new PrintWriter(new FileWriter(properties.getProperty("General.Output") + File.separator
365                    + "info.txt"));
366            logStat
367                    .println("testNr;nrVars;nrVals;density[%];tightness[%];time[s];iters;speed[it/s];unassConstr;assigned;assigned[%]"
368                            + (mpp ? ";perts;perts[%]" : "") + ";value;totalValue");
369            logAvgStat
370                    .println("nrVars;nrVals;density[%];tightness[%];time[s];RMStime[s];iters;RMSiters;speed[it/s];unassConst;assigned;RMSassigned;assigned[%]"
371                            + (mpp ? ";perts;RMSperts;perts[%]" : "") + ";value;RMSvalue;totalValue;RMStotalValue");
372            System.out.println("Number of variables: " + nrVariablesMin + " .. " + nrVariablesMax + "  (step="
373                    + nrVariablesStep + ")");
374            System.out.println("Density:             " + densityMin + " .. " + densityMax + "  (step=" + densityStep + ")");
375            System.out.println("Tightness:           " + tightnessMin + " .. " + tightnessMax + "  (step=" + tightnessStep
376                    + ")");
377            for (int nrVariables = nrVariablesMin; nrVariables <= nrVariablesMax; nrVariables += nrVariablesStep) {
378                if (nrValuesRatio > 0.0)
379                    nrValues = (int) Math.round(nrValuesRatio * nrVariables);
380                for (float density = densityMin; density <= densityMax; density += densityStep) {
381                    for (float tightness = tightnessMin; tightness <= tightnessMax; tightness += tightnessStep) {
382                        log.println("CSP{#Var=" + nrVariables + ", #Val=" + nrValues + ", P(density)="
383                                + sDoubleFormat.format(100.0 * density) + "%, P(tighness)="
384                                + sDoubleFormat.format(100.0 * tightness) + ", " + nrKernels + "x Kernel{#Var="
385                                + nrKernelVariables + ", P(density)=" + sDoubleFormat.format(100.0 * kernelDensity)
386                                + "%, P(tighness)=" + sDoubleFormat.format(100.0 * kernelTightness) + "%}}");
387                        double sumTime = 0;
388                        double sumTime2 = 0;
389                        int sumIters = 0;
390                        int sumIters2 = 0;
391                        int sumConfl = 0;
392                        int sumAssign = 0;
393                        int sumAssign2 = 0;
394                        int sumPert = 0;
395                        int sumPert2 = 0;
396                        int sumVal = 0;
397                        int sumVal2 = 0;
398                        int sumTotalVal = 0;
399                        int sumTotalVal2 = 0;
400                        for (int test = 1; test <= nrTests; test++) {
401                            log.println("  " + test + ". test");
402                            log.flush();
403                            properties.setProperty("CSP.NrVariables", String.valueOf(nrVariables));
404                            properties.setProperty("CSP.Tightness", String.valueOf(tightness));
405                            properties.setProperty("CSP.Density", String.valueOf(density));
406    
407                            long currentSeed = (seed * 1000000L)
408                                    + (1000 * (long) ((sameProblemStep ? densityMin : density) * 1000.0))
409                                    + ((long) ((sameProblemStep ? tightnessMin : tightness) * 1000.0));
410                            currentSeed = (currentSeed * nrTests) + (sameProblemTest ? 0 : test - 1);
411    
412                            sLogger.debug("Seed: " + currentSeed);
413                            StructuredCSPModel csp = new StructuredCSPModel(properties, currentSeed);
414    
415                            Solver<CSPVariable, CSPValue> s = new Solver<CSPVariable, CSPValue>(properties);
416                            s.setInitalSolution(csp);
417                            s.currentSolution().clearBest();
418                            s.start();
419    
420                            try {
421                                s.getSolverThread().join();
422                            } catch (NullPointerException npe) {
423                            }
424    
425                            if (s.lastSolution().getBestInfo() == null)
426                                sLogger.error("No solution found :-(");
427                            sLogger.debug("Last solution:" + s.lastSolution().getInfo());
428                            Solution<CSPVariable, CSPValue> best = s.lastSolution();
429                            sLogger.debug("Best solution:" + s.lastSolution().getBestInfo());
430                            best.restoreBest();
431                            int val = 0;
432                            for (Iterator<CSPVariable> iv = best.getModel().assignedVariables().iterator(); iv.hasNext();)
433                                val += (int) iv.next().getAssignment().toDouble();
434                            int totalVal = val + (best.getModel().unassignedVariables().size() * nrValues);
435                            sLogger.debug("Last solution:" + best.getInfo());
436                            logStat.println(test
437                                    + ";"
438                                    + nrVariables
439                                    + ";"
440                                    + nrValues
441                                    + ";"
442                                    + sDoubleFormat.format(density)
443                                    + ";"
444                                    + sDoubleFormat.format(tightness)
445                                    + ";"
446                                    + sDoubleFormat.format(best.getTime())
447                                    + ";"
448                                    + best.getIteration()
449                                    + ";"
450                                    + sDoubleFormat.format((best.getIteration()) / best.getTime())
451                                    + ";"
452                                    + best.getModel().unassignedHardConstraints().size()
453                                    + ";"
454                                    + best.getModel().assignedVariables().size()
455                                    + ";"
456                                    + sDoubleFormat.format(100.0 * best.getModel().assignedVariables().size()
457                                            / best.getModel().variables().size())
458                                    + (mpp ? ";"
459                                            + (best.getModel().perturbVariables().size() + best.getModel()
460                                                    .unassignedVariables().size())
461                                            + ";"
462                                            + sDoubleFormat.format(100.0
463                                                    * (best.getModel().perturbVariables().size() + best.getModel()
464                                                            .unassignedVariables().size())
465                                                    / best.getModel().variables().size()) : "") + ";" + val + ";"
466                                    + totalVal);
467                            log.println("    seed:         " + currentSeed);
468                            log.println("    constraints:  " + best.getModel().constraints().size());
469                            for (Iterator<Constraint<CSPVariable, CSPValue>> i = best.getModel().constraints().iterator(); i
470                                    .hasNext();) {
471                                CSPBinaryConstraint c = (CSPBinaryConstraint) i.next();
472                                log.println("      " + c.getName() + " (" + c.first().getName() + ","
473                                        + c.second().getName() + ")");
474                                for (CSPValue v0 : c.first().values()) {
475                                    log.print("        ");
476                                    for (CSPValue v1 : c.second().values())
477                                        log.print(c.isConsistent(v0, v1) ? "1 " : "0 ");
478                                }
479                                log.println();
480                            }
481                            log.println("    time:         " + sDoubleFormat.format(best.getTime()) + " s");
482                            log.println("    iteration:    " + best.getIteration());
483                            log.println("    speed:        " + sDoubleFormat.format((best.getIteration()) / best.getTime())
484                                    + " it/s");
485                            log.println("    assigned:     "
486                                    + best.getModel().assignedVariables().size()
487                                    + " ("
488                                    + sDoubleFormat.format(100.0 * best.getModel().assignedVariables().size()
489                                            / best.getModel().variables().size()) + "%)");
490                            log.println("    total value:  " + val);
491                            if (mpp)
492                                log.println("    perturbations:"
493                                        + (best.getModel().perturbVariables().size() + best.getModel()
494                                                .unassignedVariables().size())
495                                        + " ("
496                                        + sDoubleFormat
497                                                .format(100.0
498                                                        * (best.getModel().perturbVariables().size() + best.getModel()
499                                                                .unassignedVariables().size())
500                                                        / best.getModel().variables().size()) + "%)");
501                            log.print("    solution:     ");
502                            for (CSPVariable v : ((CSPModel) best.getModel()).variables()) {
503                                if (v.getBestAssignment() == null)
504                                    continue;
505                                log.print(v.getName() + "=" + v.getBestAssignment().getName());
506                                log.print(", ");
507                            }
508                            log.println();
509                            sumTime += best.getTime();
510                            sumTime2 += best.getTime() * best.getTime();
511                            sumIters += best.getIteration();
512                            sumIters2 += best.getIteration() * best.getIteration();
513                            sumConfl += best.getModel().unassignedHardConstraints().size();
514                            sumAssign += best.getModel().assignedVariables().size();
515                            sumAssign2 += best.getModel().assignedVariables().size()
516                                    * best.getModel().assignedVariables().size();
517                            sumVal += val;
518                            sumVal2 += val * val;
519                            sumTotalVal += totalVal;
520                            sumTotalVal2 += totalVal * totalVal;
521                            if (mpp) {
522                                sumPert += (best.getModel().perturbVariables().size() + best.getModel()
523                                        .unassignedVariables().size());
524                                sumPert2 += (best.getModel().perturbVariables().size() + best.getModel()
525                                        .unassignedVariables().size())
526                                        * (best.getModel().perturbVariables().size() + best.getModel()
527                                                .unassignedVariables().size());
528                            }
529                            log.flush();
530                            logStat.flush();
531                        }
532                        logAvgStat.println(nrVariables
533                                + ";"
534                                + nrValues
535                                + ";"
536                                + sDoubleFormat.format(density)
537                                + ";"
538                                + sDoubleFormat.format(tightness)
539                                + ";"
540                                + sDoubleFormat.format(sumTime / nrTests)
541                                + ";"
542                                + sDoubleFormat.format(ToolBox.rms(nrTests, sumTime, sumTime2))
543                                + ";"
544                                + sDoubleFormat.format(((double) sumIters) / nrTests)
545                                + ";"
546                                + sDoubleFormat.format(ToolBox.rms(nrTests, sumIters, sumIters2))
547                                + ";"
548                                + sDoubleFormat.format((sumIters) / sumTime)
549                                + ";"
550                                + sDoubleFormat.format(((double) sumConfl) / nrTests)
551                                + ";"
552                                + sDoubleFormat.format(((double) sumAssign) / nrTests)
553                                + ";"
554                                + sDoubleFormat.format(ToolBox.rms(nrTests, sumAssign, sumAssign2))
555                                + ";"
556                                + sDoubleFormat.format(100.0 * (sumAssign) / (nrVariables * nrTests))
557                                + (mpp ? ";" + sDoubleFormat.format(((double) sumPert) / nrTests) + ";"
558                                        + sDoubleFormat.format(ToolBox.rms(nrTests, sumPert, sumPert2)) + ";"
559                                        + sDoubleFormat.format(100.0 * (sumPert) / (nrVariables * nrTests)) : "")
560                                + ";"
561                                + sDoubleFormat.format(((double) sumVal) / (nrTests * nrVariables))
562                                + ";"
563                                + sDoubleFormat.format(ToolBox.rms(nrTests, (double) sumVal / nrVariables, (double) sumVal2
564                                        / (nrVariables * nrVariables))) + ";"
565                                + sDoubleFormat.format(((double) sumTotalVal) / nrTests) + ";"
566                                + sDoubleFormat.format(ToolBox.rms(nrTests, sumTotalVal, sumTotalVal2)));
567                        logAvgStat.flush();
568                    }
569                }
570            }
571            log.flush();
572            log.close();
573            logStat.flush();
574            logStat.close();
575            logAvgStat.flush();
576            logAvgStat.close();
577        }
578    
579        private static void test(File inputCfg, String name, String include, String regexp, String outDir) throws Exception {
580            if (regexp != null) {
581                String incFile;
582    
583                if (regexp.indexOf(';') > 0) {
584                    incFile = regexp.substring(0, regexp.indexOf(';'));
585                    regexp = regexp.substring(regexp.indexOf(';') + 1);
586                } else {
587                    incFile = regexp;
588                    regexp = null;
589                }
590                if (incFile.startsWith("[") && incFile.endsWith("]")) {
591                    test(inputCfg, name, include, regexp, outDir);
592                    incFile = incFile.substring(1, incFile.length() - 1);
593                }
594                if (incFile.indexOf('{') >= 0 && incFile.indexOf('}') >= 0) {
595                    String prefix = incFile.substring(0, incFile.indexOf('{'));
596                    StringTokenizer middle = new StringTokenizer(incFile.substring(incFile.indexOf('{') + 1, incFile
597                            .indexOf('}')), "|");
598                    String sufix = incFile.substring(incFile.indexOf('}') + 1);
599    
600                    while (middle.hasMoreTokens()) {
601                        String m = middle.nextToken();
602    
603                        test(inputCfg, (name == null ? "" : name + "_") + m, (include == null ? "" : include + ";")
604                                + prefix + m + sufix, regexp, outDir);
605                    }
606                } else {
607                    test(inputCfg, name, (include == null ? "" : include + ";") + incFile, regexp, outDir);
608                }
609            } else {
610                DataProperties properties = ToolBox.loadProperties(inputCfg);
611                StringTokenizer inc = new StringTokenizer(include, ";");
612    
613                while (inc.hasMoreTokens()) {
614                    String aFile = inc.nextToken();
615    
616                    System.out.println("  Loading included file '" + aFile + "' ... ");
617                    FileInputStream is = null;
618    
619                    if ((new File(aFile)).exists()) {
620                        is = new FileInputStream(aFile);
621                    }
622                    if ((new File(inputCfg.getParent() + File.separator + aFile)).exists()) {
623                        is = new FileInputStream(inputCfg.getParent() + File.separator + aFile);
624                    }
625                    if (is == null) {
626                        System.err.println("Unable to find include file '" + aFile + "'.");
627                    }
628                    properties.load(is);
629                    is.close();
630                }
631                String outDirThisTest = (outDir == null ? properties.getProperty("General.Output", ".") : outDir)
632                        + File.separator + name + File.separator + sDateFormat.format(new Date());
633                properties.setProperty("General.Output", outDirThisTest.toString());
634                System.out.println("Output folder: " + properties.getProperty("General.Output"));
635                (new File(outDirThisTest)).mkdirs();
636                ToolBox.configureLogging(outDirThisTest, null);
637                FileOutputStream fos = new FileOutputStream(outDirThisTest + File.separator + "rcsp.conf");
638    
639                properties.store(fos, "Random CSP problem configuration file");
640                fos.flush();
641                fos.close();
642                test(properties);
643            }
644        }
645    
646        public static void main(String[] args) {
647            try {
648                Progress.getInstance().addProgressListener(new ProgressWriter(System.out));
649                File inputCfg = new File(args[0]);
650                DataProperties properties = ToolBox.loadProperties(inputCfg);
651                if (properties.getProperty("INCLUDE_REGEXP") != null) {
652                    test(inputCfg, null, null, properties.getProperty("INCLUDE_REGEXP"), (args.length > 1 ? args[1] : null));
653                } else {
654                    String outDir = properties.getProperty("General.Output", ".") + File.separator
655                            + inputCfg.getName().substring(0, inputCfg.getName().lastIndexOf('.')) + File.separator
656                            + sDateFormat.format(new Date());
657                    if (args.length > 1)
658                        outDir = args[1] + File.separator + (sDateFormat.format(new Date()));
659                    properties.setProperty("General.Output", outDir.toString());
660                    System.out.println("Output folder: " + properties.getProperty("General.Output"));
661                    (new File(outDir)).mkdirs();
662                    ToolBox.configureLogging(outDir, null);
663                    test(properties);
664                }
665            } catch (Exception e) {
666                e.printStackTrace();
667            }
668        }
669    }