diff --git a/NeatSim/src/main/java/neatsim/Evaluator.java b/NeatSim/src/main/java/neatsim/Evaluator.java index 0f6dd3b..8d33aa8 100644 --- a/NeatSim/src/main/java/neatsim/Evaluator.java +++ b/NeatSim/src/main/java/neatsim/Evaluator.java @@ -1,10 +1,15 @@ package neatsim; +import static com.google.common.collect.Maps.newLinkedHashMap; + import java.io.BufferedWriter; +import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; import neatsim.core.blackbox.BlackBox; import neatsim.core.evaluators.gendreau.GendreauEvaluator; @@ -13,6 +18,8 @@ import rinde.evo4mas.common.ResultDTO; import rinde.evo4mas.gendreau06.GSimulationTask.SolutionType; import rinde.sim.problem.common.StatsTracker.StatisticsDTO; + +import com.google.common.base.Joiner; //import rinde.evo4mas.gendreau06.GSimulationTask.SolutionType; public class Evaluator { @@ -43,9 +50,20 @@ private void evaluateGendreau() throws IOException { return; final BufferedWriter writer = new BufferedWriter(new FileWriter(config.getOutputFile())); + + writer.append(Joiner.on(",").join("genome_id", "scenario_id", "travel_time","tardiness","over_time","total_cost")); + writer.newLine(); + if (gconfig.getBatch()) { // batch mode System.out.println("Batch mode"); System.out.println("Using data provider: " + gconfig.isUseDataprovider()); + + final Map idBbMap = newLinkedHashMap(); + for( final BlackBox bb : config.getGenomes()){ + idBbMap.put(bb.getId(),bb); + } + + final ResultDTO[][] results = evaluator.evaluateGenomes( config.getGenomes(), gconfig.getNumberOfScenariosInNonfinalGenerations(), @@ -54,9 +72,17 @@ private void evaluateGendreau() throws IOException { System.out.println("Results received, writing file"); for (int m=0; m < results.length; m++) { for (int n=0; n < results[m].length; n++) { - writer.append(resultDTOToString(results[m][n])); - if (n + 1 < results[m].length) - writer.append(","); + final ResultDTO cur = results[m][n]; + + final String fileName =config.getGenomeMap().get(idBbMap.get(cur.taskDataId)); + final String fileId = new File(fileName).getName().replaceAll(" ", "").split("run")[1].split("\\.")[0]; + writer.append(fileId+","); + + writer.append(resultDTOToString(cur)); + if (n + 1 < results[m].length){ + //writer.append(","); + writer.newLine(); + } } if (m + 1 < results.length) writer.newLine(); @@ -67,20 +93,23 @@ private void evaluateGendreau() throws IOException { System.out.println("Using data provider: " + gconfig.isUseDataprovider()); // We do not evaluate all genomes before writing; just in case something goes wrong... final ArrayList genomeList = new ArrayList<>(); - final Iterator it = config.getGenomes().iterator(); + final Iterator> it = config.getGenomeMap().entrySet().iterator(); while (it.hasNext()) { - final BlackBox genome = it.next(); - genomeList.clear(); genomeList.add(genome); + final Entry genomeEntry = it.next(); + genomeList.clear(); + genomeList.add(genomeEntry.getKey()); final ResultDTO[][] results = evaluator.evaluateGenomes( genomeList, gconfig.getNumberOfScenariosInNonfinalGenerations(), gconfig.getNumberOfScenariosInFinalGeneration(), gconfig.isUseDataprovider()); assert results.length == 1; + writer.append(genomeEntry.getValue()) + .append(","); for (int i=0; i < results[0].length; i++) { writer.append(resultDTOToString(results[0][i])); - if (i + 1 < results[0].length) - writer.append(","); +// if (i + 1 < results[0].length) +// writer.append(","); } if (it.hasNext()) writer.newLine(); @@ -94,11 +123,14 @@ private void evaluateGendreau() throws IOException { private final Gendreau06ObjectiveFunction obj = new Gendreau06ObjectiveFunction(); private String resultDTOToString(final ResultDTO results) { + //final StatisticsDTO stats = results.s; final StatisticsDTO stats = results.stats; - return obj.travelTime(stats) + ";" - + obj.tardiness(stats) + ";" - + obj.overTime(stats) + ";" - + obj.computeCost(stats); + return Joiner.on(",").join( + results.scenarioKey, + obj.travelTime(stats), + obj.tardiness(stats), + obj.overTime(stats), + obj.computeCost(stats)); } } diff --git a/NeatSim/src/main/java/neatsim/EvaluatorConfig.java b/NeatSim/src/main/java/neatsim/EvaluatorConfig.java index 9d3b96e..086125b 100644 --- a/NeatSim/src/main/java/neatsim/EvaluatorConfig.java +++ b/NeatSim/src/main/java/neatsim/EvaluatorConfig.java @@ -14,9 +14,13 @@ import neatsim.util.NeuralNetworkReader; import neatsim.util.PrefixSuffixFilter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + public class EvaluatorConfig extends Config { private final Evaluator.Type type; - private final List genomes; + private final ImmutableList genomes; + private final ImmutableMap genomeMap; private final File outputPath; private final boolean dry; @@ -34,10 +38,17 @@ public Evaluator.Type getType() { return type; } - public List getGenomes() { + public ImmutableList getGenomes() { return genomes; } + /** + * @return A map of filename to {@link BlackBox}. + */ + public ImmutableMap getGenomeMap(){ + return genomeMap; + } + public File getOutputFile() { return outputPath; } @@ -51,7 +62,8 @@ public EvaluatorConfig() throws IOException { super(); logger.debug("Creating EvaluatorConfig"); type = enumToProperty(Evaluator.Type.class, TYPE); - genomes = propToBlackboxes(); + genomeMap = propToBlackboxes(); + genomes = ImmutableList.copyOf( genomeMap.keySet()); final String t1 = getProperty(OUTPUTPATH,"output.csv"); final File t2 = (new File(t1)).getCanonicalFile(); if (t2.isDirectory()) @@ -72,8 +84,10 @@ private int propToTimesteps() { //throw new InvalidConfigurationException(EVALUTATIONSTEPS); } + + // TODO change this so that this uses NeuralNetworkReader.readDirectory - public List propToBlackboxes() throws IOException { + private ImmutableMap propToBlackboxes() throws IOException { // Read data directory setting String dataDirectory = (new File(getProperty(DATADIR, ""))).getCanonicalPath(); // Read prefix setting @@ -95,7 +109,9 @@ public List propToBlackboxes() throws IOException { Collections.sort(sgenomes, NaturalOrderComparator.CASEINSENSITIVE_NUMERICAL_ORDER); printEvaluationOrder(sgenomes); //# Convert the genome path names to ANNs - final List genomes = new ArrayList(sgenomes.size()); + + final ImmutableMap.Builder mapBuilder = ImmutableMap.builder(); + final NeuralNetworkReader reader = new NeuralNetworkReader(timesteps); if (singletonGenome) { for (final String sgenome : sgenomes) { @@ -104,22 +120,25 @@ public List propToBlackboxes() throws IOException { logger.warn("File {} contains more than one genome.",sgenome); throw new RuntimeException("File " + sgenome + " contains more than one genome."); } - genomes.add(t.get(0)); + mapBuilder.put(t.get(0),sgenome); } } else { - for (final String sgenome : sgenomes) { - final List t = reader.readFile(sgenome).neuralNetworks; - for (final BlackBox bb : t) - genomes.add(bb); - } + throw new UnsupportedOperationException("This is not implemented yet, it should use a multimap instead of a map."); +// for (final String sgenome : sgenomes) { +// final List t = reader.readFile(sgenome).neuralNetworks; +// for (final BlackBox bb : t) +// genomes.add(bb); +// } } logger.info("Genomes were read from file correctly"); - return genomes; + return mapBuilder.build(); } private void printEvaluationOrder(final List sgenomes) { logger.info("Evaluating genomes in the following order: "); + //System.out.println("Evaluating genomes in the following order: "); for(final String genome : sgenomes) { + //System.out.println(genome); logger.info("\t{}",genome); } }