37 #include "ompl/tools/benchmark/Benchmark.h"
38 #include "ompl/tools/benchmark/MachineSpecs.h"
39 #include "ompl/util/Time.h"
40 #include <boost/scoped_ptr.hpp>
41 #include <boost/lexical_cast.hpp>
42 #include <boost/progress.hpp>
52 static std::string getResultsFilename(
const Benchmark::CompleteExperiment &exp)
54 return "ompl_" + exp.host +
"_" + boost::posix_time::to_iso_extended_string(exp.startTime) +
".log";
58 static std::string getConsoleFilename(
const Benchmark::CompleteExperiment &exp)
60 return "ompl_" + exp.host +
"_" + boost::posix_time::to_iso_extended_string(exp.startTime) +
".console";
74 RunPlanner(
const Benchmark *benchmark,
bool useThreads)
75 : benchmark_(benchmark), timeUsed_(0.0), memUsed_(0), useThreads_(useThreads)
83 runThread(planner, memStart + maxMem,
time::seconds(maxTime));
87 boost::thread t(boost::bind(&RunPlanner::runThread,
this, planner, memStart + maxMem,
time::seconds(maxTime)));
95 es <<
"Planner " << benchmark_->getStatus().activePlanner <<
" did not complete run " << benchmark_->getStatus().activeRun
96 <<
" within the specified amount of time (possible crash). Attempting to force termination of planning thread ..." << std::endl;
97 std::cerr << es.str();
103 std::string m =
"Planning thread cancelled";
104 std::cerr << m << std::endl;
108 if (memStart < memUsed_)
109 memUsed_ -= memStart;
114 double getTimeUsed(
void)
const
124 base::PlannerStatus getStatus(
void)
const
138 status_ = planner->solve(ptc, 0.1);
140 catch(std::runtime_error &e)
142 std::stringstream es;
143 es <<
"There was an error executing planner " << benchmark_->getStatus().activePlanner <<
", run = " << benchmark_->getStatus().activeRun << std::endl;
144 es <<
"*** " << e.what() << std::endl;
145 std::cerr << es.str();
153 const Benchmark *benchmark_;
156 base::PlannerStatus status_;
168 std::ofstream fout(filename);
172 logInform(
"Results saved to '%s'", filename);
177 if (getResultsFilename(
exp_) != std::string(filename))
180 logError(
"Unable to write results to '%s'", filename);
187 std::string filename = getResultsFilename(exp_);
188 return saveResultsToFile(filename.c_str());
193 if (exp_.planners.empty())
195 logWarn(
"There is no experimental data to save");
201 logError(
"Unable to write to stream");
205 out <<
"Experiment " << (exp_.name.empty() ?
"NO_NAME" : exp_.name) << std::endl;
206 out <<
"Running on " << (exp_.host.empty() ?
"UNKNOWN" : exp_.host) << std::endl;
207 out <<
"Starting at " << boost::posix_time::to_iso_extended_string(exp_.startTime) << std::endl;
208 out <<
"<<<|" << std::endl << exp_.setupInfo <<
"|>>>" << std::endl;
210 out << exp_.seed <<
" is the random seed" << std::endl;
211 out << exp_.maxTime <<
" seconds per run" << std::endl;
212 out << exp_.maxMem <<
" MB per run" << std::endl;
213 out << exp_.runCount <<
" runs per planner" << std::endl;
214 out << exp_.totalDuration <<
" seconds spent to collect the data" << std::endl;
217 out <<
"1 enum type" << std::endl;
223 out << exp_.planners.size() <<
" planners" << std::endl;
225 for (
unsigned int i = 0 ; i < exp_.planners.size() ; ++i)
227 out << exp_.planners[i].name << std::endl;
230 std::vector<std::string> properties;
231 for (std::map<std::string, std::string>::const_iterator mit = exp_.planners[i].common.begin() ;
232 mit != exp_.planners[i].common.end() ; ++mit)
233 properties.push_back(mit->first);
234 std::sort(properties.begin(), properties.end());
237 out << properties.size() <<
" common properties" << std::endl;
238 for (
unsigned int k = 0 ; k < properties.size() ; ++k)
240 std::map<std::string, std::string>::const_iterator it = exp_.planners[i].common.find(properties[k]);
241 out << it->first <<
" = " << it->second << std::endl;
245 std::map<std::string, bool> propSeen;
246 for (
unsigned int j = 0 ; j < exp_.planners[i].runs.size() ; ++j)
247 for (std::map<std::string, std::string>::const_iterator mit = exp_.planners[i].runs[j].begin() ;
248 mit != exp_.planners[i].runs[j].end() ; ++mit)
249 propSeen[mit->first] =
true;
253 for (std::map<std::string, bool>::iterator it = propSeen.begin() ; it != propSeen.end() ; ++it)
254 properties.push_back(it->first);
255 std::sort(properties.begin(), properties.end());
258 out << properties.size() <<
" properties for each run" << std::endl;
259 for (
unsigned int j = 0 ; j < properties.size() ; ++j)
260 out << properties[j] << std::endl;
263 out << exp_.planners[i].runs.size() <<
" runs" << std::endl;
264 for (
unsigned int j = 0 ; j < exp_.planners[i].runs.size() ; ++j)
266 for (
unsigned int k = 0 ; k < properties.size() ; ++k)
268 std::map<std::string, std::string>::const_iterator it = exp_.planners[i].runs[j].find(properties[k]);
269 if (it != exp_.planners[i].runs[j].end())
276 out <<
'.' << std::endl;
286 if (!gsetup_->getSpaceInformation()->isSetup())
287 gsetup_->getSpaceInformation()->setup();
291 if (!csetup_->getSpaceInformation()->isSetup())
292 csetup_->getSpaceInformation()->setup();
295 if (!(gsetup_ ? gsetup_->getGoal() : csetup_->getGoal()))
301 if (planners_.empty())
303 logError(
"There are no planners to benchmark");
307 status_.running =
true;
308 exp_.totalDuration = 0.0;
320 exp_.planners.clear();
321 exp_.planners.resize(planners_.size());
325 for (
unsigned int i = 0 ; i < planners_.size() ; ++i)
328 planners_[i]->setProblemDefinition(pdef);
329 if (!planners_[i]->isSetup())
330 planners_[i]->setup();
331 exp_.planners[i].name = (gsetup_ ?
"geometric_" :
"control_") + planners_[i]->getName();
332 logInform(
"Configured %s", exp_.planners[i].name.c_str());
336 logInform(
"Saving planner setup information ...");
338 std::stringstream setupInfo;
340 gsetup_->print(setupInfo);
342 csetup_->print(setupInfo);
343 setupInfo << std::endl <<
"Properties of benchmarked planners:" << std::endl;
344 for (
unsigned int i = 0 ; i < planners_.size() ; ++i)
345 planners_[i]->printProperties(setupInfo);
347 exp_.setupInfo = setupInfo.str();
353 boost::scoped_ptr<msg::OutputHandlerFile> ohf;
363 boost::scoped_ptr<boost::progress_display> progress;
366 std::cout <<
"Running experiment " << exp_.name <<
"." << std::endl;
367 std::cout <<
"Each planner will be executed " << req.
runCount <<
" times for at most " << req.
maxTime <<
" seconds. Memory is limited at "
368 << req.
maxMem <<
"MB." << std::endl;
369 progress.reset(
new boost::progress_display(100, std::cout));
375 for (
unsigned int i = 0 ; i < planners_.size() ; ++i)
377 status_.activePlanner = exp_.planners[i].name;
383 logInform(
"Executing planner-switch event for planner %s ...", status_.activePlanner.c_str());
384 plannerSwitch_(planners_[i]);
385 logInform(
"Completed execution of planner-switch event");
388 catch(std::runtime_error &e)
390 std::stringstream es;
391 es <<
"There was an error executing the planner-switch event for planner " << status_.activePlanner << std::endl;
392 es <<
"*** " << e.what() << std::endl;
393 std::cerr << es.str();
400 planners_[i]->params().getParams(exp_.planners[i].common);
401 planners_[i]->getSpaceInformation()->params().getParams(exp_.planners[i].common);
404 for (
unsigned int j = 0 ; j < req.
runCount ; ++j)
406 status_.activeRun = j;
407 status_.progressPercentage = (double)(100 * (req.
runCount * i + j)) / (
double)(planners_.size() * req.
runCount);
410 while (status_.progressPercentage > progress->count())
413 logInform(
"Preparing for run %d of %s", status_.activeRun, status_.activePlanner.c_str());
418 planners_[i]->clear();
421 gsetup_->getProblemDefinition()->clearSolutionPaths();
422 gsetup_->getSpaceInformation()->getMotionValidator()->resetMotionCounter();
426 csetup_->getProblemDefinition()->clearSolutionPaths();
427 csetup_->getSpaceInformation()->getMotionValidator()->resetMotionCounter();
430 catch(std::runtime_error &e)
432 std::stringstream es;
433 es <<
"There was an error while preparing for run " << status_.activeRun <<
" of planner " << status_.activePlanner << std::endl;
434 es <<
"*** " << e.what() << std::endl;
435 std::cerr << es.str();
444 logInform(
"Executing pre-run event for run %d of planner %s ...", status_.activeRun, status_.activePlanner.c_str());
445 preRun_(planners_[i]);
446 logInform(
"Completed execution of pre-run event");
449 catch(std::runtime_error &e)
451 std::stringstream es;
452 es <<
"There was an error executing the pre-run event for run " << status_.activeRun <<
" of planner " << status_.activePlanner << std::endl;
453 es <<
"*** " << e.what() << std::endl;
454 std::cerr << es.str();
459 rp.run(planners_[i], memStart, maxMemBytes, req.
maxTime);
460 bool solved = gsetup_ ? gsetup_->haveSolutionPath() : csetup_->haveSolutionPath();
467 run[
"time REAL"] = boost::lexical_cast<std::string>(rp.getTimeUsed());
468 run[
"memory REAL"] = boost::lexical_cast<std::string>((double)rp.getMemUsed() / (1024.0 * 1024.0));
469 run[
"status ENUM"] = boost::lexical_cast<std::string>((int)static_cast<base::PlannerStatus::StatusType>(rp.getStatus()));
472 run[
"solved BOOLEAN"] = boost::lexical_cast<std::string>(gsetup_->haveExactSolutionPath());
473 run[
"valid segment fraction REAL"] = boost::lexical_cast<std::string>(gsetup_->getSpaceInformation()->getMotionValidator()->getValidMotionFraction());
477 run[
"solved BOOLEAN"] = boost::lexical_cast<std::string>(csetup_->haveExactSolutionPath());
478 run[
"valid segment fraction REAL"] = boost::lexical_cast<std::string>(csetup_->getSpaceInformation()->getMotionValidator()->getValidMotionFraction());
485 run[
"approximate solution BOOLEAN"] = boost::lexical_cast<std::string>(gsetup_->getProblemDefinition()->hasApproximateSolution());
486 run[
"solution difference REAL"] = boost::lexical_cast<std::string>(gsetup_->getProblemDefinition()->getSolutionDifference());
487 run[
"solution length REAL"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().length());
488 run[
"solution smoothness REAL"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().smoothness());
489 run[
"solution clearance REAL"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().clearance());
490 run[
"solution segments INTEGER"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().getStateCount() - 1);
491 run[
"correct solution BOOLEAN"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().check());
493 unsigned int factor = gsetup_->getStateSpace()->getValidSegmentCountFactor();
494 gsetup_->getStateSpace()->setValidSegmentCountFactor(factor * 4);
495 run[
"correct solution strict BOOLEAN"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().check());
496 gsetup_->getStateSpace()->setValidSegmentCountFactor(factor);
500 gsetup_->simplifySolution();
502 run[
"simplification time REAL"] = boost::lexical_cast<std::string>(timeUsed);
503 run[
"simplified solution length REAL"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().length());
504 run[
"simplified solution smoothness REAL"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().smoothness());
505 run[
"simplified solution clearance REAL"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().clearance());
506 run[
"simplified solution segments INTEGER"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().getStateCount() - 1);
507 run[
"simplified correct solution BOOLEAN"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().check());
508 gsetup_->getStateSpace()->setValidSegmentCountFactor(factor * 4);
509 run[
"simplified correct solution strict BOOLEAN"] = boost::lexical_cast<std::string>(gsetup_->getSolutionPath().check());
510 gsetup_->getStateSpace()->setValidSegmentCountFactor(factor);
514 run[
"approximate solution BOOLEAN"] = boost::lexical_cast<std::string>(csetup_->getProblemDefinition()->hasApproximateSolution());
515 run[
"solution difference REAL"] = boost::lexical_cast<std::string>(csetup_->getProblemDefinition()->getSolutionDifference());
516 run[
"solution length REAL"] = boost::lexical_cast<std::string>(csetup_->getSolutionPath().length());
517 run[
"solution clearance REAL"] = boost::lexical_cast<std::string>(csetup_->getSolutionPath().asGeometric().clearance());
518 run[
"solution segments INTEGER"] = boost::lexical_cast<std::string>(csetup_->getSolutionPath().getControlCount());
519 run[
"correct solution BOOLEAN"] = boost::lexical_cast<std::string>(csetup_->getSolutionPath().check());
523 base::PlannerData pd (gsetup_ ? gsetup_->getSpaceInformation() : csetup_->getSpaceInformation());
524 planners_[i]->getPlannerData(pd);
525 run[
"graph states INTEGER"] = boost::lexical_cast<std::string>(pd.numVertices());
526 run[
"graph motions INTEGER"] = boost::lexical_cast<std::string>(pd.numEdges());
528 for (std::map<std::string, std::string>::const_iterator it = pd.properties.begin() ; it != pd.properties.end() ; ++it)
529 run[it->first] = it->second;
536 logInform(
"Executing post-run event for run %d of planner %s ...", status_.activeRun, status_.activePlanner.c_str());
537 postRun_(planners_[i], run);
538 logInform(
"Completed execution of post-run event");
541 catch(std::runtime_error &e)
543 std::stringstream es;
544 es <<
"There was an error in the execution of the post-run event for run " << status_.activeRun <<
" of planner " << status_.activePlanner << std::endl;
545 es <<
"*** " << e.what() << std::endl;
546 std::cerr << es.str();
550 exp_.planners[i].runs.push_back(run);
552 catch(std::runtime_error &e)
554 std::stringstream es;
555 es <<
"There was an error in the extraction of planner results: planner = " << status_.activePlanner <<
", run = " << status_.activePlanner << std::endl;
556 es <<
"*** " << e.what() << std::endl;
557 std::cerr << es.str();
563 status_.running =
false;
564 status_.progressPercentage = 100.0;
565 if (req.displayProgress)
567 while (status_.progressPercentage > progress->count())
569 std::cout << std::endl;