37 #include "ompl/tools/benchmark/Benchmark.h" 38 #include "ompl/tools/benchmark/MachineSpecs.h" 39 #include "ompl/util/Time.h" 40 #include "ompl/config.h" 41 #include <boost/scoped_ptr.hpp> 42 #include <boost/progress.hpp> 45 #include <condition_variable> 56 static std::string getResultsFilename(
const Benchmark::CompleteExperiment &exp)
58 return "ompl_" + exp.host +
"_" +
time::as_string(exp.startTime) +
".log";
63 static std::string getConsoleFilename(
const Benchmark::CompleteExperiment &exp)
65 return "ompl_" + exp.host +
"_" +
time::as_string(exp.startTime) +
".console";
78 RunPlanner(
const Benchmark *benchmark,
bool useThreads)
79 : benchmark_(benchmark), timeUsed_(0.0), memUsed_(0), useThreads_(useThreads)
125 double getTimeUsed()
const 135 base::PlannerStatus getStatus()
const 140 const Benchmark::RunProgressData &getRunProgressData()
const 142 return runProgressData_;
156 return terminationCondition(maxMem, endtime);
166 boost::scoped_ptr<std::thread> t;
167 if (planner->getPlannerProgressProperties().size() > 0)
168 t.reset(
new std::thread([
this, &planner, timeBetweenUpdates]
170 collectProgressProperties(planner->getPlannerProgressProperties(),
173 status_ = planner->solve(ptc, 0.1);
176 solvedCondition_.notify_all();
177 solvedFlag_.unlock();
181 catch (std::runtime_error &e)
183 std::stringstream es;
184 es <<
"There was an error executing planner " << benchmark_->getStatus().activePlanner
185 <<
", run = " << benchmark_->getStatus().activeRun << std::endl;
186 es <<
"*** " << e.what() << std::endl;
187 std::cerr << es.str();
200 std::unique_lock<std::mutex> ulock(solvedFlag_);
203 if (solvedCondition_.wait_for(ulock, timePerUpdate) == std::cv_status::no_timeout)
208 std::string timeStamp = std::to_string(timeInSeconds);
209 std::map<std::string, std::string> data;
210 data[
"time REAL"] = timeStamp;
211 for (
const auto &property : properties)
213 data[
property.first] =
property.second();
215 runProgressData_.push_back(data);
220 const Benchmark *benchmark_;
223 base::PlannerStatus status_;
225 Benchmark::RunProgressData runProgressData_;
229 std::mutex solvedFlag_;
230 std::condition_variable solvedCondition_;
240 std::ofstream fout(filename);
249 if (getResultsFilename(
exp_) != std::string(filename))
252 OMPL_ERROR(
"Unable to write results to '%s'", filename);
259 std::string filename = getResultsFilename(exp_);
260 return saveResultsToFile(filename.c_str());
265 if (exp_.planners.empty())
267 OMPL_WARN(
"There is no experimental data to save");
277 out <<
"OMPL version " << OMPL_VERSION << std::endl;
278 out <<
"Experiment " << (exp_.name.empty() ?
"NO_NAME" : exp_.name) << std::endl;
280 out << exp_.parameters.size() <<
" experiment properties" << std::endl;
281 for (
const auto ¶meter : exp_.parameters)
282 out << parameter.first <<
" = " << parameter.second << std::endl;
284 out <<
"Running on " << (exp_.host.empty() ?
"UNKNOWN" : exp_.host) << std::endl;
286 out <<
"<<<|" << std::endl
287 << exp_.setupInfo <<
"|>>>" << std::endl;
288 out <<
"<<<|" << std::endl
289 << exp_.cpuInfo <<
"|>>>" << std::endl;
291 out << exp_.seed <<
" is the random seed" << std::endl;
292 out << exp_.maxTime <<
" seconds per run" << std::endl;
293 out << exp_.maxMem <<
" MB per run" << std::endl;
294 out << exp_.runCount <<
" runs per planner" << std::endl;
295 out << exp_.totalDuration <<
" seconds spent to collect the data" << std::endl;
298 out <<
"1 enum type" << std::endl;
304 out << exp_.planners.size() <<
" planners" << std::endl;
306 for (
const auto &planner : exp_.planners)
308 out << planner.name << std::endl;
311 std::vector<std::string> properties;
312 for (
auto &property : planner.common)
313 properties.push_back(property.first);
314 std::sort(properties.begin(), properties.end());
317 out << properties.size() <<
" common properties" << std::endl;
318 for (
auto &property : properties)
320 auto it = planner.common.find(property);
321 out << it->first <<
" = " << it->second << std::endl;
325 std::map<std::string, bool> propSeen;
326 for (
auto &run : planner.runs)
327 for (
auto &property : run)
328 propSeen[
property.first] =
true;
332 for (
auto &it : propSeen)
333 properties.push_back(it.first);
334 std::sort(properties.begin(), properties.end());
337 out << properties.size() <<
" properties for each run" << std::endl;
338 for (
auto &propertie : properties)
339 out << propertie << std::endl;
342 out << planner.runs.size() <<
" runs" << std::endl;
343 for (
auto &run : planner.runs)
345 for (
auto &property : properties)
347 auto it = run.find(property);
356 if (planner.runsProgressData.size() > 0)
359 out << planner.progressPropertyNames.size() <<
" progress properties for each run" << std::endl;
361 for (
const auto &progPropName : planner.progressPropertyNames)
363 out << progPropName << std::endl;
366 out << planner.runsProgressData.size() <<
" runs" << std::endl;
367 for (
const auto &r : planner.runsProgressData)
370 for (
const auto &t : r)
373 for (
const auto &iter : t)
375 out << iter.second <<
",";
387 out <<
'.' << std::endl;
397 if (!gsetup_->getSpaceInformation()->isSetup())
398 gsetup_->getSpaceInformation()->setup();
402 if (!csetup_->getSpaceInformation()->isSetup())
403 csetup_->getSpaceInformation()->setup();
406 if (!(gsetup_ ? gsetup_->getGoal() : csetup_->getGoal()))
412 if (planners_.empty())
414 OMPL_ERROR(
"There are no planners to benchmark");
418 status_.running =
true;
419 exp_.totalDuration = 0.0;
432 exp_.planners.clear();
433 exp_.planners.resize(planners_.size());
436 gsetup_ ? gsetup_->getProblemDefinition() : csetup_->getProblemDefinition();
438 for (
unsigned int i = 0; i < planners_.size(); ++i)
441 planners_[i]->setProblemDefinition(pdef);
442 if (!planners_[i]->isSetup())
443 planners_[i]->setup();
444 exp_.planners[i].name = (gsetup_ ?
"geometric_" :
"control_") + planners_[i]->getName();
445 OMPL_INFORM(
"Configured %s", exp_.planners[i].name.c_str());
449 OMPL_INFORM(
"Saving planner setup information ...");
451 std::stringstream setupInfo;
453 gsetup_->print(setupInfo);
455 csetup_->print(setupInfo);
456 setupInfo << std::endl
457 <<
"Properties of benchmarked planners:" << std::endl;
458 for (
auto &planner : planners_)
459 planner->printProperties(setupInfo);
461 exp_.setupInfo = setupInfo.str();
467 boost::scoped_ptr<msg::OutputHandlerFile> ohf;
477 boost::scoped_ptr<boost::progress_display> progress;
480 std::cout <<
"Running experiment " << exp_.name <<
"." << std::endl;
481 std::cout <<
"Each planner will be executed " << req.
runCount <<
" times for at most " << req.
maxTime 482 <<
" seconds. Memory is limited at " << req.
maxMem <<
"MB." << std::endl;
483 progress.reset(
new boost::progress_display(100, std::cout));
489 for (
unsigned int i = 0; i < planners_.size(); ++i)
491 status_.activePlanner = exp_.planners[i].name;
497 OMPL_INFORM(
"Executing planner-switch event for planner %s ...", status_.activePlanner.c_str());
498 plannerSwitch_(planners_[i]);
499 OMPL_INFORM(
"Completed execution of planner-switch event");
502 catch (std::runtime_error &e)
504 std::stringstream es;
505 es <<
"There was an error executing the planner-switch event for planner " << status_.activePlanner
507 es <<
"*** " << e.what() << std::endl;
508 std::cerr << es.str();
515 planners_[i]->params().getParams(exp_.planners[i].common);
516 planners_[i]->getSpaceInformation()->params().getParams(exp_.planners[i].common);
519 exp_.planners[i].progressPropertyNames.emplace_back(
"time REAL");
520 base::Planner::PlannerProgressProperties::const_iterator iter;
521 for (iter = planners_[i]->getPlannerProgressProperties().begin();
522 iter != planners_[i]->getPlannerProgressProperties().end(); ++iter)
524 exp_.planners[i].progressPropertyNames.push_back(iter->first);
526 std::sort(exp_.planners[i].progressPropertyNames.begin(), exp_.planners[i].progressPropertyNames.end());
529 for (
unsigned int j = 0; j < req.
runCount; ++j)
531 status_.activeRun = j;
532 status_.progressPercentage =
533 (double)(100 * (req.
runCount * i + j)) / (
double)(planners_.size() * req.
runCount);
536 while (status_.progressPercentage > progress->count())
539 OMPL_INFORM(
"Preparing for run %d of %s", status_.activeRun, status_.activePlanner.c_str());
544 planners_[i]->clear();
547 gsetup_->getProblemDefinition()->clearSolutionPaths();
548 gsetup_->getSpaceInformation()->getMotionValidator()->resetMotionCounter();
552 csetup_->getProblemDefinition()->clearSolutionPaths();
553 csetup_->getSpaceInformation()->getMotionValidator()->resetMotionCounter();
556 catch (std::runtime_error &e)
558 std::stringstream es;
559 es <<
"There was an error while preparing for run " << status_.activeRun <<
" of planner " 560 << status_.activePlanner << std::endl;
561 es <<
"*** " << e.what() << std::endl;
562 std::cerr << es.str();
571 OMPL_INFORM(
"Executing pre-run event for run %d of planner %s ...", status_.activeRun,
572 status_.activePlanner.c_str());
573 preRun_(planners_[i]);
574 OMPL_INFORM(
"Completed execution of pre-run event");
577 catch (std::runtime_error &e)
579 std::stringstream es;
580 es <<
"There was an error executing the pre-run event for run " << status_.activeRun <<
" of planner " 581 << status_.activePlanner << std::endl;
582 es <<
"*** " << e.what() << std::endl;
583 std::cerr << es.str();
589 bool solved = gsetup_ ? gsetup_->haveSolutionPath() : csetup_->haveSolutionPath();
596 run[
"time REAL"] = std::to_string(rp.getTimeUsed());
597 run[
"memory REAL"] = std::to_string((
double)rp.getMemUsed() / (1024.0 * 1024.0));
598 run[
"status ENUM"] = std::to_string((
int)static_cast<base::PlannerStatus::StatusType>(rp.getStatus()));
601 run[
"solved BOOLEAN"] = std::to_string(gsetup_->haveExactSolutionPath());
602 run[
"valid segment fraction REAL"] =
603 std::to_string(gsetup_->getSpaceInformation()->getMotionValidator()->getValidMotionFraction());
607 run[
"solved BOOLEAN"] = std::to_string(csetup_->haveExactSolutionPath());
608 run[
"valid segment fraction REAL"] =
609 std::to_string(csetup_->getSpaceInformation()->getMotionValidator()->getValidMotionFraction());
616 run[
"approximate solution BOOLEAN"] =
617 std::to_string(gsetup_->getProblemDefinition()->hasApproximateSolution());
618 run[
"solution difference REAL"] =
619 std::to_string(gsetup_->getProblemDefinition()->getSolutionDifference());
620 run[
"solution length REAL"] = std::to_string(gsetup_->getSolutionPath().length());
621 run[
"solution smoothness REAL"] = std::to_string(gsetup_->getSolutionPath().smoothness());
622 run[
"solution clearance REAL"] = std::to_string(gsetup_->getSolutionPath().clearance());
623 run[
"solution segments INTEGER"] =
624 std::to_string(gsetup_->getSolutionPath().getStateCount() - 1);
625 run[
"correct solution BOOLEAN"] = std::to_string(gsetup_->getSolutionPath().check());
627 unsigned int factor = gsetup_->getStateSpace()->getValidSegmentCountFactor();
628 gsetup_->getStateSpace()->setValidSegmentCountFactor(factor * 4);
629 run[
"correct solution strict BOOLEAN"] = std::to_string(gsetup_->getSolutionPath().check());
630 gsetup_->getStateSpace()->setValidSegmentCountFactor(factor);
636 gsetup_->simplifySolution();
638 run[
"simplification time REAL"] = std::to_string(timeUsed);
639 run[
"simplified solution length REAL"] =
640 std::to_string(gsetup_->getSolutionPath().length());
641 run[
"simplified solution smoothness REAL"] =
642 std::to_string(gsetup_->getSolutionPath().smoothness());
643 run[
"simplified solution clearance REAL"] =
644 std::to_string(gsetup_->getSolutionPath().clearance());
645 run[
"simplified solution segments INTEGER"] =
646 std::to_string(gsetup_->getSolutionPath().getStateCount() - 1);
647 run[
"simplified correct solution BOOLEAN"] =
648 std::to_string(gsetup_->getSolutionPath().check());
649 gsetup_->getStateSpace()->setValidSegmentCountFactor(factor * 4);
650 run[
"simplified correct solution strict BOOLEAN"] =
651 std::to_string(gsetup_->getSolutionPath().check());
652 gsetup_->getStateSpace()->setValidSegmentCountFactor(factor);
657 run[
"approximate solution BOOLEAN"] =
658 std::to_string(csetup_->getProblemDefinition()->hasApproximateSolution());
659 run[
"solution difference REAL"] =
660 std::to_string(csetup_->getProblemDefinition()->getSolutionDifference());
661 run[
"solution length REAL"] = std::to_string(csetup_->getSolutionPath().length());
662 run[
"solution clearance REAL"] =
663 std::to_string(csetup_->getSolutionPath().asGeometric().clearance());
664 run[
"solution segments INTEGER"] = std::to_string(csetup_->getSolutionPath().getControlCount());
665 run[
"correct solution BOOLEAN"] = std::to_string(csetup_->getSolutionPath().check());
669 base::PlannerData pd(gsetup_ ? gsetup_->getSpaceInformation() : csetup_->getSpaceInformation());
670 planners_[i]->getPlannerData(pd);
671 run[
"graph states INTEGER"] = std::to_string(pd.numVertices());
672 run[
"graph motions INTEGER"] = std::to_string(pd.numEdges());
674 for (std::map<std::string, std::string>::const_iterator it = pd.properties.begin();
675 it != pd.properties.end(); ++it)
676 run[it->first] = it->second;
683 OMPL_INFORM(
"Executing post-run event for run %d of planner %s ...", status_.activeRun,
684 status_.activePlanner.c_str());
685 postRun_(planners_[i], run);
686 OMPL_INFORM(
"Completed execution of post-run event");
689 catch (std::runtime_error &e)
691 std::stringstream es;
692 es <<
"There was an error in the execution of the post-run event for run " << status_.activeRun
693 <<
" of planner " << status_.activePlanner << std::endl;
694 es <<
"*** " << e.what() << std::endl;
695 std::cerr << es.str();
699 exp_.planners[i].runs.push_back(run);
703 if (planners_[i]->getPlannerProgressProperties().size() > 0)
705 exp_.planners[i].runsProgressData.push_back(rp.getRunProgressData());
708 catch (std::runtime_error &e)
710 std::stringstream es;
711 es <<
"There was an error in the extraction of planner results: planner = " << status_.activePlanner
712 <<
", run = " << status_.activePlanner << std::endl;
713 es <<
"*** " << e.what() << std::endl;
714 std::cerr << es.str();
720 status_.running =
false;
721 status_.progressPercentage = 100.0;
722 if (req.displayProgress)
724 while (status_.progressPercentage > progress->count())
726 std::cout << std::endl;
Object containing planner generated vertex and edge data. It is assumed that all vertices are unique...
std::string getCPUInfo()
Get information about the CPU of the machine in use.
A shared pointer wrapper for ompl::base::ProblemDefinition.
std::function< bool()> PlannerTerminationConditionFn
Signature for functions that decide whether termination conditions have been met for a planner...
void noOutputHandler()
This function instructs ompl that no messages should be outputted. Equivalent to useOutputHandler(nul...
std::string as_string(const point &p)
Return string representation of point in time.
duration seconds(double sec)
Return the time duration representing a given number of seconds.
Main namespace. Contains everything in this library.
The number of possible status values.
#define OMPL_ERROR(fmt,...)
Log a formatted error string.
std::chrono::system_clock::duration duration
Representation of a time duration.
A class to store the exit status of Planner::solve()
Generic class to handle output from a piece of code.
static std::uint_fast32_t getSeed()
Get the seed used to generate the seeds of each RNG instance. Passing the returned value to setSeed()...
#define OMPL_WARN(fmt,...)
Log a formatted warning string.
MemUsage_t getProcessMemoryUsage()
Get the amount of memory the current process is using. This should work on major platforms (Windows...
point now()
Get the current time point.
OutputHandler * getOutputHandler()
Get the instance of the OutputHandler currently used. This is nullptr in case there is no output hand...
Implementation of OutputHandler that saves messages in a file.
unsigned long long MemUsage_t
Amount of memory used, in bytes.
std::string getHostname()
Get the hostname of the machine in use.
std::chrono::system_clock::time_point point
Representation of a point in time.
std::string asString() const
Return a string representation.
void useOutputHandler(OutputHandler *oh)
Specify the instance of the OutputHandler to use. By default, this is OutputHandlerSTD.
std::map< std::string, PlannerProgressProperty > PlannerProgressProperties
A dictionary which maps the name of a progress property to the function to be used for querying that ...
#define OMPL_INFORM(fmt,...)
Log a formatted information string.