diff options
Diffstat (limited to 'third_party/include/hayai')
17 files changed, 3641 insertions, 0 deletions
diff --git a/third_party/include/hayai/LICENSE.md b/third_party/include/hayai/LICENSE.md new file mode 100755 index 0000000..1ace0da --- /dev/null +++ b/third_party/include/hayai/LICENSE.md @@ -0,0 +1,21 @@ +Copyright (c) 2011 - Nick Bruun. + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. +3. If you meet (any of) the author(s), you're encouraged to buy them a beer, + a drink or whatever is suited to the situation, given that you like the + software. +4. This notice may not be removed or altered from any source + distribution. diff --git a/third_party/include/hayai/hayai.hpp b/third_party/include/hayai/hayai.hpp new file mode 100755 index 0000000..0396751 --- /dev/null +++ b/third_party/include/hayai/hayai.hpp @@ -0,0 +1,136 @@ +#ifndef __HAYAI +#define __HAYAI + +#include "hayai_benchmarker.hpp" +#include "hayai_test.hpp" +#include "hayai_default_test_factory.hpp" +#include "hayai_fixture.hpp" +#include "hayai_console_outputter.hpp" +#include "hayai_json_outputter.hpp" +#include "hayai_junit_xml_outputter.hpp" + + +#define HAYAI_VERSION "1.0.1" + + +#define BENCHMARK_CLASS_NAME_(fixture_name, benchmark_name) \ + fixture_name ## _ ## benchmark_name ## _Benchmark + +#define BENCHMARK_(fixture_name, \ + benchmark_name, \ + fixture_class_name, \ + runs, \ + iterations) \ + class BENCHMARK_CLASS_NAME_(fixture_name, benchmark_name) \ + : public fixture_class_name \ + { \ + public: \ + BENCHMARK_CLASS_NAME_(fixture_name, benchmark_name)() \ + { \ + \ + } \ + protected: \ + virtual void TestBody(); \ + private: \ + static const ::hayai::TestDescriptor* _descriptor; \ + }; \ + \ + const ::hayai::TestDescriptor* \ + BENCHMARK_CLASS_NAME_(fixture_name, benchmark_name)::_descriptor = \ + ::hayai::Benchmarker::Instance().RegisterTest( \ + #fixture_name, \ + #benchmark_name, \ + runs, \ + iterations, \ + new ::hayai::TestFactoryDefault< \ + BENCHMARK_CLASS_NAME_(fixture_name, benchmark_name) \ + >(), \ + ::hayai::TestParametersDescriptor()); \ + \ + void BENCHMARK_CLASS_NAME_(fixture_name, benchmark_name)::TestBody() + +#define BENCHMARK_F(fixture_name, \ + benchmark_name, \ + runs, \ + iterations) \ + BENCHMARK_(fixture_name, \ + benchmark_name, \ + fixture_name, \ + runs, \ + iterations) + +#define BENCHMARK(fixture_name, \ + benchmark_name, \ + runs, \ + iterations) \ + BENCHMARK_(fixture_name, \ + benchmark_name, \ + ::hayai::Test, \ + runs, \ + iterations) + +// Parametrized benchmarks. +#define BENCHMARK_P_(fixture_name, \ + benchmark_name, \ + fixture_class_name, \ + runs, \ + iterations, \ + arguments) \ + class BENCHMARK_CLASS_NAME_(fixture_name, benchmark_name) \ + : public fixture_class_name { \ + public: \ + BENCHMARK_CLASS_NAME_(fixture_name, benchmark_name) () {} \ + virtual ~ BENCHMARK_CLASS_NAME_(fixture_name, benchmark_name) () {} \ + static const std::size_t _runs = runs; \ + static const std::size_t _iterations = iterations; \ + static const char* _argumentsDeclaration() { return #arguments; } \ + protected: \ + inline void TestPayload arguments; \ + }; \ + void BENCHMARK_CLASS_NAME_(fixture_name, benchmark_name)::TestPayload arguments + +#define BENCHMARK_P(fixture_name, \ + benchmark_name, \ + runs, \ + iterations, \ + arguments) \ + BENCHMARK_P_(fixture_name, \ + benchmark_name, \ + hayai::Fixture, \ + runs, \ + iterations, \ + arguments) + +#define BENCHMARK_P_F(fixture_name, benchmark_name, runs, iterations, arguments) \ + BENCHMARK_P_(fixture_name, benchmark_name, fixture_name, runs, iterations, arguments) + +#define BENCHMARK_P_CLASS_NAME_(fixture_name, benchmark_name, id) \ + fixture_name ## _ ## benchmark_name ## _Benchmark_ ## id + +#define BENCHMARK_P_INSTANCE1(fixture_name, benchmark_name, arguments, id) \ + class BENCHMARK_P_CLASS_NAME_(fixture_name, benchmark_name, id): \ + public BENCHMARK_CLASS_NAME_(fixture_name, benchmark_name) { \ + protected: \ + virtual void TestBody() { this->TestPayload arguments; } \ + private: \ + static const ::hayai::TestDescriptor* _descriptor; \ + }; \ + const ::hayai::TestDescriptor* BENCHMARK_P_CLASS_NAME_(fixture_name, benchmark_name, id)::_descriptor = \ + ::hayai::Benchmarker::Instance().RegisterTest( \ + #fixture_name, #benchmark_name, \ + BENCHMARK_CLASS_NAME_(fixture_name, benchmark_name)::_runs, \ + BENCHMARK_CLASS_NAME_(fixture_name, benchmark_name)::_iterations, \ + new ::hayai::TestFactoryDefault< BENCHMARK_P_CLASS_NAME_(fixture_name, benchmark_name, id) >(), \ + ::hayai::TestParametersDescriptor(BENCHMARK_CLASS_NAME_(fixture_name, benchmark_name)::_argumentsDeclaration(), #arguments)) + +#if defined(__COUNTER__) +# define BENCHMARK_P_ID_ __COUNTER__ +#else +# define BENCHMARK_P_ID_ __LINE__ +#endif + +#define BENCHMARK_P_INSTANCE(fixture_name, benchmark_name, arguments) \ + BENCHMARK_P_INSTANCE1(fixture_name, benchmark_name, arguments, BENCHMARK_P_ID_) + + +#endif diff --git a/third_party/include/hayai/hayai_benchmarker.hpp b/third_party/include/hayai/hayai_benchmarker.hpp new file mode 100755 index 0000000..33f005c --- /dev/null +++ b/third_party/include/hayai/hayai_benchmarker.hpp @@ -0,0 +1,552 @@ +#ifndef __HAYAI_BENCHMARKER +#define __HAYAI_BENCHMARKER +#include <algorithm> +#include <vector> +#include <limits> +#include <iomanip> +#if __cplusplus > 201100L +#include <random> +#endif +#include <string> +#include <cstring> + +#include "hayai_test_factory.hpp" +#include "hayai_test_descriptor.hpp" +#include "hayai_test_result.hpp" +#include "hayai_console_outputter.hpp" + + +namespace hayai +{ + /// Benchmarking execution controller singleton. + class Benchmarker + { + public: + /// Get the singleton instance of @ref Benchmarker. + + /// @returns a reference to the singleton instance of the + /// benchmarker execution controller. + static Benchmarker& Instance() + { + static Benchmarker singleton; + return singleton; + } + + + /// Register a test with the benchmarker instance. + + /// @param fixtureName Name of the fixture. + /// @param testName Name of the test. + /// @param runs Number of runs for the test. + /// @param iterations Number of iterations per run. + /// @param testFactory Test factory implementation for the test. + /// @returns a pointer to a @ref TestDescriptor instance + /// representing the given test. + static TestDescriptor* RegisterTest( + const char* fixtureName, + const char* testName, + std::size_t runs, + std::size_t iterations, + TestFactory* testFactory, + TestParametersDescriptor parameters + ) + { + // Determine if the test has been disabled. + static const char* disabledPrefix = "DISABLED_"; + bool isDisabled = ((::strlen(testName) >= 9) && + (!::memcmp(testName, disabledPrefix, 9))); + + if (isDisabled) + testName += 9; + + // Add the descriptor. + TestDescriptor* descriptor = new TestDescriptor(fixtureName, + testName, + runs, + iterations, + testFactory, + parameters, + isDisabled); + + Instance()._tests.push_back(descriptor); + + return descriptor; + } + + + /// Add an outputter. + + /// @param outputter Outputter. The caller must ensure that the + /// outputter remains in existence for the entire benchmark run. + static void AddOutputter(Outputter& outputter) + { + Instance()._outputters.push_back(&outputter); + } + + + /// Apply a pattern filter to the tests. + + /// --gtest_filter-compatible pattern: + /// + /// https://code.google.com/p/googletest/wiki/AdvancedGuide + /// + /// @param pattern Filter pattern compatible with gtest. + static void ApplyPatternFilter(const char* pattern) + { + Benchmarker& instance = Instance(); + + // Split the filter at '-' if it exists. + const char* const dash = strchr(pattern, '-'); + + std::string positive; + std::string negative; + + if (dash == NULL) + positive = pattern; + else + { + positive = std::string(pattern, dash); + negative = std::string(dash + 1); + if (positive.empty()) + positive = "*"; + } + + // Iterate across all tests and test them against the patterns. + std::size_t index = 0; + while (index < instance._tests.size()) + { + TestDescriptor* desc = instance._tests[index]; + + if ((!FilterMatchesString(positive.c_str(), + desc->CanonicalName)) || + (FilterMatchesString(negative.c_str(), + desc->CanonicalName))) + { + instance._tests.erase( + instance._tests.begin() + + std::vector<TestDescriptor*>::difference_type(index) + ); + delete desc; + } + else + ++index; + } + } + + + /// Run all benchmarking tests. + static void RunAllTests() + { + ConsoleOutputter defaultOutputter; + std::vector<Outputter*> defaultOutputters; + defaultOutputters.push_back(&defaultOutputter); + + Benchmarker& instance = Instance(); + std::vector<Outputter*>& outputters = + (instance._outputters.empty() ? + defaultOutputters : + instance._outputters); + + // Get the tests for execution. + std::vector<TestDescriptor*> tests = instance.GetTests(); + + const std::size_t totalCount = tests.size(); + std::size_t disabledCount = 0; + + std::vector<TestDescriptor*>::const_iterator testsIt = + tests.begin(); + + while (testsIt != tests.end()) + { + if ((*testsIt)->IsDisabled) + ++disabledCount; + ++testsIt; + } + + const std::size_t enabledCount = totalCount - disabledCount; + + // Calibrate the tests. + const CalibrationModel calibrationModel = GetCalibrationModel(); + + // Begin output. + for (std::size_t outputterIndex = 0; + outputterIndex < outputters.size(); + outputterIndex++) + outputters[outputterIndex]->Begin(enabledCount, disabledCount); + + // Run through all the tests in ascending order. + std::size_t index = 0; + + while (index < tests.size()) + { + // Get the test descriptor. + TestDescriptor* descriptor = tests[index++]; + + // Check if test matches include filters + if (instance._include.size() > 0) + { + bool included = false; + std::string name = + descriptor->FixtureName + "." + + descriptor->TestName; + + for (std::size_t i = 0; i < instance._include.size(); i++) + { + if (name.find(instance._include[i]) != + std::string::npos) + { + included = true; + break; + } + } + + if (!included) + continue; + } + + // Check if test is not disabled. + if (descriptor->IsDisabled) + { + for (std::size_t outputterIndex = 0; + outputterIndex < outputters.size(); + outputterIndex++) + outputters[outputterIndex]->SkipDisabledTest( + descriptor->FixtureName, + descriptor->TestName, + descriptor->Parameters, + descriptor->Runs, + descriptor->Iterations + ); + + continue; + } + + // Describe the beginning of the run. + for (std::size_t outputterIndex = 0; + outputterIndex < outputters.size(); + outputterIndex++) + outputters[outputterIndex]->BeginTest( + descriptor->FixtureName, + descriptor->TestName, + descriptor->Parameters, + descriptor->Runs, + descriptor->Iterations + ); + + // Execute each individual run. + std::vector<uint64_t> runTimes(descriptor->Runs); + uint64_t overheadCalibration = + calibrationModel.GetCalibration(descriptor->Iterations); + + std::size_t run = 0; + while (run < descriptor->Runs) + { + // Construct a test instance. + Test* test = descriptor->Factory->CreateTest(); + + // Run the test. + uint64_t time = test->Run(descriptor->Iterations); + + // Store the test time. + runTimes[run] = (time > overheadCalibration ? + time - overheadCalibration : + 0); + + // Dispose of the test instance. + delete test; + + ++run; + } + + // Calculate the test result. + TestResult testResult(runTimes, descriptor->Iterations); + + // Describe the end of the run. + for (std::size_t outputterIndex = 0; + outputterIndex < outputters.size(); + outputterIndex++) + outputters[outputterIndex]->EndTest( + descriptor->FixtureName, + descriptor->TestName, + descriptor->Parameters, + testResult + ); + + } + + // End output. + for (std::size_t outputterIndex = 0; + outputterIndex < outputters.size(); + outputterIndex++) + outputters[outputterIndex]->End(enabledCount, + disabledCount); + } + + + /// List tests. + static std::vector<const TestDescriptor*> ListTests() + { + std::vector<const TestDescriptor*> tests; + Benchmarker& instance = Instance(); + + std::size_t index = 0; + while (index < instance._tests.size()) + tests.push_back(instance._tests[index++]); + + return tests; + } + + + /// Shuffle tests. + + /// Randomly shuffles the order of tests. + static void ShuffleTests() + { + Benchmarker& instance = Instance(); +#if __cplusplus > 201100L + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(instance._tests.begin(), + instance._tests.end(), + g); +#else + std::random_shuffle(instance._tests.begin(), + instance._tests.end()); +#endif + } + private: + /// Calibration model. + + /// Describes a linear calibration model for test runs. + struct CalibrationModel + { + public: + CalibrationModel(std::size_t scale, + uint64_t slope, + uint64_t yIntercept) + : Scale(scale), + Slope(slope), + YIntercept(yIntercept) + { + + } + + + /// Scale. + + /// Number of iterations per slope unit. + const std::size_t Scale; + + + /// Slope. + const uint64_t Slope; + + + /// Y-intercept; + const uint64_t YIntercept; + + + /// Get calibration value for a run. + int64_t GetCalibration(std::size_t iterations) const + { + return YIntercept + (iterations * Slope) / Scale; + } + }; + + + /// Private constructor. + Benchmarker() + { + + } + + + /// Private destructor. + ~Benchmarker() + { + // Release all test descriptors. + std::size_t index = _tests.size(); + while (index--) + delete _tests[index]; + } + + + /// Get the tests to be executed. + std::vector<TestDescriptor*> GetTests() const + { + std::vector<TestDescriptor*> tests; + + std::size_t index = 0; + while (index < _tests.size()) + tests.push_back(_tests[index++]); + + return tests; + } + + + /// Test if a filter matches a string. + + /// Adapted from gtest. All rights reserved by original authors. + static bool FilterMatchesString(const char* filter, + const std::string& str) + { + const char *patternStart = filter; + + while (true) + { + if (PatternMatchesString(patternStart, str.c_str())) + return true; + + // Finds the next pattern in the filter. + patternStart = strchr(patternStart, ':'); + + // Returns if no more pattern can be found. + if (!patternStart) + return false; + + // Skips the pattern separater (the ':' character). + patternStart++; + } + } + + + /// Test if pattern matches a string. + + /// Adapted from gtest. All rights reserved by original authors. + static bool PatternMatchesString(const char* pattern, const char *str) + { + switch (*pattern) + { + case '\0': + case ':': + return (*str == '\0'); + case '?': // Matches any single character. + return ((*str != '\0') && + (PatternMatchesString(pattern + 1, str + 1))); + case '*': // Matches any string (possibly empty) of characters. + return (((*str != '\0') && + (PatternMatchesString(pattern, str + 1))) || + (PatternMatchesString(pattern + 1, str))); + default: + return ((*pattern == *str) && + (PatternMatchesString(pattern + 1, str + 1))); + } + } + + + /// Get calibration model. + + /// Returns an average linear calibration model. + static CalibrationModel GetCalibrationModel() + { + // We perform a number of runs of varying iterations with an empty + // test body. The assumption here is, that the time taken for the + // test run is linear with regards to the number of iterations, ie. + // some constant overhead with a per-iteration overhead. This + // hypothesis has been manually validated by linear regression over + // sample data. + // + // In order to avoid losing too much precision, we are going to + // calibrate in terms of the overhead of some x n iterations, + // where n must be a sufficiently large number to produce some + // significant runtime. On a high-end 2012 Retina MacBook Pro with + // -O3 on clang-602.0.53 (LLVM 6.1.0) n = 1,000,000 produces + // run times of ~1.9 ms, which should be sufficiently precise. + // + // However, as the constant overhead is mostly related to + // retrieving the system clock, which under the same conditions + // clocks in at around 17 ms, we run the risk of winding up with + // a negative y-intercept if we do not fix the y-intercept. This + // intercept is therefore fixed by a large number of runs of 0 + // iterations. + ::hayai::Test* test = new Test(); + +#define HAYAI_CALIBRATION_INTERESECT_RUNS 10000 + +#define HAYAI_CALIBRATION_RUNS 10 +#define HAYAI_CALIBRATION_SCALE 1000000 +#define HAYAI_CALIBRATION_PPR 6 + + // Determine the intercept. + uint64_t + interceptSum = 0, + interceptMin = std::numeric_limits<uint64_t>::min(), + interceptMax = 0; + + for (std::size_t run = 0; + run < HAYAI_CALIBRATION_INTERESECT_RUNS; + ++run) + { + uint64_t intercept = test->Run(0); + interceptSum += intercept; + if (intercept < interceptMin) + interceptMin = intercept; + if (intercept > interceptMax) + interceptMax = intercept; + } + + uint64_t interceptAvg = + interceptSum / HAYAI_CALIBRATION_INTERESECT_RUNS; + + // Produce a series of sample points. + std::vector<uint64_t> x(HAYAI_CALIBRATION_RUNS * + HAYAI_CALIBRATION_PPR); + std::vector<uint64_t> t(HAYAI_CALIBRATION_RUNS * + HAYAI_CALIBRATION_PPR); + + std::size_t point = 0; + + for (std::size_t run = 0; run < HAYAI_CALIBRATION_RUNS; ++run) + { +#define HAYAI_CALIBRATION_POINT(_x) \ + x[point] = _x; \ + t[point++] = \ + test->Run(_x * std::size_t(HAYAI_CALIBRATION_SCALE)) + + HAYAI_CALIBRATION_POINT(1); + HAYAI_CALIBRATION_POINT(2); + HAYAI_CALIBRATION_POINT(5); + HAYAI_CALIBRATION_POINT(10); + HAYAI_CALIBRATION_POINT(15); + HAYAI_CALIBRATION_POINT(20); + +#undef HAYAI_CALIBRATION_POINT + } + + // As we have a fixed y-intercept, b, the optimal slope for a line + // fitting the sample points will be + // $\frac {\sum_{i=1}^{n} x_n \cdot (y_n - b)} + // {\sum_{i=1}^{n} {x_n}^2}$. + uint64_t + sumProducts = 0, + sumXSquared = 0; + + std::size_t p = x.size(); + while (p--) + { + sumXSquared += x[p] * x[p]; + sumProducts += x[p] * (t[p] - interceptAvg); + } + + uint64_t slope = sumProducts / sumXSquared; + + delete test; + + return CalibrationModel(HAYAI_CALIBRATION_SCALE, + slope, + interceptAvg); + +#undef HAYAI_CALIBRATION_INTERESECT_RUNS + +#undef HAYAI_CALIBRATION_RUNS +#undef HAYAI_CALIBRATION_SCALE +#undef HAYAI_CALIBRATION_PPR + } + + + std::vector<Outputter*> _outputters; ///< Registered outputters. + std::vector<TestDescriptor*> _tests; ///< Registered tests. + std::vector<std::string> _include; ///< Test filters. + }; +} +#endif diff --git a/third_party/include/hayai/hayai_clock.hpp b/third_party/include/hayai/hayai_clock.hpp new file mode 100755 index 0000000..c6d4e41 --- /dev/null +++ b/third_party/include/hayai/hayai_clock.hpp @@ -0,0 +1,367 @@ +// +// System-specific implementation of the clock functions. +// +// Copyright (C) 2011 Nick Bruun <nick@bruun.co> +// Copyright (C) 2013 Vlad Lazarenko <vlad@lazarenko.me> +// Copyright (C) 2014 Nicolas Pauss <nicolas.pauss@gmail.com> +// +// Implementation notes: +// +// On Windows, QueryPerformanceCounter() is used. It gets +// real-time clock with up to nanosecond precision. +// +// On Apple (OS X, iOS), mach_absolute_time() is used. It gets +// CPU/bus dependent real-time clock with up to nanosecond precision. +// +// On Unix, gethrtime() is used with HP-UX and Solaris. Otherwise, +// clock_gettime() is used to access monotonic real-time clock +// with up to nanosecond precision. On kernels 2.6.28 and newer, the ticks +// are also raw and are not subject to NTP and/or adjtime(3) adjustments. +// +// Other POSIX compliant platforms resort to using gettimeofday(). It is +// subject to clock adjustments, does not allow for higher than microsecond +// resolution and is also declared obsolete by POSIX.1-2008. +// +// Note on C++11: +// +// Starting with C++11, we could use std::chrono. However, the details of +// what clock is really being used is implementation-specific. For example, +// Visual Studio 2012 defines "high_resolution_clock" as system clock with +// ~1 millisecond precision that is not acceptable for performance +// measurements. Therefore, we are much better off having full control of what +// mechanism we use to obtain the system clock. +// +// Note on durations: it is assumed that end times passed to the clock methods +// are all after the start time. Wrap-around of clocks is not tested, as +// nanosecond precision of unsigned 64-bit integers would require an uptime of +// almost 585 years for this to happen. Let's call ourselves safe on that one. +// +#ifndef __HAYAI_CLOCK +#define __HAYAI_CLOCK + +#include "hayai_compatibility.hpp" + + +// POSIX +#if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__)) +#include <unistd.h> +#endif + +// Win32 +#if defined(_WIN32) +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include <windows.h> + +// Apple +#elif defined(__APPLE__) && defined(__MACH__) +#include <mach/mach_time.h> + +// Unix +#elif defined(__unix__) || defined(__unix) || defined(unix) + +// gethrtime +# if (defined(__hpux) || defined(hpux)) || ((defined(__sun__) || defined(__sun) || defined(sun)) && (defined(__SVR4) || defined(__svr4__))) +# include <sys/time.h> + +// clock_gettime +# elif defined(_POSIX_TIMERS) && (_POSIX_TIMERS > 0) +# include <time.h> + +// gettimeofday +# else +# include <sys/time.h> + +# endif +#else +#error "Unable to define high resolution timer for an unknown OS." +#endif + +#include <stdexcept> +#include <stdint.h> + + +namespace hayai +{ +// Win32 +#if defined(_WIN32) + class Clock + { + public: + /// Time point. + + /// Opaque representation of a point in time. + typedef LARGE_INTEGER TimePoint; + + + /// Get the current time as a time point. + + /// @returns the current time point. + static TimePoint Now() + { + TimePoint result; + QueryPerformanceCounter(&result); + return result; + } + + + /// Get the duration between two time points. + + /// @param startTime Start time point. + /// @param endTime End time point. + /// @returns the number of nanoseconds elapsed between the two time + /// points. + static uint64_t Duration(const TimePoint& startTime, + const TimePoint& endTime) + { + const static double performanceFrequencyNs = + PerformanceFrequencyNs(); + + return static_cast<uint64_t>( + (endTime.QuadPart - startTime.QuadPart) + * performanceFrequencyNs + ); + } + + + /// Clock implementation description. + + /// @returns a description of the clock implementation used. + static const char* Description() + { + return "QueryPerformanceCounter"; + } + private: + static double PerformanceFrequencyNs() + { + TimePoint result; + QueryPerformanceFrequency(&result); + return 1e9 / static_cast<double>(result.QuadPart); + } + }; + +// Mach kernel. +#elif defined(__APPLE__) && defined(__MACH__) + class Clock + { + public: + /// Time point. + + /// Opaque representation of a point in time. + typedef uint64_t TimePoint; + + + /// Get the current time as a time point. + + /// @returns the current time point. + static TimePoint Now() __hayai_noexcept + { + return mach_absolute_time(); + } + + + /// Get the duration between two time points. + + /// @param startTime Start time point. + /// @param endTime End time point. + /// @returns the number of nanoseconds elapsed between the two time + /// points. + static uint64_t Duration(const TimePoint& startTime, + const TimePoint& endTime) __hayai_noexcept + { + mach_timebase_info_data_t time_info; + mach_timebase_info(&time_info); + + return (endTime - startTime) * time_info.numer / time_info.denom; + } + + + /// Clock implementation description. + + /// @returns a description of the clock implementation used. + static const char* Description() + { + return "mach_absolute_time"; + } + }; + +// Unix +#elif defined(__unix__) || defined(__unix) || defined(unix) + +// gethrtime +# if (defined(__hpux) || defined(hpux)) || ((defined(__sun__) || defined(__sun) || defined(sun)) && (defined(__SVR4) || defined(__svr4__))) + class Clock + { + public: + /// Time point. + + /// Opaque representation of a point in time. + typedef hrtime_t TimePoint; + + + /// Get the current time as a time point. + + /// @returns the current time point. + static TimePoint Now() __hayai_noexcept + { + return gethrtime(); + } + + + /// Get the duration between two time points. + + /// @param startTime Start time point. + /// @param endTime End time point. + /// @returns the number of nanoseconds elapsed between the two time + /// points. + static uint64_t Duration(const TimePoint& startTime, + const TimePoint& endTime) __hayai_noexcept + { + return static_cast<uint64_t>(endTime - startTime); + } + + + /// Clock implementation description. + + /// @returns a description of the clock implementation used. + static const char* Description() + { + return "gethrtime"; + } + }; + + +// clock_gettime +# elif defined(_POSIX_TIMERS) && (_POSIX_TIMERS > 0) + class Clock + { + public: + /// Time point. + + /// Opaque representation of a point in time. + typedef struct timespec TimePoint; + + + /// Get the current time as a time point. + + /// @returns the current time point. + static TimePoint Now() __hayai_noexcept + { + TimePoint result; +# if defined(CLOCK_MONOTONIC_RAW) + clock_gettime(CLOCK_MONOTONIC_RAW, &result); +# elif defined(CLOCK_MONOTONIC) + clock_gettime(CLOCK_MONOTONIC, &result); +# elif defined(CLOCK_REALTIME) + clock_gettime(CLOCK_REALTIME, &result); +# else + clock_gettime((clocId_t)-1, &result); +# endif + return result; + } + + + /// Get the duration between two time points. + + /// @param startTime Start time point. + /// @param endTime End time point. + /// @returns the number of nanoseconds elapsed between the two time + /// points. + static uint64_t Duration(const TimePoint& startTime, + const TimePoint& endTime) __hayai_noexcept + { + TimePoint timeDiff; + + timeDiff.tv_sec = endTime.tv_sec - startTime.tv_sec; + if (endTime.tv_nsec < startTime.tv_nsec) + { + timeDiff.tv_nsec = endTime.tv_nsec + 1000000000LL - + startTime.tv_nsec; + timeDiff.tv_sec--; + } + else + timeDiff.tv_nsec = endTime.tv_nsec - startTime.tv_nsec; + + return static_cast<uint64_t>(timeDiff.tv_sec * 1000000000LL + + timeDiff.tv_nsec); + } + + + /// Clock implementation description. + + /// @returns a description of the clock implementation used. + static const char* Description() + { +# if defined(CLOCK_MONOTONIC_RAW) + return "clock_gettime(CLOCK_MONOTONIC_RAW)"; +# elif defined(CLOCK_MONOTONIC) + return "clock_gettime(CLOCK_MONOTONIC)"; +# elif defined(CLOCK_REALTIME) + return "clock_gettime(CLOCK_REALTIME)"; +# else + return "clock_gettime(-1)"; +# endif + } + }; + +// gettimeofday +# else + class Clock + { + public: + /// Time point. + + /// Opaque representation of a point in time. + typedef struct timeval TimePoint; + + + /// Get the current time as a time point. + + /// @returns the current time point. + static TimePoint Now() __hayai_noexcept + { + TimePoint result; + gettimeofday(&result, NULL); + return result; + } + + + /// Get the duration between two time points. + + /// @param startTime Start time point. + /// @param endTime End time point. + /// @returns the number of nanoseconds elapsed between the two time + /// points. + static uint64_t Duration(const TimePoint& startTime, + const TimePoint& endTime) __hayai_noexcept + { + TimePoint timeDiff; + + timeDiff.tv_sec = endTime.tv_sec - startTime.tv_sec; + if (endTime.tv_usec < startTime.tv_usec) + { + timeDiff.tv_usec = endTime.tv_usec + 1000000L - + startTime.tv_usec; + timeDiff.tv_sec--; + } + else + timeDiff.tv_usec = endTime.tv_usec - startTime.tv_usec; + + return static_cast<uint64_t>(timeDiff.tv_sec * 1000000000LL + + timeDiff.tv_usec * 1000); + } + + + /// Clock implementation description. + + /// @returns a description of the clock implementation used. + static const char* Description() + { + return "gettimeofday"; + } + }; +# endif +#endif +} +#endif diff --git a/third_party/include/hayai/hayai_compatibility.hpp b/third_party/include/hayai/hayai_compatibility.hpp new file mode 100755 index 0000000..8cb5307 --- /dev/null +++ b/third_party/include/hayai/hayai_compatibility.hpp @@ -0,0 +1,10 @@ +#ifndef __HAYAI_COMPATIBILITY +#define __HAYAI_COMPATIBILITY + +# if __cplusplus > 201100L +# define __hayai_noexcept noexcept +# else +# define __hayai_noexcept +# endif + +#endif diff --git a/third_party/include/hayai/hayai_console.hpp b/third_party/include/hayai/hayai_console.hpp new file mode 100755 index 0000000..6639890 --- /dev/null +++ b/third_party/include/hayai/hayai_console.hpp @@ -0,0 +1,199 @@ +#ifndef __HAYAI_CONSOLE +#define __HAYAI_CONSOLE + +#include <iostream> + +#if !defined(HAYAI_NO_COLOR) +# if defined(_WIN32) +# ifndef NOMINMAX +# define NOMINMAX +# endif +# include <windows.h> +# else +# include <unistd.h> +# include <cstdio> +# endif +#endif + + +namespace hayai +{ + /// Static helper class for outputting to a terminal/console. + class Console + { + public: + /// Console text colors. + enum TextColor + { + /// Default console color. Used for resets. + TextDefault, + + /// Black. + /// + /// @warning Avoid using black unless it is absolutely necesssary. + TextBlack, + + /// Blue. + TextBlue, + + /// Green. + TextGreen, + + /// Cyan. + TextCyan, + + /// Red. + TextRed, + + /// Purple. + TextPurple, + + /// Yellow. + TextYellow, + + /// White. + /// + /// @warning Avoid using white unless it is absolutely necessary. + TextWhite + }; + + + /// Get the singleton instance of @ref Console. + + /// @returns a reference to the singleton instance of the + /// benchmarker execution controller. + inline static Console& Instance() + { + static Console singleton; + return singleton; + } + + + /// Test if formatting is enabled. + inline static bool IsFormattingEnabled() + { + return Instance()._formattingEnabled; + } + + + /// Set whether formatting is enabled. + inline static void SetFormattingEnabled(bool enabled) + { + Instance()._formattingEnabled = enabled; + } + private: + inline Console() + : _formattingEnabled(true) + { + + } + + + bool _formattingEnabled; + }; + +#if defined(_WIN32) && !defined(HAYAI_NO_COLOR) // Windows + static inline WORD GetConsoleAttributes() + { + CONSOLE_SCREEN_BUFFER_INFO consoleInfo; + GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), + &consoleInfo); + return consoleInfo.wAttributes; + } + + inline std::ostream& operator <<(std::ostream& stream, + const Console::TextColor& color) + { + static const WORD defaultConsoleAttributes = + GetConsoleAttributes(); + WORD newColor; + + if ((!Console::IsFormattingEnabled()) || + ((stream.rdbuf() != std::cout.rdbuf()) && + (stream.rdbuf() != std::cerr.rdbuf()))) + return stream; + + switch(color) + { + case Console::TextDefault: + newColor = defaultConsoleAttributes; + break; + case Console::TextBlack: + newColor = 0; + break; + case Console::TextBlue: + newColor = FOREGROUND_BLUE; + break; + case Console::TextGreen: + newColor = FOREGROUND_GREEN; + break; + case Console::TextCyan: + newColor = FOREGROUND_GREEN | FOREGROUND_BLUE; + break; + case Console::TextRed: + newColor = FOREGROUND_RED; + break; + case Console::TextPurple: + newColor = FOREGROUND_RED | FOREGROUND_BLUE; + break; + case Console::TextYellow: + newColor = + FOREGROUND_RED | + FOREGROUND_GREEN | + FOREGROUND_INTENSITY; + break; + case Console::TextWhite: + newColor = + FOREGROUND_RED | + FOREGROUND_GREEN | + FOREGROUND_BLUE | + FOREGROUND_INTENSITY; + break; + } + SetConsoleTextAttribute(GetStdHandle(STD_OUTPUT_HANDLE), newColor); + return stream; + } +#elif !defined(HAYAI_NO_COLOR) // Linux or others + inline std::ostream& operator <<(std::ostream& stream, + const Console::TextColor& color) + { + static const bool outputNoColor = isatty(fileno(stdout)) != 1; + + if ((!Console::IsFormattingEnabled()) || + (outputNoColor) || + ((stream.rdbuf() != std::cout.rdbuf()) && + (stream.rdbuf() != std::cerr.rdbuf()))) + return stream; + + const char* value = ""; + switch(color) { + case Console::TextDefault: + value = "\033[m"; break; + case Console::TextBlack: + value = "\033[0;30m"; break; + case Console::TextBlue: + value = "\033[0;34m"; break; + case Console::TextGreen: + value = "\033[0;32m"; break; + case Console::TextCyan: + value = "\033[0;36m"; break; + case Console::TextRed: + value = "\033[0;31m"; break; + case Console::TextPurple: + value = "\033[0;35m"; break; + case Console::TextYellow: + value = "\033[0;33m"; break; + case Console::TextWhite: + value = "\033[0;37m"; break; + } + return stream << value; + } +#else // No color + inline std::ostream& operator <<(std::ostream& stream, + const Console::TextColor&) + { + return stream; + } +#endif +} +#endif diff --git a/third_party/include/hayai/hayai_console_outputter.hpp b/third_party/include/hayai/hayai_console_outputter.hpp new file mode 100755 index 0000000..f7cddd5 --- /dev/null +++ b/third_party/include/hayai/hayai_console_outputter.hpp @@ -0,0 +1,284 @@ +#ifndef __HAYAI_CONSOLEOUTPUTTER +#define __HAYAI_CONSOLEOUTPUTTER +#include "hayai_outputter.hpp" +#include "hayai_console.hpp" + + +namespace hayai +{ + /// Console outputter. + + /// Prints the result to standard output. + class ConsoleOutputter + : public Outputter + { + public: + /// Initialize console outputter. + + /// @param stream Output stream. Must exist for the entire duration of + /// the outputter's use. + ConsoleOutputter(std::ostream& stream = std::cout) + : _stream(stream) + { + + } + + + virtual void Begin(const std::size_t& enabledCount, + const std::size_t& disabledCount) + { + _stream << std::fixed; + _stream << Console::TextGreen << "[==========]" + << Console::TextDefault << " Running " + << enabledCount + << (enabledCount == 1 ? " benchmark." : " benchmarks"); + + if (disabledCount) + _stream << ", skipping " + << disabledCount + << (disabledCount == 1 ? + " benchmark." : + " benchmarks"); + else + _stream << "."; + + _stream << std::endl; + } + + + virtual void End(const std::size_t& executedCount, + const std::size_t& disabledCount) + { + _stream << Console::TextGreen << "[==========]" + << Console::TextDefault << " Ran " << executedCount + << (executedCount == 1 ? + " benchmark." : + " benchmarks"); + + if (disabledCount) + _stream << ", skipped " + << disabledCount + << (disabledCount == 1 ? + " benchmark." : + " benchmarks"); + else + _stream << "."; + + _stream << std::endl; + } + + + inline void BeginOrSkipTest(const std::string& fixtureName, + const std::string& testName, + const TestParametersDescriptor& parameters, + const std::size_t& runsCount, + const std::size_t& iterationsCount, + const bool skip) + { + if (skip) + _stream << Console::TextCyan << "[ DISABLED ]"; + else + _stream << Console::TextGreen << "[ RUN ]"; + + _stream << Console::TextYellow << " "; + WriteTestNameToStream(_stream, fixtureName, testName, parameters); + _stream << Console::TextDefault + << " (" << runsCount + << (runsCount == 1 ? " run, " : " runs, ") + << iterationsCount + << (iterationsCount == 1 ? + " iteration per run)" : + " iterations per run)") + << std::endl; + } + + + virtual void BeginTest(const std::string& fixtureName, + const std::string& testName, + const TestParametersDescriptor& parameters, + const std::size_t& runsCount, + const std::size_t& iterationsCount) + { + BeginOrSkipTest(fixtureName, + testName, + parameters, + runsCount, + iterationsCount, + false); + } + + + virtual void SkipDisabledTest( + const std::string& fixtureName, + const std::string& testName, + const TestParametersDescriptor& parameters, + const std::size_t& runsCount, + const std::size_t& iterationsCount + ) + { + BeginOrSkipTest(fixtureName, + testName, + parameters, + runsCount, + iterationsCount, + true); + } + + + virtual void EndTest(const std::string& fixtureName, + const std::string& testName, + const TestParametersDescriptor& parameters, + const TestResult& result) + { +#define PAD(x) _stream << std::setw(34) << x << std::endl; +#define PAD_DEVIATION(description, \ + deviated, \ + average, \ + unit) \ + { \ + double _d_ = \ + double(deviated) - double(average); \ + \ + PAD(description << \ + deviated << " " << unit << " (" << \ + (deviated < average ? \ + Console::TextRed : \ + Console::TextGreen) << \ + (deviated > average ? "+" : "") << \ + _d_ << " " << unit << " / " << \ + (deviated > average ? "+" : "") << \ + (_d_ * 100.0 / average) << " %" << \ + Console::TextDefault << ")"); \ + } +#define PAD_DEVIATION_INVERSE(description, \ + deviated, \ + average, \ + unit) \ + { \ + double _d_ = \ + double(deviated) - double(average); \ + \ + PAD(description << \ + deviated << " " << unit << " (" << \ + (deviated > average ? \ + Console::TextRed : \ + Console::TextGreen) << \ + (deviated > average ? "+" : "") << \ + _d_ << " " << unit << " / " << \ + (deviated > average ? "+" : "") << \ + (_d_ * 100.0 / average) << " %" << \ + Console::TextDefault << ")"); \ + } + + _stream << Console::TextGreen << "[ DONE ]" + << Console::TextYellow << " "; + WriteTestNameToStream(_stream, fixtureName, testName, parameters); + _stream << Console::TextDefault << " (" + << std::setprecision(6) + << (result.TimeTotal() / 1000000.0) << " ms)" + << std::endl; + + _stream << Console::TextBlue << "[ RUNS ] " + << Console::TextDefault + << " Average time: " + << std::setprecision(3) + << result.RunTimeAverage() / 1000.0 << " us " + << "(" << Console::TextBlue << "~" + << result.RunTimeStdDev() / 1000.0 << " us" + << Console::TextDefault << ")" + << std::endl; + + PAD_DEVIATION_INVERSE("Fastest time: ", + (result.RunTimeMinimum() / 1000.0), + (result.RunTimeAverage() / 1000.0), + "us"); + PAD_DEVIATION_INVERSE("Slowest time: ", + (result.RunTimeMaximum() / 1000.0), + (result.RunTimeAverage() / 1000.0), + "us"); + PAD("Median time: " << + result.RunTimeMedian() / 1000.0 << " us (" << + Console::TextCyan << "1st quartile: " << + result.RunTimeQuartile1() / 1000.0 << " us | 3rd quartile: " << + result.RunTimeQuartile3() / 1000.0 << " us" << + Console::TextDefault << ")"); + + _stream << std::setprecision(5); + + PAD(""); + PAD("Average performance: " << + result.RunsPerSecondAverage() << " runs/s"); + PAD_DEVIATION("Best performance: ", + result.RunsPerSecondMaximum(), + result.RunsPerSecondAverage(), + "runs/s"); + PAD_DEVIATION("Worst performance: ", + result.RunsPerSecondMinimum(), + result.RunsPerSecondAverage(), + "runs/s"); + PAD("Median performance: " << + result.RunsPerSecondMedian() << " runs/s (" << + Console::TextCyan << "1st quartile: " << + result.RunsPerSecondQuartile1() << " | 3rd quartile: " << + result.RunsPerSecondQuartile3() << + Console::TextDefault << ")"); + + PAD(""); + _stream << Console::TextBlue << "[ITERATIONS] " + << Console::TextDefault + << std::setprecision(3) + << " Average time: " + << result.IterationTimeAverage() / 1000.0 << " us " + << "(" << Console::TextBlue << "~" + << result.IterationTimeStdDev() / 1000.0 << " us" + << Console::TextDefault << ")" + << std::endl; + + PAD_DEVIATION_INVERSE("Fastest time: ", + (result.IterationTimeMinimum() / 1000.0), + (result.IterationTimeAverage() / 1000.0), + "us"); + PAD_DEVIATION_INVERSE("Slowest time: ", + (result.IterationTimeMaximum() / 1000.0), + (result.IterationTimeAverage() / 1000.0), + "us"); + PAD("Median time: " << + result.IterationTimeMedian() / 1000.0 << " us (" << + Console::TextCyan << "1st quartile: " << + result.IterationTimeQuartile1() / 1000.0 << + " us | 3rd quartile: " << + result.IterationTimeQuartile3() / 1000.0 << " us" << + Console::TextDefault << ")"); + + _stream << std::setprecision(5); + + PAD(""); + PAD("Average performance: " << + result.IterationsPerSecondAverage() << + " iterations/s"); + PAD_DEVIATION("Best performance: ", + (result.IterationsPerSecondMaximum()), + (result.IterationsPerSecondAverage()), + "iterations/s"); + PAD_DEVIATION("Worst performance: ", + (result.IterationsPerSecondMinimum()), + (result.IterationsPerSecondAverage()), + "iterations/s"); + PAD("Median performance: " << + result.IterationsPerSecondMedian() << " iterations/s (" << + Console::TextCyan << "1st quartile: " << + result.IterationsPerSecondQuartile1() << + " | 3rd quartile: " << + result.IterationsPerSecondQuartile3() << + Console::TextDefault << ")"); + +#undef PAD_DEVIATION_INVERSE +#undef PAD_DEVIATION +#undef PAD + } + + + std::ostream& _stream; + }; +} +#endif diff --git a/third_party/include/hayai/hayai_default_test_factory.hpp b/third_party/include/hayai/hayai_default_test_factory.hpp new file mode 100755 index 0000000..34f66ac --- /dev/null +++ b/third_party/include/hayai/hayai_default_test_factory.hpp @@ -0,0 +1,27 @@ +#ifndef __HAYAI_DEFAULTTESTFACTORY +#define __HAYAI_DEFAULTTESTFACTORY +#include "hayai_test_factory.hpp" + +namespace hayai +{ + /// Default test factory implementation. + + /// Simply constructs an instance of a the test of class @ref T with no + /// constructor parameters. + /// + /// @tparam T Test class. + template<class T> + class TestFactoryDefault + : public TestFactory + { + public: + /// Create a test instance with no constructor parameters. + + /// @returns a pointer to an initialized test. + virtual Test* CreateTest() + { + return new T(); + } + }; +} +#endif diff --git a/third_party/include/hayai/hayai_fixture.hpp b/third_party/include/hayai/hayai_fixture.hpp new file mode 100755 index 0000000..315c234 --- /dev/null +++ b/third_party/include/hayai/hayai_fixture.hpp @@ -0,0 +1,9 @@ +#ifndef __HAYAI_FIXTURE +#define __HAYAI_FIXTURE +#include "hayai_test.hpp" + +namespace hayai +{ + typedef Test Fixture; +} +#endif diff --git a/third_party/include/hayai/hayai_json_outputter.hpp b/third_party/include/hayai/hayai_json_outputter.hpp new file mode 100755 index 0000000..0f172a5 --- /dev/null +++ b/third_party/include/hayai/hayai_json_outputter.hpp @@ -0,0 +1,355 @@ +#ifndef __HAYAI_JSONOUTPUTTER +#define __HAYAI_JSONOUTPUTTER +#include <iomanip> +#include <ostream> + +#include "hayai_outputter.hpp" + + +#define JSON_OBJECT_BEGIN "{" +#define JSON_OBJECT_END "}" +#define JSON_ARRAY_BEGIN "[" +#define JSON_ARRAY_END "]" +#define JSON_STRING_BEGIN "\"" +#define JSON_STRING_END "\"" +#define JSON_NAME_SEPARATOR ":" +#define JSON_VALUE_SEPARATOR "," +#define JSON_TRUE "true" +#define JSON_FALSE "false" + +namespace hayai +{ + /// JSON outputter. + + /// Outputs the result of benchmarks in JSON format with the following + /// structure: + /// + /// { + /// "format_version": 1, + /// "benchmarks": [{ + /// "fixture": "DeliveryMan", + /// "name": "DeliverPackage", + /// "parameters": { + /// "declaration": "std::size_t distance", + /// "value": "1" + /// }, + /// "iterations_per_run": 10, + /// "disabled": false, + /// "runs": [{ + /// "duration": 3801.889831 + /// }, ..] + /// }, { + /// "fixture": "DeliveryMan", + /// "name": "DisabledTest", + /// "iterations_per_run": 10, + /// "disabled": true + /// }, ..] + /// } + /// + /// All durations are represented as milliseconds. + class JsonOutputter + : public Outputter + { + public: + /// Initialize JSON outputter. + + /// @param stream Output stream. Must exist for the entire duration of + /// the outputter's use. + JsonOutputter(std::ostream& stream) + : _stream(stream), + _firstTest(true) + { + + } + + + virtual void Begin(const std::size_t& enabledCount, + const std::size_t& disabledCount) + { + (void)enabledCount; + (void)disabledCount; + + _stream << + JSON_OBJECT_BEGIN + + JSON_STRING_BEGIN "format_version" JSON_STRING_END + JSON_NAME_SEPARATOR + "1" + + JSON_VALUE_SEPARATOR + + JSON_STRING_BEGIN "benchmarks" JSON_STRING_END + JSON_NAME_SEPARATOR + JSON_ARRAY_BEGIN; + } + + + virtual void End(const std::size_t& executedCount, + const std::size_t& disabledCount) + { + (void)executedCount; + (void)disabledCount; + + _stream << + JSON_ARRAY_END + JSON_OBJECT_END; + } + + + virtual void BeginTest(const std::string& fixtureName, + const std::string& testName, + const TestParametersDescriptor& parameters, + const std::size_t& runsCount, + const std::size_t& iterationsCount) + { + BeginTestObject(fixtureName, + testName, + parameters, + runsCount, + iterationsCount, + false); + } + + + virtual void SkipDisabledTest(const std::string& fixtureName, + const std::string& testName, + const TestParametersDescriptor& parameters, + const std::size_t& runsCount, + const std::size_t& iterationsCount) + { + BeginTestObject(fixtureName, + testName, + parameters, + runsCount, + iterationsCount, + true); + EndTestObject(); + } + + + virtual void EndTest(const std::string& fixtureName, + const std::string& testName, + const TestParametersDescriptor& parameters, + const TestResult& result) + { + (void)fixtureName; + (void)testName; + (void)parameters; + + _stream << + JSON_VALUE_SEPARATOR + + JSON_STRING_BEGIN "runs" JSON_STRING_END + JSON_NAME_SEPARATOR + JSON_ARRAY_BEGIN; + + const std::vector<uint64_t>& runTimes = result.RunTimes(); + + for (std::vector<uint64_t>::const_iterator it = runTimes.begin(); + it != runTimes.end(); + ++it) + { + if (it != runTimes.begin()) + _stream << JSON_VALUE_SEPARATOR; + + _stream << JSON_OBJECT_BEGIN + JSON_STRING_BEGIN "duration" JSON_STRING_END + JSON_NAME_SEPARATOR + << std::fixed + << std::setprecision(6) + << (double(*it) / 1000000.0) + << JSON_OBJECT_END; + } + + _stream << + JSON_ARRAY_END; + + WriteDoubleProperty("mean", result.RunTimeAverage()); + WriteDoubleProperty("std_dev", result.RunTimeStdDev()); + WriteDoubleProperty("median", result.RunTimeMedian()); + WriteDoubleProperty("quartile_1", result.RunTimeQuartile1()); + WriteDoubleProperty("quartile_3", result.RunTimeQuartile3()); + + EndTestObject(); + } + private: + void BeginTestObject(const std::string& fixtureName, + const std::string& testName, + const TestParametersDescriptor& parameters, + const std::size_t& runsCount, + const std::size_t& iterationsCount, + bool disabled) + { + (void)runsCount; + + if (_firstTest) + _firstTest = false; + else + _stream << JSON_VALUE_SEPARATOR; + + _stream << + JSON_OBJECT_BEGIN + + JSON_STRING_BEGIN "fixture" JSON_STRING_END + JSON_NAME_SEPARATOR; + + WriteString(fixtureName); + + _stream << + JSON_VALUE_SEPARATOR + + JSON_STRING_BEGIN "name" JSON_STRING_END + JSON_NAME_SEPARATOR; + + WriteString(testName); + + _stream << + JSON_VALUE_SEPARATOR; + + const std::vector<TestParameterDescriptor>& descs = + parameters.Parameters(); + + if (!descs.empty()) + { + _stream << + JSON_STRING_BEGIN "parameters" JSON_STRING_END + JSON_NAME_SEPARATOR + JSON_ARRAY_BEGIN; + + for (std::size_t i = 0; i < descs.size(); ++i) + { + if (i) + _stream << JSON_VALUE_SEPARATOR; + + const TestParameterDescriptor& desc = descs[i]; + + _stream << + JSON_OBJECT_BEGIN + + JSON_STRING_BEGIN "declaration" JSON_STRING_END + JSON_NAME_SEPARATOR; + + WriteString(desc.Declaration); + + _stream << + JSON_VALUE_SEPARATOR + + JSON_STRING_BEGIN "value" JSON_STRING_END + JSON_NAME_SEPARATOR; + + WriteString(desc.Value); + + _stream << + JSON_OBJECT_END; + } + + _stream << + JSON_ARRAY_END + JSON_VALUE_SEPARATOR; + } + + _stream << + JSON_STRING_BEGIN "iterations_per_run" JSON_STRING_END + JSON_NAME_SEPARATOR << iterationsCount << + + JSON_VALUE_SEPARATOR + + JSON_STRING_BEGIN "disabled" JSON_STRING_END + JSON_NAME_SEPARATOR << (disabled ? JSON_TRUE : JSON_FALSE); + } + + + inline void EndTestObject() + { + _stream << + JSON_OBJECT_END; + } + + + /// Write an escaped string. + + /// The escaping is currently very rudimentary and assumes that names, + /// parameters etc. are ASCII. + /// + /// @param str String to write. + void WriteString(const std::string& str) + { + _stream << JSON_STRING_BEGIN; + + std::string::const_iterator it = str.begin(); + while (it != str.end()) + { + char c = *it++; + + switch (c) + { + case '\\': + case '"': + case '/': + _stream << "\\" << c; + break; + + case '\b': + _stream << "\\b"; + break; + + case '\f': + _stream << "\\f"; + break; + + case '\n': + _stream << "\\n"; + break; + + case '\r': + _stream << "\\r"; + break; + + case '\t': + _stream << "\\t"; + break; + + default: + _stream << c; + break; + } + } + + _stream << JSON_STRING_END; + } + + + /// Write a property with a double value. + + /// @param key Property key. + /// @param value Property value. + void WriteDoubleProperty(const std::string& key, const double value) + { + _stream << JSON_VALUE_SEPARATOR + << JSON_STRING_BEGIN + << key + << JSON_STRING_END + << JSON_NAME_SEPARATOR + << std::fixed + << std::setprecision(6) + << (value / 1000000.0); + } + + + std::ostream& _stream; + bool _firstTest; + }; +} + +#undef JSON_OBJECT_BEGIN +#undef JSON_OBJECT_END +#undef JSON_ARRAY_BEGIN +#undef JSON_ARRAY_END +#undef JSON_STRING_BEGIN +#undef JSON_STRING_END +#undef JSON_NAME_SEPARATOR +#undef JSON_VALUE_SEPARATOR +#undef JSON_TRUE +#undef JSON_FALSE + +#endif diff --git a/third_party/include/hayai/hayai_junit_xml_outputter.hpp b/third_party/include/hayai/hayai_junit_xml_outputter.hpp new file mode 100755 index 0000000..e7e1164 --- /dev/null +++ b/third_party/include/hayai/hayai_junit_xml_outputter.hpp @@ -0,0 +1,260 @@ +#ifndef __HAYAI_JUNITXMLOUTPUTTER +#define __HAYAI_JUNITXMLOUTPUTTER +#include <iomanip> +#include <ostream> +#include <vector> +#include <sstream> +#include <map> + +#include "hayai_outputter.hpp" + + +namespace hayai +{ + /// JUnit-compatible XML outputter. + class JUnitXmlOutputter + : public Outputter + { + private: + /// Test case. + class TestCase + { + public: + TestCase(const std::string& fixtureName, + const std::string& testName, + const TestParametersDescriptor& parameters, + const TestResult* result) + { + // Derive a pretty name. + std::stringstream nameStream; + WriteTestNameToStream(nameStream, + fixtureName, + testName, + parameters); + Name = nameStream.str(); + + // Derive the result. + Skipped = !result; + + if (result) + { + std::stringstream timeStream; + timeStream << std::fixed + << std::setprecision(9) + << (result->IterationTimeAverage() / 1e9); + Time = timeStream.str(); + } + } + + + std::string Name; + std::string Time; + bool Skipped; + }; + + + /// Test suite map. + typedef std::map<std::string, std::vector<TestCase> > TestSuiteMap; + public: + /// Initialize outputter. + + /// @param stream Output stream. Must exist for the entire duration of + /// the outputter's use. + JUnitXmlOutputter(std::ostream& stream) + : _stream(stream) + { + + } + + + virtual void Begin(const std::size_t& enabledCount, + const std::size_t& disabledCount) + { + (void)enabledCount; + (void)disabledCount; + } + + + virtual void End(const std::size_t& executedCount, + const std::size_t& disabledCount) + { + (void)executedCount; + (void)disabledCount; + + // Write the header. + _stream << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + << std::endl + << "<testsuites>" << std::endl; + + // Write out each test suite (fixture.) + for (TestSuiteMap::iterator testSuiteIt = _testSuites.begin(); + testSuiteIt != _testSuites.end(); + ++testSuiteIt) + { + _stream << " <testsuite name=\"" << testSuiteIt->first + << "\" tests=\"" << testSuiteIt->second.size() << "\">" + << std::endl; + + // Write out each test case. + for (std::vector<TestCase>::iterator testCaseIt = + testSuiteIt->second.begin(); + testCaseIt != testSuiteIt->second.end(); + ++testCaseIt) + { + _stream << " <testcase name=\""; + WriteEscapedString(testCaseIt->Name); + _stream << "\""; + + if (!testCaseIt->Skipped) + _stream << " time=\"" << testCaseIt->Time << "\" />" + << std::endl; + else + { + _stream << ">" << std::endl + << " <skipped />" << std::endl + << " </testcase>" << std::endl; + } + } + + _stream << " </testsuite>" << std::endl; + } + + _stream << "</testsuites>" << std::endl; + } + + + virtual void BeginTest(const std::string& fixtureName, + const std::string& testName, + const TestParametersDescriptor& parameters, + const std::size_t& runsCount, + const std::size_t& iterationsCount) + { + (void)fixtureName; + (void)testName; + (void)parameters; + (void)runsCount; + (void)iterationsCount; + } + + + virtual void SkipDisabledTest(const std::string& fixtureName, + const std::string& testName, + const TestParametersDescriptor& parameters, + const std::size_t& runsCount, + const std::size_t& iterationsCount) + { + (void)fixtureName; + (void)testName; + (void)parameters; + (void)runsCount; + (void)iterationsCount; + } + + + virtual void EndTest(const std::string& fixtureName, + const std::string& testName, + const TestParametersDescriptor& parameters, + const TestResult& result) + { + (void)fixtureName; + (void)testName; + (void)parameters; + + TestSuiteMap::iterator fixtureIt = + _testSuites.find(fixtureName); + if (fixtureIt == _testSuites.end()) + { + _testSuites[fixtureName] = std::vector<TestCase>(); + fixtureIt = _testSuites.find(fixtureName); + } + + std::vector<TestCase>& testCases = fixtureIt->second; + + testCases.push_back(TestCase(fixtureName, + testName, + parameters, + &result)); + + /* + _stream << + JSON_VALUE_SEPARATOR + + JSON_STRING_BEGIN "runs" JSON_STRING_END + JSON_NAME_SEPARATOR + JSON_ARRAY_BEGIN; + + const std::vector<uint64_t>& runTimes = result.RunTimes(); + + for (std::vector<uint64_t>::const_iterator it = runTimes.begin(); + it != runTimes.end(); + ++it) + { + if (it != runTimes.begin()) + _stream << JSON_VALUE_SEPARATOR; + + _stream << JSON_OBJECT_BEGIN + + JSON_STRING_BEGIN "duration" JSON_STRING_END + JSON_NAME_SEPARATOR + << std::fixed + << std::setprecision(6) + << (double(*it) / 1000000.0) + << JSON_OBJECT_END; + } + + _stream << + JSON_ARRAY_END; + + EndTestObject(); + */ + } + private: + /// Write an escaped string. + + /// The escaping is currently very rudimentary and assumes that names, + /// parameters etc. are ASCII. + /// + /// @param str String to write. + void WriteEscapedString(const std::string& str) + { + std::string::const_iterator it = str.begin(); + while (it != str.end()) + { + char c = *it++; + + switch (c) + { + case '"': + _stream << """; + break; + + case '\'': + _stream << "'"; + break; + + case '<': + _stream << "<"; + break; + + case '>': + _stream << ">"; + break; + + case '&': + _stream << "&"; + break; + + default: + _stream << c; + break; + } + } + } + + + std::ostream& _stream; + TestSuiteMap _testSuites; + }; +} + +#endif diff --git a/third_party/include/hayai/hayai_main.hpp b/third_party/include/hayai/hayai_main.hpp new file mode 100755 index 0000000..d6d7cd3 --- /dev/null +++ b/third_party/include/hayai/hayai_main.hpp @@ -0,0 +1,530 @@ +#ifndef __HAYAI_MAIN +#define __HAYAI_MAIN +#include <algorithm> +#include <cstdlib> +#include <cstring> +#include <ctime> +#include <errno.h> +#include <fstream> +#include <set> +#include <vector> + +#include "hayai.hpp" + + +#if defined(_WIN32) +# define PATH_SEPARATOR '\\' +#else +# define PATH_SEPARATOR '/' +#endif + + +#define HAYAI_MAIN_FORMAT_FLAG(_desc) \ + ::hayai::Console::TextGreen << _desc << ::hayai::Console::TextDefault +#define HAYAI_MAIN_FORMAT_ARGUMENT(_desc) \ + ::hayai::Console::TextYellow << _desc << ::hayai::Console::TextDefault +#define HAYAI_MAIN_FORMAT_ERROR(_desc) \ + ::hayai::Console::TextRed << "Error:" << \ + ::hayai::Console::TextDefault << " " << _desc +#define HAYAI_MAIN_USAGE_ERROR(_desc) \ + { \ + std::cerr << HAYAI_MAIN_FORMAT_ERROR(_desc) << std::endl \ + << std::endl; \ + ShowUsage(argv[0]); \ + return EXIT_FAILURE; \ + } + + +namespace hayai +{ + /// Execution mode. + enum MainExecutionMode + { + /// Run benchmarks. + MainRunBenchmarks, + + + /// List benchmarks but do not execute them. + MainListBenchmarks + }; + + + /// File outputter. + class FileOutputter + { + public: + /// File outputter. + + /// @param path Output path. Expected to be available during the life + /// time of the outputter. + FileOutputter(const char* path) + : _path(path), + _outputter(NULL) + { + + } + + + virtual ~FileOutputter() + { + if (_outputter) + delete _outputter; + + _stream.close(); + } + + + /// Set up. + + /// Opens the output file for writing and initializes the outputter. + virtual void SetUp() + { + _stream.open(_path, + std::ios_base::out | + std::ios_base::trunc | + std::ios_base::binary); + if (_stream.bad()) + { + std::stringstream error; + error << "failed to open " << _path << " for writing: " + << strerror(errno); + throw std::runtime_error(error.str()); + } + + _outputter = CreateOutputter(_stream); + } + + + /// Outputter. + virtual ::hayai::Outputter& Outputter() + { + if (!_outputter) + throw std::runtime_error("outputter has not been set up"); + + return *_outputter; + } + protected: + /// Create outputter from output stream. + + /// @param stream Output stream for the outputter. + /// @returns the outputter for the given format. + virtual ::hayai::Outputter* CreateOutputter(std::ostream& stream) = 0; + private: + const char* _path; + std::ofstream _stream; + ::hayai::Outputter* _outputter; + }; + + +#define FILE_OUTPUTTER_IMPLEMENTATION(_prefix) \ + class _prefix ## FileOutputter \ + : public FileOutputter \ + { \ + public: \ + _prefix ## FileOutputter(const char* path) \ + : FileOutputter(path) \ + {} \ + protected: \ + virtual ::hayai::Outputter* CreateOutputter(std::ostream& stream) \ + { \ + return new ::hayai::_prefix ## Outputter(stream); \ + } \ + } + + + FILE_OUTPUTTER_IMPLEMENTATION(Json); + FILE_OUTPUTTER_IMPLEMENTATION(Console); + FILE_OUTPUTTER_IMPLEMENTATION(JUnitXml); + +#undef FILE_OUTPUTTER_IMPLEMENTATION + + + /// Default main executable runner for Hayai. + class MainRunner + { + public: + MainRunner() + : ExecutionMode(MainRunBenchmarks), + ShuffleBenchmarks(false), + StdoutOutputter(NULL) + { + + } + + + ~MainRunner() + { + // Clean up the outputters. + for (std::vector<FileOutputter*>::iterator it = + FileOutputters.begin(); + it != FileOutputters.end(); + ++it) + delete *it; + + if (StdoutOutputter) + delete StdoutOutputter; + } + + + /// Execution mode. + MainExecutionMode ExecutionMode; + + + /// Shuffle benchmarks. + bool ShuffleBenchmarks; + + + /// File outputters. + /// + /// Outputter will be freed by the class on destruction. + std::vector<FileOutputter*> FileOutputters; + + + /// Standard output outputter. + /// + /// Will be freed by the class on destruction. + Outputter* StdoutOutputter; + + + /// Parse arguments. + + /// @param argc Argument count including the executable name. + /// @param argv Arguments. + /// @param residualArgs Pointer to vector to hold residual arguments + /// after parsing. If not NULL, the parser will not fail upon + /// encounting an unknown argument but will instead add it to the list + /// of residual arguments and return a success code. Note: the parser + /// will still fail if an invalid value is provided to a known + /// argument. + /// @returns 0 on success, otherwise the exit status code to be + /// returned from the executable. + int ParseArgs(int argc, + char** argv, + std::vector<char*>* residualArgs = NULL) + { + int argI = 1; + while (argI < argc) + { + char* arg = argv[argI++]; + bool argLast = (argI == argc); + std::size_t argLen = strlen(arg); + + if (argLen == 0) + continue; + + // List flag. + if ((!strcmp(arg, "-l")) || (!strcmp(arg, "--list"))) + ExecutionMode = ::hayai::MainListBenchmarks; + // Shuffle flag. + else if ((!strcmp(arg, "-s")) || (!strcmp(arg, "--shuffle"))) + ShuffleBenchmarks = true; + // Filter flag. + else if ((!strcmp(arg, "-f")) || (!strcmp(arg, "--filter"))) + { + if ((argLast) || (*argv[argI] == 0)) + HAYAI_MAIN_USAGE_ERROR(HAYAI_MAIN_FORMAT_FLAG(arg) << + " requires a pattern to be specified"); + char* pattern = argv[argI++]; + + ::hayai::Benchmarker::ApplyPatternFilter(pattern); + } + // Output flag. + else if ((!strcmp(arg, "-o")) || (!strcmp(arg, "--output"))) + { + if (argLast) + HAYAI_MAIN_USAGE_ERROR(HAYAI_MAIN_FORMAT_FLAG(arg) << + " requires a format to be specified"); + char* formatSpecifier = argv[argI++]; + + char* format = formatSpecifier; + char* path = strchr(formatSpecifier, ':'); + if (path) + { + *(path++) = 0; + if (!strlen(path)) + path = NULL; + } + +#define ADD_OUTPUTTER(_prefix) \ + { \ + if (path) \ + FileOutputters.push_back( \ + new ::hayai::_prefix ## FileOutputter(path) \ + ); \ + else \ + { \ + if (StdoutOutputter) \ + delete StdoutOutputter; \ + StdoutOutputter = \ + new ::hayai::_prefix ## Outputter(std::cout); \ + } \ + } + + if (!strcmp(format, "console")) + ADD_OUTPUTTER(Console) + else if (!strcmp(format, "json")) + ADD_OUTPUTTER(Json) + else if (!strcmp(format, "junit")) + ADD_OUTPUTTER(JUnitXml) + else + HAYAI_MAIN_USAGE_ERROR("invalid format: " << format); + +#undef ADD_OUTPUTTER + } + // Console coloring flag. + else if ((!strcmp(arg, "-c")) || (!strcmp(arg, "--color"))) + { + if (argLast) + HAYAI_MAIN_USAGE_ERROR( + HAYAI_MAIN_FORMAT_FLAG(arg) << + " requires an argument " << + "of either " << HAYAI_MAIN_FORMAT_FLAG("yes") << + " or " << HAYAI_MAIN_FORMAT_FLAG("no") + ); + + char* choice = argv[argI++]; + bool enabled; + + if ((!strcmp(choice, "yes")) || + (!strcmp(choice, "true")) || + (!strcmp(choice, "on")) || + (!strcmp(choice, "1"))) + enabled = true; + else if ((!strcmp(choice, "no")) || + (!strcmp(choice, "false")) || + (!strcmp(choice, "off")) || + (!strcmp(choice, "0"))) + enabled = false; + else + HAYAI_MAIN_USAGE_ERROR( + "invalid argument to " << + HAYAI_MAIN_FORMAT_FLAG(arg) << + ": " << choice + ); + + ::hayai::Console::SetFormattingEnabled(enabled); + } + // Help. + else if ((!strcmp(arg, "-?")) || + (!strcmp(arg, "-h")) || + (!strcmp(arg, "--help"))) + { + ShowUsage(argv[0]); + return EXIT_FAILURE; + } + else if (residualArgs) + { + residualArgs->push_back(arg); + } + else + { + HAYAI_MAIN_USAGE_ERROR("unknown option: " << arg); + } + } + + return EXIT_SUCCESS; + } + + + /// Run the selected execution mode. + + /// @returns the exit status code to be returned from the executable. + int Run() + { + // Execute based on the selected mode. + switch (ExecutionMode) + { + case ::hayai::MainRunBenchmarks: + return RunBenchmarks(); + + case ::hayai::MainListBenchmarks: + return ListBenchmarks(); + + default: + std::cerr << HAYAI_MAIN_FORMAT_ERROR( + "invalid execution mode: " << ExecutionMode + ) << std::endl; + return EXIT_FAILURE; + } + } + private: + /// Run benchmarks. + + /// @returns the exit status code to be returned from the executable. + int RunBenchmarks() + { + // Hook up the outputs. + if (StdoutOutputter) + ::hayai::Benchmarker::AddOutputter(*StdoutOutputter); + + for (std::vector< ::hayai::FileOutputter*>::iterator it = + FileOutputters.begin(); + it < FileOutputters.end(); + ++it) + { + ::hayai::FileOutputter& fileOutputter = **it; + + try + { + fileOutputter.SetUp(); + } + catch (std::exception& e) + { + std::cerr << HAYAI_MAIN_FORMAT_ERROR(e.what()) << std::endl; + return EXIT_FAILURE; + } + + ::hayai::Benchmarker::AddOutputter(fileOutputter.Outputter()); + } + + // Run the benchmarks. + if (ShuffleBenchmarks) + { + std::srand(static_cast<unsigned>(std::time(0))); + ::hayai::Benchmarker::ShuffleTests(); + } + + ::hayai::Benchmarker::RunAllTests(); + + return EXIT_SUCCESS; + } + + + /// List benchmarks. + + /// @returns the exit status code to be returned from the executable. + int ListBenchmarks() + { + // List out the unique benchmark names. + std::vector<const ::hayai::TestDescriptor*> tests = + ::hayai::Benchmarker::ListTests(); + std::vector<std::string> testNames; + std::set<std::string> uniqueTestNames; + + for (std::vector<const ::hayai::TestDescriptor*>::iterator it = + tests.begin(); + it < tests.end(); + ++it) + { + if (uniqueTestNames.find((*it)->CanonicalName) != + uniqueTestNames.end()) + continue; + + testNames.push_back((*it)->CanonicalName); + uniqueTestNames.insert((*it)->CanonicalName); + } + + // Sort the benchmark names. + std::sort(testNames.begin(), testNames.end()); + + // Dump the list. + for (std::vector<std::string>::iterator it = testNames.begin(); + it < testNames.end(); + ++it) + std::cout << *it << std::endl; + + return EXIT_SUCCESS; + } + + + /// Show usage. + + /// @param execName Executable name. + void ShowUsage(const char* execName) + { + const char* baseName = strrchr(execName, PATH_SEPARATOR); + + std::cerr << "Usage: " << (baseName ? baseName + 1 : execName) + << " " << HAYAI_MAIN_FORMAT_FLAG("[OPTIONS]") << std::endl + << std::endl + + << " Runs the benchmarks for this project." << std::endl + << std::endl + + << "Benchmark selection options:" << std::endl + << " " << HAYAI_MAIN_FORMAT_FLAG("-l") << ", " + << HAYAI_MAIN_FORMAT_FLAG("--list") + << std::endl + << " List the names of all benchmarks instead of " + << "running them." << std::endl + << " " << HAYAI_MAIN_FORMAT_FLAG("-f") << ", " + << HAYAI_MAIN_FORMAT_FLAG("--filter") + << " <" << HAYAI_MAIN_FORMAT_ARGUMENT("pattern") << ">" + << std::endl + << " Run only the tests whose name matches one of the " + << "positive patterns but" << std::endl + << " none of the negative patterns. '?' matches any " + << "single character; '*'" << std::endl + << " matches any substring; ':' separates two " + << "patterns." + << std::endl + + << "Benchmark execution options:" << std::endl + << " " << HAYAI_MAIN_FORMAT_FLAG("-s") << ", " + << HAYAI_MAIN_FORMAT_FLAG("--shuffle") + << std::endl + << " Randomize benchmark execution order." + << std::endl + << std::endl + + << "Benchmark output options:" << std::endl + << " " << HAYAI_MAIN_FORMAT_FLAG("-o") << ", " + << HAYAI_MAIN_FORMAT_FLAG("--output") + << " <" << HAYAI_MAIN_FORMAT_ARGUMENT("format") << ">[:" + << HAYAI_MAIN_FORMAT_ARGUMENT("<path>") << "]" + << std::endl + << " Output results in a specific format. If no " + << "path is specified, the output" << std::endl + << " will be presented on stdout. Can be specified " + << "multiple times to get output" << std::endl + << " in different formats. The supported formats are:" + << std::endl + << std::endl + << " " << HAYAI_MAIN_FORMAT_ARGUMENT("console") + << std::endl + << " Standard console output." << std::endl + << " " << HAYAI_MAIN_FORMAT_ARGUMENT("json") + << std::endl + << " JSON." << std::endl + << " " << HAYAI_MAIN_FORMAT_ARGUMENT("junit") + << std::endl + << " JUnit-compatible XML (very restrictive.)" + << std::endl + << std::endl + << " If multiple output formats are provided without " + << "a path, only the last" << std::endl + << " provided format will be output to stdout." + << std::endl + << " " << HAYAI_MAIN_FORMAT_FLAG("--c") << ", " + << HAYAI_MAIN_FORMAT_FLAG("--color") << " (" + << ::hayai::Console::TextGreen << "yes" + << ::hayai::Console::TextDefault << "|" + << ::hayai::Console::TextGreen << "no" + << ::hayai::Console::TextDefault << ")" << std::endl + << " Enable colored output when available. Default " + << ::hayai::Console::TextGreen << "yes" + << ::hayai::Console::TextDefault << "." << std::endl + << std::endl + + << "Miscellaneous options:" << std::endl + << " " << HAYAI_MAIN_FORMAT_FLAG("-?") << ", " + << HAYAI_MAIN_FORMAT_FLAG("-h") << ", " + << HAYAI_MAIN_FORMAT_FLAG("--help") << std::endl + << " Show this help information." << std::endl + << std::endl + + << "hayai version: " << HAYAI_VERSION << std::endl + << "Clock implementation: " + << ::hayai::Clock::Description() + << std::endl; + } + }; +} + + +#undef HAYAI_MAIN_FORMAT_FLAG +#undef HAYAI_MAIN_FORMAT_ARGUMENT +#undef HAYAI_MAIN_FORMAT_ERROR +#undef HAYAI_MAIN_USAGE_ERROR + +#endif diff --git a/third_party/include/hayai/hayai_outputter.hpp b/third_party/include/hayai/hayai_outputter.hpp new file mode 100755 index 0000000..94055b2 --- /dev/null +++ b/third_party/include/hayai/hayai_outputter.hpp @@ -0,0 +1,113 @@ +#ifndef __HAYAI_OUTPUTTER +#define __HAYAI_OUTPUTTER +#include <iostream> +#include <cstddef> + +#include "hayai_test_result.hpp" + + +namespace hayai +{ + /// Outputter. + + /// Abstract base class for outputters. + class Outputter + { + public: + /// Begin benchmarking. + + /// The total number of benchmarks registred is the sum of the two + /// counts passed to the outputter. + /// + /// @param enabledCount Number of benchmarks to be executed. + /// @param disabledCount Number of disabled benchmarks to be skipped. + virtual void Begin(const std::size_t& enabledCount, + const std::size_t& disabledCount) = 0; + + + /// End benchmarking. + + /// @param executedCount Number of benchmarks that have been executed. + /// @param disabledCount Number of benchmarks that have been skipped + /// because they are disabled. + virtual void End(const std::size_t& executedCount, + const std::size_t& disabledCount) = 0; + + + /// Begin benchmark test run. + + /// @param fixtureName Fixture name. + /// @param testName Test name. + /// @param parameters Test parameter description. + /// @param runsCount Number of runs to be executed. + /// @param iterationsCount Number of iterations per run. + virtual void BeginTest(const std::string& fixtureName, + const std::string& testName, + const TestParametersDescriptor& parameters, + const std::size_t& runsCount, + const std::size_t& iterationsCount) = 0; + + + /// End benchmark test run. + + /// @param fixtureName Fixture name. + /// @param testName Test name. + /// @param parameters Test parameter description. + /// @param result Test result. + virtual void EndTest(const std::string& fixtureName, + const std::string& testName, + const TestParametersDescriptor& parameters, + const TestResult& result) = 0; + + + /// Skip disabled benchmark test run. + + /// @param fixtureName Fixture name. + /// @param testName Test name. + /// @param parameters Test parameter description. + /// @param runsCount Number of runs to be executed. + /// @param iterationsCount Number of iterations per run. + virtual void SkipDisabledTest(const std::string& fixtureName, + const std::string& testName, + const TestParametersDescriptor& + parameters, + const std::size_t& runsCount, + const std::size_t& iterationsCount) = 0; + + + virtual ~Outputter() + { + + } + protected: + /// Write a nicely formatted test name to a stream. + static void WriteTestNameToStream(std::ostream& stream, + const std::string& fixtureName, + const std::string& testName, + const TestParametersDescriptor& + parameters) + { + stream << fixtureName << "." << testName; + + const std::vector<TestParameterDescriptor>& descs = + parameters.Parameters(); + + if (descs.empty()) + return; + + stream << "("; + + for (std::size_t i = 0; i < descs.size(); ++i) + { + if (i) + stream << ", "; + + const TestParameterDescriptor& desc = descs[i]; + stream << desc.Declaration << " = " << desc.Value; + } + + stream << ")"; + } + }; +} +#endif diff --git a/third_party/include/hayai/hayai_test.hpp b/third_party/include/hayai/hayai_test.hpp new file mode 100755 index 0000000..36d25fd --- /dev/null +++ b/third_party/include/hayai/hayai_test.hpp @@ -0,0 +1,83 @@ +#ifndef __HAYAI_TEST +#define __HAYAI_TEST +#include <cstddef> + +#include "hayai_clock.hpp" +#include "hayai_test_result.hpp" + + +namespace hayai +{ + /// Base test class. + + /// @ref SetUp is invoked before each run, and @ref TearDown is invoked + /// once the run is finished. Iterations rely on the same fixture + /// for every run. + /// + /// The default test class does not contain any actual code in the + /// SetUp and TearDown methods, which means that tests can inherit + /// this class directly for non-fixture based benchmarking tests. + class Test + { + public: + /// Set up the testing fixture for execution of a run. + virtual void SetUp() + { + + } + + + /// Tear down the previously set up testing fixture after the + /// execution run. + virtual void TearDown() + { + + } + + + /// Run the test. + + /// @param iterations Number of iterations to gather data for. + /// @returns the number of nanoseconds the run took. + uint64_t Run(std::size_t iterations) + { + std::size_t iteration = iterations; + + // Set up the testing fixture. + SetUp(); + + // Get the starting time. + Clock::TimePoint startTime, endTime; + + startTime = Clock::Now(); + + // Run the test body for each iteration. + while (iteration--) + TestBody(); + + // Get the ending time. + endTime = Clock::Now(); + + // Tear down the testing fixture. + TearDown(); + + // Return the duration in nanoseconds. + return Clock::Duration(startTime, endTime); + } + + + virtual ~Test() + { + + } + protected: + /// Test body. + + /// Executed for each iteration the benchmarking test is run. + virtual void TestBody() + { + + } + }; +} +#endif diff --git a/third_party/include/hayai/hayai_test_descriptor.hpp b/third_party/include/hayai/hayai_test_descriptor.hpp new file mode 100755 index 0000000..529744e --- /dev/null +++ b/third_party/include/hayai/hayai_test_descriptor.hpp @@ -0,0 +1,365 @@ +#ifndef __HAYAI_TESTDESCRIPTOR +#define __HAYAI_TESTDESCRIPTOR +#include <cstring> +#include <sstream> +#include <string> +#include <vector> + +#include "hayai_test.hpp" +#include "hayai_test_factory.hpp" + + +namespace hayai +{ + /// Parameter declaration. + + /// Describes parameter type and name. + class TestParameterDescriptor + { + public: + TestParameterDescriptor(std::string declaration, + std::string value) + : Declaration(declaration), + Value(value) + { + + } + + + /// Declaration. + std::string Declaration; + + + /// Value. + std::string Value; + }; + + + /// Test parameters descriptor. + class TestParametersDescriptor + { + private: + /// Quoting state. + enum QuotingState + { + /// Unquoted. + Unquoted, + + + /// Single quoted. + SingleQuoted, + + + /// Double quoted. + DoubleQuoted + }; + + + /// Trimmed string. + + /// @param start Start character. + /// @param end Character one position beyond end. + inline static std::string TrimmedString(const char* start, + const char* end) + { + while (start < end) + { + if ((*start == ' ') || + (*start == '\r') || + (*start == '\n') || + (*start == '\t')) + ++start; + else + break; + } + + while (end > start) + { + const char c = *(end - 1); + + if ((c != ' ') && + (c != '\r') && + (c != '\n') && + (c != '\t')) + break; + + --end; + } + + return std::string(start, std::string::size_type(end - start)); + } + + + /// Parse comma separated parentherized value. + + /// @param separated Separated values as "(..[, ..])". + /// @returns the individual values with white space trimmed. + static std::vector<std::string> + ParseCommaSeparated(const char* separated) + { + std::vector<std::string> result; + + if (*separated) + ++separated; + + while ((*separated) && (*separated != ')')) + { + std::size_t escapeCounter = 0; + const char* start = separated; + QuotingState state = Unquoted; + bool escaped = false; + + while (*separated) + { + const char c = *separated++; + + if (state == Unquoted) + { + if ((c == '"') || (c == '\'')) + { + state = (c == '"' ? DoubleQuoted : SingleQuoted); + escaped = false; + } + else if ((c == '<') || + (c == '(') || + (c == '[') || + (c == '{')) + ++escapeCounter; + else if ((escapeCounter) && + ((c == '>') || + (c == ')') || + (c == ']') || + (c == '}'))) + --escapeCounter; + else if ((!escapeCounter) && + ((c == ',') || (c == ')'))) + { + result.push_back(TrimmedString(start, + separated - 1)); + break; + } + } + else + { + if (escaped) + escaped = false; + else if (c == '\\') + escaped = true; + else if (c == (state == DoubleQuoted ? '"' : '\'')) + state = Unquoted; + } + } + } + + return result; + } + + + /// Parse parameter declaration. + + /// @param raw Raw declaration. + TestParameterDescriptor ParseDescriptor(const std::string& raw) + { + const char* position = raw.c_str(); + + // Split the declaration into its declaration and its default + // type. + const char* equalPosition = NULL; + std::size_t escapeCounter = 0; + QuotingState state = Unquoted; + bool escaped = false; + + while (*position) + { + const char c = *position++; + + if (state == Unquoted) + { + if ((c == '"') || (c == '\'')) + { + state = (c == '"' ? DoubleQuoted : SingleQuoted); + escaped = false; + } + else if ((c == '<') || + (c == '(') || + (c == '[') || + (c == '{')) + ++escapeCounter; + else if ((escapeCounter) && + ((c == '>') || + (c == ')') || + (c == ']') || + (c == '}'))) + --escapeCounter; + else if ((!escapeCounter) && + (c == '=')) + { + equalPosition = position; + break; + } + } + else + { + if (escaped) + escaped = false; + else if (c == '\\') + escaped = true; + else if (c == (state == DoubleQuoted ? '"' : '\'')) + state = Unquoted; + } + } + + // Construct the parameter descriptor. + if (equalPosition) + { + const char* start = raw.c_str(); + const char* end = start + raw.length(); + + return TestParameterDescriptor( + std::string(TrimmedString(start, + equalPosition - 1)), + std::string(TrimmedString(equalPosition, + end)) + ); + } + else + return TestParameterDescriptor(raw, std::string()); + } + public: + TestParametersDescriptor() + { + + } + + + TestParametersDescriptor(const char* rawDeclarations, + const char* rawValues) + { + // Parse the declarations. + std::vector<std::string> declarations = + ParseCommaSeparated(rawDeclarations); + + for (std::vector<std::string>::const_iterator it = + declarations.begin(); + it != declarations.end(); + ++it) + _parameters.push_back(ParseDescriptor(*it)); + + // Parse the values. + std::vector<std::string> values = ParseCommaSeparated(rawValues); + + std::size_t + straightValues = (_parameters.size() > values.size() ? + values.size() : + _parameters.size()), + variadicValues = 0; + + if (values.size() > _parameters.size()) + { + if (straightValues > 0) + --straightValues; + variadicValues = values.size() - _parameters.size() + 1; + } + + for (std::size_t i = 0; i < straightValues; ++i) + _parameters[i].Value = values[i]; + + if (variadicValues) + { + std::stringstream variadic; + + for (std::size_t i = 0; i < variadicValues; ++i) + { + if (i) + variadic << ", "; + variadic << values[straightValues + i]; + } + + _parameters[_parameters.size() - 1].Value = variadic.str(); + } + } + + + inline const std::vector<TestParameterDescriptor>& Parameters() const + { + return _parameters; + } + private: + std::vector<TestParameterDescriptor> _parameters; + }; + + + /// Test descriptor. + class TestDescriptor + { + public: + /// Initialize a new test descriptor. + + /// @param fixtureName Name of the fixture. + /// @param testName Name of the test. + /// @param runs Number of runs for the test. + /// @param iterations Number of iterations per run. + /// @param testFactory Test factory implementation for the test. + /// @param parameters Parametrized test parameters. + TestDescriptor(const char* fixtureName, + const char* testName, + std::size_t runs, + std::size_t iterations, + TestFactory* testFactory, + TestParametersDescriptor parameters, + bool isDisabled = false) + : FixtureName(fixtureName), + TestName(testName), + CanonicalName(std::string(fixtureName) + "." + testName), + Runs(runs), + Iterations(iterations), + Factory(testFactory), + Parameters(parameters), + IsDisabled(isDisabled) + { + + } + + + /// Dispose of a test descriptor. + ~TestDescriptor() + { + delete this->Factory; + } + + + /// Fixture name. + std::string FixtureName; + + + /// Test name. + std::string TestName; + + + /// Canonical name. + + /// As: <FixtureName>.<TestName>. + std::string CanonicalName; + + + /// Test runs. + std::size_t Runs; + + + /// Iterations per test run. + std::size_t Iterations; + + + /// Test factory. + TestFactory* Factory; + + + /// Parameters for parametrized tests + TestParametersDescriptor Parameters; + + + /// Disabled. + bool IsDisabled; + }; +} +#endif diff --git a/third_party/include/hayai/hayai_test_factory.hpp b/third_party/include/hayai/hayai_test_factory.hpp new file mode 100755 index 0000000..1c5f469 --- /dev/null +++ b/third_party/include/hayai/hayai_test_factory.hpp @@ -0,0 +1,26 @@ +#ifndef __HAYAI_TESTFACTORY +#define __HAYAI_TESTFACTORY +#include "hayai_test.hpp" + +namespace hayai +{ + /// Base class for test factory implementations. + class TestFactory + { + public: + /// Virtual destructor + + /// Has no function in the base class. + virtual ~TestFactory() + { + + } + + + /// Creates a test instance to run. + + /// @returns a pointer to an initialized test. + virtual Test* CreateTest() = 0; + }; +} +#endif diff --git a/third_party/include/hayai/hayai_test_result.hpp b/third_party/include/hayai/hayai_test_result.hpp new file mode 100755 index 0000000..6c5368d --- /dev/null +++ b/third_party/include/hayai/hayai_test_result.hpp @@ -0,0 +1,304 @@ +#ifndef __HAYAI_TESTRESULT +#define __HAYAI_TESTRESULT +#include <vector> +#include <stdexcept> +#include <limits> +#include <cmath> + +#include "hayai_clock.hpp" + + +namespace hayai +{ + /// Test result descriptor. + + /// All durations are expressed in nanoseconds. + struct TestResult + { + public: + /// Initialize test result descriptor. + + /// @param runTimes Timing for the individual runs. + /// @param iterations Number of iterations per run. + TestResult(const std::vector<uint64_t>& runTimes, + std::size_t iterations) + : _runTimes(runTimes), + _iterations(iterations), + _timeTotal(0), + _timeRunMin(std::numeric_limits<uint64_t>::max()), + _timeRunMax(std::numeric_limits<uint64_t>::min()), + _timeStdDev(0.0), + _timeMedian(0.0), + _timeQuartile1(0.0), + _timeQuartile3(0.0) + { + // Summarize under the assumption of values being accessed more + // than once. + std::vector<uint64_t>::iterator runIt = _runTimes.begin(); + + while (runIt != _runTimes.end()) + { + const uint64_t run = *runIt; + + _timeTotal += run; + if ((runIt == _runTimes.begin()) || (run > _timeRunMax)) + _timeRunMax = run; + if ((runIt == _runTimes.begin()) || (run < _timeRunMin)) + _timeRunMin = run; + + ++runIt; + } + + // Calculate standard deviation. + const double mean = RunTimeAverage(); + double accu = 0.0; + + runIt = _runTimes.begin(); + + while (runIt != _runTimes.end()) + { + const uint64_t run = *runIt; + const double diff = double(run) - mean; + accu += (diff * diff); + + ++runIt; + } + + _timeStdDev = std::sqrt(accu / (_runTimes.size() - 1)); + + // Calculate quartiles. + std::vector<uint64_t> sortedRunTimes(_runTimes); + std::sort(sortedRunTimes.begin(), sortedRunTimes.end()); + + const std::size_t sortedSize = sortedRunTimes.size(); + const std::size_t sortedSizeHalf = sortedSize / 2; + + if (sortedSize >= 2) + { + const std::size_t quartile = sortedSizeHalf / 2; + + if ((sortedSize % 2) == 0) + { + _timeMedian = + (double(sortedRunTimes[sortedSizeHalf - 1]) + + double(sortedRunTimes[sortedSizeHalf])) / 2; + + _timeQuartile1 = + double(sortedRunTimes[quartile]); + _timeQuartile3 = + double(sortedRunTimes[sortedSizeHalf + quartile]); + } + else + { + _timeMedian = double(sortedRunTimes[sortedSizeHalf]); + + _timeQuartile1 = + (double(sortedRunTimes[quartile - 1]) + + double(sortedRunTimes[quartile])) / 2; + _timeQuartile3 = ( + double( + sortedRunTimes[sortedSizeHalf + (quartile - 1)] + ) + + double( + sortedRunTimes[sortedSizeHalf + quartile] + ) + ) / 2; + } + } + else if (sortedSize > 0) + { + _timeQuartile1 = double(sortedRunTimes[0]); + _timeQuartile3 = _timeQuartile1; + } + } + + + /// Total time. + inline double TimeTotal() const + { + return double(_timeTotal); + } + + + /// Run times. + inline const std::vector<uint64_t>& RunTimes() const + { + return _runTimes; + } + + + /// Average time per run. + inline double RunTimeAverage() const + { + return double(_timeTotal) / double(_runTimes.size()); + } + + /// Standard deviation time per run. + inline double RunTimeStdDev() const + { + return _timeStdDev; + } + + /// Median (2nd Quartile) time per run. + inline double RunTimeMedian() const + { + return _timeMedian; + } + + /// 1st Quartile time per run. + inline double RunTimeQuartile1() const + { + return _timeQuartile1; + } + + /// 3rd Quartile time per run. + inline double RunTimeQuartile3() const + { + return _timeQuartile3; + } + + /// Maximum time per run. + inline double RunTimeMaximum() const + { + return double(_timeRunMax); + } + + + /// Minimum time per run. + inline double RunTimeMinimum() const + { + return double(_timeRunMin); + } + + + /// Average runs per second. + inline double RunsPerSecondAverage() const + { + return 1000000000.0 / RunTimeAverage(); + } + + /// Median (2nd Quartile) runs per second. + inline double RunsPerSecondMedian() const + { + return 1000000000.0 / RunTimeMedian(); + } + + /// 1st Quartile runs per second. + inline double RunsPerSecondQuartile1() const + { + return 1000000000.0 / RunTimeQuartile1(); + } + + /// 3rd Quartile runs per second. + inline double RunsPerSecondQuartile3() const + { + return 1000000000.0 / RunTimeQuartile3(); + } + + /// Maximum runs per second. + inline double RunsPerSecondMaximum() const + { + return 1000000000.0 / _timeRunMin; + } + + + /// Minimum runs per second. + inline double RunsPerSecondMinimum() const + { + return 1000000000.0 / _timeRunMax; + } + + + /// Average time per iteration. + inline double IterationTimeAverage() const + { + return RunTimeAverage() / double(_iterations); + } + + /// Standard deviation time per iteration. + inline double IterationTimeStdDev() const + { + return RunTimeStdDev() / double(_iterations); + } + + /// Median (2nd Quartile) time per iteration. + inline double IterationTimeMedian() const + { + return RunTimeMedian() / double(_iterations); + } + + /// 1st Quartile time per iteration. + inline double IterationTimeQuartile1() const + { + return RunTimeQuartile1() / double(_iterations); + } + + /// 3rd Quartile time per iteration. + inline double IterationTimeQuartile3() const + { + return RunTimeQuartile3() / double(_iterations); + } + + /// Minimum time per iteration. + inline double IterationTimeMinimum() const + { + return _timeRunMin / double(_iterations); + } + + + /// Maximum time per iteration. + inline double IterationTimeMaximum() const + { + return _timeRunMax / double(_iterations); + } + + + /// Average iterations per second. + inline double IterationsPerSecondAverage() const + { + return 1000000000.0 / IterationTimeAverage(); + } + + /// Median (2nd Quartile) iterations per second. + inline double IterationsPerSecondMedian() const + { + return 1000000000.0 / IterationTimeMedian(); + } + + /// 1st Quartile iterations per second. + inline double IterationsPerSecondQuartile1() const + { + return 1000000000.0 / IterationTimeQuartile1(); + } + + /// 3rd Quartile iterations per second. + inline double IterationsPerSecondQuartile3() const + { + return 1000000000.0 / IterationTimeQuartile3(); + } + + /// Minimum iterations per second. + inline double IterationsPerSecondMinimum() const + { + return 1000000000.0 / IterationTimeMaximum(); + } + + + /// Maximum iterations per second. + inline double IterationsPerSecondMaximum() const + { + return 1000000000.0 / IterationTimeMinimum(); + } + private: + std::vector<uint64_t> _runTimes; + std::size_t _iterations; + uint64_t _timeTotal; + uint64_t _timeRunMin; + uint64_t _timeRunMax; + double _timeStdDev; + double _timeMedian; + double _timeQuartile1; + double _timeQuartile3; + }; +} +#endif |