Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 1 | /* |
| 2 | ** Copyright 2010 The Android Open Source Project |
| 3 | ** |
| 4 | ** Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | ** you may not use this file except in compliance with the License. |
| 6 | ** You may obtain a copy of the License at |
| 7 | ** |
| 8 | ** http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | ** |
| 10 | ** Unless required by applicable law or agreed to in writing, software |
| 11 | ** distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | ** See the License for the specific language governing permissions and |
| 14 | ** limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | /* |
Christopher Ferris | 25ada90 | 2013-04-02 13:28:16 -0700 | [diff] [blame] | 18 | * Micro-benchmarking of sleep/cpu speed/memcpy/memset/memory reads/strcmp. |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 19 | */ |
| 20 | |
| 21 | #include <stdio.h> |
| 22 | #include <stdlib.h> |
Elliott Hughes | 7e2587f | 2015-01-28 11:18:24 -0800 | [diff] [blame] | 23 | #include <string.h> |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 24 | #include <ctype.h> |
| 25 | #include <math.h> |
| 26 | #include <sched.h> |
| 27 | #include <sys/resource.h> |
| 28 | #include <time.h> |
| 29 | #include <unistd.h> |
| 30 | |
| 31 | // The default size of data that will be manipulated in each iteration of |
| 32 | // a memory benchmark. Can be modified with the --data_size option. |
| 33 | #define DEFAULT_DATA_SIZE 1000000000 |
| 34 | |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 35 | // The amount of memory allocated for the cold benchmarks to use. |
Chih-Hung Hsieh | 7b8ef7a | 2016-05-18 14:46:31 -0700 | [diff] [blame] | 36 | #define DEFAULT_COLD_DATA_SIZE (128*1024*1024) |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 37 | |
| 38 | // The default size of the stride between each buffer for cold benchmarks. |
| 39 | #define DEFAULT_COLD_STRIDE_SIZE 4096 |
| 40 | |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 41 | // Number of nanoseconds in a second. |
| 42 | #define NS_PER_SEC 1000000000 |
| 43 | |
| 44 | // The maximum number of arguments that a benchmark will accept. |
| 45 | #define MAX_ARGS 2 |
| 46 | |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 47 | // Default memory alignment of malloc. |
| 48 | #define DEFAULT_MALLOC_MEMORY_ALIGNMENT 8 |
| 49 | |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 50 | // Contains information about benchmark options. |
| 51 | typedef struct { |
| 52 | bool print_average; |
| 53 | bool print_each_iter; |
| 54 | |
| 55 | int dst_align; |
Christopher Ferris | 25ada90 | 2013-04-02 13:28:16 -0700 | [diff] [blame] | 56 | int dst_or_mask; |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 57 | int src_align; |
Christopher Ferris | 25ada90 | 2013-04-02 13:28:16 -0700 | [diff] [blame] | 58 | int src_or_mask; |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 59 | |
| 60 | int cpu_to_lock; |
| 61 | |
| 62 | int data_size; |
Christopher Ferris | 7401bc1 | 2013-07-10 18:55:57 -0700 | [diff] [blame] | 63 | int dst_str_size; |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 64 | int cold_data_size; |
| 65 | int cold_stride_size; |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 66 | |
| 67 | int args[MAX_ARGS]; |
| 68 | int num_args; |
| 69 | } command_data_t; |
| 70 | |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 71 | typedef void *(*void_func_t)(); |
| 72 | typedef void *(*memcpy_func_t)(void *, const void *, size_t); |
| 73 | typedef void *(*memset_func_t)(void *, int, size_t); |
| 74 | typedef int (*strcmp_func_t)(const char *, const char *); |
Christopher Ferris | 7401bc1 | 2013-07-10 18:55:57 -0700 | [diff] [blame] | 75 | typedef char *(*str_func_t)(char *, const char *); |
| 76 | typedef size_t (*strlen_func_t)(const char *); |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 77 | |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 78 | // Struct that contains a mapping of benchmark name to benchmark function. |
| 79 | typedef struct { |
| 80 | const char *name; |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 81 | int (*ptr)(const char *, const command_data_t &, void_func_t func); |
| 82 | void_func_t func; |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 83 | } function_t; |
| 84 | |
| 85 | // Get the current time in nanoseconds. |
| 86 | uint64_t nanoTime() { |
| 87 | struct timespec t; |
| 88 | |
| 89 | t.tv_sec = t.tv_nsec = 0; |
| 90 | clock_gettime(CLOCK_MONOTONIC, &t); |
| 91 | return static_cast<uint64_t>(t.tv_sec) * NS_PER_SEC + t.tv_nsec; |
| 92 | } |
| 93 | |
Chih-Hung Hsieh | ae4366e | 2016-02-02 10:49:50 -0800 | [diff] [blame] | 94 | // Static analyzer warns about potential memory leak of orig_ptr |
| 95 | // in getAlignedMemory. That is true and the callers in this program |
| 96 | // do not free orig_ptr. But, we don't care about that in this |
| 97 | // going-obsolete test program. So, here is a hack to trick the |
| 98 | // static analyzer. |
| 99 | static void *saved_orig_ptr; |
| 100 | |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 101 | // Allocate memory with a specific alignment and return that pointer. |
| 102 | // This function assumes an alignment value that is a power of 2. |
| 103 | // If the alignment is 0, then use the pointer returned by malloc. |
Christopher Ferris | 25ada90 | 2013-04-02 13:28:16 -0700 | [diff] [blame] | 104 | uint8_t *getAlignedMemory(uint8_t *orig_ptr, int alignment, int or_mask) { |
| 105 | uint64_t ptr = reinterpret_cast<uint64_t>(orig_ptr); |
Chih-Hung Hsieh | ae4366e | 2016-02-02 10:49:50 -0800 | [diff] [blame] | 106 | saved_orig_ptr = orig_ptr; |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 107 | if (alignment > 0) { |
| 108 | // When setting the alignment, set it to exactly the alignment chosen. |
| 109 | // The pointer returned will be guaranteed not to be aligned to anything |
| 110 | // more than that. |
| 111 | ptr += alignment - (ptr & (alignment - 1)); |
Christopher Ferris | 25ada90 | 2013-04-02 13:28:16 -0700 | [diff] [blame] | 112 | ptr |= alignment | or_mask; |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 113 | } |
| 114 | |
| 115 | return reinterpret_cast<uint8_t*>(ptr); |
| 116 | } |
| 117 | |
Christopher Ferris | 25ada90 | 2013-04-02 13:28:16 -0700 | [diff] [blame] | 118 | // Allocate memory with a specific alignment and return that pointer. |
| 119 | // This function assumes an alignment value that is a power of 2. |
| 120 | // If the alignment is 0, then use the pointer returned by malloc. |
| 121 | uint8_t *allocateAlignedMemory(size_t size, int alignment, int or_mask) { |
| 122 | uint64_t ptr = reinterpret_cast<uint64_t>(malloc(size + 3 * alignment)); |
| 123 | if (!ptr) |
| 124 | return NULL; |
| 125 | return getAlignedMemory((uint8_t*)ptr, alignment, or_mask); |
| 126 | } |
| 127 | |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 128 | void initString(uint8_t *buf, size_t size) { |
| 129 | for (size_t i = 0; i < size - 1; i++) { |
| 130 | buf[i] = static_cast<char>(32 + (i % 96)); |
Christopher Ferris | 7401bc1 | 2013-07-10 18:55:57 -0700 | [diff] [blame] | 131 | } |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 132 | buf[size-1] = '\0'; |
Christopher Ferris | 7401bc1 | 2013-07-10 18:55:57 -0700 | [diff] [blame] | 133 | } |
| 134 | |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 135 | static inline double computeAverage(uint64_t time_ns, size_t size, size_t copies) { |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 136 | return ((size/1024.0) * copies) / ((double)time_ns/NS_PER_SEC); |
| 137 | } |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 138 | |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 139 | static inline double computeRunningAvg(double avg, double running_avg, size_t cur_idx) { |
| 140 | return (running_avg / (cur_idx + 1)) * cur_idx + (avg / (cur_idx + 1)); |
| 141 | } |
| 142 | |
| 143 | static inline double computeRunningSquareAvg(double avg, double square_avg, size_t cur_idx) { |
| 144 | return (square_avg / (cur_idx + 1)) * cur_idx + (avg / (cur_idx + 1)) * avg; |
| 145 | } |
| 146 | |
| 147 | static inline double computeStdDev(double square_avg, double running_avg) { |
| 148 | return sqrt(square_avg - running_avg * running_avg); |
| 149 | } |
| 150 | |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 151 | static inline void printIter(uint64_t time_ns, const char *name, size_t size, size_t copies, double avg) { |
Mark Salyzyn | 8ec652c | 2014-03-20 12:46:42 -0700 | [diff] [blame] | 152 | printf("%s %zux%zu bytes took %.06f seconds (%f MB/s)\n", |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 153 | name, copies, size, (double)time_ns/NS_PER_SEC, avg/1024.0); |
| 154 | } |
| 155 | |
Mark Salyzyn | 8ec652c | 2014-03-20 12:46:42 -0700 | [diff] [blame] | 156 | static inline void printSummary(uint64_t /*time_ns*/, const char *name, size_t size, size_t copies, double running_avg, double std_dev, double min, double max) { |
| 157 | printf(" %s %zux%zu bytes average %.2f MB/s std dev %.4f min %.2f MB/s max %.2f MB/s\n", |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 158 | name, copies, size, running_avg/1024.0, std_dev/1024.0, min/1024.0, |
| 159 | max/1024.0); |
| 160 | } |
| 161 | |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 162 | // For the cold benchmarks, a large buffer will be created which |
| 163 | // contains many "size" buffers. This function will figure out the increment |
| 164 | // needed between each buffer so that each one is aligned to "alignment". |
| 165 | int getAlignmentIncrement(size_t size, int alignment) { |
| 166 | if (alignment == 0) { |
| 167 | alignment = DEFAULT_MALLOC_MEMORY_ALIGNMENT; |
| 168 | } |
| 169 | alignment *= 2; |
| 170 | return size + alignment - (size % alignment); |
| 171 | } |
| 172 | |
| 173 | uint8_t *getColdBuffer(int num_buffers, size_t incr, int alignment, int or_mask) { |
| 174 | uint8_t *buffers = reinterpret_cast<uint8_t*>(malloc(num_buffers * incr + 3 * alignment)); |
| 175 | if (!buffers) { |
| 176 | return NULL; |
| 177 | } |
| 178 | return getAlignedMemory(buffers, alignment, or_mask); |
| 179 | } |
| 180 | |
| 181 | static inline double computeColdAverage(uint64_t time_ns, size_t size, size_t copies, size_t num_buffers) { |
| 182 | return ((size/1024.0) * copies * num_buffers) / ((double)time_ns/NS_PER_SEC); |
| 183 | } |
| 184 | |
| 185 | static void inline printColdIter(uint64_t time_ns, const char *name, size_t size, size_t copies, size_t num_buffers, double avg) { |
Mark Salyzyn | 8ec652c | 2014-03-20 12:46:42 -0700 | [diff] [blame] | 186 | printf("%s %zux%zux%zu bytes took %.06f seconds (%f MB/s)\n", |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 187 | name, copies, num_buffers, size, (double)time_ns/NS_PER_SEC, avg/1024.0); |
| 188 | } |
| 189 | |
| 190 | static void inline printColdSummary( |
Mark Salyzyn | 8ec652c | 2014-03-20 12:46:42 -0700 | [diff] [blame] | 191 | uint64_t /*time_ns*/, const char *name, size_t size, size_t copies, size_t num_buffers, |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 192 | double running_avg, double square_avg, double min, double max) { |
Mark Salyzyn | 8ec652c | 2014-03-20 12:46:42 -0700 | [diff] [blame] | 193 | printf(" %s %zux%zux%zu bytes average %.2f MB/s std dev %.4f min %.2f MB/s max %.2f MB/s\n", |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 194 | name, copies, num_buffers, size, running_avg/1024.0, |
| 195 | computeStdDev(running_avg, square_avg)/1024.0, min/1024.0, max/1024.0); |
| 196 | } |
| 197 | |
| 198 | #define MAINLOOP(cmd_data, BENCH, COMPUTE_AVG, PRINT_ITER, PRINT_AVG) \ |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 199 | uint64_t time_ns; \ |
Chih-Hung Hsieh | 7b8ef7a | 2016-05-18 14:46:31 -0700 | [diff] [blame] | 200 | int iters = (cmd_data).args[1]; \ |
| 201 | bool print_average = (cmd_data).print_average; \ |
| 202 | bool print_each_iter = (cmd_data).print_each_iter; \ |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 203 | double min = 0.0, max = 0.0, running_avg = 0.0, square_avg = 0.0; \ |
| 204 | double avg; \ |
| 205 | for (int i = 0; iters == -1 || i < iters; i++) { \ |
| 206 | time_ns = nanoTime(); \ |
| 207 | BENCH; \ |
| 208 | time_ns = nanoTime() - time_ns; \ |
| 209 | avg = COMPUTE_AVG; \ |
| 210 | if (print_average) { \ |
| 211 | running_avg = computeRunningAvg(avg, running_avg, i); \ |
| 212 | square_avg = computeRunningSquareAvg(avg, square_avg, i); \ |
| 213 | if (min == 0.0 || avg < min) { \ |
| 214 | min = avg; \ |
| 215 | } \ |
| 216 | if (avg > max) { \ |
| 217 | max = avg; \ |
| 218 | } \ |
| 219 | } \ |
| 220 | if (print_each_iter) { \ |
| 221 | PRINT_ITER; \ |
| 222 | } \ |
| 223 | } \ |
| 224 | if (print_average) { \ |
| 225 | PRINT_AVG; \ |
| 226 | } |
| 227 | |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 228 | #define MAINLOOP_DATA(name, cmd_data, size, BENCH) \ |
Chih-Hung Hsieh | 7b8ef7a | 2016-05-18 14:46:31 -0700 | [diff] [blame] | 229 | size_t copies = (cmd_data).data_size/(size); \ |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 230 | size_t j; \ |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 231 | MAINLOOP(cmd_data, \ |
| 232 | for (j = 0; j < copies; j++) { \ |
| 233 | BENCH; \ |
| 234 | }, \ |
| 235 | computeAverage(time_ns, size, copies), \ |
| 236 | printIter(time_ns, name, size, copies, avg), \ |
| 237 | double std_dev = computeStdDev(square_avg, running_avg); \ |
| 238 | printSummary(time_ns, name, size, copies, running_avg, \ |
| 239 | std_dev, min, max)); |
| 240 | |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 241 | #define MAINLOOP_COLD(name, cmd_data, size, num_incrs, BENCH) \ |
Chih-Hung Hsieh | 7b8ef7a | 2016-05-18 14:46:31 -0700 | [diff] [blame] | 242 | size_t num_strides = num_buffers / (num_incrs); \ |
| 243 | if ((num_buffers % (num_incrs)) != 0) { \ |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 244 | num_strides--; \ |
| 245 | } \ |
| 246 | size_t copies = 1; \ |
Chih-Hung Hsieh | 7b8ef7a | 2016-05-18 14:46:31 -0700 | [diff] [blame] | 247 | num_buffers = (num_incrs) * num_strides; \ |
| 248 | if (num_buffers * (size) < static_cast<size_t>((cmd_data).data_size)) { \ |
| 249 | copies = (cmd_data).data_size / (num_buffers * (size)); \ |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 250 | } \ |
| 251 | if (num_strides == 0) { \ |
| 252 | printf("%s: Chosen options lead to no copies, aborting.\n", name); \ |
| 253 | return -1; \ |
| 254 | } \ |
| 255 | size_t j, k; \ |
| 256 | MAINLOOP(cmd_data, \ |
| 257 | for (j = 0; j < copies; j++) { \ |
Chih-Hung Hsieh | 7b8ef7a | 2016-05-18 14:46:31 -0700 | [diff] [blame] | 258 | for (k = 0; k < (num_incrs); k++) { \ |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 259 | BENCH; \ |
| 260 | } \ |
| 261 | }, \ |
| 262 | computeColdAverage(time_ns, size, copies, num_buffers), \ |
| 263 | printColdIter(time_ns, name, size, copies, num_buffers, avg), \ |
| 264 | printColdSummary(time_ns, name, size, copies, num_buffers, \ |
| 265 | running_avg, square_avg, min, max)); |
| 266 | |
| 267 | // This version of the macro creates a single buffer of the given size and |
| 268 | // alignment. The variable "buf" will be a pointer to the buffer and should |
| 269 | // be used by the BENCH code. |
| 270 | // INIT - Any specialized code needed to initialize the data. This will only |
| 271 | // be executed once. |
| 272 | // BENCH - The actual code to benchmark and is timed. |
| 273 | #define BENCH_ONE_BUF(name, cmd_data, INIT, BENCH) \ |
Chih-Hung Hsieh | 7b8ef7a | 2016-05-18 14:46:31 -0700 | [diff] [blame] | 274 | size_t size = (cmd_data).args[0]; \ |
| 275 | uint8_t *buf = allocateAlignedMemory(size, (cmd_data).dst_align, (cmd_data).dst_or_mask); \ |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 276 | if (!buf) \ |
| 277 | return -1; \ |
| 278 | INIT; \ |
| 279 | MAINLOOP_DATA(name, cmd_data, size, BENCH); |
| 280 | |
| 281 | // This version of the macro creates two buffers of the given sizes and |
| 282 | // alignments. The variables "buf1" and "buf2" will be pointers to the |
| 283 | // buffers and should be used by the BENCH code. |
| 284 | // INIT - Any specialized code needed to initialize the data. This will only |
| 285 | // be executed once. |
| 286 | // BENCH - The actual code to benchmark and is timed. |
| 287 | #define BENCH_TWO_BUFS(name, cmd_data, INIT, BENCH) \ |
Chih-Hung Hsieh | 7b8ef7a | 2016-05-18 14:46:31 -0700 | [diff] [blame] | 288 | size_t size = (cmd_data).args[0]; \ |
| 289 | uint8_t *buf1 = allocateAlignedMemory(size, (cmd_data).src_align, (cmd_data).src_or_mask); \ |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 290 | if (!buf1) \ |
| 291 | return -1; \ |
| 292 | size_t total_size = size; \ |
Chih-Hung Hsieh | 7b8ef7a | 2016-05-18 14:46:31 -0700 | [diff] [blame] | 293 | if ((cmd_data).dst_str_size > 0) \ |
| 294 | total_size += (cmd_data).dst_str_size; \ |
| 295 | uint8_t *buf2 = allocateAlignedMemory(total_size, (cmd_data).dst_align, (cmd_data).dst_or_mask); \ |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 296 | if (!buf2) \ |
| 297 | return -1; \ |
| 298 | INIT; \ |
| 299 | MAINLOOP_DATA(name, cmd_data, size, BENCH); |
| 300 | |
| 301 | // This version of the macro attempts to benchmark code when the data |
| 302 | // being manipulated is not in the cache, thus the cache is cold. It does |
| 303 | // this by creating a single large buffer that is designed to be larger than |
| 304 | // the largest cache in the system. The variable "buf" will be one slice |
| 305 | // of the buffer that the BENCH code should use that is of the correct size |
| 306 | // and alignment. In order to avoid any algorithms that prefetch past the end |
| 307 | // of their "buf" and into the next sequential buffer, the code strides |
| 308 | // through the buffer. Specifically, as "buf" values are iterated in BENCH |
| 309 | // code, the end of "buf" is guaranteed to be at least "stride_size" away |
| 310 | // from the next "buf". |
| 311 | // INIT - Any specialized code needed to initialize the data. This will only |
| 312 | // be executed once. |
| 313 | // BENCH - The actual code to benchmark and is timed. |
| 314 | #define COLD_ONE_BUF(name, cmd_data, INIT, BENCH) \ |
Chih-Hung Hsieh | 7b8ef7a | 2016-05-18 14:46:31 -0700 | [diff] [blame] | 315 | size_t size = (cmd_data).args[0]; \ |
| 316 | size_t incr = getAlignmentIncrement(size, (cmd_data).dst_align); \ |
| 317 | size_t num_buffers = (cmd_data).cold_data_size / incr; \ |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 318 | size_t buffer_size = num_buffers * incr; \ |
Chih-Hung Hsieh | 7b8ef7a | 2016-05-18 14:46:31 -0700 | [diff] [blame] | 319 | uint8_t *buffer = getColdBuffer(num_buffers, incr, (cmd_data).dst_align, (cmd_data).dst_or_mask); \ |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 320 | if (!buffer) \ |
| 321 | return -1; \ |
Chih-Hung Hsieh | 7b8ef7a | 2016-05-18 14:46:31 -0700 | [diff] [blame] | 322 | size_t num_incrs = (cmd_data).cold_stride_size / incr + 1; \ |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 323 | size_t stride_incr = incr * num_incrs; \ |
| 324 | uint8_t *buf; \ |
| 325 | size_t l; \ |
| 326 | INIT; \ |
Chih-Hung Hsieh | 7b8ef7a | 2016-05-18 14:46:31 -0700 | [diff] [blame] | 327 | MAINLOOP_COLD(name, (cmd_data), size, num_incrs, \ |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 328 | buf = buffer + k * incr; \ |
| 329 | for (l = 0; l < num_strides; l++) { \ |
| 330 | BENCH; \ |
| 331 | buf += stride_incr; \ |
| 332 | }); |
| 333 | |
| 334 | // This version of the macro attempts to benchmark code when the data |
| 335 | // being manipulated is not in the cache, thus the cache is cold. It does |
| 336 | // this by creating two large buffers each of which is designed to be |
| 337 | // larger than the largest cache in the system. Two variables "buf1" and |
| 338 | // "buf2" will be the two buffers that BENCH code should use. In order |
| 339 | // to avoid any algorithms that prefetch past the end of either "buf1" |
| 340 | // or "buf2" and into the next sequential buffer, the code strides through |
| 341 | // both buffers. Specifically, as "buf1" and "buf2" values are iterated in |
| 342 | // BENCH code, the end of "buf1" and "buf2" is guaranteed to be at least |
| 343 | // "stride_size" away from the next "buf1" and "buf2". |
| 344 | // INIT - Any specialized code needed to initialize the data. This will only |
| 345 | // be executed once. |
| 346 | // BENCH - The actual code to benchmark and is timed. |
| 347 | #define COLD_TWO_BUFS(name, cmd_data, INIT, BENCH) \ |
Chih-Hung Hsieh | 7b8ef7a | 2016-05-18 14:46:31 -0700 | [diff] [blame] | 348 | size_t size = (cmd_data).args[0]; \ |
| 349 | size_t buf1_incr = getAlignmentIncrement(size, (cmd_data).src_align); \ |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 350 | size_t total_size = size; \ |
Chih-Hung Hsieh | 7b8ef7a | 2016-05-18 14:46:31 -0700 | [diff] [blame] | 351 | if ((cmd_data).dst_str_size > 0) \ |
| 352 | total_size += (cmd_data).dst_str_size; \ |
| 353 | size_t buf2_incr = getAlignmentIncrement(total_size, (cmd_data).dst_align); \ |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 354 | size_t max_incr = (buf1_incr > buf2_incr) ? buf1_incr : buf2_incr; \ |
Chih-Hung Hsieh | 7b8ef7a | 2016-05-18 14:46:31 -0700 | [diff] [blame] | 355 | size_t num_buffers = (cmd_data).cold_data_size / max_incr; \ |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 356 | size_t buffer1_size = num_buffers * buf1_incr; \ |
| 357 | size_t buffer2_size = num_buffers * buf2_incr; \ |
Chih-Hung Hsieh | 7b8ef7a | 2016-05-18 14:46:31 -0700 | [diff] [blame] | 358 | uint8_t *buffer1 = getColdBuffer(num_buffers, buf1_incr, (cmd_data).src_align, (cmd_data).src_or_mask); \ |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 359 | if (!buffer1) \ |
| 360 | return -1; \ |
Chih-Hung Hsieh | 7b8ef7a | 2016-05-18 14:46:31 -0700 | [diff] [blame] | 361 | uint8_t *buffer2 = getColdBuffer(num_buffers, buf2_incr, (cmd_data).dst_align, (cmd_data).dst_or_mask); \ |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 362 | if (!buffer2) \ |
| 363 | return -1; \ |
| 364 | size_t min_incr = (buf1_incr < buf2_incr) ? buf1_incr : buf2_incr; \ |
Chih-Hung Hsieh | 7b8ef7a | 2016-05-18 14:46:31 -0700 | [diff] [blame] | 365 | size_t num_incrs = (cmd_data).cold_stride_size / min_incr + 1; \ |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 366 | size_t buf1_stride_incr = buf1_incr * num_incrs; \ |
| 367 | size_t buf2_stride_incr = buf2_incr * num_incrs; \ |
| 368 | size_t l; \ |
| 369 | uint8_t *buf1; \ |
| 370 | uint8_t *buf2; \ |
| 371 | INIT; \ |
Chih-Hung Hsieh | 7b8ef7a | 2016-05-18 14:46:31 -0700 | [diff] [blame] | 372 | MAINLOOP_COLD(name, (cmd_data), size, num_incrs, \ |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 373 | buf1 = buffer1 + k * buf1_incr; \ |
| 374 | buf2 = buffer2 + k * buf2_incr; \ |
| 375 | for (l = 0; l < num_strides; l++) { \ |
| 376 | BENCH; \ |
| 377 | buf1 += buf1_stride_incr; \ |
| 378 | buf2 += buf2_stride_incr; \ |
| 379 | }); |
| 380 | |
Mark Salyzyn | 8ec652c | 2014-03-20 12:46:42 -0700 | [diff] [blame] | 381 | int benchmarkSleep(const char* /*name*/, const command_data_t &cmd_data, void_func_t /*func*/) { |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 382 | int delay = cmd_data.args[0]; |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 383 | MAINLOOP(cmd_data, sleep(delay), |
| 384 | (double)time_ns/NS_PER_SEC, |
| 385 | printf("sleep(%d) took %.06f seconds\n", delay, avg);, |
| 386 | printf(" sleep(%d) average %.06f seconds std dev %f min %.06f seconds max %0.6f seconds\n", \ |
| 387 | delay, running_avg, computeStdDev(square_avg, running_avg), \ |
| 388 | min, max)); |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 389 | |
| 390 | return 0; |
| 391 | } |
| 392 | |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 393 | int benchmarkMemset(const char *name, const command_data_t &cmd_data, void_func_t func) { |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 394 | memset_func_t memset_func = reinterpret_cast<memset_func_t>(func); |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 395 | BENCH_ONE_BUF(name, cmd_data, ;, memset_func(buf, i, size)); |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 396 | |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 397 | return 0; |
| 398 | } |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 399 | |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 400 | int benchmarkMemsetCold(const char *name, const command_data_t &cmd_data, void_func_t func) { |
| 401 | memset_func_t memset_func = reinterpret_cast<memset_func_t>(func); |
| 402 | COLD_ONE_BUF(name, cmd_data, ;, memset_func(buf, l, size)); |
Chih-Hung Hsieh | 373d3c7 | 2017-10-18 16:28:14 -0700 | [diff] [blame] | 403 | (void)buffer_size; |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 404 | |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 405 | return 0; |
| 406 | } |
| 407 | |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 408 | int benchmarkMemcpy(const char *name, const command_data_t &cmd_data, void_func_t func) { |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 409 | memcpy_func_t memcpy_func = reinterpret_cast<memcpy_func_t>(func); |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 410 | |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 411 | BENCH_TWO_BUFS(name, cmd_data, |
| 412 | memset(buf1, 0xff, size); \ |
| 413 | memset(buf2, 0, size), |
| 414 | memcpy_func(buf2, buf1, size)); |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 415 | |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 416 | return 0; |
| 417 | } |
Christopher Ferris | 25ada90 | 2013-04-02 13:28:16 -0700 | [diff] [blame] | 418 | |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 419 | int benchmarkMemcpyCold(const char *name, const command_data_t &cmd_data, void_func_t func) { |
| 420 | memcpy_func_t memcpy_func = reinterpret_cast<memcpy_func_t>(func); |
| 421 | |
| 422 | COLD_TWO_BUFS(name, cmd_data, |
| 423 | memset(buffer1, 0xff, buffer1_size); \ |
| 424 | memset(buffer2, 0x0, buffer2_size), |
| 425 | memcpy_func(buf2, buf1, size)); |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 426 | |
Christopher Ferris | 25ada90 | 2013-04-02 13:28:16 -0700 | [diff] [blame] | 427 | return 0; |
| 428 | } |
| 429 | |
Christopher Ferris | 867c8c8 | 2015-04-08 15:21:42 -0700 | [diff] [blame] | 430 | int benchmarkMemmoveBackwards(const char *name, const command_data_t &cmd_data, void_func_t func) { |
| 431 | memcpy_func_t memmove_func = reinterpret_cast<memcpy_func_t>(func); |
| 432 | |
| 433 | size_t size = cmd_data.args[0]; |
Chih-Hung Hsieh | 373d3c7 | 2017-10-18 16:28:14 -0700 | [diff] [blame] | 434 | size_t alloc_size = size * 2 + 3 * cmd_data.dst_align; // should alloc_size be used? |
Christopher Ferris | 867c8c8 | 2015-04-08 15:21:42 -0700 | [diff] [blame] | 435 | uint8_t* src = allocateAlignedMemory(size, cmd_data.src_align, cmd_data.src_or_mask); |
| 436 | if (!src) |
| 437 | return -1; |
| 438 | // Force memmove to do a backwards copy by getting a pointer into the source buffer. |
| 439 | uint8_t* dst = getAlignedMemory(src+1, cmd_data.dst_align, cmd_data.dst_or_mask); |
| 440 | if (!dst) |
| 441 | return -1; |
| 442 | MAINLOOP_DATA(name, cmd_data, size, memmove_func(dst, src, size)); |
| 443 | return 0; |
| 444 | } |
| 445 | |
Mark Salyzyn | 8ec652c | 2014-03-20 12:46:42 -0700 | [diff] [blame] | 446 | int benchmarkMemread(const char *name, const command_data_t &cmd_data, void_func_t /*func*/) { |
Christopher Ferris | 25ada90 | 2013-04-02 13:28:16 -0700 | [diff] [blame] | 447 | int size = cmd_data.args[0]; |
Christopher Ferris | 25ada90 | 2013-04-02 13:28:16 -0700 | [diff] [blame] | 448 | |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 449 | uint32_t *src = reinterpret_cast<uint32_t*>(malloc(size)); |
| 450 | if (!src) |
Christopher Ferris | 25ada90 | 2013-04-02 13:28:16 -0700 | [diff] [blame] | 451 | return -1; |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 452 | memset(src, 0xff, size); |
Christopher Ferris | 25ada90 | 2013-04-02 13:28:16 -0700 | [diff] [blame] | 453 | |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 454 | // Use volatile so the compiler does not optimize away the reads. |
| 455 | volatile int foo; |
| 456 | size_t k; |
| 457 | MAINLOOP_DATA(name, cmd_data, size, |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 458 | for (k = 0; k < size/sizeof(uint32_t); k++) foo = src[k]); |
Chih-Hung Hsieh | ae4366e | 2016-02-02 10:49:50 -0800 | [diff] [blame] | 459 | free(src); |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 460 | |
| 461 | return 0; |
| 462 | } |
| 463 | |
| 464 | int benchmarkStrcmp(const char *name, const command_data_t &cmd_data, void_func_t func) { |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 465 | strcmp_func_t strcmp_func = reinterpret_cast<strcmp_func_t>(func); |
| 466 | |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 467 | int retval; |
| 468 | BENCH_TWO_BUFS(name, cmd_data, |
| 469 | initString(buf1, size); \ |
| 470 | initString(buf2, size), |
| 471 | retval = strcmp_func(reinterpret_cast<char*>(buf1), reinterpret_cast<char*>(buf2)); \ |
| 472 | if (retval != 0) printf("%s failed, return value %d\n", name, retval)); |
| 473 | |
| 474 | return 0; |
| 475 | } |
| 476 | |
| 477 | int benchmarkStrcmpCold(const char *name, const command_data_t &cmd_data, void_func_t func) { |
| 478 | strcmp_func_t strcmp_func = reinterpret_cast<strcmp_func_t>(func); |
Christopher Ferris | 25ada90 | 2013-04-02 13:28:16 -0700 | [diff] [blame] | 479 | |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 480 | int retval; |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 481 | COLD_TWO_BUFS(name, cmd_data, |
| 482 | memset(buffer1, 'a', buffer1_size); \ |
| 483 | memset(buffer2, 'a', buffer2_size); \ |
| 484 | for (size_t i =0; i < num_buffers; i++) { \ |
| 485 | buffer1[size-1+buf1_incr*i] = '\0'; \ |
| 486 | buffer2[size-1+buf2_incr*i] = '\0'; \ |
| 487 | }, |
| 488 | retval = strcmp_func(reinterpret_cast<char*>(buf1), reinterpret_cast<char*>(buf2)); \ |
| 489 | if (retval != 0) printf("%s failed, return value %d\n", name, retval)); |
Christopher Ferris | 7401bc1 | 2013-07-10 18:55:57 -0700 | [diff] [blame] | 490 | |
| 491 | return 0; |
| 492 | } |
| 493 | |
| 494 | int benchmarkStrlen(const char *name, const command_data_t &cmd_data, void_func_t func) { |
Christopher Ferris | 7401bc1 | 2013-07-10 18:55:57 -0700 | [diff] [blame] | 495 | size_t real_size; |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 496 | strlen_func_t strlen_func = reinterpret_cast<strlen_func_t>(func); |
| 497 | BENCH_ONE_BUF(name, cmd_data, |
| 498 | initString(buf, size), |
| 499 | real_size = strlen_func(reinterpret_cast<char*>(buf)); \ |
Christopher Ferris | 7401bc1 | 2013-07-10 18:55:57 -0700 | [diff] [blame] | 500 | if (real_size + 1 != size) { \ |
Mark Salyzyn | 8ec652c | 2014-03-20 12:46:42 -0700 | [diff] [blame] | 501 | printf("%s failed, expected %zu, got %zu\n", name, size, real_size); \ |
Christopher Ferris | 7401bc1 | 2013-07-10 18:55:57 -0700 | [diff] [blame] | 502 | return -1; \ |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 503 | }); |
Christopher Ferris | 7401bc1 | 2013-07-10 18:55:57 -0700 | [diff] [blame] | 504 | |
| 505 | return 0; |
| 506 | } |
| 507 | |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 508 | int benchmarkStrlenCold(const char *name, const command_data_t &cmd_data, void_func_t func) { |
| 509 | strlen_func_t strlen_func = reinterpret_cast<strlen_func_t>(func); |
| 510 | size_t real_size; |
| 511 | COLD_ONE_BUF(name, cmd_data, |
| 512 | memset(buffer, 'a', buffer_size); \ |
| 513 | for (size_t i = 0; i < num_buffers; i++) { \ |
| 514 | buffer[size-1+incr*i] = '\0'; \ |
| 515 | }, |
| 516 | real_size = strlen_func(reinterpret_cast<char*>(buf)); \ |
| 517 | if (real_size + 1 != size) { \ |
Mark Salyzyn | 8ec652c | 2014-03-20 12:46:42 -0700 | [diff] [blame] | 518 | printf("%s failed, expected %zu, got %zu\n", name, size, real_size); \ |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 519 | return -1; \ |
| 520 | }); |
| 521 | return 0; |
| 522 | } |
| 523 | |
Christopher Ferris | 7401bc1 | 2013-07-10 18:55:57 -0700 | [diff] [blame] | 524 | int benchmarkStrcat(const char *name, const command_data_t &cmd_data, void_func_t func) { |
Christopher Ferris | 7401bc1 | 2013-07-10 18:55:57 -0700 | [diff] [blame] | 525 | str_func_t str_func = reinterpret_cast<str_func_t>(func); |
| 526 | |
| 527 | int dst_str_size = cmd_data.dst_str_size; |
| 528 | if (dst_str_size <= 0) { |
| 529 | printf("%s requires --dst_str_size to be set to a non-zero value.\n", |
| 530 | name); |
| 531 | return -1; |
| 532 | } |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 533 | BENCH_TWO_BUFS(name, cmd_data, |
| 534 | initString(buf1, size); \ |
| 535 | initString(buf2, dst_str_size), |
| 536 | str_func(reinterpret_cast<char*>(buf2), reinterpret_cast<char*>(buf1)); buf2[dst_str_size-1] = '\0'); |
Christopher Ferris | 25ada90 | 2013-04-02 13:28:16 -0700 | [diff] [blame] | 537 | |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 538 | return 0; |
| 539 | } |
| 540 | |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 541 | int benchmarkStrcatCold(const char *name, const command_data_t &cmd_data, void_func_t func) { |
Christopher Ferris | 7401bc1 | 2013-07-10 18:55:57 -0700 | [diff] [blame] | 542 | str_func_t str_func = reinterpret_cast<str_func_t>(func); |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 543 | |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 544 | int dst_str_size = cmd_data.dst_str_size; |
| 545 | if (dst_str_size <= 0) { |
| 546 | printf("%s requires --dst_str_size to be set to a non-zero value.\n", |
| 547 | name); |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 548 | return -1; |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 549 | } |
| 550 | COLD_TWO_BUFS(name, cmd_data, |
| 551 | memset(buffer1, 'a', buffer1_size); \ |
| 552 | memset(buffer2, 'b', buffer2_size); \ |
| 553 | for (size_t i = 0; i < num_buffers; i++) { \ |
| 554 | buffer1[size-1+buf1_incr*i] = '\0'; \ |
| 555 | buffer2[dst_str_size-1+buf2_incr*i] = '\0'; \ |
| 556 | }, |
| 557 | str_func(reinterpret_cast<char*>(buf2), reinterpret_cast<char*>(buf1)); buf2[dst_str_size-1] = '\0'); |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 558 | |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 559 | return 0; |
| 560 | } |
| 561 | |
| 562 | |
| 563 | int benchmarkStrcpy(const char *name, const command_data_t &cmd_data, void_func_t func) { |
| 564 | str_func_t str_func = reinterpret_cast<str_func_t>(func); |
| 565 | |
| 566 | BENCH_TWO_BUFS(name, cmd_data, |
| 567 | initString(buf1, size); \ |
| 568 | memset(buf2, 0, size), |
| 569 | str_func(reinterpret_cast<char*>(buf2), reinterpret_cast<char*>(buf1))); |
| 570 | |
| 571 | return 0; |
| 572 | } |
| 573 | |
| 574 | int benchmarkStrcpyCold(const char *name, const command_data_t &cmd_data, void_func_t func) { |
| 575 | str_func_t str_func = reinterpret_cast<str_func_t>(func); |
| 576 | |
| 577 | COLD_TWO_BUFS(name, cmd_data, |
| 578 | memset(buffer1, 'a', buffer1_size); \ |
| 579 | for (size_t i = 0; i < num_buffers; i++) { \ |
| 580 | buffer1[size-1+buf1_incr*i] = '\0'; \ |
| 581 | } \ |
| 582 | memset(buffer2, 0, buffer2_size), |
| 583 | str_func(reinterpret_cast<char*>(buf2), reinterpret_cast<char*>(buf1))); |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 584 | |
| 585 | return 0; |
| 586 | } |
| 587 | |
| 588 | // Create the mapping structure. |
| 589 | function_t function_table[] = { |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 590 | { "memcpy", benchmarkMemcpy, reinterpret_cast<void_func_t>(memcpy) }, |
| 591 | { "memcpy_cold", benchmarkMemcpyCold, reinterpret_cast<void_func_t>(memcpy) }, |
Christopher Ferris | 867c8c8 | 2015-04-08 15:21:42 -0700 | [diff] [blame] | 592 | { "memmove_forward", benchmarkMemcpy, reinterpret_cast<void_func_t>(memmove) }, |
| 593 | { "memmove_backward", benchmarkMemmoveBackwards, reinterpret_cast<void_func_t>(memmove) }, |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 594 | { "memread", benchmarkMemread, NULL }, |
| 595 | { "memset", benchmarkMemset, reinterpret_cast<void_func_t>(memset) }, |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 596 | { "memset_cold", benchmarkMemsetCold, reinterpret_cast<void_func_t>(memset) }, |
| 597 | { "sleep", benchmarkSleep, NULL }, |
Christopher Ferris | 7401bc1 | 2013-07-10 18:55:57 -0700 | [diff] [blame] | 598 | { "strcat", benchmarkStrcat, reinterpret_cast<void_func_t>(strcat) }, |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 599 | { "strcat_cold", benchmarkStrcatCold, reinterpret_cast<void_func_t>(strcat) }, |
| 600 | { "strcmp", benchmarkStrcmp, reinterpret_cast<void_func_t>(strcmp) }, |
| 601 | { "strcmp_cold", benchmarkStrcmpCold, reinterpret_cast<void_func_t>(strcmp) }, |
| 602 | { "strcpy", benchmarkStrcpy, reinterpret_cast<void_func_t>(strcpy) }, |
| 603 | { "strcpy_cold", benchmarkStrcpyCold, reinterpret_cast<void_func_t>(strcpy) }, |
| 604 | { "strlen", benchmarkStrlen, reinterpret_cast<void_func_t>(strlen) }, |
| 605 | { "strlen_cold", benchmarkStrlenCold, reinterpret_cast<void_func_t>(strlen) }, |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 606 | }; |
| 607 | |
| 608 | void usage() { |
| 609 | printf("Usage:\n"); |
| 610 | printf(" micro_bench [--data_size DATA_BYTES] [--print_average]\n"); |
| 611 | printf(" [--no_print_each_iter] [--lock_to_cpu CORE]\n"); |
Christopher Ferris | 7401bc1 | 2013-07-10 18:55:57 -0700 | [diff] [blame] | 612 | printf(" [--src_align ALIGN] [--src_or_mask OR_MASK]\n"); |
| 613 | printf(" [--dst_align ALIGN] [--dst_or_mask OR_MASK]\n"); |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 614 | printf(" [--dst_str_size SIZE] [--cold_data_size DATA_BYTES]\n"); |
| 615 | printf(" [--cold_stride_size SIZE]\n"); |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 616 | printf(" --data_size DATA_BYTES\n"); |
| 617 | printf(" For the data benchmarks (memcpy/memset/memread) the approximate\n"); |
| 618 | printf(" size of data, in bytes, that will be manipulated in each iteration.\n"); |
| 619 | printf(" --print_average\n"); |
| 620 | printf(" Print the average and standard deviation of all iterations.\n"); |
| 621 | printf(" --no_print_each_iter\n"); |
| 622 | printf(" Do not print any values in each iteration.\n"); |
| 623 | printf(" --lock_to_cpu CORE\n"); |
| 624 | printf(" Lock to the specified CORE. The default is to use the last core found.\n"); |
Christopher Ferris | 7401bc1 | 2013-07-10 18:55:57 -0700 | [diff] [blame] | 625 | printf(" --dst_align ALIGN\n"); |
| 626 | printf(" If the command supports it, align the destination pointer to ALIGN.\n"); |
| 627 | printf(" The default is to use the value returned by malloc.\n"); |
| 628 | printf(" --dst_or_mask OR_MASK\n"); |
| 629 | printf(" If the command supports it, or in the OR_MASK on to the destination pointer.\n"); |
| 630 | printf(" The OR_MASK must be smaller than the dst_align value.\n"); |
| 631 | printf(" The default value is 0.\n"); |
| 632 | |
| 633 | printf(" --src_align ALIGN\n"); |
| 634 | printf(" If the command supports it, align the source pointer to ALIGN. The default is to use the\n"); |
| 635 | printf(" value returned by malloc.\n"); |
| 636 | printf(" --src_or_mask OR_MASK\n"); |
| 637 | printf(" If the command supports it, or in the OR_MASK on to the source pointer.\n"); |
| 638 | printf(" The OR_MASK must be smaller than the src_align value.\n"); |
| 639 | printf(" The default value is 0.\n"); |
| 640 | printf(" --dst_str_size SIZE\n"); |
| 641 | printf(" If the command supports it, create a destination string of this length.\n"); |
| 642 | printf(" The default is to not update the destination string.\n"); |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 643 | printf(" --cold_data_size DATA_SIZE\n"); |
| 644 | printf(" For _cold benchmarks, use this as the total amount of memory to use.\n"); |
| 645 | printf(" The default is 128MB, and the number should be larger than the cache on the chip.\n"); |
| 646 | printf(" This value is specified in bytes.\n"); |
| 647 | printf(" --cold_stride_size SIZE\n"); |
| 648 | printf(" For _cold benchmarks, use this as the minimum stride between iterations.\n"); |
| 649 | printf(" The default is 4096 bytes and the number should be larger than the amount of data\n"); |
| 650 | printf(" pulled in to the cache by each run of the benchmark.\n"); |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 651 | printf(" ITERS\n"); |
| 652 | printf(" The number of iterations to execute each benchmark. If not\n"); |
| 653 | printf(" passed in then run forever.\n"); |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 654 | printf(" micro_bench cpu UNUSED [ITERS]\n"); |
| 655 | printf(" micro_bench [--dst_align ALIGN] [--dst_or_mask OR_MASK] memcpy NUM_BYTES [ITERS]\n"); |
| 656 | printf(" micro_bench memread NUM_BYTES [ITERS]\n"); |
| 657 | printf(" micro_bench [--dst_align ALIGN] [--dst_or_mask OR_MASK] memset NUM_BYTES [ITERS]\n"); |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 658 | printf(" micro_bench sleep TIME_TO_SLEEP [ITERS]\n"); |
| 659 | printf(" TIME_TO_SLEEP\n"); |
| 660 | printf(" The time in seconds to sleep.\n"); |
Christopher Ferris | 7401bc1 | 2013-07-10 18:55:57 -0700 | [diff] [blame] | 661 | printf(" micro_bench [--src_align ALIGN] [--src_or_mask OR_MASK] [--dst_align ALIGN] [--dst_or_mask] [--dst_str_size SIZE] strcat NUM_BYTES [ITERS]\n"); |
| 662 | printf(" micro_bench [--src_align ALIGN] [--src_or_mask OR_MASK] [--dst_align ALIGN] [--dst_or_mask OR_MASK] strcmp NUM_BYTES [ITERS]\n"); |
| 663 | printf(" micro_bench [--src_align ALIGN] [--src_or_mask OR_MASK] [--dst_align ALIGN] [--dst_or_mask] strcpy NUM_BYTES [ITERS]\n"); |
| 664 | printf(" micro_bench [--dst_align ALIGN] [--dst_or_mask OR_MASK] strlen NUM_BYTES [ITERS]\n"); |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 665 | printf("\n"); |
| 666 | printf(" In addition, memcpy/memcpy/memset/strcat/strcpy/strlen have _cold versions\n"); |
| 667 | printf(" that will execute the function on a buffer not in the cache.\n"); |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 668 | } |
| 669 | |
| 670 | function_t *processOptions(int argc, char **argv, command_data_t *cmd_data) { |
| 671 | function_t *command = NULL; |
| 672 | |
| 673 | // Initialize the command_flags. |
| 674 | cmd_data->print_average = false; |
| 675 | cmd_data->print_each_iter = true; |
| 676 | cmd_data->dst_align = 0; |
| 677 | cmd_data->src_align = 0; |
Christopher Ferris | 25ada90 | 2013-04-02 13:28:16 -0700 | [diff] [blame] | 678 | cmd_data->src_or_mask = 0; |
| 679 | cmd_data->dst_or_mask = 0; |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 680 | cmd_data->num_args = 0; |
| 681 | cmd_data->cpu_to_lock = -1; |
| 682 | cmd_data->data_size = DEFAULT_DATA_SIZE; |
Christopher Ferris | 7401bc1 | 2013-07-10 18:55:57 -0700 | [diff] [blame] | 683 | cmd_data->dst_str_size = -1; |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 684 | cmd_data->cold_data_size = DEFAULT_COLD_DATA_SIZE; |
| 685 | cmd_data->cold_stride_size = DEFAULT_COLD_STRIDE_SIZE; |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 686 | for (int i = 0; i < MAX_ARGS; i++) { |
| 687 | cmd_data->args[i] = -1; |
| 688 | } |
| 689 | |
| 690 | for (int i = 1; i < argc; i++) { |
| 691 | if (argv[i][0] == '-') { |
| 692 | int *save_value = NULL; |
| 693 | if (strcmp(argv[i], "--print_average") == 0) { |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 694 | cmd_data->print_average = true; |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 695 | } else if (strcmp(argv[i], "--no_print_each_iter") == 0) { |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 696 | cmd_data->print_each_iter = false; |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 697 | } else if (strcmp(argv[i], "--dst_align") == 0) { |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 698 | save_value = &cmd_data->dst_align; |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 699 | } else if (strcmp(argv[i], "--src_align") == 0) { |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 700 | save_value = &cmd_data->src_align; |
Christopher Ferris | 25ada90 | 2013-04-02 13:28:16 -0700 | [diff] [blame] | 701 | } else if (strcmp(argv[i], "--dst_or_mask") == 0) { |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 702 | save_value = &cmd_data->dst_or_mask; |
Christopher Ferris | 25ada90 | 2013-04-02 13:28:16 -0700 | [diff] [blame] | 703 | } else if (strcmp(argv[i], "--src_or_mask") == 0) { |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 704 | save_value = &cmd_data->src_or_mask; |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 705 | } else if (strcmp(argv[i], "--lock_to_cpu") == 0) { |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 706 | save_value = &cmd_data->cpu_to_lock; |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 707 | } else if (strcmp(argv[i], "--data_size") == 0) { |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 708 | save_value = &cmd_data->data_size; |
Christopher Ferris | 7401bc1 | 2013-07-10 18:55:57 -0700 | [diff] [blame] | 709 | } else if (strcmp(argv[i], "--dst_str_size") == 0) { |
Christopher Ferris | 991bcd4 | 2013-07-24 15:34:13 -0700 | [diff] [blame] | 710 | save_value = &cmd_data->dst_str_size; |
| 711 | } else if (strcmp(argv[i], "--cold_data_size") == 0) { |
| 712 | save_value = &cmd_data->cold_data_size; |
| 713 | } else if (strcmp(argv[i], "--cold_stride_size") == 0) { |
| 714 | save_value = &cmd_data->cold_stride_size; |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 715 | } else { |
| 716 | printf("Unknown option %s\n", argv[i]); |
| 717 | return NULL; |
| 718 | } |
| 719 | if (save_value) { |
| 720 | // Checking both characters without a strlen() call should be |
| 721 | // safe since as long as the argument exists, one character will |
| 722 | // be present (\0). And if the first character is '-', then |
| 723 | // there will always be a second character (\0 again). |
| 724 | if (i == argc - 1 || (argv[i + 1][0] == '-' && !isdigit(argv[i + 1][1]))) { |
| 725 | printf("The option %s requires one argument.\n", |
| 726 | argv[i]); |
| 727 | return NULL; |
| 728 | } |
Christopher Ferris | 25ada90 | 2013-04-02 13:28:16 -0700 | [diff] [blame] | 729 | *save_value = (int)strtol(argv[++i], NULL, 0); |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 730 | } |
| 731 | } else if (!command) { |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 732 | for (size_t j = 0; j < sizeof(function_table)/sizeof(function_t); j++) { |
| 733 | if (strcmp(argv[i], function_table[j].name) == 0) { |
| 734 | command = &function_table[j]; |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 735 | break; |
| 736 | } |
| 737 | } |
| 738 | if (!command) { |
| 739 | printf("Uknown command %s\n", argv[i]); |
| 740 | return NULL; |
| 741 | } |
| 742 | } else if (cmd_data->num_args > MAX_ARGS) { |
| 743 | printf("More than %d number arguments passed in.\n", MAX_ARGS); |
| 744 | return NULL; |
| 745 | } else { |
| 746 | cmd_data->args[cmd_data->num_args++] = atoi(argv[i]); |
| 747 | } |
| 748 | } |
| 749 | |
| 750 | // Check the arguments passed in make sense. |
| 751 | if (cmd_data->num_args != 1 && cmd_data->num_args != 2) { |
| 752 | printf("Not enough arguments passed in.\n"); |
| 753 | return NULL; |
| 754 | } else if (cmd_data->dst_align < 0) { |
| 755 | printf("The --dst_align option must be greater than or equal to 0.\n"); |
| 756 | return NULL; |
| 757 | } else if (cmd_data->src_align < 0) { |
| 758 | printf("The --src_align option must be greater than or equal to 0.\n"); |
| 759 | return NULL; |
| 760 | } else if (cmd_data->data_size <= 0) { |
| 761 | printf("The --data_size option must be a positive number.\n"); |
| 762 | return NULL; |
| 763 | } else if ((cmd_data->dst_align & (cmd_data->dst_align - 1))) { |
| 764 | printf("The --dst_align option must be a power of 2.\n"); |
| 765 | return NULL; |
| 766 | } else if ((cmd_data->src_align & (cmd_data->src_align - 1))) { |
| 767 | printf("The --src_align option must be a power of 2.\n"); |
| 768 | return NULL; |
Christopher Ferris | 25ada90 | 2013-04-02 13:28:16 -0700 | [diff] [blame] | 769 | } else if (!cmd_data->src_align && cmd_data->src_or_mask) { |
| 770 | printf("The --src_or_mask option requires that --src_align be set.\n"); |
| 771 | return NULL; |
| 772 | } else if (!cmd_data->dst_align && cmd_data->dst_or_mask) { |
| 773 | printf("The --dst_or_mask option requires that --dst_align be set.\n"); |
| 774 | return NULL; |
| 775 | } else if (cmd_data->src_or_mask > cmd_data->src_align) { |
| 776 | printf("The value of --src_or_mask cannot be larger that --src_align.\n"); |
| 777 | return NULL; |
| 778 | } else if (cmd_data->dst_or_mask > cmd_data->dst_align) { |
| 779 | printf("The value of --src_or_mask cannot be larger that --src_align.\n"); |
| 780 | return NULL; |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 781 | } |
| 782 | |
| 783 | return command; |
| 784 | } |
| 785 | |
| 786 | bool raisePriorityAndLock(int cpu_to_lock) { |
| 787 | cpu_set_t cpuset; |
| 788 | |
| 789 | if (setpriority(PRIO_PROCESS, 0, -20)) { |
| 790 | perror("Unable to raise priority of process.\n"); |
| 791 | return false; |
| 792 | } |
| 793 | |
| 794 | CPU_ZERO(&cpuset); |
| 795 | if (sched_getaffinity(0, sizeof(cpuset), &cpuset) != 0) { |
| 796 | perror("sched_getaffinity failed"); |
| 797 | return false; |
| 798 | } |
| 799 | |
| 800 | if (cpu_to_lock < 0) { |
| 801 | // Lock to the last active core we find. |
| 802 | for (int i = 0; i < CPU_SETSIZE; i++) { |
| 803 | if (CPU_ISSET(i, &cpuset)) { |
| 804 | cpu_to_lock = i; |
| 805 | } |
| 806 | } |
| 807 | } else if (!CPU_ISSET(cpu_to_lock, &cpuset)) { |
| 808 | printf("Cpu %d does not exist.\n", cpu_to_lock); |
| 809 | return false; |
| 810 | } |
| 811 | |
| 812 | if (cpu_to_lock < 0) { |
| 813 | printf("Cannot find any valid cpu to lock.\n"); |
| 814 | return false; |
| 815 | } |
| 816 | |
| 817 | CPU_ZERO(&cpuset); |
| 818 | CPU_SET(cpu_to_lock, &cpuset); |
| 819 | if (sched_setaffinity(0, sizeof(cpuset), &cpuset) != 0) { |
| 820 | perror("sched_setaffinity failed"); |
| 821 | return false; |
| 822 | } |
| 823 | |
| 824 | return true; |
| 825 | } |
| 826 | |
| 827 | int main(int argc, char **argv) { |
| 828 | command_data_t cmd_data; |
| 829 | |
| 830 | function_t *command = processOptions(argc, argv, &cmd_data); |
| 831 | if (!command) { |
| 832 | usage(); |
| 833 | return -1; |
| 834 | } |
| 835 | |
| 836 | if (!raisePriorityAndLock(cmd_data.cpu_to_lock)) { |
| 837 | return -1; |
| 838 | } |
| 839 | |
| 840 | printf("%s\n", command->name); |
Christopher Ferris | 014cf9d | 2013-06-26 13:42:18 -0700 | [diff] [blame] | 841 | return (*command->ptr)(command->name, cmd_data, command->func); |
Christopher Ferris | 82ac1af | 2013-02-15 12:27:58 -0800 | [diff] [blame] | 842 | } |