1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
|
#!../libguile/guile \
-e main -s
!#
;;;; guile-benchmark --- run the Guile benchmark suite
;;;; Adapted from code by Jim Blandy <jimb@red-bean.com> --- May 1999
;;;;
;;;; Copyright (C) 2002, 2006, 2010 Free Software Foundation, Inc.
;;;;
;;;; This program is free software; you can redistribute it and/or
;;;; modify it under the terms of the GNU Lesser General Public
;;;; License as published by the Free Software Foundation; either
;;;; version 3, or (at your option) any later version.
;;;;
;;;; This program is distributed in the hope that it will be useful,
;;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;;;; GNU Lesser General Public License for more details.
;;;;
;;;; You should have received a copy of the GNU Lesser General Public
;;;; License along with this software; see the file COPYING.LESSER.
;;;; If not, write to the Free Software Foundation, Inc., 51 Franklin
;;;; Street, Fifth Floor, Boston, MA 02110-1301 USA
;;;; Usage: [guile -e main -s] guile-benchmark [OPTIONS] [BENCHMARK ...]
;;;;
;;;; Run benchmarks from the Guile benchmark suite. Report timing
;;;; results to the standard output, along with a summary of all
;;;; the results. Record each reported benchmark outcome in the log
;;;; file, `benchmarks.log'.
;;;;
;;;; Normally, guile-benchmark scans the benchmark directory, and
;;;; executes all files whose names end in `.bm'. (It assumes they contain
;;;; Scheme code.) However, you can have it execute specific benchmarks by
;;;; listing their filenames on the command line.
;;;;
;;;; The option `--benchmark-suite' can be given to specify the benchmark
;;;; directory. If no such option is given, the benchmark directory is
;;;; taken from the environment variable BENCHMARK_SUITE_DIR (if defined),
;;;; otherwise a default directory that is hardcoded in this file is
;;;; used (see "Installation" below).
;;;;
;;;; If present, the `--iteration-factor FACTOR' option tells
;;;; `guile-benchmark' to multiply the number of iterations given with
;;;; each single benchmark by the value of FACTOR. This allows to
;;;; reduce or increase the total time for benchmarking.
;;;;
;;;; If present, the `--log-file LOG' option tells `guile-benchmark' to put
;;;; the log output in a file named LOG.
;;;;
;;;; If present, the `--debug' option will enable a debugging mode.
;;;;
;;;;
;;;; Installation:
;;;;
;;;; If you change the #! line at the top of this script to point at
;;;; the Guile interpreter you want to run, you can call this script
;;;; as an executable instead of having to pass it as a parameter to
;;;; guile via "guile -e main -s guile-benchmark". Further, you can edit
;;;; the definition of default-benchmark-suite to point to the parent
;;;; directory of the `benchmarks' tree, which makes it unnecessary to set
;;;; the environment variable `BENCHMARK_SUITE_DIR'.
;;;;
;;;;
;;;; Shortcomings:
;;;;
;;;; At the moment, due to a simple-minded implementation, benchmark files
;;;; must live in the benchmark directory, and you must specify their names
;;;; relative to the top of the benchmark directory. If you want to send
;;;; me a patch that fixes this, but still leaves sane benchmark names in
;;;; the log file, that would be great. At the moment, all the benchmarks
;;;; I care about are in the benchmark directory, though.
;;;;
;;;; It would be nice if you could specify the Guile interpreter you
;;;; want to benchmark on the command line. As it stands, if you want to
;;;; change which Guile interpreter you're benchmarking, you need to edit
;;;; the #! line at the top of this file, which is stupid.
;;; User configurable settings:
(define default-benchmark-suite
(string-append (getenv "HOME") "/bogus-path/benchmark-suite"))
(use-modules (benchmark-suite lib)
(ice-9 getopt-long)
(ice-9 and-let-star)
(ice-9 rdelim))
;;; Variables that will receive their actual values later.
(define benchmark-suite default-benchmark-suite)
(define tmp-dir #f)
;;; General utilities, that probably should be in a library somewhere.
;;; Enable debugging
(define (enable-debug-mode)
(write-line %load-path)
(set! %load-verbosely #t)
(debug-enable 'backtrace 'debug))
;;; Traverse the directory tree at ROOT, applying F to the name of
;;; each file in the tree, including ROOT itself. For a subdirectory
;;; SUB, if (F SUB) is true, we recurse into SUB. Do not follow
;;; symlinks.
(define (for-each-file f root)
;; A "hard directory" is a path that denotes a directory and is not a
;; symlink.
(define (file-is-hard-directory? filename)
(eq? (stat:type (lstat filename)) 'directory))
(let visit ((root root))
(let ((should-recur (f root)))
(if (and should-recur (file-is-hard-directory? root))
(let ((dir (opendir root)))
(let loop ()
(let ((entry (readdir dir)))
(cond
((eof-object? entry) #f)
((or (string=? entry ".")
(string=? entry "..")
(string=? entry "CVS")
(string=? entry "RCS"))
(loop))
(else
(visit (string-append root "/" entry))
(loop))))))))))
;;; The benchmark driver.
;;; Localizing benchmark files and temporary data files.
(define (data-file-name filename)
(in-vicinity tmp-dir filename))
(define (benchmark-file-name benchmark)
(in-vicinity benchmark-suite benchmark))
;;; Return a list of all the benchmark files in the benchmark tree.
(define (enumerate-benchmarks benchmark-dir)
(let ((root-len (+ 1 (string-length benchmark-dir)))
(benchmarks '()))
(for-each-file (lambda (file)
(if (string-suffix? ".bm" file)
(let ((short-name
(substring file root-len)))
(set! benchmarks (cons short-name benchmarks))))
#t)
benchmark-dir)
;; for-each-file presents the files in whatever order it finds
;; them in the directory. We sort them here, so they'll always
;; appear in the same order. This makes it easier to compare benchmark
;; log files mechanically.
(sort benchmarks string<?)))
(define (main args)
(let ((options (getopt-long args
`((benchmark-suite
(single-char #\t)
(value #t))
(iteration-factor
(single-char #\t)
(value #t))
(log-file
(single-char #\l)
(value #t))
(debug
(single-char #\d))))))
(define (opt tag default)
(let ((pair (assq tag options)))
(if pair (cdr pair) default)))
(if (opt 'debug #f)
(enable-debug-mode))
(set! benchmark-suite
(or (opt 'benchmark-suite #f)
(getenv "BENCHMARK_SUITE_DIR")
default-benchmark-suite))
(set! iteration-factor
(string->number (opt 'iteration-factor "1")))
;; directory where temporary files are created.
(set! tmp-dir (getcwd))
(let* ((benchmarks
(let ((foo (opt '() '())))
(if (null? foo)
(enumerate-benchmarks benchmark-suite)
foo)))
(log-file
(opt 'log-file "benchmarks.log")))
;; Open the log file.
(let ((log-port (open-output-file log-file)))
;; Register some reporters.
(register-reporter (make-log-reporter log-port))
(register-reporter user-reporter)
;; Run the benchmarks.
(for-each (lambda (benchmark)
(with-benchmark-prefix benchmark
(load (benchmark-file-name benchmark))))
benchmarks)
(close-port log-port)))))
;;; Local Variables:
;;; mode: scheme
;;; End:
|
