Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 1 | #!/usr/bin/env ruby |
Shinichiro Hamaji | b69bf8a | 2015-06-10 14:52:06 +0900 | [diff] [blame] | 2 | # |
| 3 | # Copyright 2015 Google Inc. All rights reserved |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 16 | |
| 17 | require 'fileutils' |
| 18 | |
Shinichiro Hamaji | 9f4f5d3 | 2015-04-28 03:53:49 +0900 | [diff] [blame] | 19 | if ARGV[0] == '-s' |
| 20 | test_serialization = true |
Shinichiro Hamaji | 776ca30 | 2015-06-06 03:52:48 +0900 | [diff] [blame] | 21 | elsif ARGV[0] == '-c' |
| 22 | ckati = true |
| 23 | ARGV.shift |
Shinichiro Hamaji | 9f4f5d3 | 2015-04-28 03:53:49 +0900 | [diff] [blame] | 24 | end |
| 25 | |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 26 | def get_output_filenames |
| 27 | files = Dir.glob('*') |
| 28 | files.delete('Makefile') |
Shinichiro Hamaji | 65489db | 2015-06-29 20:44:26 +0900 | [diff] [blame] | 29 | files.delete('gmon.out') |
Shinichiro Hamaji | b674d58 | 2015-04-28 03:40:57 +0900 | [diff] [blame] | 30 | files.reject!{|f|f =~ /\.json$/} |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 31 | files |
| 32 | end |
| 33 | |
| 34 | def cleanup |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 35 | (get_output_filenames + Dir.glob('.*')).each do |fname| |
| 36 | next if fname == '.' || fname == '..' |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 37 | FileUtils.rm_rf fname |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 38 | end |
| 39 | end |
| 40 | |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 41 | def move_circular_dep(l) |
| 42 | # We don't care when circular dependency detection happens. |
| 43 | circ = '' |
| 44 | while l.sub!(/Circular .* dropped\.\n/, '') do |
| 45 | circ += $& |
| 46 | end |
| 47 | circ + l |
| 48 | end |
| 49 | |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 50 | expected_failures = [] |
| 51 | unexpected_passes = [] |
| 52 | failures = [] |
| 53 | passes = [] |
| 54 | |
Shinichiro Hamaji | ec42b4e | 2015-05-14 16:20:03 +0900 | [diff] [blame] | 55 | if !ARGV.empty? |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 56 | test_files = ARGV.map do |test| |
| 57 | "testcase/#{File.basename(test)}" |
Shinichiro Hamaji | ec42b4e | 2015-05-14 16:20:03 +0900 | [diff] [blame] | 58 | end |
| 59 | else |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 60 | test_files = Dir.glob('testcase/*.mk').sort |
| 61 | test_files += Dir.glob('testcase/*.sh').sort |
Shinichiro Hamaji | ec42b4e | 2015-05-14 16:20:03 +0900 | [diff] [blame] | 62 | end |
| 63 | |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 64 | def run_in_testdir(test_filename) |
| 65 | c = File.read(test_filename) |
| 66 | name = File.basename(test_filename) |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 67 | dir = "out/#{name}" |
Shinichiro Hamaji | 52e4203 | 2015-03-31 23:20:13 +0900 | [diff] [blame] | 68 | |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 69 | FileUtils.mkdir_p(dir) |
Shinichiro Hamaji | 52e4203 | 2015-03-31 23:20:13 +0900 | [diff] [blame] | 70 | Dir.glob("#{dir}/*").each do |fname| |
| 71 | FileUtils.rm_rf(fname) |
| 72 | end |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 73 | |
| 74 | Dir.chdir(dir) do |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 75 | yield name |
| 76 | end |
| 77 | end |
| 78 | |
| 79 | def normalize_make_log(expected) |
| 80 | expected.gsub!(/^make(?:\[\d+\])?: (Entering|Leaving) directory.*\n/, '') |
| 81 | expected.gsub!(/^make(?:\[\d+\])?: /, '') |
| 82 | expected = move_circular_dep(expected) |
| 83 | |
| 84 | # Normalizations for old/new GNU make. |
| 85 | expected.gsub!(/[`'"]/, '"') |
| 86 | expected.gsub!(/ (?:commands|recipe) for target /, |
| 87 | ' commands for target ') |
| 88 | expected.gsub!(/ (?:commands|recipe) commences /, |
| 89 | ' commands commence ') |
| 90 | expected.gsub!(' (did you mean TAB instead of 8 spaces?)', '') |
| 91 | expected.gsub!('Extraneous text after', 'extraneous text after') |
| 92 | # Not sure if this is useful. |
| 93 | expected.gsub!(/\s+Stop\.$/, '') |
| 94 | # GNU make 4.0 has this output. |
| 95 | expected.gsub!(/Makefile:\d+: commands for target ".*?" failed\n/, '') |
| 96 | # We treat some warnings as errors. |
| 97 | expected.gsub!(/Nothing to be done for "test"\.\n/, '') |
| 98 | |
| 99 | expected |
| 100 | end |
| 101 | |
| 102 | def normalize_kati_log(output) |
| 103 | output = move_circular_dep(output) |
| 104 | # kati specific log messages. |
| 105 | output.gsub!(/^\*kati\*.*\n/, '') |
| 106 | output.gsub!(/[`'"]/, '"') |
Shinichiro Hamaji | 861bd64 | 2015-06-19 16:59:13 +0900 | [diff] [blame] | 107 | output.gsub!(/(: )(?:open )?(\S+): [Nn](o such file or directory)\nNOTE:.*/, |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 108 | "\\1\\2: N\\3\n*** No rule to make target \"\\2\".") |
| 109 | output |
| 110 | end |
| 111 | |
| 112 | run_make_test = proc do |mk| |
| 113 | c = File.read(mk) |
Shinichiro Hamaji | e919d00 | 2015-06-26 01:15:55 +0900 | [diff] [blame] | 114 | expected_failure = c =~ /\A# TODO(?:\((go|c)\))?/ |
| 115 | if $1 |
| 116 | if $1 == 'go' && ckati |
| 117 | expected_failure = false |
| 118 | elsif $1 == 'c' && !ckati |
| 119 | expected_failure = false |
| 120 | end |
| 121 | end |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 122 | |
| 123 | run_in_testdir(mk) do |name| |
Shinichiro Hamaji | 776ca30 | 2015-06-06 03:52:48 +0900 | [diff] [blame] | 124 | # TODO: Fix |
| 125 | if name =~ /eval_assign/ && ckati |
| 126 | next |
| 127 | end |
| 128 | |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 129 | File.open("Makefile", 'w') do |ofile| |
| 130 | ofile.print(c) |
| 131 | end |
| 132 | |
| 133 | expected = '' |
| 134 | output = '' |
| 135 | |
Shinichiro Hamaji | 491e73f | 2015-04-07 12:41:59 +0900 | [diff] [blame] | 136 | testcases = c.scan(/^test\d*/).sort.uniq |
Shinichiro Hamaji | 497754d | 2015-03-31 02:02:11 +0900 | [diff] [blame] | 137 | if testcases.empty? |
| 138 | testcases = [''] |
| 139 | end |
Shinichiro Hamaji | e5be014 | 2015-03-30 19:26:01 +0900 | [diff] [blame] | 140 | |
| 141 | cleanup |
| 142 | testcases.each do |tc| |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 143 | res = `make #{tc} 2>&1` |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 144 | res = normalize_make_log(res) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 145 | expected += "=== #{tc} ===\n" + res |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 146 | expected_files = get_output_filenames |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 147 | expected += "\n=== FILES ===\n#{expected_files * "\n"}\n" |
Shinichiro Hamaji | e5be014 | 2015-03-30 19:26:01 +0900 | [diff] [blame] | 148 | end |
| 149 | |
| 150 | cleanup |
| 151 | testcases.each do |tc| |
Shinichiro Hamaji | c48d9ce | 2015-04-28 03:47:39 +0900 | [diff] [blame] | 152 | json = "#{tc.empty? ? 'test' : tc}" |
| 153 | cmd = "../../kati -save_json=#{json}.json -kati_log #{tc} 2>&1" |
Shinichiro Hamaji | 776ca30 | 2015-06-06 03:52:48 +0900 | [diff] [blame] | 154 | if ckati |
| 155 | cmd = "../../ckati #{tc} 2>&1" |
| 156 | end |
Shinichiro Hamaji | b674d58 | 2015-04-28 03:40:57 +0900 | [diff] [blame] | 157 | res = IO.popen(cmd, 'r:binary', &:read) |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 158 | res = normalize_kati_log(res) |
Shinichiro Hamaji | 42f42e1 | 2015-04-13 14:26:45 +0900 | [diff] [blame] | 159 | output += "=== #{tc} ===\n" + res |
Shinichiro Hamaji | e5be014 | 2015-03-30 19:26:01 +0900 | [diff] [blame] | 160 | output_files = get_output_filenames |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 161 | output += "\n=== FILES ===\n#{output_files * "\n"}\n" |
| 162 | end |
| 163 | |
| 164 | File.open('out.make', 'w'){|ofile|ofile.print(expected)} |
| 165 | File.open('out.kati', 'w'){|ofile|ofile.print(output)} |
| 166 | |
Shinichiro Hamaji | 74c832f | 2015-04-09 14:54:39 +0900 | [diff] [blame] | 167 | if expected =~ /FAIL/ |
| 168 | puts %Q(#{name} has a string "FAIL" in its expectation) |
| 169 | exit 1 |
| 170 | end |
| 171 | |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 172 | if expected != output |
Shinichiro Hamaji | 91b105c | 2015-03-30 18:39:05 +0900 | [diff] [blame] | 173 | if expected_failure |
| 174 | puts "#{name}: FAIL (expected)" |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 175 | expected_failures << name |
Shinichiro Hamaji | 91b105c | 2015-03-30 18:39:05 +0900 | [diff] [blame] | 176 | else |
| 177 | puts "#{name}: FAIL" |
| 178 | puts `diff -u out.make out.kati` |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 179 | failures << name |
Shinichiro Hamaji | 91b105c | 2015-03-30 18:39:05 +0900 | [diff] [blame] | 180 | end |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 181 | else |
Shinichiro Hamaji | 91b105c | 2015-03-30 18:39:05 +0900 | [diff] [blame] | 182 | if expected_failure |
| 183 | puts "#{name}: PASS (unexpected)" |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 184 | unexpected_passes << name |
Shinichiro Hamaji | 91b105c | 2015-03-30 18:39:05 +0900 | [diff] [blame] | 185 | else |
| 186 | puts "#{name}: PASS" |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 187 | passes << name |
Shinichiro Hamaji | 91b105c | 2015-03-30 18:39:05 +0900 | [diff] [blame] | 188 | end |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 189 | end |
Shinichiro Hamaji | c48d9ce | 2015-04-28 03:47:39 +0900 | [diff] [blame] | 190 | |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 191 | if name !~ /^err_/ && test_serialization && !expected_failure |
Shinichiro Hamaji | c48d9ce | 2015-04-28 03:47:39 +0900 | [diff] [blame] | 192 | testcases.each do |tc| |
| 193 | json = "#{tc.empty? ? 'test' : tc}" |
Shinichiro Hamaji | 9f4f5d3 | 2015-04-28 03:53:49 +0900 | [diff] [blame] | 194 | cmd = "../../kati -save_json=#{json}_2.json -load_json=#{json}.json -n -kati_log #{tc} 2>&1" |
Shinichiro Hamaji | c48d9ce | 2015-04-28 03:47:39 +0900 | [diff] [blame] | 195 | res = IO.popen(cmd, 'r:binary', &:read) |
| 196 | if !File.exist?("#{json}.json") || !File.exist?("#{json}_2.json") |
| 197 | puts "#{name}##{json}: Serialize failure (not exist)" |
Shinichiro Hamaji | 9f4f5d3 | 2015-04-28 03:53:49 +0900 | [diff] [blame] | 198 | puts res |
Shinichiro Hamaji | c48d9ce | 2015-04-28 03:47:39 +0900 | [diff] [blame] | 199 | else |
| 200 | json1 = File.read("#{json}.json") |
| 201 | json2 = File.read("#{json}_2.json") |
| 202 | if json1 != json2 |
| 203 | puts "#{name}##{json}: Serialize failure" |
Shinichiro Hamaji | 9f4f5d3 | 2015-04-28 03:53:49 +0900 | [diff] [blame] | 204 | puts res |
Shinichiro Hamaji | c48d9ce | 2015-04-28 03:47:39 +0900 | [diff] [blame] | 205 | end |
| 206 | end |
| 207 | end |
| 208 | end |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 209 | end |
| 210 | end |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 211 | |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 212 | run_shell_test = proc do |sh| |
| 213 | run_in_testdir(sh) do |name| |
| 214 | cleanup |
| 215 | cmd = "sh ../../#{sh} make" |
| 216 | expected = IO.popen(cmd, 'r:binary', &:read) |
| 217 | cleanup |
| 218 | cmd = "sh ../../#{sh} ../../kati --use_cache --kati_log" |
| 219 | output = IO.popen(cmd, 'r:binary', &:read) |
| 220 | |
| 221 | expected = normalize_make_log(expected) |
| 222 | output = normalize_kati_log(output) |
| 223 | File.open('out.make', 'w'){|ofile|ofile.print(expected)} |
| 224 | File.open('out.kati', 'w'){|ofile|ofile.print(output)} |
| 225 | |
| 226 | if expected != output |
| 227 | puts "#{name}: FAIL" |
Shinichiro Hamaji | 39e6240 | 2015-05-20 19:34:58 +0900 | [diff] [blame] | 228 | puts `diff -u out.make out.kati` |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 229 | failures << name |
| 230 | else |
| 231 | puts "#{name}: PASS" |
| 232 | passes << name |
| 233 | end |
| 234 | end |
| 235 | end |
| 236 | |
| 237 | test_files.each do |test| |
| 238 | if /\.mk$/ =~ test |
| 239 | run_make_test.call(test) |
| 240 | elsif /\.sh$/ =~ test |
| 241 | run_shell_test.call(test) |
| 242 | else |
| 243 | raise "Unknown test type: #{test}" |
| 244 | end |
| 245 | end |
| 246 | |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 247 | puts |
| 248 | |
| 249 | if !expected_failures.empty? |
| 250 | puts "=== Expected failures ===" |
| 251 | expected_failures.each do |n| |
| 252 | puts n |
| 253 | end |
| 254 | end |
| 255 | |
| 256 | if !unexpected_passes.empty? |
| 257 | puts "=== Unexpected passes ===" |
| 258 | unexpected_passes.each do |n| |
| 259 | puts n |
| 260 | end |
| 261 | end |
| 262 | |
| 263 | if !failures.empty? |
| 264 | puts "=== Failures ===" |
| 265 | failures.each do |n| |
| 266 | puts n |
| 267 | end |
| 268 | end |
| 269 | |
| 270 | puts |
| 271 | |
| 272 | if !unexpected_passes.empty? || !failures.empty? |
Shinichiro Hamaji | 776ca30 | 2015-06-06 03:52:48 +0900 | [diff] [blame] | 273 | puts "FAIL! (#{failures.size + unexpected_passes.size} fails #{passes.size} passes)" |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 274 | else |
| 275 | puts 'PASS!' |
| 276 | end |