Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 1 | #!/usr/bin/env ruby |
| 2 | |
| 3 | require 'fileutils' |
| 4 | |
Shinichiro Hamaji | 9f4f5d3 | 2015-04-28 03:53:49 +0900 | [diff] [blame] | 5 | if ARGV[0] == '-s' |
| 6 | test_serialization = true |
| 7 | end |
| 8 | |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 9 | def get_output_filenames |
| 10 | files = Dir.glob('*') |
| 11 | files.delete('Makefile') |
Shinichiro Hamaji | b674d58 | 2015-04-28 03:40:57 +0900 | [diff] [blame] | 12 | files.reject!{|f|f =~ /\.json$/} |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 13 | files |
| 14 | end |
| 15 | |
| 16 | def cleanup |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame^] | 17 | (get_output_filenames + Dir.glob('.*')).each do |fname| |
| 18 | next if fname == '.' || fname == '..' |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 19 | FileUtils.rm_rf fname |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 20 | end |
| 21 | end |
| 22 | |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 23 | def move_circular_dep(l) |
| 24 | # We don't care when circular dependency detection happens. |
| 25 | circ = '' |
| 26 | while l.sub!(/Circular .* dropped\.\n/, '') do |
| 27 | circ += $& |
| 28 | end |
| 29 | circ + l |
| 30 | end |
| 31 | |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 32 | expected_failures = [] |
| 33 | unexpected_passes = [] |
| 34 | failures = [] |
| 35 | passes = [] |
| 36 | |
Shinichiro Hamaji | ec42b4e | 2015-05-14 16:20:03 +0900 | [diff] [blame] | 37 | if !ARGV.empty? |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame^] | 38 | test_files = ARGV.map do |test| |
| 39 | "testcase/#{File.basename(test)}" |
Shinichiro Hamaji | ec42b4e | 2015-05-14 16:20:03 +0900 | [diff] [blame] | 40 | end |
| 41 | else |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame^] | 42 | test_files = Dir.glob('testcase/*.mk').sort |
| 43 | test_files += Dir.glob('testcase/*.sh').sort |
Shinichiro Hamaji | ec42b4e | 2015-05-14 16:20:03 +0900 | [diff] [blame] | 44 | end |
| 45 | |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame^] | 46 | def run_in_testdir(test_filename) |
| 47 | c = File.read(test_filename) |
| 48 | name = File.basename(test_filename) |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 49 | dir = "out/#{name}" |
Shinichiro Hamaji | 52e4203 | 2015-03-31 23:20:13 +0900 | [diff] [blame] | 50 | |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 51 | FileUtils.mkdir_p(dir) |
Shinichiro Hamaji | 52e4203 | 2015-03-31 23:20:13 +0900 | [diff] [blame] | 52 | Dir.glob("#{dir}/*").each do |fname| |
| 53 | FileUtils.rm_rf(fname) |
| 54 | end |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 55 | |
| 56 | Dir.chdir(dir) do |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame^] | 57 | yield name |
| 58 | end |
| 59 | end |
| 60 | |
| 61 | def normalize_make_log(expected) |
| 62 | expected.gsub!(/^make(?:\[\d+\])?: (Entering|Leaving) directory.*\n/, '') |
| 63 | expected.gsub!(/^make(?:\[\d+\])?: /, '') |
| 64 | expected = move_circular_dep(expected) |
| 65 | |
| 66 | # Normalizations for old/new GNU make. |
| 67 | expected.gsub!(/[`'"]/, '"') |
| 68 | expected.gsub!(/ (?:commands|recipe) for target /, |
| 69 | ' commands for target ') |
| 70 | expected.gsub!(/ (?:commands|recipe) commences /, |
| 71 | ' commands commence ') |
| 72 | expected.gsub!(' (did you mean TAB instead of 8 spaces?)', '') |
| 73 | expected.gsub!('Extraneous text after', 'extraneous text after') |
| 74 | # Not sure if this is useful. |
| 75 | expected.gsub!(/\s+Stop\.$/, '') |
| 76 | # GNU make 4.0 has this output. |
| 77 | expected.gsub!(/Makefile:\d+: commands for target ".*?" failed\n/, '') |
| 78 | # We treat some warnings as errors. |
| 79 | expected.gsub!(/Nothing to be done for "test"\.\n/, '') |
| 80 | |
| 81 | expected |
| 82 | end |
| 83 | |
| 84 | def normalize_kati_log(output) |
| 85 | output = move_circular_dep(output) |
| 86 | # kati specific log messages. |
| 87 | output.gsub!(/^\*kati\*.*\n/, '') |
| 88 | output.gsub!(/[`'"]/, '"') |
| 89 | output.gsub!(/(: )open (\S+): n(o such file or directory)\nNOTE:.*/, |
| 90 | "\\1\\2: N\\3\n*** No rule to make target \"\\2\".") |
| 91 | output |
| 92 | end |
| 93 | |
| 94 | run_make_test = proc do |mk| |
| 95 | c = File.read(mk) |
| 96 | expected_failure = c =~ /\A# TODO/ |
| 97 | |
| 98 | run_in_testdir(mk) do |name| |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 99 | File.open("Makefile", 'w') do |ofile| |
| 100 | ofile.print(c) |
| 101 | end |
| 102 | |
| 103 | expected = '' |
| 104 | output = '' |
| 105 | |
Shinichiro Hamaji | 491e73f | 2015-04-07 12:41:59 +0900 | [diff] [blame] | 106 | testcases = c.scan(/^test\d*/).sort.uniq |
Shinichiro Hamaji | 497754d | 2015-03-31 02:02:11 +0900 | [diff] [blame] | 107 | if testcases.empty? |
| 108 | testcases = [''] |
| 109 | end |
Shinichiro Hamaji | e5be014 | 2015-03-30 19:26:01 +0900 | [diff] [blame] | 110 | |
| 111 | cleanup |
| 112 | testcases.each do |tc| |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 113 | res = `make #{tc} 2>&1` |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame^] | 114 | res = normalize_make_log(res) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 115 | expected += "=== #{tc} ===\n" + res |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 116 | expected_files = get_output_filenames |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 117 | expected += "\n=== FILES ===\n#{expected_files * "\n"}\n" |
Shinichiro Hamaji | e5be014 | 2015-03-30 19:26:01 +0900 | [diff] [blame] | 118 | end |
| 119 | |
| 120 | cleanup |
| 121 | testcases.each do |tc| |
Shinichiro Hamaji | c48d9ce | 2015-04-28 03:47:39 +0900 | [diff] [blame] | 122 | json = "#{tc.empty? ? 'test' : tc}" |
| 123 | cmd = "../../kati -save_json=#{json}.json -kati_log #{tc} 2>&1" |
Shinichiro Hamaji | b674d58 | 2015-04-28 03:40:57 +0900 | [diff] [blame] | 124 | res = IO.popen(cmd, 'r:binary', &:read) |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame^] | 125 | res = normalize_kati_log(res) |
Shinichiro Hamaji | 42f42e1 | 2015-04-13 14:26:45 +0900 | [diff] [blame] | 126 | output += "=== #{tc} ===\n" + res |
Shinichiro Hamaji | e5be014 | 2015-03-30 19:26:01 +0900 | [diff] [blame] | 127 | output_files = get_output_filenames |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 128 | output += "\n=== FILES ===\n#{output_files * "\n"}\n" |
| 129 | end |
| 130 | |
| 131 | File.open('out.make', 'w'){|ofile|ofile.print(expected)} |
| 132 | File.open('out.kati', 'w'){|ofile|ofile.print(output)} |
| 133 | |
Shinichiro Hamaji | 74c832f | 2015-04-09 14:54:39 +0900 | [diff] [blame] | 134 | if expected =~ /FAIL/ |
| 135 | puts %Q(#{name} has a string "FAIL" in its expectation) |
| 136 | exit 1 |
| 137 | end |
| 138 | |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 139 | if expected != output |
Shinichiro Hamaji | 91b105c | 2015-03-30 18:39:05 +0900 | [diff] [blame] | 140 | if expected_failure |
| 141 | puts "#{name}: FAIL (expected)" |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 142 | expected_failures << name |
Shinichiro Hamaji | 91b105c | 2015-03-30 18:39:05 +0900 | [diff] [blame] | 143 | else |
| 144 | puts "#{name}: FAIL" |
| 145 | puts `diff -u out.make out.kati` |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 146 | failures << name |
Shinichiro Hamaji | 91b105c | 2015-03-30 18:39:05 +0900 | [diff] [blame] | 147 | end |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 148 | else |
Shinichiro Hamaji | 91b105c | 2015-03-30 18:39:05 +0900 | [diff] [blame] | 149 | if expected_failure |
| 150 | puts "#{name}: PASS (unexpected)" |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 151 | unexpected_passes << name |
Shinichiro Hamaji | 91b105c | 2015-03-30 18:39:05 +0900 | [diff] [blame] | 152 | else |
| 153 | puts "#{name}: PASS" |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 154 | passes << name |
Shinichiro Hamaji | 91b105c | 2015-03-30 18:39:05 +0900 | [diff] [blame] | 155 | end |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 156 | end |
Shinichiro Hamaji | c48d9ce | 2015-04-28 03:47:39 +0900 | [diff] [blame] | 157 | |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 158 | if name !~ /^err_/ && test_serialization && !expected_failure |
Shinichiro Hamaji | c48d9ce | 2015-04-28 03:47:39 +0900 | [diff] [blame] | 159 | testcases.each do |tc| |
| 160 | json = "#{tc.empty? ? 'test' : tc}" |
Shinichiro Hamaji | 9f4f5d3 | 2015-04-28 03:53:49 +0900 | [diff] [blame] | 161 | cmd = "../../kati -save_json=#{json}_2.json -load_json=#{json}.json -n -kati_log #{tc} 2>&1" |
Shinichiro Hamaji | c48d9ce | 2015-04-28 03:47:39 +0900 | [diff] [blame] | 162 | res = IO.popen(cmd, 'r:binary', &:read) |
| 163 | if !File.exist?("#{json}.json") || !File.exist?("#{json}_2.json") |
| 164 | puts "#{name}##{json}: Serialize failure (not exist)" |
Shinichiro Hamaji | 9f4f5d3 | 2015-04-28 03:53:49 +0900 | [diff] [blame] | 165 | puts res |
Shinichiro Hamaji | c48d9ce | 2015-04-28 03:47:39 +0900 | [diff] [blame] | 166 | else |
| 167 | json1 = File.read("#{json}.json") |
| 168 | json2 = File.read("#{json}_2.json") |
| 169 | if json1 != json2 |
| 170 | puts "#{name}##{json}: Serialize failure" |
Shinichiro Hamaji | 9f4f5d3 | 2015-04-28 03:53:49 +0900 | [diff] [blame] | 171 | puts res |
Shinichiro Hamaji | c48d9ce | 2015-04-28 03:47:39 +0900 | [diff] [blame] | 172 | end |
| 173 | end |
| 174 | end |
| 175 | end |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 176 | end |
| 177 | end |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 178 | |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame^] | 179 | run_shell_test = proc do |sh| |
| 180 | run_in_testdir(sh) do |name| |
| 181 | cleanup |
| 182 | cmd = "sh ../../#{sh} make" |
| 183 | expected = IO.popen(cmd, 'r:binary', &:read) |
| 184 | cleanup |
| 185 | cmd = "sh ../../#{sh} ../../kati --use_cache --kati_log" |
| 186 | output = IO.popen(cmd, 'r:binary', &:read) |
| 187 | |
| 188 | expected = normalize_make_log(expected) |
| 189 | output = normalize_kati_log(output) |
| 190 | File.open('out.make', 'w'){|ofile|ofile.print(expected)} |
| 191 | File.open('out.kati', 'w'){|ofile|ofile.print(output)} |
| 192 | |
| 193 | if expected != output |
| 194 | puts "#{name}: FAIL" |
| 195 | failures << name |
| 196 | else |
| 197 | puts "#{name}: PASS" |
| 198 | passes << name |
| 199 | end |
| 200 | end |
| 201 | end |
| 202 | |
| 203 | test_files.each do |test| |
| 204 | if /\.mk$/ =~ test |
| 205 | run_make_test.call(test) |
| 206 | elsif /\.sh$/ =~ test |
| 207 | run_shell_test.call(test) |
| 208 | else |
| 209 | raise "Unknown test type: #{test}" |
| 210 | end |
| 211 | end |
| 212 | |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 213 | puts |
| 214 | |
| 215 | if !expected_failures.empty? |
| 216 | puts "=== Expected failures ===" |
| 217 | expected_failures.each do |n| |
| 218 | puts n |
| 219 | end |
| 220 | end |
| 221 | |
| 222 | if !unexpected_passes.empty? |
| 223 | puts "=== Unexpected passes ===" |
| 224 | unexpected_passes.each do |n| |
| 225 | puts n |
| 226 | end |
| 227 | end |
| 228 | |
| 229 | if !failures.empty? |
| 230 | puts "=== Failures ===" |
| 231 | failures.each do |n| |
| 232 | puts n |
| 233 | end |
| 234 | end |
| 235 | |
| 236 | puts |
| 237 | |
| 238 | if !unexpected_passes.empty? || !failures.empty? |
| 239 | puts 'FAIL!' |
| 240 | else |
| 241 | puts 'PASS!' |
| 242 | end |