From 06f4eec91e9f850197f3611f3c6951e12cbe173e Mon Sep 17 00:00:00 2001 From: Jon Ross-Perkins Date: Wed, 23 Oct 2024 11:56:41 -0700 Subject: [PATCH] Modify lex yaml output to elide FileStart/End in tests. (#4433) Trying to make split file tests of lex functionality shorter and easier to read. numeric_literals.carbon in particular has an example of why I'm interested in this (at the bottom). This also switches from `[]` list format to `-` list format so that the trailing `]` is removed. Trimming comments in tokenized_buffer.h because (1) it feels like it's giving too much detail about what's printed, which has drifted slightly and (2) it also feels like it's trying to justify YAML output, when that's just what we're doing in general. --------- Co-authored-by: Geoff Romer --- toolchain/driver/compile_subcommand.cpp | 13 +- toolchain/driver/compile_subcommand.h | 1 + toolchain/lex/testdata/basic_syntax.carbon | 32 +++-- .../fail_bad_comment_introducers.carbon | 7 +- ...introducers_mid_block_indent_change.carbon | 7 +- .../testdata/fail_bad_raw_identifier.carbon | 57 ++++---- .../fail_block_string_second_line.carbon | 20 ++- .../testdata/fail_mismatched_brackets.carbon | 16 +-- .../fail_mismatched_brackets_2.carbon | 32 ++--- toolchain/lex/testdata/fail_multifile.carbon | 15 +- .../testdata/fail_trailing_comments.carbon | 48 +++---- toolchain/lex/testdata/keywords.carbon | 42 +++--- toolchain/lex/testdata/multifile.carbon | 23 ++-- .../lex/testdata/numeric_literals.carbon | 128 ++++++++---------- .../testdata/printing_digit_padding.carbon | 14 +- .../testdata/printing_integer_literal.carbon | 8 +- .../lex/testdata/printing_real_literal.carbon | 8 +- toolchain/lex/testdata/printing_token.carbon | 8 +- toolchain/lex/testdata/raw_identifier.carbon | 18 +-- toolchain/lex/testdata/string_literals.carbon | Bin 3722 -> 2575 bytes toolchain/lex/tokenized_buffer.cpp | 20 +-- toolchain/lex/tokenized_buffer.h | 25 +--- toolchain/testing/file_test.cpp | 2 +- 23 files changed, 233 insertions(+), 311 deletions(-) diff --git a/toolchain/driver/compile_subcommand.cpp b/toolchain/driver/compile_subcommand.cpp index 729f4e3243ed0..79b4d93b68bc1 100644 --- a/toolchain/driver/compile_subcommand.cpp +++ b/toolchain/driver/compile_subcommand.cpp @@ -176,6 +176,16 @@ Dump the tokens to stdout when lexed. )""", }, [&](auto& arg_b) { arg_b.Set(&dump_tokens); }); + + b.AddFlag( + { + .name = "omit-file-boundary-tokens", + .help = R"""( +For `--dump-tokens`, omit file start and end boundary tokens. +)""", + }, + [&](auto& arg_b) { arg_b.Set(&omit_file_boundary_tokens); }); + b.AddFlag( { .name = "dump-parse-tree", @@ -356,7 +366,8 @@ class CompilationUnit { [&] { tokens_ = Lex::Lex(value_stores_, *source_, *consumer_); }); if (options_.dump_tokens && IncludeInDumps()) { consumer_->Flush(); - driver_env_->output_stream << tokens_; + tokens_->Print(driver_env_->output_stream, + options_.omit_file_boundary_tokens); } if (mem_usage_) { mem_usage_->Collect("tokens_", *tokens_); diff --git a/toolchain/driver/compile_subcommand.h b/toolchain/driver/compile_subcommand.h index af01d07bbb16d..fce7052434bd5 100644 --- a/toolchain/driver/compile_subcommand.h +++ b/toolchain/driver/compile_subcommand.h @@ -45,6 +45,7 @@ struct CompileOptions { bool force_obj_output = false; bool dump_shared_values = false; bool dump_tokens = false; + bool omit_file_boundary_tokens = false; bool dump_parse_tree = false; bool dump_raw_sem_ir = false; bool dump_sem_ir = false; diff --git a/toolchain/lex/testdata/basic_syntax.carbon b/toolchain/lex/testdata/basic_syntax.carbon index 4a68514ca1df6..1f0b2a996a2cd 100644 --- a/toolchain/lex/testdata/basic_syntax.carbon +++ b/toolchain/lex/testdata/basic_syntax.carbon @@ -2,29 +2,31 @@ // Exceptions. See /LICENSE for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // +// Dump all tokens so that we include the file start and end. +// ARGS: compile --phase=lex --dump-tokens %s +// // AUTOUPDATE // TIP: To test this file alone, run: // TIP: bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/lex/testdata/basic_syntax.carbon // TIP: To dump output, run: // TIP: bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/lex/testdata/basic_syntax.carbon // CHECK:STDOUT: - filename: basic_syntax.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, +// CHECK:STDOUT: tokens: +// CHECK:STDOUT: - { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' } fn run(String program) { -// CHECK:STDOUT: { index: 1, kind: 'Fn', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'fn', has_leading_space: true }, -// CHECK:STDOUT: { index: 2, kind: 'Identifier', line: {{ *}}[[@LINE-2]], column: 4, indent: 1, spelling: 'run', identifier: 0, has_leading_space: true }, -// CHECK:STDOUT: { index: 3, kind: 'OpenParen', line: {{ *}}[[@LINE-3]], column: 7, indent: 1, spelling: '(', closing_token: 6 }, -// CHECK:STDOUT: { index: 4, kind: 'StringTypeLiteral', line: {{ *}}[[@LINE-4]], column: 8, indent: 1, spelling: 'String' }, -// CHECK:STDOUT: { index: 5, kind: 'Identifier', line: {{ *}}[[@LINE-5]], column: 15, indent: 1, spelling: 'program', identifier: 1, has_leading_space: true }, -// CHECK:STDOUT: { index: 6, kind: 'CloseParen', line: {{ *}}[[@LINE-6]], column: 22, indent: 1, spelling: ')', opening_token: 3 }, -// CHECK:STDOUT: { index: 7, kind: 'OpenCurlyBrace', line: {{ *}}[[@LINE-7]], column: 24, indent: 1, spelling: '{', closing_token: 11, has_leading_space: true }, +// CHECK:STDOUT: - { index: 1, kind: 'Fn', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'fn', has_leading_space: true } +// CHECK:STDOUT: - { index: 2, kind: 'Identifier', line: {{ *}}[[@LINE-2]], column: 4, indent: 1, spelling: 'run', identifier: 0, has_leading_space: true } +// CHECK:STDOUT: - { index: 3, kind: 'OpenParen', line: {{ *}}[[@LINE-3]], column: 7, indent: 1, spelling: '(', closing_token: 6 } +// CHECK:STDOUT: - { index: 4, kind: 'StringTypeLiteral', line: {{ *}}[[@LINE-4]], column: 8, indent: 1, spelling: 'String' } +// CHECK:STDOUT: - { index: 5, kind: 'Identifier', line: {{ *}}[[@LINE-5]], column: 15, indent: 1, spelling: 'program', identifier: 1, has_leading_space: true } +// CHECK:STDOUT: - { index: 6, kind: 'CloseParen', line: {{ *}}[[@LINE-6]], column: 22, indent: 1, spelling: ')', opening_token: 3 } +// CHECK:STDOUT: - { index: 7, kind: 'OpenCurlyBrace', line: {{ *}}[[@LINE-7]], column: 24, indent: 1, spelling: '{', closing_token: 11, has_leading_space: true } return True; - // CHECK:STDOUT: { index: 8, kind: 'Return', line: {{ *}}[[@LINE-1]], column: 3, indent: 3, spelling: 'return', has_leading_space: true }, - // CHECK:STDOUT: { index: 9, kind: 'Identifier', line: {{ *}}[[@LINE-2]], column: 10, indent: 3, spelling: 'True', identifier: 2, has_leading_space: true }, - // CHECK:STDOUT: { index: 10, kind: 'Semi', line: {{ *}}[[@LINE-3]], column: 14, indent: 3, spelling: ';' }, + // CHECK:STDOUT: - { index: 8, kind: 'Return', line: {{ *}}[[@LINE-1]], column: 3, indent: 3, spelling: 'return', has_leading_space: true } + // CHECK:STDOUT: - { index: 9, kind: 'Identifier', line: {{ *}}[[@LINE-2]], column: 10, indent: 3, spelling: 'True', identifier: 2, has_leading_space: true } + // CHECK:STDOUT: - { index: 10, kind: 'Semi', line: {{ *}}[[@LINE-3]], column: 14, indent: 3, spelling: ';' } } -// CHECK:STDOUT: { index: 11, kind: 'CloseCurlyBrace', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '}', opening_token: 7, has_leading_space: true }, +// CHECK:STDOUT: - { index: 11, kind: 'CloseCurlyBrace', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '}', opening_token: 7, has_leading_space: true } -// CHECK:STDOUT: { index: 12, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] +// CHECK:STDOUT: - { index: 12, kind: 'FileEnd', line: {{ *}}[[@LINE+0]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true } diff --git a/toolchain/lex/testdata/fail_bad_comment_introducers.carbon b/toolchain/lex/testdata/fail_bad_comment_introducers.carbon index fa3009d9b43f0..d08be7075680b 100644 --- a/toolchain/lex/testdata/fail_bad_comment_introducers.carbon +++ b/toolchain/lex/testdata/fail_bad_comment_introducers.carbon @@ -7,9 +7,6 @@ // TIP: bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/lex/testdata/fail_bad_comment_introducers.carbon // TIP: To dump output, run: // TIP: bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/lex/testdata/fail_bad_comment_introducers.carbon -// CHECK:STDOUT: - filename: fail_bad_comment_introducers.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, // // Comments have to have whitespace after `//` currently. @@ -58,5 +55,5 @@ // An extra un-indented comment line to anchor the end of the file checks. -// CHECK:STDOUT: { index: 1, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] +// CHECK:STDOUT: - filename: fail_bad_comment_introducers.carbon +// CHECK:STDOUT: tokens: diff --git a/toolchain/lex/testdata/fail_bad_comment_introducers_mid_block_indent_change.carbon b/toolchain/lex/testdata/fail_bad_comment_introducers_mid_block_indent_change.carbon index c7de18913ffca..bc472077fad1d 100644 --- a/toolchain/lex/testdata/fail_bad_comment_introducers_mid_block_indent_change.carbon +++ b/toolchain/lex/testdata/fail_bad_comment_introducers_mid_block_indent_change.carbon @@ -34,9 +34,6 @@ // CHECK:STDERR: fail_bad_comment_introducers_mid_block_indent_change.carbon:[[@LINE-19]]:3: error: whitespace is required after '//' [NoWhitespaceAfterCommentIntroducer] // CHECK:STDERR: //abc // CHECK:STDERR: ^ -// CHECK:STDOUT: - filename: fail_bad_comment_introducers_mid_block_indent_change.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, -// CHECK:STDOUT: { index: 1, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] +// CHECK:STDOUT: - filename: fail_bad_comment_introducers_mid_block_indent_change.carbon +// CHECK:STDOUT: tokens: diff --git a/toolchain/lex/testdata/fail_bad_raw_identifier.carbon b/toolchain/lex/testdata/fail_bad_raw_identifier.carbon index 4d21edb2c6396..3e2ee2a25e58c 100644 --- a/toolchain/lex/testdata/fail_bad_raw_identifier.carbon +++ b/toolchain/lex/testdata/fail_bad_raw_identifier.carbon @@ -10,8 +10,7 @@ // --- fail_bad_raw_identifier.carbon // CHECK:STDOUT: - filename: fail_bad_raw_identifier.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, +// CHECK:STDOUT: tokens: // Missing the character after `#`. @@ -20,8 +19,8 @@ // CHECK:STDERR: ^ // CHECK:STDERR: r# -// CHECK:STDOUT: { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true }, -// CHECK:STDOUT: { index: 2, kind: 'Error', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: '#' }, +// CHECK:STDOUT: - { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true } +// CHECK:STDOUT: - { index: 2, kind: 'Error', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: '#' } // Not a valid identifier. // CHECK:STDERR: fail_bad_raw_identifier.carbon:[[@LINE+4]]:2: error: encountered unrecognized characters while parsing [UnrecognizedCharacters] @@ -29,9 +28,9 @@ r# // CHECK:STDERR: ^ // CHECK:STDERR: r#3 -// CHECK:STDOUT: { index: 3, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true }, -// CHECK:STDOUT: { index: 4, kind: 'Error', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: '#' }, -// CHECK:STDOUT: { index: 5, kind: 'IntLiteral', line: {{ *}}[[@LINE-3]], column: 3, indent: 1, spelling: '3', value: `3` }, +// CHECK:STDOUT: - { index: 3, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true } +// CHECK:STDOUT: - { index: 4, kind: 'Error', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: '#' } +// CHECK:STDOUT: - { index: 5, kind: 'IntLiteral', line: {{ *}}[[@LINE-3]], column: 3, indent: 1, spelling: '3', value: `3` } // Non ascii start to identifier. // CHECK:STDERR: fail_bad_raw_identifier.carbon:[[@LINE+4]]:2: error: encountered unrecognized characters while parsing [UnrecognizedCharacters] @@ -39,8 +38,8 @@ r#3 // CHECK:STDERR: ^ // CHECK:STDERR: r#á -// CHECK:STDOUT: { index: 6, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true }, -// CHECK:STDOUT: { index: 7, kind: 'Error', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: '#á' }, +// CHECK:STDOUT: - { index: 6, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true } +// CHECK:STDOUT: - { index: 7, kind: 'Error', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: '#á' } // Raw `r` identifier doesn't start a second raw identifier. // CHECK:STDERR: fail_bad_raw_identifier.carbon:[[@LINE+4]]:4: error: encountered unrecognized characters while parsing [UnrecognizedCharacters] @@ -48,9 +47,9 @@ r#á // CHECK:STDERR: ^ // CHECK:STDERR: r#r#foo -// CHECK:STDOUT: { index: 8, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true }, -// CHECK:STDOUT: { index: 9, kind: 'Error', line: {{ *}}[[@LINE-2]], column: 4, indent: 1, spelling: '#' }, -// CHECK:STDOUT: { index: 10, kind: 'Identifier', line: {{ *}}[[@LINE-3]], column: 5, indent: 1, spelling: 'foo', identifier: 1 }, +// CHECK:STDOUT: - { index: 8, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true } +// CHECK:STDOUT: - { index: 9, kind: 'Error', line: {{ *}}[[@LINE-2]], column: 4, indent: 1, spelling: '#' } +// CHECK:STDOUT: - { index: 10, kind: 'Identifier', line: {{ *}}[[@LINE-3]], column: 5, indent: 1, spelling: 'foo', identifier: 1 } // Other identifier characters don't start a raw identifier. // CHECK:STDERR: fail_bad_raw_identifier.carbon:[[@LINE+4]]:2: error: encountered unrecognized characters while parsing [UnrecognizedCharacters] @@ -58,9 +57,9 @@ r#r#foo // CHECK:STDERR: ^ // CHECK:STDERR: s#foo -// CHECK:STDOUT: { index: 11, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 's', identifier: 2, has_leading_space: true }, -// CHECK:STDOUT: { index: 12, kind: 'Error', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: '#' }, -// CHECK:STDOUT: { index: 13, kind: 'Identifier', line: {{ *}}[[@LINE-3]], column: 3, indent: 1, spelling: 'foo', identifier: 1 }, +// CHECK:STDOUT: - { index: 11, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 's', identifier: 2, has_leading_space: true } +// CHECK:STDOUT: - { index: 12, kind: 'Error', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: '#' } +// CHECK:STDOUT: - { index: 13, kind: 'Identifier', line: {{ *}}[[@LINE-3]], column: 3, indent: 1, spelling: 'foo', identifier: 1 } // Identifier ending in `r` doesn't start a raw identifier. // CHECK:STDERR: fail_bad_raw_identifier.carbon:[[@LINE+4]]:4: error: encountered unrecognized characters while parsing [UnrecognizedCharacters] @@ -68,9 +67,9 @@ s#foo // CHECK:STDERR: ^ // CHECK:STDERR: arr#foo -// CHECK:STDOUT: { index: 14, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'arr', identifier: 3, has_leading_space: true }, -// CHECK:STDOUT: { index: 15, kind: 'Error', line: {{ *}}[[@LINE-2]], column: 4, indent: 1, spelling: '#' }, -// CHECK:STDOUT: { index: 16, kind: 'Identifier', line: {{ *}}[[@LINE-3]], column: 5, indent: 1, spelling: 'foo', identifier: 1 }, +// CHECK:STDOUT: - { index: 14, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'arr', identifier: 3, has_leading_space: true } +// CHECK:STDOUT: - { index: 15, kind: 'Error', line: {{ *}}[[@LINE-2]], column: 4, indent: 1, spelling: '#' } +// CHECK:STDOUT: - { index: 16, kind: 'Identifier', line: {{ *}}[[@LINE-3]], column: 5, indent: 1, spelling: 'foo', identifier: 1 } // Whitespace between `r` and `#` isn't allowed. // CHECK:STDERR: fail_bad_raw_identifier.carbon:[[@LINE+4]]:3: error: encountered unrecognized characters while parsing [UnrecognizedCharacters] @@ -78,29 +77,23 @@ arr#foo // CHECK:STDERR: ^ // CHECK:STDERR: r #foo -// CHECK:STDOUT: { index: 17, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true }, -// CHECK:STDOUT: { index: 18, kind: 'Error', line: {{ *}}[[@LINE-2]], column: 3, indent: 1, spelling: '#', has_leading_space: true }, -// CHECK:STDOUT: { index: 19, kind: 'Identifier', line: {{ *}}[[@LINE-3]], column: 4, indent: 1, spelling: 'foo', identifier: 1 }, +// CHECK:STDOUT: - { index: 17, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true } +// CHECK:STDOUT: - { index: 18, kind: 'Error', line: {{ *}}[[@LINE-2]], column: 3, indent: 1, spelling: '#', has_leading_space: true } +// CHECK:STDOUT: - { index: 19, kind: 'Identifier', line: {{ *}}[[@LINE-3]], column: 4, indent: 1, spelling: 'foo', identifier: 1 } // This is an `r` identifier followed by a string literal. r#"hello"# -// CHECK:STDOUT: { index: 20, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true }, -// CHECK:STDOUT: { index: 21, kind: 'StringLiteral', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: '#"hello"#', value: `hello` }, +// CHECK:STDOUT: - { index: 20, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'r', identifier: 0, has_leading_space: true } +// CHECK:STDOUT: - { index: 21, kind: 'StringLiteral', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: '#"hello"#', value: `hello` } -// CHECK:STDOUT: { index: 22, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] // --- fail_hash_at_start_of_file.carbon // CHECK:STDOUT: - filename: fail_hash_at_start_of_file.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, +// CHECK:STDOUT: tokens: // Ensure that we correctly handle a `#` as the first token in the file. // CHECK:STDERR: fail_hash_at_start_of_file.carbon:[[@LINE+3]]:1: error: encountered unrecognized characters while parsing [UnrecognizedCharacters] // CHECK:STDERR: #foo // CHECK:STDERR: ^ #foo -// CHECK:STDOUT: { index: 1, kind: 'Error', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '#', has_leading_space: true }, -// CHECK:STDOUT: { index: 2, kind: 'Identifier', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: 'foo', identifier: 0 }, - -// CHECK:STDOUT: { index: 3, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] +// CHECK:STDOUT: - { index: 1, kind: 'Error', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '#', has_leading_space: true } +// CHECK:STDOUT: - { index: 2, kind: 'Identifier', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: 'foo', identifier: 0 } diff --git a/toolchain/lex/testdata/fail_block_string_second_line.carbon b/toolchain/lex/testdata/fail_block_string_second_line.carbon index 82bb9d7137c86..9b4178dad4740 100644 --- a/toolchain/lex/testdata/fail_block_string_second_line.carbon +++ b/toolchain/lex/testdata/fail_block_string_second_line.carbon @@ -17,15 +17,11 @@ var s: String = ''' // CHECK:STDERR: error here: ''' // CHECK:STDERR: ^ // CHECK:STDOUT: - filename: fail_block_string_second_line.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, -// CHECK:STDOUT: { index: 1, kind: 'Var', line: {{ *}}[[@LINE-17]], column: 1, indent: 1, spelling: 'var', has_leading_space: true }, -// CHECK:STDOUT: { index: 2, kind: 'Identifier', line: {{ *}}[[@LINE-18]], column: 5, indent: 1, spelling: 's', identifier: 0, has_leading_space: true }, -// CHECK:STDOUT: { index: 3, kind: 'Colon', line: {{ *}}[[@LINE-19]], column: 6, indent: 1, spelling: ':' }, -// CHECK:STDOUT: { index: 4, kind: 'StringTypeLiteral', line: {{ *}}[[@LINE-20]], column: 8, indent: 1, spelling: 'String', has_leading_space: true }, -// CHECK:STDOUT: { index: 5, kind: 'Equal', line: {{ *}}[[@LINE-21]], column: 15, indent: 1, spelling: '=', has_leading_space: true }, -// CHECK:STDOUT: { index: 6, kind: 'StringLiteral', line: {{ *}}[[@LINE-22]], column: 17, indent: 1, spelling: '''' -// CHECK:STDOUT: error here: '''', value: `error here: `, has_leading_space: true }, - -// CHECK:STDOUT: { index: 7, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] +// CHECK:STDOUT: tokens: +// CHECK:STDOUT: - { index: 1, kind: 'Var', line: {{ *}}[[@LINE-16]], column: 1, indent: 1, spelling: 'var', has_leading_space: true } +// CHECK:STDOUT: - { index: 2, kind: 'Identifier', line: {{ *}}[[@LINE-17]], column: 5, indent: 1, spelling: 's', identifier: 0, has_leading_space: true } +// CHECK:STDOUT: - { index: 3, kind: 'Colon', line: {{ *}}[[@LINE-18]], column: 6, indent: 1, spelling: ':' } +// CHECK:STDOUT: - { index: 4, kind: 'StringTypeLiteral', line: {{ *}}[[@LINE-19]], column: 8, indent: 1, spelling: 'String', has_leading_space: true } +// CHECK:STDOUT: - { index: 5, kind: 'Equal', line: {{ *}}[[@LINE-20]], column: 15, indent: 1, spelling: '=', has_leading_space: true } +// CHECK:STDOUT: - { index: 6, kind: 'StringLiteral', line: {{ *}}[[@LINE-21]], column: 17, indent: 1, spelling: '''' +// CHECK:STDOUT: error here: '''', value: `error here: `, has_leading_space: true } diff --git a/toolchain/lex/testdata/fail_mismatched_brackets.carbon b/toolchain/lex/testdata/fail_mismatched_brackets.carbon index 68956128254ac..f2ab84db8113e 100644 --- a/toolchain/lex/testdata/fail_mismatched_brackets.carbon +++ b/toolchain/lex/testdata/fail_mismatched_brackets.carbon @@ -8,30 +8,26 @@ // TIP: To dump output, run: // TIP: bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/lex/testdata/fail_mismatched_brackets.carbon // CHECK:STDOUT: - filename: fail_mismatched_brackets.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, +// CHECK:STDOUT: tokens: // CHECK:STDERR: fail_mismatched_brackets.carbon:[[@LINE+4]]:1: error: closing symbol without a corresponding opening symbol [UnmatchedClosing] // CHECK:STDERR: } // CHECK:STDERR: ^ // CHECK:STDERR: } -// CHECK:STDOUT: { index: 1, kind: 'Error', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '}', has_leading_space: true }, +// CHECK:STDOUT: - { index: 1, kind: 'Error', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '}', has_leading_space: true } // CHECK:STDERR: fail_mismatched_brackets.carbon:[[@LINE+4]]:3: error: closing symbol without a corresponding opening symbol [UnmatchedClosing] // CHECK:STDERR: ( } ) // CHECK:STDERR: ^ // CHECK:STDERR: ( } ) -// CHECK:STDOUT: { index: 2, kind: 'OpenParen', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '(', closing_token: 4, has_leading_space: true }, -// CHECK:STDOUT: { index: 3, kind: 'Error', line: {{ *}}[[@LINE-2]], column: 3, indent: 1, spelling: '}', has_leading_space: true }, -// CHECK:STDOUT: { index: 4, kind: 'CloseParen', line: {{ *}}[[@LINE-3]], column: 5, indent: 1, spelling: ')', opening_token: 2, has_leading_space: true }, +// CHECK:STDOUT: - { index: 2, kind: 'OpenParen', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '(', closing_token: 4, has_leading_space: true } +// CHECK:STDOUT: - { index: 3, kind: 'Error', line: {{ *}}[[@LINE-2]], column: 3, indent: 1, spelling: '}', has_leading_space: true } +// CHECK:STDOUT: - { index: 4, kind: 'CloseParen', line: {{ *}}[[@LINE-3]], column: 5, indent: 1, spelling: ')', opening_token: 2, has_leading_space: true } // CHECK:STDERR: fail_mismatched_brackets.carbon:[[@LINE+3]]:1: error: opening symbol without a corresponding closing symbol [UnmatchedOpening] // CHECK:STDERR: [ // CHECK:STDERR: ^ [ -// CHECK:STDOUT: { index: 5, kind: 'Error', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '[', has_leading_space: true }, - -// CHECK:STDOUT: { index: 6, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] +// CHECK:STDOUT: - { index: 5, kind: 'Error', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '[', has_leading_space: true } diff --git a/toolchain/lex/testdata/fail_mismatched_brackets_2.carbon b/toolchain/lex/testdata/fail_mismatched_brackets_2.carbon index 6a80913a9cff7..af574bdaab9a5 100644 --- a/toolchain/lex/testdata/fail_mismatched_brackets_2.carbon +++ b/toolchain/lex/testdata/fail_mismatched_brackets_2.carbon @@ -22,21 +22,17 @@ fn F() { // CHECK:STDERR: } // CHECK:STDERR: ^ // CHECK:STDOUT: - filename: fail_mismatched_brackets_2.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, -// CHECK:STDOUT: { index: 1, kind: 'Fn', line: {{ *}}[[@LINE-19]], column: 1, indent: 1, spelling: 'fn', has_leading_space: true }, -// CHECK:STDOUT: { index: 2, kind: 'Identifier', line: {{ *}}[[@LINE-20]], column: 4, indent: 1, spelling: 'F', identifier: 0, has_leading_space: true }, -// CHECK:STDOUT: { index: 3, kind: 'OpenParen', line: {{ *}}[[@LINE-21]], column: 5, indent: 1, spelling: '(', closing_token: 4 }, -// CHECK:STDOUT: { index: 4, kind: 'CloseParen', line: {{ *}}[[@LINE-22]], column: 6, indent: 1, spelling: ')', opening_token: 3 }, -// CHECK:STDOUT: { index: 5, kind: 'OpenCurlyBrace', line: {{ *}}[[@LINE-23]], column: 8, indent: 1, spelling: '{', closing_token: 12, has_leading_space: true }, -// CHECK:STDOUT: { index: 6, kind: 'If', line: {{ *}}[[@LINE-23]], column: 3, indent: 3, spelling: 'if', has_leading_space: true }, -// CHECK:STDOUT: { index: 7, kind: 'OpenParen', line: {{ *}}[[@LINE-24]], column: 6, indent: 3, spelling: '(', closing_token: 9, has_leading_space: true }, -// CHECK:STDOUT: { index: 8, kind: 'Identifier', line: {{ *}}[[@LINE-25]], column: 7, indent: 3, spelling: 'thing1', identifier: 1 }, -// CHECK:STDOUT: { index: 9, kind: 'CloseParen', line: {{ *}}[[@LINE-26]], column: 13, indent: 3, spelling: ')', opening_token: 7 }, -// CHECK:STDOUT: { index: 10, kind: 'Identifier', line: {{ *}}[[@LINE-26]], column: 5, indent: 5, spelling: 'thing2', identifier: 2, has_leading_space: true }, -// CHECK:STDOUT: { index: 11, kind: 'Semi', line: {{ *}}[[@LINE-27]], column: 11, indent: 5, spelling: ';' }, -// CHECK:STDOUT: { index: 12, kind: 'CloseCurlyBrace', line: {{ *}}[[@LINE-27]], column: 3, indent: 3, spelling: '}', opening_token: 5, has_leading_space: true }, -// CHECK:STDOUT: { index: 13, kind: 'Error', line: {{ *}}[[@LINE-27]], column: 1, indent: 1, spelling: '}', has_leading_space: true }, - -// CHECK:STDOUT: { index: 14, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] +// CHECK:STDOUT: tokens: +// CHECK:STDOUT: - { index: 1, kind: 'Fn', line: {{ *}}[[@LINE-18]], column: 1, indent: 1, spelling: 'fn', has_leading_space: true } +// CHECK:STDOUT: - { index: 2, kind: 'Identifier', line: {{ *}}[[@LINE-19]], column: 4, indent: 1, spelling: 'F', identifier: 0, has_leading_space: true } +// CHECK:STDOUT: - { index: 3, kind: 'OpenParen', line: {{ *}}[[@LINE-20]], column: 5, indent: 1, spelling: '(', closing_token: 4 } +// CHECK:STDOUT: - { index: 4, kind: 'CloseParen', line: {{ *}}[[@LINE-21]], column: 6, indent: 1, spelling: ')', opening_token: 3 } +// CHECK:STDOUT: - { index: 5, kind: 'OpenCurlyBrace', line: {{ *}}[[@LINE-22]], column: 8, indent: 1, spelling: '{', closing_token: 12, has_leading_space: true } +// CHECK:STDOUT: - { index: 6, kind: 'If', line: {{ *}}[[@LINE-22]], column: 3, indent: 3, spelling: 'if', has_leading_space: true } +// CHECK:STDOUT: - { index: 7, kind: 'OpenParen', line: {{ *}}[[@LINE-23]], column: 6, indent: 3, spelling: '(', closing_token: 9, has_leading_space: true } +// CHECK:STDOUT: - { index: 8, kind: 'Identifier', line: {{ *}}[[@LINE-24]], column: 7, indent: 3, spelling: 'thing1', identifier: 1 } +// CHECK:STDOUT: - { index: 9, kind: 'CloseParen', line: {{ *}}[[@LINE-25]], column: 13, indent: 3, spelling: ')', opening_token: 7 } +// CHECK:STDOUT: - { index: 10, kind: 'Identifier', line: {{ *}}[[@LINE-25]], column: 5, indent: 5, spelling: 'thing2', identifier: 2, has_leading_space: true } +// CHECK:STDOUT: - { index: 11, kind: 'Semi', line: {{ *}}[[@LINE-26]], column: 11, indent: 5, spelling: ';' } +// CHECK:STDOUT: - { index: 12, kind: 'CloseCurlyBrace', line: {{ *}}[[@LINE-26]], column: 3, indent: 3, spelling: '}', opening_token: 5, has_leading_space: true } +// CHECK:STDOUT: - { index: 13, kind: 'Error', line: {{ *}}[[@LINE-26]], column: 1, indent: 1, spelling: '}', has_leading_space: true } diff --git a/toolchain/lex/testdata/fail_multifile.carbon b/toolchain/lex/testdata/fail_multifile.carbon index 4d891fef74a39..58b26f3719b91 100644 --- a/toolchain/lex/testdata/fail_multifile.carbon +++ b/toolchain/lex/testdata/fail_multifile.carbon @@ -10,26 +10,19 @@ // --- fail_a.carbon // CHECK:STDOUT: - filename: fail_a.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, +// CHECK:STDOUT: tokens: // CHECK:STDERR: fail_a.carbon:[[@LINE+4]]:3: error: empty digit sequence in numeric literal [EmptyDigitSequence] // CHECK:STDERR: 1.a // CHECK:STDERR: ^ // CHECK:STDERR: 1.a -// CHECK:STDOUT: { index: 1, kind: 'Error', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '1.a', has_leading_space: true }, +// CHECK:STDOUT: - { index: 1, kind: 'Error', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '1.a', has_leading_space: true } -// CHECK:STDOUT: { index: 2, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] // --- fail_b.carbon // CHECK:STDOUT: - filename: fail_b.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, +// CHECK:STDOUT: tokens: // CHECK:STDERR: fail_b.carbon:[[@LINE+3]]:3: error: empty digit sequence in numeric literal [EmptyDigitSequence] // CHECK:STDERR: 2.b // CHECK:STDERR: ^ 2.b -// CHECK:STDOUT: { index: 1, kind: 'Error', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '2.b', has_leading_space: true }, - -// CHECK:STDOUT: { index: 2, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] +// CHECK:STDOUT: - { index: 1, kind: 'Error', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '2.b', has_leading_space: true } diff --git a/toolchain/lex/testdata/fail_trailing_comments.carbon b/toolchain/lex/testdata/fail_trailing_comments.carbon index 5a9a1e9c5bf08..2ca924cae03c4 100644 --- a/toolchain/lex/testdata/fail_trailing_comments.carbon +++ b/toolchain/lex/testdata/fail_trailing_comments.carbon @@ -28,29 +28,25 @@ var c: i32 = 0.4; // still more trailing comment // CHECK:STDERR: var c: i32 = 0.4; // still more trailing comment // CHECK:STDERR: ^ // CHECK:STDOUT: - filename: fail_trailing_comments.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, -// CHECK:STDOUT: { index: 1, kind: 'Var', line: {{ *}}[[@LINE-25]], column: 1, indent: 1, spelling: 'var', has_leading_space: true }, -// CHECK:STDOUT: { index: 2, kind: 'Identifier', line: {{ *}}[[@LINE-26]], column: 5, indent: 1, spelling: 'a', identifier: 0, has_leading_space: true }, -// CHECK:STDOUT: { index: 3, kind: 'Colon', line: {{ *}}[[@LINE-27]], column: 6, indent: 1, spelling: ':' }, -// CHECK:STDOUT: { index: 4, kind: 'IntTypeLiteral', line: {{ *}}[[@LINE-28]], column: 8, indent: 1, spelling: 'i32', has_leading_space: true }, -// CHECK:STDOUT: { index: 5, kind: 'Equal', line: {{ *}}[[@LINE-29]], column: 12, indent: 1, spelling: '=', has_leading_space: true }, -// CHECK:STDOUT: { index: 6, kind: 'IntLiteral', line: {{ *}}[[@LINE-30]], column: 14, indent: 1, spelling: '1', value: `1`, has_leading_space: true }, -// CHECK:STDOUT: { index: 7, kind: 'Semi', line: {{ *}}[[@LINE-31]], column: 15, indent: 1, spelling: ';' }, -// CHECK:STDOUT: { index: 8, kind: 'Var', line: {{ *}}[[@LINE-31]], column: 1, indent: 1, spelling: 'var', has_leading_space: true }, -// CHECK:STDOUT: { index: 9, kind: 'Identifier', line: {{ *}}[[@LINE-32]], column: 5, indent: 1, spelling: 'b', identifier: 1, has_leading_space: true }, -// CHECK:STDOUT: { index: 10, kind: 'Colon', line: {{ *}}[[@LINE-33]], column: 6, indent: 1, spelling: ':' }, -// CHECK:STDOUT: { index: 11, kind: 'IntLiteral', line: {{ *}}[[@LINE-34]], column: 8, indent: 1, spelling: '32', value: `32`, has_leading_space: true }, -// CHECK:STDOUT: { index: 12, kind: 'Equal', line: {{ *}}[[@LINE-35]], column: 11, indent: 1, spelling: '=', has_leading_space: true }, -// CHECK:STDOUT: { index: 13, kind: 'IntLiteral', line: {{ *}}[[@LINE-36]], column: 13, indent: 1, spelling: '13', value: `13`, has_leading_space: true }, -// CHECK:STDOUT: { index: 14, kind: 'Semi', line: {{ *}}[[@LINE-37]], column: 15, indent: 1, spelling: ';' }, -// CHECK:STDOUT: { index: 15, kind: 'Var', line: {{ *}}[[@LINE-37]], column: 1, indent: 1, spelling: 'var', has_leading_space: true }, -// CHECK:STDOUT: { index: 16, kind: 'Identifier', line: {{ *}}[[@LINE-38]], column: 5, indent: 1, spelling: 'c', identifier: 2, has_leading_space: true }, -// CHECK:STDOUT: { index: 17, kind: 'Colon', line: {{ *}}[[@LINE-39]], column: 6, indent: 1, spelling: ':' }, -// CHECK:STDOUT: { index: 18, kind: 'IntTypeLiteral', line: {{ *}}[[@LINE-40]], column: 8, indent: 1, spelling: 'i32', has_leading_space: true }, -// CHECK:STDOUT: { index: 19, kind: 'Equal', line: {{ *}}[[@LINE-41]], column: 12, indent: 1, spelling: '=', has_leading_space: true }, -// CHECK:STDOUT: { index: 20, kind: 'RealLiteral', line: {{ *}}[[@LINE-42]], column: 14, indent: 1, spelling: '0.4', value: `4*10^-1`, has_leading_space: true }, -// CHECK:STDOUT: { index: 21, kind: 'Semi', line: {{ *}}[[@LINE-43]], column: 17, indent: 1, spelling: ';' }, - -// CHECK:STDOUT: { index: 22, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] +// CHECK:STDOUT: tokens: +// CHECK:STDOUT: - { index: 1, kind: 'Var', line: {{ *}}[[@LINE-24]], column: 1, indent: 1, spelling: 'var', has_leading_space: true } +// CHECK:STDOUT: - { index: 2, kind: 'Identifier', line: {{ *}}[[@LINE-25]], column: 5, indent: 1, spelling: 'a', identifier: 0, has_leading_space: true } +// CHECK:STDOUT: - { index: 3, kind: 'Colon', line: {{ *}}[[@LINE-26]], column: 6, indent: 1, spelling: ':' } +// CHECK:STDOUT: - { index: 4, kind: 'IntTypeLiteral', line: {{ *}}[[@LINE-27]], column: 8, indent: 1, spelling: 'i32', has_leading_space: true } +// CHECK:STDOUT: - { index: 5, kind: 'Equal', line: {{ *}}[[@LINE-28]], column: 12, indent: 1, spelling: '=', has_leading_space: true } +// CHECK:STDOUT: - { index: 6, kind: 'IntLiteral', line: {{ *}}[[@LINE-29]], column: 14, indent: 1, spelling: '1', value: `1`, has_leading_space: true } +// CHECK:STDOUT: - { index: 7, kind: 'Semi', line: {{ *}}[[@LINE-30]], column: 15, indent: 1, spelling: ';' } +// CHECK:STDOUT: - { index: 8, kind: 'Var', line: {{ *}}[[@LINE-30]], column: 1, indent: 1, spelling: 'var', has_leading_space: true } +// CHECK:STDOUT: - { index: 9, kind: 'Identifier', line: {{ *}}[[@LINE-31]], column: 5, indent: 1, spelling: 'b', identifier: 1, has_leading_space: true } +// CHECK:STDOUT: - { index: 10, kind: 'Colon', line: {{ *}}[[@LINE-32]], column: 6, indent: 1, spelling: ':' } +// CHECK:STDOUT: - { index: 11, kind: 'IntLiteral', line: {{ *}}[[@LINE-33]], column: 8, indent: 1, spelling: '32', value: `32`, has_leading_space: true } +// CHECK:STDOUT: - { index: 12, kind: 'Equal', line: {{ *}}[[@LINE-34]], column: 11, indent: 1, spelling: '=', has_leading_space: true } +// CHECK:STDOUT: - { index: 13, kind: 'IntLiteral', line: {{ *}}[[@LINE-35]], column: 13, indent: 1, spelling: '13', value: `13`, has_leading_space: true } +// CHECK:STDOUT: - { index: 14, kind: 'Semi', line: {{ *}}[[@LINE-36]], column: 15, indent: 1, spelling: ';' } +// CHECK:STDOUT: - { index: 15, kind: 'Var', line: {{ *}}[[@LINE-36]], column: 1, indent: 1, spelling: 'var', has_leading_space: true } +// CHECK:STDOUT: - { index: 16, kind: 'Identifier', line: {{ *}}[[@LINE-37]], column: 5, indent: 1, spelling: 'c', identifier: 2, has_leading_space: true } +// CHECK:STDOUT: - { index: 17, kind: 'Colon', line: {{ *}}[[@LINE-38]], column: 6, indent: 1, spelling: ':' } +// CHECK:STDOUT: - { index: 18, kind: 'IntTypeLiteral', line: {{ *}}[[@LINE-39]], column: 8, indent: 1, spelling: 'i32', has_leading_space: true } +// CHECK:STDOUT: - { index: 19, kind: 'Equal', line: {{ *}}[[@LINE-40]], column: 12, indent: 1, spelling: '=', has_leading_space: true } +// CHECK:STDOUT: - { index: 20, kind: 'RealLiteral', line: {{ *}}[[@LINE-41]], column: 14, indent: 1, spelling: '0.4', value: `4*10^-1`, has_leading_space: true } +// CHECK:STDOUT: - { index: 21, kind: 'Semi', line: {{ *}}[[@LINE-42]], column: 17, indent: 1, spelling: ';' } diff --git a/toolchain/lex/testdata/keywords.carbon b/toolchain/lex/testdata/keywords.carbon index 740d2b18d9af8..17d56eadb8705 100644 --- a/toolchain/lex/testdata/keywords.carbon +++ b/toolchain/lex/testdata/keywords.carbon @@ -10,41 +10,31 @@ // --- indented.carbon // CHECK:STDOUT: - filename: indented.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, +// CHECK:STDOUT: tokens: fn - // CHECK:STDOUT: { index: 1, kind: 'Fn', line: {{ *}}[[@LINE-1]], column: 4, indent: 4, spelling: 'fn', has_leading_space: true }, + // CHECK:STDOUT: - { index: 1, kind: 'Fn', line: {{ *}}[[@LINE-1]], column: 4, indent: 4, spelling: 'fn', has_leading_space: true } -// CHECK:STDOUT: { index: 2, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] // --- chain.carbon // CHECK:STDOUT: - filename: chain.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, +// CHECK:STDOUT: tokens: and or not if else for return var break continue _ -// CHECK:STDOUT: { index: 1, kind: 'And', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'and', has_leading_space: true }, -// CHECK:STDOUT: { index: 2, kind: 'Or', line: {{ *}}[[@LINE-2]], column: 5, indent: 1, spelling: 'or', has_leading_space: true }, -// CHECK:STDOUT: { index: 3, kind: 'Not', line: {{ *}}[[@LINE-3]], column: 8, indent: 1, spelling: 'not', has_leading_space: true }, -// CHECK:STDOUT: { index: 4, kind: 'If', line: {{ *}}[[@LINE-4]], column: 12, indent: 1, spelling: 'if', has_leading_space: true }, -// CHECK:STDOUT: { index: 5, kind: 'Else', line: {{ *}}[[@LINE-5]], column: 15, indent: 1, spelling: 'else', has_leading_space: true }, -// CHECK:STDOUT: { index: 6, kind: 'For', line: {{ *}}[[@LINE-6]], column: 20, indent: 1, spelling: 'for', has_leading_space: true }, -// CHECK:STDOUT: { index: 7, kind: 'Return', line: {{ *}}[[@LINE-7]], column: 24, indent: 1, spelling: 'return', has_leading_space: true }, -// CHECK:STDOUT: { index: 8, kind: 'Var', line: {{ *}}[[@LINE-8]], column: 31, indent: 1, spelling: 'var', has_leading_space: true }, -// CHECK:STDOUT: { index: 9, kind: 'Break', line: {{ *}}[[@LINE-9]], column: 35, indent: 1, spelling: 'break', has_leading_space: true }, -// CHECK:STDOUT: { index: 10, kind: 'Continue', line: {{ *}}[[@LINE-10]], column: 41, indent: 1, spelling: 'continue', has_leading_space: true }, -// CHECK:STDOUT: { index: 11, kind: 'Underscore', line: {{ *}}[[@LINE-11]], column: 50, indent: 1, spelling: '_', has_leading_space: true }, +// CHECK:STDOUT: - { index: 1, kind: 'And', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'and', has_leading_space: true } +// CHECK:STDOUT: - { index: 2, kind: 'Or', line: {{ *}}[[@LINE-2]], column: 5, indent: 1, spelling: 'or', has_leading_space: true } +// CHECK:STDOUT: - { index: 3, kind: 'Not', line: {{ *}}[[@LINE-3]], column: 8, indent: 1, spelling: 'not', has_leading_space: true } +// CHECK:STDOUT: - { index: 4, kind: 'If', line: {{ *}}[[@LINE-4]], column: 12, indent: 1, spelling: 'if', has_leading_space: true } +// CHECK:STDOUT: - { index: 5, kind: 'Else', line: {{ *}}[[@LINE-5]], column: 15, indent: 1, spelling: 'else', has_leading_space: true } +// CHECK:STDOUT: - { index: 6, kind: 'For', line: {{ *}}[[@LINE-6]], column: 20, indent: 1, spelling: 'for', has_leading_space: true } +// CHECK:STDOUT: - { index: 7, kind: 'Return', line: {{ *}}[[@LINE-7]], column: 24, indent: 1, spelling: 'return', has_leading_space: true } +// CHECK:STDOUT: - { index: 8, kind: 'Var', line: {{ *}}[[@LINE-8]], column: 31, indent: 1, spelling: 'var', has_leading_space: true } +// CHECK:STDOUT: - { index: 9, kind: 'Break', line: {{ *}}[[@LINE-9]], column: 35, indent: 1, spelling: 'break', has_leading_space: true } +// CHECK:STDOUT: - { index: 10, kind: 'Continue', line: {{ *}}[[@LINE-10]], column: 41, indent: 1, spelling: 'continue', has_leading_space: true } +// CHECK:STDOUT: - { index: 11, kind: 'Underscore', line: {{ *}}[[@LINE-11]], column: 50, indent: 1, spelling: '_', has_leading_space: true } -// CHECK:STDOUT: { index: 12, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] // --- notakeyword.carbon // CHECK:STDOUT: - filename: notakeyword.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, +// CHECK:STDOUT: tokens: notakeyword -// CHECK:STDOUT: { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'notakeyword', identifier: 0, has_leading_space: true }, - -// CHECK:STDOUT: { index: 2, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] +// CHECK:STDOUT: - { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'notakeyword', identifier: 0, has_leading_space: true } diff --git a/toolchain/lex/testdata/multifile.carbon b/toolchain/lex/testdata/multifile.carbon index 024bef9bffb7e..0fbcabbece735 100644 --- a/toolchain/lex/testdata/multifile.carbon +++ b/toolchain/lex/testdata/multifile.carbon @@ -10,24 +10,17 @@ // --- a.carbon // CHECK:STDOUT: - filename: a.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, +// CHECK:STDOUT: tokens: a; -// CHECK:STDOUT: { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'a', identifier: 0, has_leading_space: true }, -// CHECK:STDOUT: { index: 2, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: ';' }, +// CHECK:STDOUT: - { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'a', identifier: 0, has_leading_space: true } +// CHECK:STDOUT: - { index: 2, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: ';' } -// CHECK:STDOUT: { index: 3, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] // --- b.carbon // CHECK:STDOUT: - filename: b.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, +// CHECK:STDOUT: tokens: b; -// CHECK:STDOUT: { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'b', identifier: 0, has_leading_space: true }, -// CHECK:STDOUT: { index: 2, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: ';' }, +// CHECK:STDOUT: - { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'b', identifier: 0, has_leading_space: true } +// CHECK:STDOUT: - { index: 2, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: ';' } a; -// CHECK:STDOUT: { index: 3, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'a', identifier: 1, has_leading_space: true }, -// CHECK:STDOUT: { index: 4, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: ';' }, - -// CHECK:STDOUT: { index: 5, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] +// CHECK:STDOUT: - { index: 3, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'a', identifier: 1, has_leading_space: true } +// CHECK:STDOUT: - { index: 4, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 2, indent: 1, spelling: ';' } diff --git a/toolchain/lex/testdata/numeric_literals.carbon b/toolchain/lex/testdata/numeric_literals.carbon index 2d0e838ce4241..9c6df916c1ada 100644 --- a/toolchain/lex/testdata/numeric_literals.carbon +++ b/toolchain/lex/testdata/numeric_literals.carbon @@ -10,112 +10,102 @@ // --- valid.carbon // CHECK:STDOUT: - filename: valid.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, +// CHECK:STDOUT: tokens: fn F() { -// CHECK:STDOUT: { index: 1, kind: 'Fn', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'fn', has_leading_space: true }, -// CHECK:STDOUT: { index: 2, kind: 'Identifier', line: {{ *}}[[@LINE-2]], column: 4, indent: 1, spelling: 'F', identifier: 0, has_leading_space: true }, -// CHECK:STDOUT: { index: 3, kind: 'OpenParen', line: {{ *}}[[@LINE-3]], column: 5, indent: 1, spelling: '(', closing_token: 4 }, -// CHECK:STDOUT: { index: 4, kind: 'CloseParen', line: {{ *}}[[@LINE-4]], column: 6, indent: 1, spelling: ')', opening_token: 3 }, -// CHECK:STDOUT: { index: 5, kind: 'OpenCurlyBrace', line: {{ *}}[[@LINE-5]], column: 8, indent: 1, spelling: '{', closing_token: 54, has_leading_space: true }, +// CHECK:STDOUT: - { index: 1, kind: 'Fn', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'fn', has_leading_space: true } +// CHECK:STDOUT: - { index: 2, kind: 'Identifier', line: {{ *}}[[@LINE-2]], column: 4, indent: 1, spelling: 'F', identifier: 0, has_leading_space: true } +// CHECK:STDOUT: - { index: 3, kind: 'OpenParen', line: {{ *}}[[@LINE-3]], column: 5, indent: 1, spelling: '(', closing_token: 4 } +// CHECK:STDOUT: - { index: 4, kind: 'CloseParen', line: {{ *}}[[@LINE-4]], column: 6, indent: 1, spelling: ')', opening_token: 3 } +// CHECK:STDOUT: - { index: 5, kind: 'OpenCurlyBrace', line: {{ *}}[[@LINE-5]], column: 8, indent: 1, spelling: '{', closing_token: 54, has_leading_space: true } // 8 and 9 trigger special behavior in APInt when mishandling signed versus // unsigned, so we pay extra attention to those. var ints: [i32; 5] = ( - // CHECK:STDOUT: { index: 6, kind: 'Var', line: {{ *}}[[@LINE-1]], column: 3, indent: 3, spelling: 'var', has_leading_space: true }, - // CHECK:STDOUT: { index: 7, kind: 'Identifier', line: {{ *}}[[@LINE-2]], column: 7, indent: 3, spelling: 'ints', identifier: 1, has_leading_space: true }, - // CHECK:STDOUT: { index: 8, kind: 'Colon', line: {{ *}}[[@LINE-3]], column: 11, indent: 3, spelling: ':' }, - // CHECK:STDOUT: { index: 9, kind: 'OpenSquareBracket', line: {{ *}}[[@LINE-4]], column: 13, indent: 3, spelling: '[', closing_token: 13, has_leading_space: true }, - // CHECK:STDOUT: { index: 10, kind: 'IntTypeLiteral', line: {{ *}}[[@LINE-5]], column: 14, indent: 3, spelling: 'i32' }, - // CHECK:STDOUT: { index: 11, kind: 'Semi', line: {{ *}}[[@LINE-6]], column: 17, indent: 3, spelling: ';' }, - // CHECK:STDOUT: { index: 12, kind: 'IntLiteral', line: {{ *}}[[@LINE-7]], column: 19, indent: 3, spelling: '5', value: `5`, has_leading_space: true }, - // CHECK:STDOUT: { index: 13, kind: 'CloseSquareBracket', line: {{ *}}[[@LINE-8]], column: 20, indent: 3, spelling: ']', opening_token: 9 }, - // CHECK:STDOUT: { index: 14, kind: 'Equal', line: {{ *}}[[@LINE-9]], column: 22, indent: 3, spelling: '=', has_leading_space: true }, - // CHECK:STDOUT: { index: 15, kind: 'OpenParen', line: {{ *}}[[@LINE-10]], column: 24, indent: 3, spelling: '(', closing_token: 26, has_leading_space: true }, + // CHECK:STDOUT: - { index: 6, kind: 'Var', line: {{ *}}[[@LINE-1]], column: 3, indent: 3, spelling: 'var', has_leading_space: true } + // CHECK:STDOUT: - { index: 7, kind: 'Identifier', line: {{ *}}[[@LINE-2]], column: 7, indent: 3, spelling: 'ints', identifier: 1, has_leading_space: true } + // CHECK:STDOUT: - { index: 8, kind: 'Colon', line: {{ *}}[[@LINE-3]], column: 11, indent: 3, spelling: ':' } + // CHECK:STDOUT: - { index: 9, kind: 'OpenSquareBracket', line: {{ *}}[[@LINE-4]], column: 13, indent: 3, spelling: '[', closing_token: 13, has_leading_space: true } + // CHECK:STDOUT: - { index: 10, kind: 'IntTypeLiteral', line: {{ *}}[[@LINE-5]], column: 14, indent: 3, spelling: 'i32' } + // CHECK:STDOUT: - { index: 11, kind: 'Semi', line: {{ *}}[[@LINE-6]], column: 17, indent: 3, spelling: ';' } + // CHECK:STDOUT: - { index: 12, kind: 'IntLiteral', line: {{ *}}[[@LINE-7]], column: 19, indent: 3, spelling: '5', value: `5`, has_leading_space: true } + // CHECK:STDOUT: - { index: 13, kind: 'CloseSquareBracket', line: {{ *}}[[@LINE-8]], column: 20, indent: 3, spelling: ']', opening_token: 9 } + // CHECK:STDOUT: - { index: 14, kind: 'Equal', line: {{ *}}[[@LINE-9]], column: 22, indent: 3, spelling: '=', has_leading_space: true } + // CHECK:STDOUT: - { index: 15, kind: 'OpenParen', line: {{ *}}[[@LINE-10]], column: 24, indent: 3, spelling: '(', closing_token: 26, has_leading_space: true } 8, - // CHECK:STDOUT: { index: 16, kind: 'IntLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '8', value: `8`, has_leading_space: true }, - // CHECK:STDOUT: { index: 17, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 6, indent: 5, spelling: ',' }, + // CHECK:STDOUT: - { index: 16, kind: 'IntLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '8', value: `8`, has_leading_space: true } + // CHECK:STDOUT: - { index: 17, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 6, indent: 5, spelling: ',' } 9, - // CHECK:STDOUT: { index: 18, kind: 'IntLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '9', value: `9`, has_leading_space: true }, - // CHECK:STDOUT: { index: 19, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 6, indent: 5, spelling: ',' }, + // CHECK:STDOUT: - { index: 18, kind: 'IntLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '9', value: `9`, has_leading_space: true } + // CHECK:STDOUT: - { index: 19, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 6, indent: 5, spelling: ',' } 0x8, - // CHECK:STDOUT: { index: 20, kind: 'IntLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '0x8', value: `8`, has_leading_space: true }, - // CHECK:STDOUT: { index: 21, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 8, indent: 5, spelling: ',' }, + // CHECK:STDOUT: - { index: 20, kind: 'IntLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '0x8', value: `8`, has_leading_space: true } + // CHECK:STDOUT: - { index: 21, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 8, indent: 5, spelling: ',' } 0b1000, - // CHECK:STDOUT: { index: 22, kind: 'IntLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '0b1000', value: `8`, has_leading_space: true }, - // CHECK:STDOUT: { index: 23, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 11, indent: 5, spelling: ',' }, + // CHECK:STDOUT: - { index: 22, kind: 'IntLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '0b1000', value: `8`, has_leading_space: true } + // CHECK:STDOUT: - { index: 23, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 11, indent: 5, spelling: ',' } 39999999999999999993, - // CHECK:STDOUT: { index: 24, kind: 'IntLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '39999999999999999993', value: `39999999999999999993`, has_leading_space: true }, - // CHECK:STDOUT: { index: 25, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 25, indent: 5, spelling: ',' }, + // CHECK:STDOUT: - { index: 24, kind: 'IntLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '39999999999999999993', value: `39999999999999999993`, has_leading_space: true } + // CHECK:STDOUT: - { index: 25, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 25, indent: 5, spelling: ',' } ); - // CHECK:STDOUT: { index: 26, kind: 'CloseParen', line: {{ *}}[[@LINE-1]], column: 3, indent: 3, spelling: ')', opening_token: 15, has_leading_space: true }, - // CHECK:STDOUT: { index: 27, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 4, indent: 3, spelling: ';' }, + // CHECK:STDOUT: - { index: 26, kind: 'CloseParen', line: {{ *}}[[@LINE-1]], column: 3, indent: 3, spelling: ')', opening_token: 15, has_leading_space: true } + // CHECK:STDOUT: - { index: 27, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 4, indent: 3, spelling: ';' } var floats: [f64; 7] = ( - // CHECK:STDOUT: { index: 28, kind: 'Var', line: {{ *}}[[@LINE-1]], column: 3, indent: 3, spelling: 'var', has_leading_space: true }, - // CHECK:STDOUT: { index: 29, kind: 'Identifier', line: {{ *}}[[@LINE-2]], column: 7, indent: 3, spelling: 'floats', identifier: 2, has_leading_space: true }, - // CHECK:STDOUT: { index: 30, kind: 'Colon', line: {{ *}}[[@LINE-3]], column: 13, indent: 3, spelling: ':' }, - // CHECK:STDOUT: { index: 31, kind: 'OpenSquareBracket', line: {{ *}}[[@LINE-4]], column: 15, indent: 3, spelling: '[', closing_token: 35, has_leading_space: true }, - // CHECK:STDOUT: { index: 32, kind: 'FloatTypeLiteral', line: {{ *}}[[@LINE-5]], column: 16, indent: 3, spelling: 'f64' }, - // CHECK:STDOUT: { index: 33, kind: 'Semi', line: {{ *}}[[@LINE-6]], column: 19, indent: 3, spelling: ';' }, - // CHECK:STDOUT: { index: 34, kind: 'IntLiteral', line: {{ *}}[[@LINE-7]], column: 21, indent: 3, spelling: '7', value: `7`, has_leading_space: true }, - // CHECK:STDOUT: { index: 35, kind: 'CloseSquareBracket', line: {{ *}}[[@LINE-8]], column: 22, indent: 3, spelling: ']', opening_token: 31 }, - // CHECK:STDOUT: { index: 36, kind: 'Equal', line: {{ *}}[[@LINE-9]], column: 24, indent: 3, spelling: '=', has_leading_space: true }, - // CHECK:STDOUT: { index: 37, kind: 'OpenParen', line: {{ *}}[[@LINE-10]], column: 26, indent: 3, spelling: '(', closing_token: 52, has_leading_space: true }, + // CHECK:STDOUT: - { index: 28, kind: 'Var', line: {{ *}}[[@LINE-1]], column: 3, indent: 3, spelling: 'var', has_leading_space: true } + // CHECK:STDOUT: - { index: 29, kind: 'Identifier', line: {{ *}}[[@LINE-2]], column: 7, indent: 3, spelling: 'floats', identifier: 2, has_leading_space: true } + // CHECK:STDOUT: - { index: 30, kind: 'Colon', line: {{ *}}[[@LINE-3]], column: 13, indent: 3, spelling: ':' } + // CHECK:STDOUT: - { index: 31, kind: 'OpenSquareBracket', line: {{ *}}[[@LINE-4]], column: 15, indent: 3, spelling: '[', closing_token: 35, has_leading_space: true } + // CHECK:STDOUT: - { index: 32, kind: 'FloatTypeLiteral', line: {{ *}}[[@LINE-5]], column: 16, indent: 3, spelling: 'f64' } + // CHECK:STDOUT: - { index: 33, kind: 'Semi', line: {{ *}}[[@LINE-6]], column: 19, indent: 3, spelling: ';' } + // CHECK:STDOUT: - { index: 34, kind: 'IntLiteral', line: {{ *}}[[@LINE-7]], column: 21, indent: 3, spelling: '7', value: `7`, has_leading_space: true } + // CHECK:STDOUT: - { index: 35, kind: 'CloseSquareBracket', line: {{ *}}[[@LINE-8]], column: 22, indent: 3, spelling: ']', opening_token: 31 } + // CHECK:STDOUT: - { index: 36, kind: 'Equal', line: {{ *}}[[@LINE-9]], column: 24, indent: 3, spelling: '=', has_leading_space: true } + // CHECK:STDOUT: - { index: 37, kind: 'OpenParen', line: {{ *}}[[@LINE-10]], column: 26, indent: 3, spelling: '(', closing_token: 52, has_leading_space: true } 0.9, - // CHECK:STDOUT: { index: 38, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '0.9', value: `9*10^-1`, has_leading_space: true }, - // CHECK:STDOUT: { index: 39, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 8, indent: 5, spelling: ',' }, + // CHECK:STDOUT: - { index: 38, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '0.9', value: `9*10^-1`, has_leading_space: true } + // CHECK:STDOUT: - { index: 39, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 8, indent: 5, spelling: ',' } 8.0, - // CHECK:STDOUT: { index: 40, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '8.0', value: `80*10^-1`, has_leading_space: true }, - // CHECK:STDOUT: { index: 41, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 8, indent: 5, spelling: ',' }, + // CHECK:STDOUT: - { index: 40, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '8.0', value: `80*10^-1`, has_leading_space: true } + // CHECK:STDOUT: - { index: 41, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 8, indent: 5, spelling: ',' } 80.0, - // CHECK:STDOUT: { index: 42, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '80.0', value: `800*10^-1`, has_leading_space: true }, - // CHECK:STDOUT: { index: 43, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 9, indent: 5, spelling: ',' }, + // CHECK:STDOUT: - { index: 42, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '80.0', value: `800*10^-1`, has_leading_space: true } + // CHECK:STDOUT: - { index: 43, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 9, indent: 5, spelling: ',' } 1.0e7, - // CHECK:STDOUT: { index: 44, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '1.0e7', value: `10*10^6`, has_leading_space: true }, - // CHECK:STDOUT: { index: 45, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 10, indent: 5, spelling: ',' }, + // CHECK:STDOUT: - { index: 44, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '1.0e7', value: `10*10^6`, has_leading_space: true } + // CHECK:STDOUT: - { index: 45, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 10, indent: 5, spelling: ',' } 1.0e8, - // CHECK:STDOUT: { index: 46, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '1.0e8', value: `10*10^7`, has_leading_space: true }, - // CHECK:STDOUT: { index: 47, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 10, indent: 5, spelling: ',' }, + // CHECK:STDOUT: - { index: 46, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '1.0e8', value: `10*10^7`, has_leading_space: true } + // CHECK:STDOUT: - { index: 47, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 10, indent: 5, spelling: ',' } 1.0e-8, - // CHECK:STDOUT: { index: 48, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '1.0e-8', value: `10*10^-9`, has_leading_space: true }, - // CHECK:STDOUT: { index: 49, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 11, indent: 5, spelling: ',' }, + // CHECK:STDOUT: - { index: 48, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '1.0e-8', value: `10*10^-9`, has_leading_space: true } + // CHECK:STDOUT: - { index: 49, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 11, indent: 5, spelling: ',' } 39999999999999999993.0e39999999999999999993, - // CHECK:STDOUT: { index: 50, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '39999999999999999993.0e39999999999999999993', value: `399999999999999999930*10^39999999999999999992`, has_leading_space: true }, - // CHECK:STDOUT: { index: 51, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 48, indent: 5, spelling: ',' }, + // CHECK:STDOUT: - { index: 50, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column: 5, indent: 5, spelling: '39999999999999999993.0e39999999999999999993', value: `399999999999999999930*10^39999999999999999992`, has_leading_space: true } + // CHECK:STDOUT: - { index: 51, kind: 'Comma', line: {{ *}}[[@LINE-2]], column: 48, indent: 5, spelling: ',' } ); - // CHECK:STDOUT: { index: 52, kind: 'CloseParen', line: {{ *}}[[@LINE-1]], column: 3, indent: 3, spelling: ')', opening_token: 37, has_leading_space: true }, - // CHECK:STDOUT: { index: 53, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 4, indent: 3, spelling: ';' }, + // CHECK:STDOUT: - { index: 52, kind: 'CloseParen', line: {{ *}}[[@LINE-1]], column: 3, indent: 3, spelling: ')', opening_token: 37, has_leading_space: true } + // CHECK:STDOUT: - { index: 53, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 4, indent: 3, spelling: ';' } } -// CHECK:STDOUT: { index: 54, kind: 'CloseCurlyBrace', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '}', opening_token: 5, has_leading_space: true }, +// CHECK:STDOUT: - { index: 54, kind: 'CloseCurlyBrace', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '}', opening_token: 5, has_leading_space: true } -// CHECK:STDOUT: { index: 55, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] // --- fail_binary_real.carbon // CHECK:STDOUT: - filename: fail_binary_real.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, +// CHECK:STDOUT: tokens: // CHECK:STDERR: fail_binary_real.carbon:[[@LINE+4]]:4: error: binary real number literals are not supported [BinaryRealLiteral] // CHECK:STDERR: 0b1.0 // CHECK:STDERR: ^ // CHECK:STDERR: 0b1.0 -// CHECK:STDOUT: { index: 1, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '0b1.0', value: `2*2^-1`, has_leading_space: true }, +// CHECK:STDOUT: - { index: 1, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '0b1.0', value: `2*2^-1`, has_leading_space: true } -// CHECK:STDOUT: { index: 2, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] // --- fail_wrong_real_exponent.carbon // CHECK:STDOUT: - filename: fail_wrong_real_exponent.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, +// CHECK:STDOUT: tokens: // CHECK:STDERR: fail_wrong_real_exponent.carbon:[[@LINE+3]]:4: error: expected 'e' to introduce exponent [WrongRealLiteralExponent] // CHECK:STDERR: 1.0r3 // CHECK:STDERR: ^ 1.0r3 -// CHECK:STDOUT: { index: 1, kind: 'Error', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '1.0r3', has_leading_space: true }, - -// CHECK:STDOUT: { index: 2, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] +// CHECK:STDOUT: - { index: 1, kind: 'Error', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '1.0r3', has_leading_space: true } diff --git a/toolchain/lex/testdata/printing_digit_padding.carbon b/toolchain/lex/testdata/printing_digit_padding.carbon index 0766eebe362f2..aeef12c761b9d 100644 --- a/toolchain/lex/testdata/printing_digit_padding.carbon +++ b/toolchain/lex/testdata/printing_digit_padding.carbon @@ -8,17 +8,13 @@ // TIP: To dump output, run: // TIP: bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/lex/testdata/printing_digit_padding.carbon // CHECK:STDOUT: - filename: printing_digit_padding.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, +// CHECK:STDOUT: tokens: ; -// CHECK:STDOUT: { index: 1, kind: 'Semi', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: ';', has_leading_space: true }, +// CHECK:STDOUT: - { index: 1, kind: 'Semi', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: ';', has_leading_space: true } // Test digit padding with values of 9, 10, and 11. ;;; - // CHECK:STDOUT: { index: 2, kind: 'Semi', line: {{ *}}[[@LINE-1]], column: 9, indent: 9, spelling: ';', has_leading_space: true }, - // CHECK:STDOUT: { index: 3, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 10, indent: 9, spelling: ';' }, - // CHECK:STDOUT: { index: 4, kind: 'Semi', line: {{ *}}[[@LINE-3]], column: 11, indent: 9, spelling: ';' }, - -// CHECK:STDOUT: { index: 5, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] + // CHECK:STDOUT: - { index: 2, kind: 'Semi', line: {{ *}}[[@LINE-1]], column: 9, indent: 9, spelling: ';', has_leading_space: true } + // CHECK:STDOUT: - { index: 3, kind: 'Semi', line: {{ *}}[[@LINE-2]], column: 10, indent: 9, spelling: ';' } + // CHECK:STDOUT: - { index: 4, kind: 'Semi', line: {{ *}}[[@LINE-3]], column: 11, indent: 9, spelling: ';' } diff --git a/toolchain/lex/testdata/printing_integer_literal.carbon b/toolchain/lex/testdata/printing_integer_literal.carbon index a5801ae761c1b..c11e4c25b7790 100644 --- a/toolchain/lex/testdata/printing_integer_literal.carbon +++ b/toolchain/lex/testdata/printing_integer_literal.carbon @@ -8,11 +8,7 @@ // TIP: To dump output, run: // TIP: bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/lex/testdata/printing_integer_literal.carbon // CHECK:STDOUT: - filename: printing_integer_literal.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, +// CHECK:STDOUT: tokens: 123 -// CHECK:STDOUT: { index: 1, kind: 'IntLiteral', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '123', value: `123`, has_leading_space: true }, - -// CHECK:STDOUT: { index: 2, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] +// CHECK:STDOUT: - { index: 1, kind: 'IntLiteral', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '123', value: `123`, has_leading_space: true } diff --git a/toolchain/lex/testdata/printing_real_literal.carbon b/toolchain/lex/testdata/printing_real_literal.carbon index 726c206c7c512..e3097c5b67112 100644 --- a/toolchain/lex/testdata/printing_real_literal.carbon +++ b/toolchain/lex/testdata/printing_real_literal.carbon @@ -8,11 +8,7 @@ // TIP: To dump output, run: // TIP: bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/lex/testdata/printing_real_literal.carbon // CHECK:STDOUT: - filename: printing_real_literal.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, +// CHECK:STDOUT: tokens: 2.5 -// CHECK:STDOUT: { index: 1, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '2.5', value: `25*10^-1`, has_leading_space: true }, - -// CHECK:STDOUT: { index: 2, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] +// CHECK:STDOUT: - { index: 1, kind: 'RealLiteral', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '2.5', value: `25*10^-1`, has_leading_space: true } diff --git a/toolchain/lex/testdata/printing_token.carbon b/toolchain/lex/testdata/printing_token.carbon index 606c819b45774..dfb3491fedf53 100644 --- a/toolchain/lex/testdata/printing_token.carbon +++ b/toolchain/lex/testdata/printing_token.carbon @@ -8,11 +8,7 @@ // TIP: To dump output, run: // TIP: bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/lex/testdata/printing_token.carbon // CHECK:STDOUT: - filename: printing_token.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, +// CHECK:STDOUT: tokens: 0x9 -// CHECK:STDOUT: { index: 1, kind: 'IntLiteral', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '0x9', value: `9`, has_leading_space: true }, - -// CHECK:STDOUT: { index: 2, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] +// CHECK:STDOUT: - { index: 1, kind: 'IntLiteral', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: '0x9', value: `9`, has_leading_space: true } diff --git a/toolchain/lex/testdata/raw_identifier.carbon b/toolchain/lex/testdata/raw_identifier.carbon index 593d6c3ff99ec..11ddc8da130c4 100644 --- a/toolchain/lex/testdata/raw_identifier.carbon +++ b/toolchain/lex/testdata/raw_identifier.carbon @@ -8,32 +8,28 @@ // TIP: To dump output, run: // TIP: bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/lex/testdata/raw_identifier.carbon // CHECK:STDOUT: - filename: raw_identifier.carbon -// CHECK:STDOUT: tokens: [ -// CHECK:STDOUT: { index: 0, kind: 'FileStart', line: {{ *\d+}}, column: 1, indent: 1, spelling: '' }, +// CHECK:STDOUT: tokens: // A non-keyword identifier. r#foo -// CHECK:STDOUT: { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'foo', identifier: 1, has_leading_space: true }, +// CHECK:STDOUT: - { index: 1, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'foo', identifier: 1, has_leading_space: true } // The same non-keyword identifier, for comparison. foo -// CHECK:STDOUT: { index: 2, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'foo', identifier: 1, has_leading_space: true }, +// CHECK:STDOUT: - { index: 2, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'foo', identifier: 1, has_leading_space: true } // A keyword as a raw identifier. r#self -// CHECK:STDOUT: { index: 3, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'self', identifier: 2, has_leading_space: true }, +// CHECK:STDOUT: - { index: 3, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'self', identifier: 2, has_leading_space: true } // The same keyword, for comparison. self -// CHECK:STDOUT: { index: 4, kind: 'SelfValueIdentifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'self', has_leading_space: true }, +// CHECK:STDOUT: - { index: 4, kind: 'SelfValueIdentifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'self', has_leading_space: true } // A type literal as a raw identifier. r#i32 -// CHECK:STDOUT: { index: 5, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'i32', identifier: 3, has_leading_space: true }, +// CHECK:STDOUT: - { index: 5, kind: 'Identifier', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'i32', identifier: 3, has_leading_space: true } // The same type literal, for comparison. i32 -// CHECK:STDOUT: { index: 6, kind: 'IntTypeLiteral', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'i32', has_leading_space: true }, - -// CHECK:STDOUT: { index: 7, kind: 'FileEnd', line: {{ *}}[[@LINE+1]], column: {{ *\d+}}, indent: 1, spelling: '', has_leading_space: true }, -// CHECK:STDOUT: ] +// CHECK:STDOUT: - { index: 6, kind: 'IntTypeLiteral', line: {{ *}}[[@LINE-1]], column: 1, indent: 1, spelling: 'i32', has_leading_space: true } diff --git a/toolchain/lex/testdata/string_literals.carbon b/toolchain/lex/testdata/string_literals.carbon index 8c54e214cff8a91b6f2459616166a175a77c3f42..db2ca7889a3ab96d928ae9a4f692980082a68632 100644 GIT binary patch delta 132 zcmeB@?HAe5z_j@jlQbiv?qq*n*U3U$(v$O9bQrlNd$ATy{>33RIg(XyvN|X43jl$IBk%wK delta 770 zcmeAd>5|>hz{I2wJ=u^$#7;q>S|KwpCAGpz!9YhL8_2X$00DKk%$(HVlEk7CbsdGA z%)C@9h3aYrt(X+;+FBij2QHe zS3)w~2-$R~`L1~>lO5T`#I+4$V-beJ?U}rgRf17{azDE?qYlV8#@NZ4EQL^)1D%QK zauQuVxq(SyvKfo; void { - if (tokens().begin() == tokens().end()) { - return; - } - +auto TokenizedBuffer::Print(llvm::raw_ostream& output_stream, + bool omit_file_boundary_tokens) const -> void { output_stream << "- filename: " << source_->filename() << "\n" - << " tokens: [\n"; + << " tokens:\n"; PrintWidths widths = {}; widths.index = ComputeDecimalPrintedWidth((token_infos_.size())); @@ -230,10 +227,15 @@ auto TokenizedBuffer::Print(llvm::raw_ostream& output_stream) const -> void { } for (TokenIndex token : tokens()) { + if (omit_file_boundary_tokens) { + auto kind = GetKind(token); + if (kind == TokenKind::FileStart || kind == TokenKind::FileEnd) { + continue; + } + } PrintToken(output_stream, token, widths); output_stream << "\n"; } - output_stream << " ]\n"; } auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, @@ -254,7 +256,7 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, // justification manually in order to use the dynamically computed widths // and get the quotes included. output_stream << llvm::formatv( - " { index: {0}, kind: {1}, line: {2}, column: {3}, indent: {4}, " + " - { index: {0}, kind: {1}, line: {2}, column: {3}, indent: {4}, " "spelling: '{5}'", llvm::format_decimal(token_index, widths.index), llvm::right_justify( @@ -304,7 +306,7 @@ auto TokenizedBuffer::PrintToken(llvm::raw_ostream& output_stream, output_stream << ", recovery: true"; } - output_stream << " },"; + output_stream << " }"; } // Find the line index corresponding to a specific byte offset within the source diff --git a/toolchain/lex/tokenized_buffer.h b/toolchain/lex/tokenized_buffer.h index 7074f1e792561..766b49df58cde 100644 --- a/toolchain/lex/tokenized_buffer.h +++ b/toolchain/lex/tokenized_buffer.h @@ -173,26 +173,15 @@ class TokenizedBuffer : public Printable { // Returns the comment's full text range. auto GetCommentText(CommentIndex comment_index) const -> llvm::StringRef; - // Prints a description of the tokenized stream to the provided `raw_ostream`. + // Returns tokens as YAML. This prints the tracked token information on a + // single line for each token. We use the single-line format so that output is + // compact, and so that tools like `grep` are compatible. // - // It prints one line of information for each token in the buffer, including - // the kind of token, where it occurs within the source file, indentation for - // the associated line, the spelling of the token in source, and any - // additional information tracked such as which unique identifier it is or any - // matched grouping token. + // An example token looks like: // - // Each line is formatted as a YAML record: - // - // clang-format off - // ``` - // token: { index: 0, kind: 'Semi', line: 1, column: 1, indent: 1, spelling: ';' } - // ``` - // clang-format on - // - // This can be parsed as YAML using tools like `python-yq` combined with `jq` - // on the command line. The format is also reasonably amenable to other - // line-oriented shell tools from `grep` to `awk`. - auto Print(llvm::raw_ostream& output_stream) const -> void; + // - { index: 1, kind: 'Semi', line: 1, column: 1, indent: 1, spelling: ';' } + auto Print(llvm::raw_ostream& out, + bool omit_file_boundary_tokens = false) const -> void; // Prints a description of a single token. See `Print` for details on the // format. diff --git a/toolchain/testing/file_test.cpp b/toolchain/testing/file_test.cpp index 0339af978827f..5012b90b9078e 100644 --- a/toolchain/testing/file_test.cpp +++ b/toolchain/testing/file_test.cpp @@ -67,7 +67,7 @@ class ToolchainFileTest : public FileTestBase { "compile", "--include-diagnostic-kind", "--phase=" + component_.str()}; if (component_ == "lex") { - args.push_back("--dump-tokens"); + args.insert(args.end(), {"--dump-tokens", "--omit-file-boundary-tokens"}); } else if (component_ == "parse") { args.push_back("--dump-parse-tree"); } else if (component_ == "check") {