Merge nucleic/sleek-thistle-egret-fyej into dev
This commit is contained in:
@@ -0,0 +1,486 @@
|
||||
//! Reporting utilities for test results.
|
||||
|
||||
use crate::comparison::{
|
||||
DirComparison, DirComparisonEntry, ExitStatusComparison, ExpectationComparison,
|
||||
OracleComparison, SingleExpectationComparison, StringComparison, TestComparison,
|
||||
};
|
||||
use crate::config::{OutputFormat, TestOptions};
|
||||
use crate::util::{make_expectrl_output_readable, write_diff};
|
||||
use anyhow::Result;
|
||||
use colored::Colorize;
|
||||
use std::io::Write;
|
||||
|
||||
/// Result of running a single test case.
|
||||
pub struct TestCaseResult {
|
||||
/// Name of the test case.
|
||||
pub name: Option<String>,
|
||||
/// Whether the test succeeded.
|
||||
pub success: bool,
|
||||
/// Whether the test was skipped.
|
||||
pub skip: bool,
|
||||
/// Whether this is a known failure.
|
||||
pub known_failure: bool,
|
||||
/// The comparison result.
|
||||
pub comparison: TestComparison,
|
||||
}
|
||||
|
||||
impl TestCaseResult {
|
||||
/// Reports this result in pretty format.
|
||||
pub fn report_pretty(&self, options: &TestOptions) -> Result<()> {
|
||||
self.write_details(std::io::stderr(), options)
|
||||
}
|
||||
|
||||
/// Writes the details of this result to a writer.
|
||||
pub fn write_details<W: Write>(&self, mut writer: W, options: &TestOptions) -> Result<()> {
|
||||
if self.skip {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if !options.verbose {
|
||||
if (!self.comparison.is_failure() && !self.known_failure)
|
||||
|| (self.comparison.is_failure() && self.known_failure)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
write!(
|
||||
writer,
|
||||
"* {}: [{}]... ",
|
||||
"Test case".bright_yellow(),
|
||||
self.name
|
||||
.as_ref()
|
||||
.map_or_else(|| "(unnamed)", |n| n.as_str())
|
||||
.italic()
|
||||
)?;
|
||||
|
||||
if !self.comparison.is_failure() {
|
||||
if self.known_failure {
|
||||
writeln!(writer, "{}", "unexpected success.".bright_red())?;
|
||||
} else {
|
||||
writeln!(writer, "{}", "ok.".bright_green())?;
|
||||
return Ok(());
|
||||
}
|
||||
} else if self.known_failure {
|
||||
writeln!(writer, "{}", "known failure.".bright_magenta())?;
|
||||
if !options.display_known_failure_details {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
writeln!(writer)?;
|
||||
|
||||
// Report oracle comparison if present
|
||||
if let Some(oracle) = &self.comparison.oracle {
|
||||
self.write_oracle_details(&mut writer, oracle, options)?;
|
||||
}
|
||||
|
||||
// Report expectation comparison
|
||||
if self.comparison.expectation.has_any_checks() {
|
||||
self.write_expectation_details(&mut writer, &self.comparison.expectation)?;
|
||||
}
|
||||
|
||||
if !self.success {
|
||||
writeln!(writer, " {}", "FAILED.".bright_red())?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[expect(clippy::too_many_lines)]
|
||||
#[expect(clippy::unused_self)]
|
||||
fn write_oracle_details<W: Write>(
|
||||
&self,
|
||||
writer: &mut W,
|
||||
oracle: &OracleComparison,
|
||||
options: &TestOptions,
|
||||
) -> Result<()> {
|
||||
writeln!(writer, " {} comparison:", "Oracle".cyan())?;
|
||||
|
||||
match oracle.exit_status {
|
||||
ExitStatusComparison::Ignored => writeln!(writer, " status {}", "ignored".cyan())?,
|
||||
ExitStatusComparison::Same(status) => {
|
||||
writeln!(
|
||||
writer,
|
||||
" status matches ({}) {}",
|
||||
format!("{status}").green(),
|
||||
"✔️".green()
|
||||
)?;
|
||||
}
|
||||
ExitStatusComparison::TestDiffers {
|
||||
test_exit_status,
|
||||
oracle_exit_status,
|
||||
} => {
|
||||
writeln!(
|
||||
writer,
|
||||
" status mismatch: {} from oracle vs. {} from test",
|
||||
format!("{oracle_exit_status}").cyan(),
|
||||
format!("{test_exit_status}").bright_red()
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
match &oracle.stdout {
|
||||
StringComparison::Ignored {
|
||||
test_string,
|
||||
oracle_string,
|
||||
} => {
|
||||
writeln!(writer, " stdout {}", "ignored".cyan())?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"------ Oracle: stdout ---------------------------------".cyan()
|
||||
)?;
|
||||
writeln!(writer, "{}", indent::indent_all_by(10, oracle_string))?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"------ Oracle: stdout [cleaned]------------------------".cyan()
|
||||
)?;
|
||||
writeln!(
|
||||
writer,
|
||||
"{}",
|
||||
indent::indent_all_by(10, make_expectrl_output_readable(oracle_string))
|
||||
)?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"------ Test: stdout ---------------------------------".cyan()
|
||||
)?;
|
||||
writeln!(writer, "{}", indent::indent_all_by(10, test_string))?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"------ Test: stdout [cleaned]------------------------".cyan()
|
||||
)?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
"{}",
|
||||
indent::indent_all_by(10, make_expectrl_output_readable(test_string))
|
||||
)?;
|
||||
}
|
||||
StringComparison::Same(s) => {
|
||||
writeln!(writer, " stdout matches {}", "✔️".green())?;
|
||||
|
||||
if options.verbose {
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"------ Oracle <> Test: stdout ---------------------------------".cyan()
|
||||
)?;
|
||||
|
||||
writeln!(writer, "{}", indent::indent_all_by(10, s))?;
|
||||
}
|
||||
}
|
||||
StringComparison::TestDiffers {
|
||||
test_string: t,
|
||||
oracle_string: o,
|
||||
} => {
|
||||
writeln!(writer, " stdout {}", "DIFFERS:".bright_red())?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"------ Oracle <> Test: stdout ---------------------------------".cyan()
|
||||
)?;
|
||||
|
||||
write_diff(writer, 10, o.as_str(), t.as_str())?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"---------------------------------------------------------------".cyan()
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
match &oracle.stderr {
|
||||
StringComparison::Ignored { .. } => {
|
||||
writeln!(writer, " stderr {}", "ignored".cyan())?;
|
||||
}
|
||||
StringComparison::Same(s) => {
|
||||
writeln!(writer, " stderr matches {}", "✔️".green())?;
|
||||
|
||||
if options.verbose {
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"------ Oracle <> Test: stderr ---------------------------------".cyan()
|
||||
)?;
|
||||
|
||||
writeln!(writer, "{}", indent::indent_all_by(10, s))?;
|
||||
}
|
||||
}
|
||||
StringComparison::TestDiffers {
|
||||
test_string: t,
|
||||
oracle_string: o,
|
||||
} => {
|
||||
writeln!(writer, " stderr {}", "DIFFERS:".bright_red())?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"------ Oracle <> Test: stderr ---------------------------------".cyan()
|
||||
)?;
|
||||
|
||||
write_diff(writer, 10, o.as_str(), t.as_str())?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"---------------------------------------------------------------".cyan()
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
match &oracle.temp_dir {
|
||||
DirComparison::Ignored => writeln!(writer, " temp dir {}", "ignored".cyan())?,
|
||||
DirComparison::Same => writeln!(writer, " temp dir matches {}", "✔️".green())?,
|
||||
DirComparison::TestDiffers(entries) => {
|
||||
writeln!(writer, " temp dir {}", "DIFFERS".bright_red())?;
|
||||
|
||||
for entry in entries {
|
||||
const INDENT: &str = " ";
|
||||
match entry {
|
||||
DirComparisonEntry::Different(
|
||||
left_path,
|
||||
left_contents,
|
||||
right_path,
|
||||
right_contents,
|
||||
) => {
|
||||
writeln!(
|
||||
writer,
|
||||
"{INDENT}oracle file {} differs from test file {}",
|
||||
left_path.to_string_lossy(),
|
||||
right_path.to_string_lossy()
|
||||
)?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
"{INDENT}{}",
|
||||
"------ Oracle <> Test: file ---------------------------------"
|
||||
.cyan()
|
||||
)?;
|
||||
|
||||
write_diff(
|
||||
writer,
|
||||
10,
|
||||
left_contents.as_str(),
|
||||
right_contents.as_str(),
|
||||
)?;
|
||||
|
||||
writeln!(
|
||||
writer,
|
||||
" {}",
|
||||
"---------------------------------------------------------------"
|
||||
.cyan()
|
||||
)?;
|
||||
}
|
||||
DirComparisonEntry::LeftOnly(p) => {
|
||||
writeln!(
|
||||
writer,
|
||||
"{INDENT}file missing from test dir: {}",
|
||||
p.to_string_lossy()
|
||||
)?;
|
||||
}
|
||||
DirComparisonEntry::RightOnly(p) => {
|
||||
writeln!(
|
||||
writer,
|
||||
"{INDENT}unexpected file in test dir: {}",
|
||||
p.to_string_lossy()
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[expect(clippy::unused_self)]
|
||||
fn write_expectation_details<W: Write>(
|
||||
&self,
|
||||
writer: &mut W,
|
||||
expectation: &ExpectationComparison,
|
||||
) -> Result<()> {
|
||||
writeln!(writer, " {} check:", "Expectation".cyan())?;
|
||||
|
||||
// Exit code
|
||||
match &expectation.exit_code {
|
||||
SingleExpectationComparison::NotSpecified => {}
|
||||
SingleExpectationComparison::Matches => {
|
||||
writeln!(writer, " exit code matches {}", "✔️".green())?;
|
||||
}
|
||||
SingleExpectationComparison::Differs { expected, actual } => {
|
||||
writeln!(
|
||||
writer,
|
||||
" exit code {}: expected {}, got {}",
|
||||
"DIFFERS".bright_red(),
|
||||
expected.cyan(),
|
||||
actual.bright_red()
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Stdout
|
||||
match &expectation.stdout {
|
||||
SingleExpectationComparison::NotSpecified => {}
|
||||
SingleExpectationComparison::Matches => {
|
||||
writeln!(writer, " stdout matches {}", "✔️".green())?;
|
||||
}
|
||||
SingleExpectationComparison::Differs { expected, actual } => {
|
||||
writeln!(writer, " stdout {}", "DIFFERS:".bright_red())?;
|
||||
write_diff(writer, 8, expected.as_str(), actual.as_str())?;
|
||||
}
|
||||
}
|
||||
|
||||
// Stderr
|
||||
match &expectation.stderr {
|
||||
SingleExpectationComparison::NotSpecified => {}
|
||||
SingleExpectationComparison::Matches => {
|
||||
writeln!(writer, " stderr matches {}", "✔️".green())?;
|
||||
}
|
||||
SingleExpectationComparison::Differs { expected, actual } => {
|
||||
writeln!(writer, " stderr {}", "DIFFERS:".bright_red())?;
|
||||
write_diff(writer, 8, expected.as_str(), actual.as_str())?;
|
||||
}
|
||||
}
|
||||
|
||||
// Snapshot
|
||||
if expectation.snapshot_used {
|
||||
if let Some(result) = &expectation.snapshot_result {
|
||||
match result {
|
||||
crate::comparison::SnapshotResult::Matches => {
|
||||
writeln!(writer, " snapshot matches {}", "✔️".green())?;
|
||||
}
|
||||
crate::comparison::SnapshotResult::Differs { message } => {
|
||||
writeln!(
|
||||
writer,
|
||||
" snapshot {}: {}",
|
||||
"DIFFERS".bright_red(),
|
||||
message
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Results from running a set of test cases.
|
||||
pub struct TestCaseSetResults {
|
||||
/// Name of the test case set.
|
||||
pub name: Option<String>,
|
||||
/// Name of the configuration used.
|
||||
pub config_name: String,
|
||||
/// Number of successful tests.
|
||||
pub success_count: u32,
|
||||
/// Number of skipped tests.
|
||||
pub skip_count: u32,
|
||||
/// Number of known failures.
|
||||
pub known_failure_count: u32,
|
||||
/// Number of failed tests.
|
||||
pub fail_count: u32,
|
||||
/// Individual test case results.
|
||||
pub test_case_results: Vec<TestCaseResult>,
|
||||
/// Total duration comparison for successful tests.
|
||||
pub success_duration: std::time::Duration,
|
||||
}
|
||||
|
||||
impl TestCaseSetResults {
|
||||
/// Reports these results in pretty format.
|
||||
pub fn report_pretty(&self, options: &TestOptions) -> Result<()> {
|
||||
self.write_details(std::io::stderr(), options)
|
||||
}
|
||||
|
||||
fn write_details<W: Write>(&self, mut writer: W, options: &TestOptions) -> Result<()> {
|
||||
if options.verbose {
|
||||
writeln!(
|
||||
writer,
|
||||
"=================== {}: [{}/{}] ===================",
|
||||
"Running test case set".blue(),
|
||||
self.name
|
||||
.as_ref()
|
||||
.map_or_else(|| "(unnamed)", |n| n.as_str())
|
||||
.italic(),
|
||||
self.config_name.magenta(),
|
||||
)?;
|
||||
}
|
||||
|
||||
for test_case_result in &self.test_case_results {
|
||||
test_case_result.report_pretty(options)?;
|
||||
}
|
||||
|
||||
if options.verbose {
|
||||
writeln!(
|
||||
writer,
|
||||
" successful cases ran in {:?}",
|
||||
self.success_duration
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Reports test results based on the configured output format.
|
||||
pub fn report_results(results: Vec<TestCaseSetResults>, options: &TestOptions) -> Result<()> {
|
||||
match options.format {
|
||||
OutputFormat::Pretty => report_results_pretty(results, options),
|
||||
OutputFormat::Junit => report_results_junit(results, options),
|
||||
OutputFormat::Terse => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn report_results_pretty(results: Vec<TestCaseSetResults>, options: &TestOptions) -> Result<()> {
|
||||
for result in results {
|
||||
result.report_pretty(options)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn report_results_junit(results: Vec<TestCaseSetResults>, options: &TestOptions) -> Result<()> {
|
||||
let mut report = junit_report::Report::new();
|
||||
|
||||
for result in results {
|
||||
let mut suite = junit_report::TestSuite::new(result.name.unwrap_or(String::new()).as_str());
|
||||
for r in result.test_case_results {
|
||||
let test_case_name = r.name.as_deref().unwrap_or("");
|
||||
let mut test_case: junit_report::TestCase = if r.success {
|
||||
junit_report::TestCase::success(test_case_name, r.comparison.duration.try_into()?)
|
||||
} else if r.known_failure {
|
||||
junit_report::TestCase::skipped(test_case_name)
|
||||
} else {
|
||||
junit_report::TestCase::failure(
|
||||
test_case_name,
|
||||
r.comparison.duration.try_into()?,
|
||||
"test failure",
|
||||
"failed",
|
||||
)
|
||||
};
|
||||
|
||||
let mut output_buf: Vec<u8> = vec![];
|
||||
r.write_details(&mut output_buf, options)?;
|
||||
|
||||
let output_as_string = String::from_utf8(output_buf)?;
|
||||
test_case.set_system_out(strip_ansi_escapes::strip_str(output_as_string).as_str());
|
||||
|
||||
suite.add_testcase(test_case);
|
||||
}
|
||||
|
||||
report.add_testsuite(suite);
|
||||
}
|
||||
|
||||
report.write_xml(std::io::stdout())?;
|
||||
writeln!(std::io::stdout())?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user