test: separate diagnostic apparatus from release gates
This commit is contained in:
Erik
parent
8f490240d4
commit
c1a905004a
33 changed files with 527 additions and 24 deletions
374
tools/audit-test-inventory.ps1
Normal file
374
tools/audit-test-inventory.ps1
Normal file
|
|
@ -0,0 +1,374 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
Produces a syntax-based inventory of every attributed C# test method.
|
||||
|
||||
.DESCRIPTION
|
||||
Parses tracked C# files under tests/ with the Roslyn assemblies bundled in
|
||||
the active .NET SDK. The report is evidence for the R3 test-truth audit: it
|
||||
records test attributes and traits, static skips, empty-return sites,
|
||||
assertion/throw signals (including same-file helper calls), diagnostic
|
||||
output signals, wall-clock waits, environment-variable dependencies, and
|
||||
source-text reads.
|
||||
|
||||
This is a candidate generator, not a semantic proof. In particular, an
|
||||
output-only candidate can call a failure-capable helper declared in another
|
||||
file or base class. Every removal or lane change still requires inspection.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$OutputPath = 'artifacts/test-audit/test-inventory.json'
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$repoRoot = [IO.Path]::GetFullPath((Split-Path -Parent $PSScriptRoot))
|
||||
$resolvedOutput = if ([IO.Path]::IsPathRooted($OutputPath)) {
|
||||
[IO.Path]::GetFullPath($OutputPath)
|
||||
} else {
|
||||
[IO.Path]::GetFullPath((Join-Path $repoRoot $OutputPath))
|
||||
}
|
||||
|
||||
$dotnetCommand = Get-Command dotnet -ErrorAction Stop
|
||||
$dotnetRoot = Split-Path -Parent $dotnetCommand.Source
|
||||
$sdkVersion = (& dotnet --version).Trim()
|
||||
$roslynDirectory = Join-Path $dotnetRoot "sdk/$sdkVersion/Roslyn/bincore"
|
||||
$loadedRoslyn = @([AppDomain]::CurrentDomain.GetAssemblies() | Where-Object {
|
||||
$_.GetName().Name -in @('Microsoft.CodeAnalysis', 'Microsoft.CodeAnalysis.CSharp')
|
||||
})
|
||||
if ($loadedRoslyn.Count -lt 2) {
|
||||
foreach ($assemblyName in 'Microsoft.CodeAnalysis.dll', 'Microsoft.CodeAnalysis.CSharp.dll') {
|
||||
$assemblyPath = Join-Path $roslynDirectory $assemblyName
|
||||
if (-not (Test-Path -LiteralPath $assemblyPath -PathType Leaf)) {
|
||||
throw "Roslyn assembly not found at '$assemblyPath'."
|
||||
}
|
||||
Add-Type -Path $assemblyPath
|
||||
}
|
||||
}
|
||||
$roslynVersion = [Microsoft.CodeAnalysis.CSharp.CSharpSyntaxTree].Assembly.GetName().Version.ToString()
|
||||
|
||||
function Get-ShortAttributeName {
|
||||
param([Parameter(Mandatory)]$Attribute)
|
||||
|
||||
$name = $Attribute.Name.ToString().Split('.')[-1]
|
||||
if ($name.EndsWith('Attribute', [StringComparison]::Ordinal)) {
|
||||
$name = $name.Substring(0, $name.Length - 'Attribute'.Length)
|
||||
}
|
||||
return $name
|
||||
}
|
||||
|
||||
function Get-NodeLine {
|
||||
param(
|
||||
[Parameter(Mandatory)]$Tree,
|
||||
[Parameter(Mandatory)]$Node
|
||||
)
|
||||
|
||||
return $Tree.GetLineSpan($Node.Span).StartLinePosition.Line + 1
|
||||
}
|
||||
|
||||
function Get-ContainingClassName {
|
||||
param([Parameter(Mandatory)]$Method)
|
||||
|
||||
$typeNode = Get-ContainingTypeNode $Method
|
||||
if ($null -ne $typeNode) {
|
||||
return $typeNode.Identifier.Text
|
||||
}
|
||||
return '<global>'
|
||||
}
|
||||
|
||||
function Get-ContainingTypeNode {
|
||||
param([Parameter(Mandatory)]$Method)
|
||||
|
||||
$cursor = $Method.Parent
|
||||
while ($null -ne $cursor) {
|
||||
if ($cursor.GetType().Name -in @(
|
||||
'ClassDeclarationSyntax',
|
||||
'RecordDeclarationSyntax',
|
||||
'StructDeclarationSyntax')) {
|
||||
return $cursor
|
||||
}
|
||||
$cursor = $cursor.Parent
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
function Get-ReturnGateKind {
|
||||
param([Parameter(Mandatory)][string]$Condition)
|
||||
|
||||
if ($Condition -match '(?i)(GetEnvironmentVariable|ACDREAM_)') {
|
||||
return 'OptIn'
|
||||
}
|
||||
if ($Condition -match '(?i)(OperatingSystem\.)') {
|
||||
return 'Platform'
|
||||
}
|
||||
if ($Condition -match '(?i)(datDir|datDirectory|Directory\.Exists|File\.Exists|TryOpen|TryBuildScenario|steps is null|csvPath|\.pak)') {
|
||||
return 'ExternalAsset'
|
||||
}
|
||||
return $null
|
||||
}
|
||||
|
||||
function Get-InvocationName {
|
||||
param([Parameter(Mandatory)]$Invocation)
|
||||
|
||||
$expression = $Invocation.Expression
|
||||
if ($expression.GetType().Name -eq 'MemberAccessExpressionSyntax') {
|
||||
return $expression.Name.Identifier.Text
|
||||
}
|
||||
if ($expression.GetType().Name -eq 'IdentifierNameSyntax') {
|
||||
return $expression.Identifier.Text
|
||||
}
|
||||
if ($expression.GetType().Name -eq 'GenericNameSyntax') {
|
||||
return $expression.Identifier.Text
|
||||
}
|
||||
return $expression.ToString()
|
||||
}
|
||||
|
||||
function Test-DirectFailureSignal {
|
||||
param([Parameter(Mandatory)]$Method)
|
||||
|
||||
if (@($Method.DescendantNodes() | Where-Object {
|
||||
$_.GetType().Name -in @('ThrowStatementSyntax', 'ThrowExpressionSyntax')
|
||||
}).Count -gt 0) {
|
||||
return $true
|
||||
}
|
||||
|
||||
foreach ($invocation in @($Method.DescendantNodes() | Where-Object {
|
||||
$_.GetType().Name -eq 'InvocationExpressionSyntax'
|
||||
})) {
|
||||
$expression = $invocation.Expression.ToString()
|
||||
if ($expression -match '(^|\.)Assert(\.|$)' -or
|
||||
$expression -match '(^|\.)(Should|Shouldly)(\.|$)' -or
|
||||
$expression -match '(^|\.)Verify($|\.)') {
|
||||
return $true
|
||||
}
|
||||
}
|
||||
return $false
|
||||
}
|
||||
|
||||
function Test-DirectOutputSignal {
|
||||
param([Parameter(Mandatory)]$Method)
|
||||
|
||||
foreach ($invocation in @($Method.DescendantNodes() | Where-Object {
|
||||
$_.GetType().Name -eq 'InvocationExpressionSyntax'
|
||||
})) {
|
||||
$expression = $invocation.Expression.ToString()
|
||||
if ($expression -match '(^|\.)(_?out|output|Console|Debug|Trace)\.(Write|WriteLine)$' -or
|
||||
$expression -match '(^|\.)(WriteDiagnostic|Dump|DumpSteps|Print)(\.|$)') {
|
||||
return $true
|
||||
}
|
||||
}
|
||||
return $false
|
||||
}
|
||||
|
||||
function Test-RecursiveSignal {
|
||||
param(
|
||||
[Parameter(Mandatory)][string]$MethodKey,
|
||||
[Parameter(Mandatory)][hashtable]$MethodMap,
|
||||
[Parameter(Mandatory)][hashtable]$DirectSignalMap,
|
||||
[Parameter(Mandatory)][hashtable]$CallMap,
|
||||
[Parameter(Mandatory)][hashtable]$Visited
|
||||
)
|
||||
|
||||
if ($Visited.ContainsKey($MethodKey)) {
|
||||
return $false
|
||||
}
|
||||
$Visited[$MethodKey] = $true
|
||||
if ($DirectSignalMap[$MethodKey]) {
|
||||
return $true
|
||||
}
|
||||
foreach ($calledName in @($CallMap[$MethodKey])) {
|
||||
foreach ($candidateKey in @($MethodMap.Keys | Where-Object {
|
||||
$_.EndsWith("::$calledName", [StringComparison]::Ordinal)
|
||||
})) {
|
||||
if (Test-RecursiveSignal $candidateKey $MethodMap $DirectSignalMap $CallMap $Visited) {
|
||||
return $true
|
||||
}
|
||||
}
|
||||
}
|
||||
return $false
|
||||
}
|
||||
|
||||
$trackedFiles = @(& git -C $repoRoot ls-files tests | Where-Object {
|
||||
$_.EndsWith('.cs', [StringComparison]::OrdinalIgnoreCase)
|
||||
})
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw 'git ls-files failed while enumerating test sources.'
|
||||
}
|
||||
|
||||
$records = [Collections.Generic.List[object]]::new()
|
||||
$parsedFileCount = 0
|
||||
foreach ($relativePath in $trackedFiles) {
|
||||
$absolutePath = Join-Path $repoRoot $relativePath
|
||||
if (-not (Test-Path -LiteralPath $absolutePath -PathType Leaf)) {
|
||||
continue
|
||||
}
|
||||
|
||||
$source = [IO.File]::ReadAllText($absolutePath)
|
||||
$tree = [Microsoft.CodeAnalysis.CSharp.CSharpSyntaxTree]::ParseText($source)
|
||||
$root = $tree.GetRoot()
|
||||
$parsedFileCount++
|
||||
|
||||
$allMethods = @($root.DescendantNodes() | Where-Object {
|
||||
$_.GetType().Name -eq 'MethodDeclarationSyntax'
|
||||
})
|
||||
$methodMap = @{}
|
||||
$failureMap = @{}
|
||||
$outputMap = @{}
|
||||
$callMap = @{}
|
||||
|
||||
foreach ($method in $allMethods) {
|
||||
$className = Get-ContainingClassName $method
|
||||
$key = "$className::$($method.Identifier.Text)"
|
||||
$methodMap[$key] = $method
|
||||
$failureMap[$key] = Test-DirectFailureSignal $method
|
||||
$outputMap[$key] = Test-DirectOutputSignal $method
|
||||
$callMap[$key] = @($method.DescendantNodes() |
|
||||
Where-Object { $_.GetType().Name -eq 'InvocationExpressionSyntax' } |
|
||||
ForEach-Object { Get-InvocationName $_ } |
|
||||
Sort-Object -Unique)
|
||||
}
|
||||
|
||||
foreach ($method in $allMethods) {
|
||||
$methodAttributes = @($method.AttributeLists | ForEach-Object { $_.Attributes })
|
||||
$attributeNames = @($methodAttributes | ForEach-Object {
|
||||
Get-ShortAttributeName $_
|
||||
})
|
||||
$testAttributes = @($attributeNames | Where-Object {
|
||||
$_ -in @('Fact', 'Theory', 'AvaloniaFact', 'InstalledDatFact')
|
||||
})
|
||||
if ($testAttributes.Count -eq 0) {
|
||||
continue
|
||||
}
|
||||
|
||||
$className = Get-ContainingClassName $method
|
||||
$key = "$className::$($method.Identifier.Text)"
|
||||
$hasFailureSignal = Test-RecursiveSignal `
|
||||
$key $methodMap $failureMap $callMap @{}
|
||||
$hasOutputSignal = Test-RecursiveSignal `
|
||||
$key $methodMap $outputMap $callMap @{}
|
||||
|
||||
$emptyReturns = [Collections.Generic.List[object]]::new()
|
||||
foreach ($returnNode in @($method.DescendantNodes() | Where-Object {
|
||||
$_.GetType().Name -eq 'ReturnStatementSyntax' -and
|
||||
$null -eq $_.Expression
|
||||
})) {
|
||||
$condition = '<unconditional>'
|
||||
$cursor = $returnNode.Parent
|
||||
while ($null -ne $cursor -and $cursor -ne $method) {
|
||||
if ($cursor.GetType().Name -eq 'IfStatementSyntax') {
|
||||
$condition = $cursor.Condition.ToString()
|
||||
break
|
||||
}
|
||||
$cursor = $cursor.Parent
|
||||
}
|
||||
$emptyReturns.Add([ordered]@{
|
||||
Line = Get-NodeLine $tree $returnNode
|
||||
Condition = $condition
|
||||
GateKind = Get-ReturnGateKind $condition
|
||||
})
|
||||
}
|
||||
|
||||
$traits = [Collections.Generic.List[string]]::new()
|
||||
$staticSkip = $null
|
||||
$typeNode = Get-ContainingTypeNode $method
|
||||
$traitAttributes = @($methodAttributes)
|
||||
if ($null -ne $typeNode) {
|
||||
$traitAttributes += @($typeNode.AttributeLists | ForEach-Object {
|
||||
$_.Attributes
|
||||
})
|
||||
}
|
||||
foreach ($attribute in $traitAttributes) {
|
||||
$attributeName = Get-ShortAttributeName $attribute
|
||||
$arguments = if ($null -eq $attribute.ArgumentList) {
|
||||
''
|
||||
} else {
|
||||
$attribute.ArgumentList.ToString()
|
||||
}
|
||||
if ($attributeName -eq 'Trait') {
|
||||
$traits.Add($arguments.Trim('(', ')').Replace('"', ''))
|
||||
}
|
||||
}
|
||||
foreach ($attribute in $methodAttributes) {
|
||||
$arguments = if ($null -eq $attribute.ArgumentList) {
|
||||
''
|
||||
} else {
|
||||
$attribute.ArgumentList.ToString()
|
||||
}
|
||||
if ($arguments -match '(^|[,\s])Skip\s*=') {
|
||||
$staticSkip = $arguments
|
||||
}
|
||||
}
|
||||
|
||||
$bodyText = $method.ToString()
|
||||
$environmentVariables = @([regex]::Matches(
|
||||
$bodyText,
|
||||
'GetEnvironmentVariable\s*\(\s*"([A-Za-z0-9_]+)"') |
|
||||
ForEach-Object { $_.Groups[1].Value } |
|
||||
Sort-Object -Unique)
|
||||
|
||||
$records.Add([ordered]@{
|
||||
Path = $relativePath.Replace('\', '/')
|
||||
Line = Get-NodeLine $tree $method
|
||||
Class = $className
|
||||
Method = $method.Identifier.Text
|
||||
Attributes = $testAttributes
|
||||
Traits = @($traits)
|
||||
StaticSkip = $staticSkip
|
||||
EmptyReturns = @($emptyReturns)
|
||||
PrerequisiteReturnCandidate = @($emptyReturns | Where-Object {
|
||||
$null -ne $_.GateKind
|
||||
}).Count -gt 0
|
||||
HasFailureSignal = $hasFailureSignal
|
||||
HasOutputSignal = $hasOutputSignal
|
||||
OutputOnlyCandidate = $hasOutputSignal -and -not $hasFailureSignal
|
||||
EnvironmentVariables = $environmentVariables
|
||||
HasThreadSleep = $bodyText -match '\bThread\.Sleep\s*\('
|
||||
HasTaskDelay = $bodyText -match '\bTask\.Delay\s*\('
|
||||
ReadsSourceText = $bodyText -match '(ReadAllText|ReadAllLines)\s*\(' -and
|
||||
$bodyText -match '\.cs'
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
$orderedRecords = @($records | Sort-Object Path, Line, Method)
|
||||
$workingTreeStatus = @(& git -C $repoRoot status --short --untracked-files=no)
|
||||
$summary = [ordered]@{
|
||||
GeneratedUtc = [DateTime]::UtcNow.ToString('O')
|
||||
RepositoryCommit = (& git -C $repoRoot rev-parse HEAD).Trim()
|
||||
WorkingTreeDirty = $workingTreeStatus.Count -gt 0
|
||||
DotnetSdkVersion = $sdkVersion
|
||||
RoslynAssemblyVersion = $roslynVersion
|
||||
TrackedCSharpFiles = $trackedFiles.Count
|
||||
ParsedCSharpFiles = $parsedFileCount
|
||||
AttributedTestMethods = $orderedRecords.Count
|
||||
StaticSkipMethods = @($orderedRecords | Where-Object { $null -ne $_.StaticSkip }).Count
|
||||
EmptyReturnMethods = @($orderedRecords | Where-Object { $_.EmptyReturns.Count -gt 0 }).Count
|
||||
EmptyReturnSites = @($orderedRecords | ForEach-Object { $_.EmptyReturns }).Count
|
||||
PrerequisiteReturnCandidates = @($orderedRecords | Where-Object {
|
||||
$_.PrerequisiteReturnCandidate
|
||||
}).Count
|
||||
OutputOnlyCandidates = @($orderedRecords | Where-Object { $_.OutputOnlyCandidate }).Count
|
||||
DiagnosticMethods = @($orderedRecords | Where-Object {
|
||||
$_.Traits -contains 'Purpose, Diagnostic'
|
||||
}).Count
|
||||
KnownFailureMethods = @($orderedRecords | Where-Object {
|
||||
$_.Traits -contains 'Status, KnownFailure'
|
||||
}).Count
|
||||
ThreadSleepMethods = @($orderedRecords | Where-Object { $_.HasThreadSleep }).Count
|
||||
TaskDelayMethods = @($orderedRecords | Where-Object { $_.HasTaskDelay }).Count
|
||||
DirectEnvironmentVariableMethods = @($orderedRecords | Where-Object {
|
||||
$_.EnvironmentVariables.Count -gt 0
|
||||
}).Count
|
||||
SourceTextReadMethods = @($orderedRecords | Where-Object { $_.ReadsSourceText }).Count
|
||||
}
|
||||
|
||||
$report = [ordered]@{
|
||||
Summary = $summary
|
||||
Tests = $orderedRecords
|
||||
}
|
||||
$outputDirectory = Split-Path -Parent $resolvedOutput
|
||||
[IO.Directory]::CreateDirectory($outputDirectory) | Out-Null
|
||||
$report | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $resolvedOutput -Encoding utf8
|
||||
|
||||
$summary | Format-List
|
||||
Write-Output "Inventory: $resolvedOutput"
|
||||
Loading…
Add table
Add a link
Reference in a new issue