erik/acdream
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Wiki Activity Actions

fix(vt): make the .usd cursor character-oriented for ba blobs and unknown tags

Browse source 
Fidelity blocker 6: VtankLineCursor pre-normalized every "\r\n" to "\n"
across the WHOLE document before splitting into lines, then ReadBlob
re-joined consumed lines with a single '\n'. VTank's real reader
(f6.cs:10-17) is `TextReader.Read(array, 0, num)` — it reads exactly N
raw characters straight off the stream, so an embedded CRLF inside a
"ba" blob costs 2 characters toward that length, not 1. The prior
cursor silently dropped that extra character, corrupting any blob
whose content used CRLF line breaks and desyncing the parse position
for everything that follows it in the same row/table.

Separately, VtankDatabaseReader.ReadCell's default case unconditionally
consumed a "value" line for ANY unrecognized tag. VTank's own reader
(gy.cs:50-55) and writer (gy.cs:98-101) treat every tag outside
{d,i,u,f,s,b,TABLE,ba} as void/unrecognized (y.cs:28-41 registers only
TABLE and ba as named custom types) and consume/emit ONLY the tag line
— no value line either way. The prior default case would misread the
next cell's own tag as this cell's bogus value, corrupting the rest of
the row exactly like the blob bug above.

- VtankLineCursor now indexes directly into the original (unnormalized)
  text: ReadLine() scans for '\n' and strips one trailing '\r' per line
  (matching StreamReader.ReadLine() semantics); ReadBlob(length) takes
  exactly `length` raw characters from the current position with zero
  reinterpretation.
- VtankCell.WriteTo (and Row/Table/Database) now build a single
  StringBuilder instead of a `List<string> lines` that assumed one
  entry == one line: a "ba" blob writes its length line then the RAW
  blob content with NO added line terminator (matching f6.cs's
  WriteLine(length) + Write(content), not WriteLine(content)) — the
  next structural token continues immediately after the blob's last
  character, exactly like real VTank output.
- ReadCell's default case now builds a bare unknown-tag cell (no
  ScalarText/BlobText, consuming nothing further); WriteTo mirrors this
  by emitting nothing after such a tag's own line.
- Bonus (adjacent, from the same gy.cs read): VtankCell.String now
  strips embedded '\n' on write (gy.cs:84's `text.Replace("\n", "")`),
  matching VTank's own string-cell writer exactly — a partial down
  payment on item 12's "strip \n from string cells on write" nit.

New tests (tests/AcDream.Plugins.MossTank.Tests/VtankUsdDocumentTests.cs):
BaBlobWithEmbeddedCrlfRoundTripsExactCharacterCount,
BaBlobLengthCountsEmbeddedCrAndLfAsSeparateCharacters, and
UnrecognizedTagConsumesOnlyItsOwnLineNotTheNextCellsValue. All three
were verified failing against the pre-fix VtankUsdDocument.cs (restored
from HEAD, tests re-run, then reverted back) before this change:
the first two threw FormatException "unexpected end of file", the
third read "s" (the next cell's own tag) as ScalarText instead of null.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Erik 2026-09-06 21:52:39 +02:00
parent 74283e1116
commit 07664b199b
2 changed files with 205 additions and 65 deletions
Download patch file Download diff file Expand all files Collapse all files

179
src/AcDream.Plugins.MossTank/VtankUsdDocument.cs
View file

@ -19,12 +19,22 @@ namespace AcDream.Plugins.MossTank;
/// (<c>"True"</c>/<c>"False"</c>), <c>TABLE</c>=a nested table (recurse into
/// <see cref="VtankTable"/>), <c>ba</c>=a length-prefixed raw-character blob
/// (an int line for the character count, then that many raw characters,
/// not lines, so embedded newlines survive). Any other tag is preserved
/// verbatim as an opaque single-line scalar so an unrecognized custom type
/// still round-trips byte-for-byte.
/// not lines, so embedded newlines survive — see <see cref="VtankLineCursor.ReadBlob"/>).
/// Any other tag (<c>y.cs:28-41</c> only ever registers <c>TABLE</c> and
/// <c>ba</c> as named custom types) is genuinely unrecognized to VTank
/// itself: <c>gy.cs:50-55</c> (read) consumes ONLY the tag line and
/// <c>gy.cs:98-101</c> (write) emits ONLY a tag line ("0" for a void cell)
/// — no value line either way. This is preserved verbatim so an
/// unrecognized custom type still round-trips byte-for-byte.
/// </summary>
internal sealed class VtankCell
{
// The only tags VTank's gy.a(TextReader)/gy.a(TextWriter) treat as a
// tag+value pair (gy.cs:20-56, gy.cs:60-92). TABLE and ba are each
// handled by their own dedicated branch; everything else is an
// unrecognized/void tag that carries no value line at all.
private static readonly HashSet<string> KnownScalarTags = ["d", "i", "u", "f", "s", "b"];
internal VtankCell() { }
/// <summary>The raw type tag: <c>d</c>/<c>i</c>/<c>u</c>/<c>f</c>/<c>s</c>/<c>b</c>/<c>TABLE</c>/<c>ba</c>/other.</summary>
@ -66,7 +76,10 @@ internal sealed class VtankCell
public static VtankCell String(string value) => new()
{
Tag = "s",
ScalarText = value,
// VTank strips embedded newlines from a string cell on write
// (gy.cs:84: `text.Replace("\n", "")`) — a string cell is always
// exactly one line; a blob that needs embedded newlines uses "ba".
ScalarText = value.Replace("\n", string.Empty, StringComparison.Ordinal),
};
public static VtankCell Bool(bool value) => new()
@ -121,45 +134,49 @@ internal sealed class VtankCell
internal static string FormatDouble(double value) =>
value.ToString("G15", CultureInfo.InvariantCulture);
internal void WriteTo(List<string> lines)
internal void WriteTo(StringBuilder sb)
{
lines.Add(Tag);
switch (Tag)
VtankWriter.AppendLine(sb, Tag);
if (Tag == "TABLE")
{
case "TABLE":
Table!.WriteTo(lines);
break;
case "ba":
string blob = BlobText ?? string.Empty;
lines.Add(blob.Length.ToString(CultureInfo.InvariantCulture));
AppendBlobLines(lines, blob);
break;
default:
lines.Add(ScalarText ?? string.Empty);
break;
Table!.WriteTo(sb);
}
else if (Tag == "ba")
{
// A "ba" blob is a raw character count, not a line count: the
// blob may contain embedded newlines. VTank's own writer
// (f6.cs:20-24) WriteLine()s only the length, then Write()s the
// raw characters with NO trailing line terminator — whatever
// structural token follows continues immediately after the
// blob's last character, exactly like the real client's output.
string blob = BlobText ?? string.Empty;
VtankWriter.AppendLine(sb, blob.Length.ToString(CultureInfo.InvariantCulture));
sb.Append(blob);
}
else if (KnownScalarTags.Contains(Tag))
{
VtankWriter.AppendLine(sb, ScalarText ?? string.Empty);
}
// else: an unrecognized/void tag is the WHOLE cell (gy.cs:98-101
// writes only "0", no value line) — nothing further to emit.
}
}
// "ba" blobs are a raw character count, not a line count: the blob may
// contain embedded newlines. FileLines (this reader) is line-oriented,
// so a blob is folded back into whole lines for storage and the reader
// re-joins with the writer's own newline so the character count is
// reproduced exactly on save.
private static void AppendBlobLines(List<string> lines, string blob)
{
foreach (string part in blob.Split('\n'))
lines.Add(part.EndsWith('\r') ? part[..^1] : part);
}
/// <summary>Shared line-terminator helper for every <c>WriteTo</c> in this file (CRLF, matching VTank's own writer).</summary>
internal static class VtankWriter
{
internal static void AppendLine(StringBuilder sb, string text) =>
sb.Append(text).Append("\r\n");
}
internal sealed class VtankRow
{
public List<VtankCell> Cells { get; } = [];
internal void WriteTo(List<string> lines)
internal void WriteTo(StringBuilder sb)
{
foreach (VtankCell cell in Cells)
cell.WriteTo(lines);
cell.WriteTo(sb);
}
}
@ -197,16 +214,16 @@ internal sealed class VtankTable
return table;
}
internal void WriteTo(List<string> lines)
internal void WriteTo(StringBuilder sb)
{
lines.Add(ColumnNames.Count.ToString(CultureInfo.InvariantCulture));
VtankWriter.AppendLine(sb, ColumnNames.Count.ToString(CultureInfo.InvariantCulture));
foreach (string column in ColumnNames)
lines.Add(column);
VtankWriter.AppendLine(sb, column);
foreach (bool flag in IndexFlags)
lines.Add(flag ? "y" : "n");
lines.Add(Rows.Count.ToString(CultureInfo.InvariantCulture));
VtankWriter.AppendLine(sb, flag ? "y" : "n");
VtankWriter.AppendLine(sb, Rows.Count.ToString(CultureInfo.InvariantCulture));
foreach (VtankRow row in Rows)
row.WriteTo(lines);
row.WriteTo(sb);
}
}
@ -236,19 +253,14 @@ internal sealed class VtankDatabase
public string Render()
{
var lines = new List<string>
{
Tables.Count.ToString(CultureInfo.InvariantCulture),
};
var sb = new StringBuilder();
VtankWriter.AppendLine(sb, Tables.Count.ToString(CultureInfo.InvariantCulture));
foreach ((string name, VtankTable table) in Tables)
{
lines.Add(name);
table.WriteTo(lines);
VtankWriter.AppendLine(sb, name);
table.WriteTo(sb);
}
var builder = new StringBuilder();
foreach (string line in lines)
builder.Append(line).Append("\r\n");
return builder.ToString();
return sb.ToString();
}
}
@ -266,8 +278,14 @@ internal static class VtankDatabaseReader
int length = cursor.ReadInt();
return new VtankCellBuilder(tag) { BlobText = cursor.ReadBlob(length) }
.Build();
default:
case "d" or "i" or "u" or "f" or "s" or "b":
return new VtankCellBuilder(tag) { ScalarText = cursor.ReadLine() }.Build();
default:
// Unrecognized/void tag (refs/vtank/decompiled/y.cs:28-41
// only ever registers TABLE and ba as named custom types):
// VTank's own reader (gy.cs:50-55) consumes ONLY the tag
// line here — no value line follows.
return new VtankCellBuilder(tag).Build();
}
}
@ -295,33 +313,57 @@ internal static class VtankCellFactory
}
/// <summary>
/// Line cursor over the CRLF-or-LF text of a <c>.usd</c>/<c>.ast</c> file.
/// VTank's own reader is a strict token stream (<c>StreamReader.ReadLine()</c>
/// Character-oriented cursor over the exact (unnormalized) text of a
/// <c>.usd</c>/<c>.ast</c> file. VTank's own reader is a strict token stream
/// (<c>StreamReader.ReadLine()</c>/<c>TextReader.Read(char[], int, int)</c>
/// calls in a fixed order) — there is no way to resynchronize after a parse
/// error, so every read here throws immediately with a file:line-shaped
/// message on the first unexpected token.
///
/// The cursor deliberately keeps the ORIGINAL string, never pre-splitting on
/// <c>'\n'</c>: a <c>ba</c> blob (<see cref="ReadBlob"/>) reads exactly N raw
/// characters straight out of that string, embedded <c>\r</c>/<c>\n</c>
/// included, matching <c>f6.cs:10-17</c>'s <c>TextReader.Read(array, 0, num)</c>
/// exactly — a CRLF-terminated line INSIDE a blob counts as 2 characters
/// toward that length, same as it would for VTank itself.
/// </summary>
internal sealed class VtankLineCursor
{
private readonly string[] _lines;
private int _index;
private readonly string _text;
private int _position;
private int _lineNumber = 1;
public VtankLineCursor(string text)
{
ArgumentNullException.ThrowIfNull(text);
_lines = text.Replace("\r\n", "\n", StringComparison.Ordinal).Split('\n');
_text = text;
}
public int LineNumber => _index + 1;
public int LineNumber => _lineNumber;
public string ReadLine()
{
if (_index >= _lines.Length)
if (_position >= _text.Length)
{
throw new FormatException(
$"line {LineNumber}: unexpected end of file (expected another line).");
}
return _lines[_index++];
int newlineIndex = _text.IndexOf('\n', _position);
string line;
if (newlineIndex < 0)
{
line = _text[_position..];
_position = _text.Length;
}
else
{
line = _text[_position..newlineIndex];
_position = newlineIndex + 1;
}
if (line.EndsWith('\r'))
line = line[..^1];
_lineNumber++;
return line;
}
public int ReadInt()
@ -336,22 +378,29 @@ internal sealed class VtankLineCursor
}
/// <summary>
/// Reads a <c>ba</c> blob's exact character count, re-joining folded
/// lines with <c>\n</c> until the count is satisfied (the writer folds
/// on <c>\n</c> too, so this round-trips the exact character count VTank
/// itself would report for embedded newlines).
/// Reads a <c>ba</c> blob's exact character count straight out of the
/// underlying text with no line-oriented interpretation whatsoever
/// (<c>f6.cs:10-17</c>: <c>TextReader.Read(array, 0, num)</c> starting
/// immediately after the length line's terminator) — embedded
/// <c>\r</c>/<c>\n</c> characters are counted and returned verbatim, not
/// folded or re-joined.
/// </summary>
public string ReadBlob(int length)
{
if (length <= 0)
return string.Empty;
var builder = new StringBuilder(length);
while (builder.Length < length)
if (_position + length > _text.Length)
{
if (builder.Length > 0)
builder.Append('\n');
builder.Append(ReadLine());
throw new FormatException(
$"line {LineNumber}: blob of {length} characters exceeds remaining input.");
}
return builder.ToString(0, length);
string blob = _text.Substring(_position, length);
_position += length;
foreach (char c in blob)
{
if (c == '\n')
_lineNumber++;
}
return blob;
}
}