acdream/src/AcDream.Core/Selection/RetailWorldPicker.cs

using System.Numerics;

namespace AcDream.Core.Selection;

/// <summary>
/// Pure port of retail's render-coupled mouse selection accumulator.
/// <c>Render::GfxObjUnderSelectionRay @ 0x0054C740</c> broad-phases each
/// visible part against its drawing sphere, then scans visual polygons in DAT
/// order. Any polygon hit globally outranks every sphere-only fallback.
/// </summary>
public static class RetailWorldPicker
{
    private const double RetailRayEpsilon = 0.0002;

    public static RetailSelectionHit? Pick(
        Vector3 worldOrigin,
        Vector3 worldDirection,
        IEnumerable<RetailSelectionPart> visibleParts,
        uint skipServerGuid = 0u)
    {
        if (worldDirection.LengthSquared() < 1e-10f)
            return null;

        RetailSelectionHit? closestSphere = null;
        RetailSelectionHit? closestPolygon = null;

        foreach (var part in visibleParts)
        {
            if (part.ServerGuid == 0u || part.ServerGuid == skipServerGuid)
                continue;
            if (part.Mesh.SphereRadius <= 0f
                || !Matrix4x4.Invert(part.LocalToWorld, out var worldToLocal))
                continue;

            // Keep direction unnormalised after the affine inverse. With row-vector
            // transforms this preserves the same ray parameter t in world metres even
            // when the part carries scale (retail divides by gfxobj_scale likewise).
            Vector3 localOrigin = Vector3.Transform(worldOrigin, worldToLocal);
            Vector3 localDirection = Vector3.TransformNormal(worldDirection, worldToLocal);

            if (!TryIntersectSphere(
                    localOrigin,
                    localDirection,
                    part.Mesh.SphereCenter,
                    part.Mesh.SphereRadius,
                    out double sphereT))
                continue;

            // Retail skips a part whose broad sphere starts beyond an already-found
            // polygon, because that part cannot improve the global polygon winner.
            if (closestPolygon is { } polygonWinner && sphereT > polygonWinner.Distance)
                continue;

            if (closestSphere is null || sphereT < closestSphere.Value.Distance)
                closestSphere = new RetailSelectionHit(
                    part.ServerGuid, part.LocalEntityId, part.PartIndex, sphereT, PolygonHit: false);

            // Retail stops at the FIRST hit polygon in this part's stored flat order.
            foreach (var polygon in part.Mesh.Polygons)
            {
                if (!TryIntersectPolygon(localOrigin, localDirection, polygon, out double polygonT))
                    continue;

                if (closestPolygon is null || polygonT < closestPolygon.Value.Distance)
                    closestPolygon = new RetailSelectionHit(
                        part.ServerGuid, part.LocalEntityId, part.PartIndex, polygonT, PolygonHit: true);
                break;
            }
        }

        return closestPolygon ?? closestSphere;
    }

    internal static bool TryIntersectSphere(
        Vector3 origin,
        Vector3 direction,
        Vector3 center,
        float radius,
        out double distance)
    {
        // CSphere::sphere_intersects_ray @ 0x005377A0. Retail intentionally
        // declines a broad-phase hit when the ray begins in or on the sphere.
        // The render view-cone normally keeps selectable objects in front of
        // the camera, so the routine does not separately reject a negative t.
        distance = 0d;
        Vector3 offset = origin - center;
        double c = Vector3.Dot(offset, offset) - (double)radius * radius;
        if (c <= 0d)
            return false;

        double a = Vector3.Dot(direction, direction);
        if (a < RetailRayEpsilon)
            return false;

        double b = -Vector3.Dot(offset, direction);
        double discriminant = b * b - c * a;
        if (discriminant < 0d)
            return false;

        double root = Math.Sqrt(discriminant);
        distance = b > root ? (b - root) / a : (b + root) / a;
        return true;
    }

    internal static bool TryIntersectPolygon(
        Vector3 origin,
        Vector3 direction,
        RetailSelectionPolygon polygon,
        out double distance)
    {
        distance = 0d;
        if (polygon.Vertices.Count < 3
            || !TryPlane(polygon.Vertices, out Vector3 normal, out float planeD))
            return false;

        double denominator = Vector3.Dot(direction, normal);
        // CPolygon::polygon_hits_ray @ 0x005395E0: raw sides_type 0 is
        // single-sided and rejects a ray travelling with the positive normal.
        if (polygon.SingleSided && denominator > 0d)
            return false;
        if (Math.Abs(denominator) < RetailRayEpsilon)
            return false;

        distance = -(Vector3.Dot(origin, normal) + planeD) / denominator;
        if (distance < 0d)
            return false;

        Vector3 point = origin + direction * (float)distance;
        return PointInPolygon(point, polygon.Vertices, normal);
    }

    private static bool TryPlane(
        IReadOnlyList<Vector3> vertices,
        out Vector3 normal,
        out float planeD)
    {
        // CPolygon::make_plane @ 0x005383D0 builds a triangle fan from
        // vertex zero, sums the fan normals, normalizes once, then chooses d
        // from the average signed distance of every vertex. DatReaderWriter
        // exposes vertices rather than retail's derived Plane, so reconstruct
        // that load-time result here.
        Vector3 first = vertices[0];
        Vector3 normalSum = Vector3.Zero;
        for (int i = 1; i + 1 < vertices.Count; i++)
            normalSum += Vector3.Cross(vertices[i] - first, vertices[i + 1] - first);

        if (normalSum.LengthSquared() > 1e-12f)
        {
            normal = Vector3.Normalize(normalSum);
            double averageDot = 0d;
            foreach (Vector3 vertex in vertices)
                averageDot += Vector3.Dot(normal, vertex);
            planeD = (float)-(averageDot / vertices.Count);
            return true;
        }

        normal = default;
        planeD = 0f;
        return false;
    }

    private static bool PointInPolygon(
        Vector3 point,
        IReadOnlyList<Vector3> vertices,
        Vector3 normal)
    {
        // CPolygon::point_in_polygon @ 0x00538D90. Retail visual polygons are
        // convex: the point must remain on the inward side of every ordered
        // edge. Zero is accepted, so a click exactly on an edge still hits.
        Vector3 previous = vertices[^1];
        for (int i = 0; i < vertices.Count; i++)
        {
            Vector3 current = vertices[i];
            Vector3 inward = Vector3.Cross(normal, current - previous);
            if (Vector3.Dot(point - previous, inward) < 0f)
                return false;
            previous = current;
        }
        return true;
    }
}