c49b81c237
Wires the validated item-processor into the ingestion endpoints, writing to an isolated inventory-go-db (never production): - schema.go: faithful 7-table replica of inventory-service/database.py. - ingest.go: /process-inventory (full replace), POST/DELETE single item, with the exact delete-then-insert flow, dynamic INSERT builder (quotes reserved "unique"), spell union (is_active), and item_raw_data verbatim. enhancements always inserts. - compose: isolated inventory-go-db (postgres:14, 127.0.0.1:5435) + read-write inventory-go-shadow (:8773) that owns it; schema init on boot. Validated by ingesting a recently-ingested character's items (from production's original_json) into the shadow DB and diffing vs production: byte-identical — items 243, combat 243, enhancements 243, ratings 6, requirements 19, spells 52 all match; 0 per-column mismatches across 243 items. Finding: older production normalized rows can be STALE (predate the code reading Decal keys 218103832/218103835); Go matches the CURRENT Python code, so validate ingestion against recently-ingested characters. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
434 lines
14 KiB
Go
434 lines
14 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"io"
|
|
"math"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// Item-processor: a faithful port of inventory-service extract_item_properties +
|
|
// the process_inventory column population. Given a raw item dict it produces the
|
|
// normalized rows for the 7 tables, applying the exact per-table sentinel->NULL
|
|
// rules. Validated against production's stored rows (read-only) via /debug/process.
|
|
|
|
// --- value-bag accessors (JSON object keys are strings) ---
|
|
|
|
func bag(raw map[string]any, name string) map[string]any {
|
|
if m, ok := raw[name].(map[string]any); ok {
|
|
return m
|
|
}
|
|
return map[string]any{}
|
|
}
|
|
func ivI(iv map[string]any, key string, def int) int {
|
|
if v, ok := iv[key]; ok {
|
|
if f, ok := v.(float64); ok {
|
|
return int(f)
|
|
}
|
|
}
|
|
return def
|
|
}
|
|
func dvF(dv map[string]any, key string, def float64) float64 {
|
|
if v, ok := dv[key]; ok {
|
|
if f, ok := v.(float64); ok {
|
|
return f
|
|
}
|
|
}
|
|
return def
|
|
}
|
|
func rawI(raw map[string]any, key string, def int) int {
|
|
if v, ok := raw[key]; ok {
|
|
if f, ok := v.(float64); ok {
|
|
return int(f)
|
|
}
|
|
}
|
|
return def
|
|
}
|
|
func rawF(raw map[string]any, key string, def float64) float64 {
|
|
if v, ok := raw[key]; ok {
|
|
if f, ok := v.(float64); ok {
|
|
return f
|
|
}
|
|
}
|
|
return def
|
|
}
|
|
func rawS(raw map[string]any, key string) string {
|
|
if s, ok := raw[key].(string); ok {
|
|
return s
|
|
}
|
|
return ""
|
|
}
|
|
func rawB(raw map[string]any, key string) bool {
|
|
b, _ := raw[key].(bool)
|
|
return b
|
|
}
|
|
|
|
// IV first, else top-level field, else default (e.g. max_damage).
|
|
func ivElseTopI(iv, raw map[string]any, ivKey, topKey string, def int) int {
|
|
if v, ok := iv[ivKey]; ok {
|
|
if f, ok := v.(float64); ok {
|
|
return int(f)
|
|
}
|
|
}
|
|
return rawI(raw, topKey, def)
|
|
}
|
|
func dvElseTopF(dv, raw map[string]any, dvKey, topKey string, def float64) float64 {
|
|
if v, ok := dv[dvKey]; ok {
|
|
if f, ok := v.(float64); ok {
|
|
return f
|
|
}
|
|
}
|
|
return rawF(raw, topKey, def)
|
|
}
|
|
|
|
// translateMaterial: materials[id] else "Unknown_Material_{id}".
|
|
func (s *Server) translateMaterial(id int) string {
|
|
if n, ok := s.materials[id]; ok {
|
|
return n
|
|
}
|
|
return "Unknown_Material_" + strconv.Itoa(id)
|
|
}
|
|
|
|
func toIntList(v any) []int {
|
|
arr, ok := v.([]any)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
out := make([]int, 0, len(arr))
|
|
for _, e := range arr {
|
|
if f, ok := e.(float64); ok {
|
|
out = append(out, int(f))
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// processItem produces the normalized columns for all 7 tables, post-null.
|
|
func (s *Server) processItem(raw map[string]any) map[string]any {
|
|
iv := bag(raw, "IntValues")
|
|
dv := bag(raw, "DoubleValues")
|
|
|
|
items := map[string]any{
|
|
"item_id": rawValue(raw, "Id"),
|
|
"name": rawS(raw, "Name"),
|
|
"icon": rawI(raw, "Icon", 0),
|
|
"object_class": rawI(raw, "ObjectClass", 0),
|
|
"value": rawI(raw, "Value", 0),
|
|
"burden": rawI(raw, "Burden", 0),
|
|
"has_id_data": rawB(raw, "HasIdData"),
|
|
"last_id_time": rawI(raw, "LastIdTime", 0),
|
|
"current_wielded_location": ivI(iv, "10", 0),
|
|
"container_id": rawI(raw, "ContainerId", 0),
|
|
"slot": ivI(iv, "231735296", -1),
|
|
"bonded": ivI(iv, "33", 0),
|
|
"attuned": ivI(iv, "114", 0),
|
|
"unique": ivI(iv, "279", 0) != 0,
|
|
"stack_size": ivI(iv, "12", 1),
|
|
"max_stack_size": ivI(iv, "11", 1),
|
|
"items_capacity": nilNeg(ivI(iv, "6", -1)),
|
|
"containers_capacity": nilNeg(ivI(iv, "7", -1)),
|
|
"structure": nilNeg(ivI(iv, "92", -1)),
|
|
"max_structure": nilNeg(ivI(iv, "91", -1)),
|
|
"rare_id": nilNeg(ivI(iv, "17", -1)),
|
|
"lifespan": nilNeg(ivI(iv, "267", -1)),
|
|
"remaining_lifespan": nilNeg(ivI(iv, "268", -1)),
|
|
}
|
|
|
|
// combat (sentinel defaults), then base values merged.
|
|
wt := ivI(iv, "218103835", -1)
|
|
if wt > 100 {
|
|
wt = 100
|
|
}
|
|
combat := map[string]any{
|
|
"max_damage": ivElseTopI(iv, raw, "218103842", "MaxDamage", -1),
|
|
"damage_type": ivI(iv, "218103832", -1),
|
|
"damage_bonus": dvElseTopF(dv, raw, "167772174", "DamageBonus", -1.0),
|
|
"elemental_damage_bonus": ivI(iv, "204", -1),
|
|
"elemental_damage_vs_monsters": dvF(dv, "152", -1.0),
|
|
"variance": dvF(dv, "167772171", -1.0),
|
|
"cleaving": ivI(iv, "292", -1),
|
|
"crit_damage_rating": ivI(iv, "314", -1),
|
|
"damage_over_time": ivI(iv, "318", -1),
|
|
"attack_bonus": dvElseTopF(dv, raw, "167772170", "AttackBonus", -1.0),
|
|
"weapon_time": wt,
|
|
"weapon_skill": ivI(iv, "218103840", -1),
|
|
"armor_level": topElseIvI(raw, iv, "ArmorLevel", "28", -1),
|
|
"melee_defense_bonus": dvF(dv, "29", -1.0),
|
|
"missile_defense_bonus": dvF(dv, "149", -1.0),
|
|
"magic_defense_bonus": dvF(dv, "150", -1.0),
|
|
"resist_magic": ivI(iv, "36", -1),
|
|
"crit_resist_rating": ivI(iv, "315", -1),
|
|
"crit_damage_resist_rating": ivI(iv, "316", -1),
|
|
"dot_resist_rating": ivI(iv, "350", -1),
|
|
"life_resist_rating": ivI(iv, "351", -1),
|
|
"nether_resist_rating": ivI(iv, "331", -1),
|
|
"heal_over_time": ivI(iv, "312", -1),
|
|
"healing_resist_rating": ivI(iv, "317", -1),
|
|
"mana_conversion_bonus": dvF(dv, "144", -1.0),
|
|
"pk_damage_rating": ivI(iv, "381", -1),
|
|
"pk_damage_resist_rating": ivI(iv, "382", -1),
|
|
"gear_pk_damage_rating": ivI(iv, "383", -1),
|
|
"gear_pk_damage_resist_rating": ivI(iv, "384", -1),
|
|
}
|
|
s.mergeBaseValues(raw, combat)
|
|
|
|
requirements := map[string]any{
|
|
"wield_level": rawI(raw, "WieldLevel", -1),
|
|
"skill_level": rawI(raw, "SkillLevel", -1),
|
|
"lore_requirement": rawI(raw, "LoreRequirement", -1),
|
|
"equip_skill": rawValueStr(raw, "EquipSkill"),
|
|
}
|
|
|
|
// material + item_set translated strings.
|
|
var material any
|
|
if m := rawS(raw, "Material"); m != "" {
|
|
material = m
|
|
} else if v, ok := iv["131"]; ok {
|
|
if f, ok := v.(float64); ok && int(f) != 0 {
|
|
name := s.translateMaterial(int(f))
|
|
if !strings.HasPrefix(name, "Unknown_Material_") {
|
|
material = name
|
|
}
|
|
}
|
|
}
|
|
var itemSet any
|
|
if v, ok := iv["265"]; ok {
|
|
if f, ok := v.(float64); ok && int(f) != 0 {
|
|
id := strconv.Itoa(int(f))
|
|
if n, ok := s.attributeSets[id]; ok {
|
|
itemSet = n
|
|
} else {
|
|
itemSet = id
|
|
}
|
|
}
|
|
}
|
|
enhancements := map[string]any{
|
|
"material": material,
|
|
"imbue": rawValueStr(raw, "Imbue"),
|
|
"tinks": rawI(raw, "Tinks", -1),
|
|
"workmanship": rawF(raw, "Workmanship", -1.0),
|
|
"num_times_tinkered": ivI(iv, "171", -1),
|
|
"free_tinkers_bitfield": ivI(iv, "264", -1),
|
|
"num_items_in_material": ivI(iv, "170", -1),
|
|
"imbue_attempts": ivI(iv, "205", -1),
|
|
"imbue_successes": ivI(iv, "206", -1),
|
|
"imbued_effect2": ivI(iv, "303", -1),
|
|
"imbued_effect3": ivI(iv, "304", -1),
|
|
"imbued_effect4": ivI(iv, "305", -1),
|
|
"imbued_effect5": ivI(iv, "306", -1),
|
|
"imbue_stacking_bits": ivI(iv, "311", -1),
|
|
"item_set": itemSet,
|
|
"equipment_set_extra": ivI(iv, "321", -1),
|
|
"aetheria_bitfield": ivI(iv, "322", -1),
|
|
"heritage_specific_armor": ivI(iv, "324", -1),
|
|
"shared_cooldown": ivI(iv, "280", -1),
|
|
}
|
|
|
|
ratingKeys := map[string]string{
|
|
"damage_rating": "307", "damage_resist_rating": "308", "crit_rating": "313",
|
|
"crit_resist_rating": "315", "crit_damage_rating": "314", "crit_damage_resist_rating": "316",
|
|
"heal_boost_rating": "323", "vitality_rating": "341", "healing_rating": "342",
|
|
"weakness_rating": "329", "nether_over_time": "330", "healing_resist_rating": "317",
|
|
"nether_resist_rating": "331", "dot_resist_rating": "350", "life_resist_rating": "351",
|
|
"sneak_attack_rating": "356", "recklessness_rating": "357", "deception_rating": "358",
|
|
"pk_damage_rating": "381", "pk_damage_resist_rating": "382", "gear_pk_damage_rating": "383",
|
|
"gear_pk_damage_resist_rating": "384", "gear_damage": "370", "gear_damage_resist": "371",
|
|
"gear_crit": "372", "gear_crit_resist": "373", "gear_crit_damage": "374",
|
|
"gear_crit_damage_resist": "375", "gear_healing_boost": "376", "gear_max_health": "379",
|
|
"gear_nether_resist": "377", "gear_life_resist": "378", "gear_overpower": "388",
|
|
"gear_overpower_resist": "389",
|
|
}
|
|
ratings := map[string]any{}
|
|
for col, k := range ratingKeys {
|
|
ratings[col] = ivI(iv, k, -1)
|
|
}
|
|
|
|
// spells: union of Spells + ActiveSpells, is_active = in ActiveSpells.
|
|
spells := toIntList(raw["Spells"])
|
|
active := toIntList(raw["ActiveSpells"])
|
|
activeSet := map[int]bool{}
|
|
for _, id := range active {
|
|
activeSet[id] = true
|
|
}
|
|
seen := map[int]bool{}
|
|
var spellRows []map[string]any
|
|
for _, id := range append(append([]int{}, spells...), active...) {
|
|
if seen[id] {
|
|
continue
|
|
}
|
|
seen[id] = true
|
|
spellRows = append(spellRows, map[string]any{"spell_id": id, "is_active": activeSet[id]})
|
|
}
|
|
|
|
return map[string]any{
|
|
"items": items,
|
|
"combat": nullify(combat, sentinelCombat),
|
|
"requirements": nullify(requirements, sentinelReq),
|
|
"enhancements": nullifyKeep(enhancements, sentinelEnh), // ALWAYS inserts a row
|
|
"ratings": nullify(ratings, sentinelRating),
|
|
"spells": spellRows,
|
|
}
|
|
}
|
|
|
|
func topElseIvI(raw, iv map[string]any, topKey, ivKey string, def int) int {
|
|
if v, ok := raw[topKey]; ok {
|
|
if f, ok := v.(float64); ok {
|
|
return int(f)
|
|
}
|
|
}
|
|
return ivI(iv, ivKey, def)
|
|
}
|
|
|
|
func rawValue(raw map[string]any, key string) any {
|
|
if v, ok := raw[key]; ok {
|
|
if f, ok := v.(float64); ok {
|
|
return int64(f)
|
|
}
|
|
return v
|
|
}
|
|
if v, ok := raw[strings.ToLower(key)]; ok { // Id -> id fallback
|
|
if f, ok := v.(float64); ok {
|
|
return int64(f)
|
|
}
|
|
return v
|
|
}
|
|
return nil
|
|
}
|
|
func rawValueStr(raw map[string]any, key string) any {
|
|
if s, ok := raw[key].(string); ok && s != "" {
|
|
return s
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func nilNeg(v int) any {
|
|
if v == -1 {
|
|
return nil
|
|
}
|
|
return v
|
|
}
|
|
|
|
// per-table sentinel predicates: true => value should become NULL.
|
|
func sentinelCombat(v any) bool { return isNeg1(v) || isNeg1f(v) }
|
|
func sentinelReq(v any) bool { return isNeg1(v) || v == nil || v == "" }
|
|
func sentinelEnh(v any) bool { return isNeg1(v) || isNeg1f(v) || v == nil || v == "" }
|
|
func sentinelRating(v any) bool { return isNeg1(v) || isNeg1f(v) || v == nil }
|
|
|
|
func isNeg1(v any) bool { i, ok := v.(int); return ok && i == -1 }
|
|
func isNeg1f(v any) bool { f, ok := v.(float64); return ok && f == -1.0 }
|
|
|
|
// nullify replaces sentinel values with nil. Returns nil for the whole table if
|
|
// every value is sentinel (the per-table "skip insert" guard) — EXCEPT
|
|
// enhancements, which always inserts; we keep its map even if all-null.
|
|
func nullify(m map[string]any, isSentinel func(any) bool) map[string]any {
|
|
any_ := false
|
|
out := make(map[string]any, len(m))
|
|
for k, v := range m {
|
|
if isSentinel(v) {
|
|
out[k] = nil
|
|
} else {
|
|
out[k] = v
|
|
any_ = true
|
|
}
|
|
}
|
|
if !any_ {
|
|
return nil // combat/req/ratings: skip the insert when all-sentinel
|
|
}
|
|
return out
|
|
}
|
|
|
|
// nullifyKeep is like nullify but ALWAYS returns the map (for item_enhancements,
|
|
// which inserts a row even when every value is NULL).
|
|
func nullifyKeep(m map[string]any, isSentinel func(any) bool) map[string]any {
|
|
out := make(map[string]any, len(m))
|
|
for k, v := range m {
|
|
if isSentinel(v) {
|
|
out[k] = nil
|
|
} else {
|
|
out[k] = v
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// mergeBaseValues reverses active-spell buffs into base_* columns (compute_base_values).
|
|
type spellEffect struct {
|
|
key int
|
|
change, bonus float64
|
|
}
|
|
|
|
var intEffects = map[int]spellEffect{
|
|
1616: {218103842, 20, 0}, 2096: {218103842, 22, 0}, 5183: {218103842, 24, 0}, 4395: {218103842, 24, 0}, 3688: {218103842, 300, 0},
|
|
2598: {218103842, 2, 2}, 2586: {218103842, 4, 4}, 4661: {218103842, 7, 7}, 6089: {218103842, 10, 10},
|
|
1486: {28, 200, 0}, 2108: {28, 220, 0}, 4407: {28, 240, 0},
|
|
2604: {28, 20, 20}, 2592: {28, 40, 40}, 4667: {28, 60, 60}, 6095: {28, 80, 80},
|
|
}
|
|
var doubleEffects = map[int]spellEffect{
|
|
3258: {152, 0.06, 0}, 3259: {152, 0.07, 0}, 5182: {152, 0.08, 0}, 4414: {152, 0.08, 0}, 3735: {152, 0.15, 0},
|
|
3251: {152, 0.01, 0.01}, 3250: {152, 0.03, 0.03}, 4670: {152, 0.05, 0.05}, 6098: {152, 0.07, 0.07},
|
|
1592: {167772172, 0.15, 0}, 2106: {167772172, 0.17, 0}, 4405: {167772172, 0.20, 0},
|
|
2603: {167772172, 0.03, 0.03}, 2591: {167772172, 0.05, 0.05}, 4666: {167772172, 0.07, 0.07}, 6094: {167772172, 0.09, 0.09},
|
|
1605: {29, 0.15, 0}, 2101: {29, 0.17, 0}, 4400: {29, 0.20, 0}, 3699: {29, 0.25, 0},
|
|
2600: {29, 0.03, 0.03}, 3985: {29, 0.04, 0.04}, 2588: {29, 0.05, 0.05}, 4663: {29, 0.07, 0.07}, 6091: {29, 0.09, 0.09},
|
|
1480: {144, 1.60, 0}, 2117: {144, 1.70, 0}, 4418: {144, 1.80, 0},
|
|
3201: {144, 1.05, 1.05}, 3199: {144, 1.10, 1.10}, 3202: {144, 1.15, 1.15}, 3200: {144, 1.20, 1.20}, 6086: {144, 1.25, 1.25}, 6087: {144, 1.30, 1.30},
|
|
}
|
|
|
|
func (s *Server) mergeBaseValues(raw, combat map[string]any) {
|
|
spells := toIntList(raw["Spells"])
|
|
active := toIntList(raw["ActiveSpells"])
|
|
for _, p := range []struct {
|
|
prop string
|
|
key int
|
|
}{{"max_damage", 218103842}, {"armor_level", 28}} {
|
|
val, ok := combat[p.prop].(int)
|
|
if !ok || val == -1 {
|
|
continue
|
|
}
|
|
for _, sid := range active {
|
|
if e, ok := intEffects[sid]; ok && e.key == p.key {
|
|
val -= int(e.change)
|
|
}
|
|
}
|
|
for _, sid := range spells {
|
|
if e, ok := intEffects[sid]; ok && e.key == p.key && e.bonus != 0 {
|
|
val += int(e.bonus)
|
|
}
|
|
}
|
|
combat["base_"+p.prop] = val
|
|
}
|
|
for _, p := range []struct {
|
|
prop string
|
|
key int
|
|
}{{"attack_bonus", 167772172}, {"melee_defense_bonus", 29}, {"elemental_damage_vs_monsters", 152}, {"mana_conversion_bonus", 144}} {
|
|
val, ok := combat[p.prop].(float64)
|
|
if !ok || val == -1.0 {
|
|
continue
|
|
}
|
|
for _, sid := range active {
|
|
if e, ok := doubleEffects[sid]; ok && e.key == p.key {
|
|
val -= e.change
|
|
}
|
|
}
|
|
for _, sid := range spells {
|
|
if e, ok := doubleEffects[sid]; ok && e.key == p.key && e.bonus != 0 {
|
|
val += e.bonus
|
|
}
|
|
}
|
|
combat["base_"+p.prop] = math.Round(val*10000) / 10000
|
|
}
|
|
}
|
|
|
|
// POST /debug/process — returns the normalized columns for a raw item JSON body
|
|
// (loopback validation against production's stored rows; never writes).
|
|
func (s *Server) handleDebugProcess(w http.ResponseWriter, r *http.Request) {
|
|
body, _ := io.ReadAll(io.LimitReader(r.Body, 8<<20))
|
|
var raw map[string]any
|
|
if json.Unmarshal(body, &raw) != nil {
|
|
writeJSON(w, http.StatusBadRequest, map[string]any{"detail": "invalid JSON"})
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, s.processItem(raw))
|
|
}
|