From 1b27371e049a6be534264f479efe72c16e43e45e Mon Sep 17 00:00:00 2001 From: Vynull App Date: Sat, 17 Oct 2026 21:16:47 +0000 Subject: [PATCH] smart: read SmartList XML and numeric values as rekordbox does Two link-spec suites crashed before a case ran. The dump helper's strict XML parser rejected a LogicalOperator of "wat", and fractional values in integer columns broke the import. The helper now parses SmartList XML the way rekordbox's JUCE parser does: element names in any case, a closing tag need not match, a repeated attribute keeps its first value, and anything after the first root is ignored, while a byte-order mark, leading text, an unclosed element, or a bad entity leave no rules. Operator and LogicalOperator read as integers, and anything but LogicalOperator 2 means all-of. Property names match in any case. Stored numbers are read as rekordbox reads them: truncated, BPM as a signed 32-bit integer (a negative BPM matches no rule), Rating as a signed byte, and Length, ReleaseYear, and DJPlayCount as small unsigned numbers. Rule values read as integers saturating at the int32 limits, so 0.5 is 0. Real library values are unchanged. The KEY column shows the key alone for a zero tempo and a negative tempo by its magnitude, as rekordbox does. --- api/smart_import.go | 43 +++++- api/smart_import_numeric_test.go | 42 ++++++ api/smart_playlist.go | 7 +- dbserver/menuitem.go | 11 +- dbserver/songinfo_display_test.go | 17 +++ library/rekordbox_dump_parse_test.go | 87 +++++++++++ tools/rekordbox_dump.py | 212 +++++++++++++++++++++++++-- 7 files changed, 399 insertions(+), 20 deletions(-) create mode 100644 api/smart_import_numeric_test.go create mode 100644 library/rekordbox_dump_parse_test.go diff --git a/api/smart_import.go b/api/smart_import.go index 0b5a80c..9597c96 100644 --- a/api/smart_import.go +++ b/api/smart_import.go @@ -3,7 +3,7 @@ package api import ( - "strconv" + "math" "strings" "time" @@ -44,6 +44,20 @@ var rbFieldMap = map[string]string{ "dateCreated": "date_created", } +// rbField looks up a SmartList PropertyName, which rekordbox matches in any +// letter case (smart-xml-matrix, rekordbox 7.2.19: GENRE works as genre). +func rbField(property string) (string, bool) { + if field, ok := rbFieldMap[property]; ok { + return field, true + } + for name, field := range rbFieldMap { + if strings.EqualFold(name, property) { + return field, true + } + } + return "", false +} + func isNumericField(field string) bool { switch field { case "bpm", "year", "rating", "play_count", "duration_sec", "color": @@ -66,6 +80,29 @@ func rbDate(s string) string { return t.UTC().Format(time.RFC3339) } +// rbInt reads a numeric rule value as rekordbox does (link-spec +// smart-numeric-matrix and smart-numeric-boundaries, rekordbox 7.2.19): an +// optionally signed run of leading digits, so 0.5 is 0 and blank or other +// text is 0, saturating at the int32 limits. +func rbInt(s string) int64 { + s = strings.TrimLeft(s, " \t\r\n") + neg := false + if s != "" && (s[0] == '-' || s[0] == '+') { + neg = s[0] == '-' + s = s[1:] + } + var n int64 + for i := 0; i < len(s) && s[i] >= '0' && s[i] <= '9'; i++ { + if n = n*10 + int64(s[i]-'0'); n > math.MaxInt32+1 { + n = math.MaxInt32 + 1 + } + } + if neg { + return max(-n, math.MinInt32) + } + return min(n, math.MaxInt32) +} + // smartRulesFromRekordbox translates a parsed rekordbox SmartList into the // app's SmartRules. Returns the rules and the number of conditions that // could not be mapped (unsupported property/operator) so the caller can log @@ -99,7 +136,7 @@ func smartRulesFromRekordbox(rb *library.RBSmartList) (rules *SmartRules, mapped // 11=ends-with. BPM values are stored ×100; myTag values arrive already // resolved to the tag name by the dump helper. func rbCondToSmart(c library.RBSmartCond) (SmartCond, bool) { - field, ok := rbFieldMap[c.Property] + field, ok := rbField(c.Property) if !ok { return SmartCond{}, false } @@ -109,7 +146,7 @@ func rbCondToSmart(c library.RBSmartCond) (SmartCond, bool) { // num parses a rekordbox numeric value string, scaling BPM down by 100. num := func(s string) float64 { - f, _ := strconv.ParseFloat(strings.TrimSpace(s), 64) + f := float64(rbInt(s)) if field == "bpm" { f /= 100 } diff --git a/api/smart_import_numeric_test.go b/api/smart_import_numeric_test.go new file mode 100644 index 0000000..5b9135f --- /dev/null +++ b/api/smart_import_numeric_test.go @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package api + +import ( + "testing" + "time" + + "github.com/vynulldev/vynull/library" +) + +// Numeric rule values and stored numbers compare as rekordbox's do (link-spec +// smart-numeric-matrix and smart-numeric-boundaries, rekordbox 7.2.19). +func TestSmartImportNumericLikeRekordbox(t *testing.T) { + for in, want := range map[string]int64{ + "0.5": 0, "1.5": 1, "-1": -1, "": 0, "not-a-number": 0, " 7": 7, + "2147483647": 2147483647, "2147483648": 2147483647, "4294967296": 2147483647, + "-99999999999": -2147483648, + } { + if got := rbInt(in); got != want { + t.Errorf("rbInt(%q) = %d, want %d", in, got, want) + } + } + match := func(prop string, op int, left, right string, tr *library.Track) bool { + rules, _, _ := smartRulesFromRekordbox(&library.RBSmartList{Logical: 1, + Conditions: []library.RBSmartCond{{Property: prop, Operator: op, Left: left, Right: right}}}) + return rules.Match(tr, nil, time.Now()) + } + // A stored rating of -1 (255 in a byte) is below zero. + if !match("rating", 4, "0", "", &library.Track{Rating: 255}) { + t.Errorf("rating 255 (signed -1) should be less than 0") + } + // A negative stored BPM matches no rule, not even its complement. + neg := &library.Track{BPM: -0.01} + if match("bpm", 2, "100", "", neg) || match("bpm", 5, "-100", "100", neg) { + t.Errorf("a negative BPM should match no BPM rule") + } + // 0.5 truncates to 0. + if !match("bpm", 1, "0.5", "", &library.Track{BPM: 0}) { + t.Errorf("BPM equal 0.5 should match a zero BPM") + } +} diff --git a/api/smart_playlist.go b/api/smart_playlist.go index 9be9296..ecbb5f8 100644 --- a/api/smart_playlist.go +++ b/api/smart_playlist.go @@ -79,11 +79,14 @@ func (c SmartCond) match(t *library.Track, tags TagLookup, now time.Time) bool { func evalLeaf(field, op string, value any, t *library.Track, tags TagLookup, now time.Time) bool { switch field { case "bpm": - return numericMatch(op, value, t.BPM) + // rekordbox's smart playlists skip a track whose stored BPM reads + // as negative (link-spec smart-numeric-boundaries, rekordbox 7.2.19). + return t.BPM >= 0 && numericMatch(op, value, t.BPM) case "year": return numericMatch(op, value, float64(t.Year)) case "rating": - return numericMatch(op, value, float64(t.Rating)) + // rekordbox reads the rating as a signed byte. + return numericMatch(op, value, float64(int8(t.Rating))) case "play_count": return numericMatch(op, value, float64(t.PlayCount)) case "duration_sec": diff --git a/dbserver/menuitem.go b/dbserver/menuitem.go index f8fd597..548bd1a 100644 --- a/dbserver/menuitem.go +++ b/dbserver/menuitem.go @@ -4,6 +4,7 @@ package dbserver import ( "fmt" + "math" "sort" "strconv" "strings" @@ -287,9 +288,15 @@ func (h *Handler) applyTrackDetail(m *menuItem, t *library.Track) { m.Label2 += " - " + t.Key } case "key": - // rekordbox composes " - bpm" even for an empty key. + // rekordbox composes " - bpm" even for an empty key, but + // shows the key alone when the tempo is zero, and a stored negative + // tempo by its magnitude (link-spec smart-numeric-boundaries, + // rekordbox 7.2.19). m.ParentID = h.trackKeyID(t) - m.Label2 = fmt.Sprintf("%s - %.1f bpm", t.Key, t.BPM) + m.Label2 = t.Key + if raw := math.Round(t.BPM * 100); raw != 0 { + m.Label2 = fmt.Sprintf("%s - %.1f bpm", t.Key, math.Abs(raw)/100) + } case "artist": m.ParentID = h.catID("artist", t.Artist) m.Label2 = t.Artist diff --git a/dbserver/songinfo_display_test.go b/dbserver/songinfo_display_test.go index 9c9f5c8..854d78e 100644 --- a/dbserver/songinfo_display_test.go +++ b/dbserver/songinfo_display_test.go @@ -177,6 +177,23 @@ func TestKeyColumnEmptyKey(t *testing.T) { } } +// The KEY column shows the key alone for a zero tempo and a stored negative +// tempo by its magnitude (link-spec smart-numeric-boundaries). +func TestKeyColumnZeroAndNegativeTempo(t *testing.T) { + h := &Handler{menu: keyColumnMenu{}} + for _, tc := range []struct { + bpm float64 + want string + }{ + {0, "Am"}, {0.015, "Am - 0.0 bpm"}, {-0.01, "Am - 0.0 bpm"}, + {-21474836.48, "Am - 21474836.5 bpm"}, {128, "Am - 128.0 bpm"}, + } { + if got := h.trackToStdItem(&library.Track{ID: 1, Key: "Am", BPM: tc.bpm}).Label2; got != tc.want { + t.Errorf("BPM %v: %q, want %q", tc.bpm, got, tc.want) + } + } +} + // keyColumnMenu is a MenuSource whose track lists show the KEY column. type keyColumnMenu struct{} diff --git a/library/rekordbox_dump_parse_test.go b/library/rekordbox_dump_parse_test.go new file mode 100644 index 0000000..39e9bc6 --- /dev/null +++ b/library/rekordbox_dump_parse_test.go @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package library + +import ( + "encoding/json" + "os/exec" + "testing" +) + +// The dump helper reads SmartList XML and numeric columns as rekordbox does +// (link-spec smart-xml-matrix and smart-numeric-boundaries, rekordbox +// 7.2.19). Runs the helper's pure functions under python3 with sqlcipher3 +// stubbed out, so no database or key is needed. +func TestDumpHelperParsesLikeRekordbox(t *testing.T) { + if _, err := exec.LookPath("python3"); err != nil { + t.Skip("python3 not available") + } + const script = ` +import importlib.util, json, sys, types +sys.modules["sqlcipher3"] = types.ModuleType("sqlcipher3") +spec = importlib.util.spec_from_file_location("dump", "../tools/rekordbox_dump.py") +dump = importlib.util.module_from_spec(spec) +spec.loader.exec_module(dump) +house = '' +canonical = '' + house + '' +def rule(x): + r = dump.parse_smartlist(x, {}) + if r is None: + return None + return [r["logical"]] + [c["operator"] for c in r["conditions"]] + [c["left"] for c in r["conditions"]] +out = { + "canonical": rule(canonical), + "lowercase": rule(canonical.replace("NODE", "node").replace("CONDITION", "condition")), + "mismatched_close": rule(canonical.replace("", "")), + "two_roots": rule(canonical + canonical.replace("House", "Techno")), + "trailing": rule(canonical + "trailing"), + "nul_junk": rule(canonical + "\0junk"), + "duplicate_value": rule(canonical.replace('ValueLeft="Fixture House"', 'ValueLeft="Fixture House" ValueLeft="Fixture Techno"')), + "duplicate_logical": rule(canonical.replace('LogicalOperator="1"', 'LogicalOperator="2" LogicalOperator="1"')), + "invalid_logical": rule(canonical.replace('LogicalOperator="1"', 'LogicalOperator="wat"')), + "entity": rule(canonical.replace("Fixture House", "Fixture House")), + "bom": rule("\ufeff" + canonical), + "leading_text": rule("leading" + canonical), + "unclosed": rule(canonical[:-len("")]), + "unknown_entity": rule(canonical.replace("Fixture House", "Fixture &bogus; House")), + "wrapper_root": rule("" + canonical + ""), + "ints": [dump.db_int(v) for v in (None, 0.5, 1.5, -0.5, -1, 2**31, 2**32)], + "u16": [dump.db_int(v, 16, False) for v in (-1, 2**31 - 1, 2**31, 2**32 - 1, 2**32)], + "s8": [dump.db_int(v, 8) for v in (-1, 2**31 - 1, 2**31, 1.5)], +} +json.dump(out, sys.stdout) +` + raw, err := exec.Command("python3", "-c", script).CombinedOutput() + if err != nil { + t.Fatalf("python3: %v\n%s", err, raw) + } + var got map[string]any + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatalf("decode %s: %v", raw, err) + } + want := map[string]string{ + "canonical": `[1,1,"Fixture House"]`, + "lowercase": `[1,1,"Fixture House"]`, + "mismatched_close": `[1,1,"Fixture House"]`, + "two_roots": `[1,1,"Fixture House"]`, + "trailing": `[1,1,"Fixture House"]`, + "nul_junk": `[1,1,"Fixture House"]`, + "duplicate_value": `[1,1,"Fixture House"]`, + "duplicate_logical": `[2,1,"Fixture House"]`, + "invalid_logical": `[1,1,"Fixture House"]`, + "entity": `[1,1,"Fixture House"]`, + "bom": `null`, + "leading_text": `null`, + "unclosed": `null`, + "unknown_entity": `null`, + "wrapper_root": `[1]`, + "ints": `[0,0,1,0,-1,-2147483648,0]`, + "u16": `[65535,65535,0,65535,0]`, + "s8": `[-1,-1,0,1]`, + } + for k, w := range want { + g, _ := json.Marshal(got[k]) + if string(g) != w { + t.Errorf("%s = %s, want %s", k, g, w) + } + } +} diff --git a/tools/rekordbox_dump.py b/tools/rekordbox_dump.py index ebf03fa..0af0b7c 100644 --- a/tools/rekordbox_dump.py +++ b/tools/rekordbox_dump.py @@ -26,8 +26,8 @@ import json import os +import re import sys -import xml.etree.ElementTree as ET import sqlcipher3 @@ -67,6 +67,184 @@ def open_db(db_path, provided_key): sys.exit(1) +def juce_int(s): + """Read text as rekordbox does an integer attribute (JUCE getIntValue): + an optionally signed run of leading digits after any spaces, 0 if none, + so "1.5" is 1 and "wat" is 0.""" + m = re.match(r"\s*([+-]?\d+)", s or "") + return int(m.group(1)) if m else 0 + + +def db_int(v, bits=32, signed=True): + """A stored numeric column as rekordbox reads it into a fixed-width + integer: truncated toward zero (REAL 1.5 is 1, -0.5 is 0), NULL as 0, + keeping the low `bits` bits (link-spec smart-numeric-boundaries, + rekordbox 7.2.19).""" + try: + n = int(v or 0) + except (TypeError, ValueError): + return 0 + n &= (1 << bits) - 1 + if signed and n >= 1 << (bits - 1): + n -= 1 << bits + return n + + +class _XMLError(Exception): + pass + + +_ENTITIES = {"amp": "&", "lt": "<", "gt": ">", "quot": '"', "apos": "'"} + + +def _unescape(text): + out, i = [], 0 + while True: + j = text.find("&", i) + if j < 0: + out.append(text[i:]) + return "".join(out) + out.append(text[i:j]) + k = text.find(";", j) + if k < 0: + raise _XMLError("bare &") + name = text[j + 1:k] + if name.startswith("#x") or name.startswith("#X"): + try: + out.append(chr(int(name[2:], 16))) + except ValueError: + raise _XMLError("bad entity") + elif name.startswith("#"): + try: + out.append(chr(int(name[1:]))) + except ValueError: + raise _XMLError("bad entity") + elif name in _ENTITIES: + out.append(_ENTITIES[name]) + else: + raise _XMLError("unknown entity") + i = k + 1 + + +class _Element: + def __init__(self, tag): + self.tag, self.attrib, self.children = tag, {}, [] + + +def parse_smartlist_xml(text): + """Parse a SmartList blob the way rekordbox's JUCE XmlDocument does + (link-spec smart-xml-matrix, rekordbox 7.2.19), which is more lenient + than a conforming XML parser: a closing tag need not match its opening + tag, a repeated attribute keeps its first value, and anything after the + first root element (another root, text, a NUL) is ignored. A byte-order + mark or text before the root, an unclosed element, or a bad entity or + attribute makes it unreadable. Returns the root element, or None.""" + if text.startswith("\ufeff"): + return None + text = text.split("\0", 1)[0] + pos = 0 + n = len(text) + + def skip_misc(p): + while True: + while p < n and text[p] in " \t\r\n": + p += 1 + if text.startswith("", p) + if e < 0: + raise _XMLError("unclosed PI") + p = e + 2 + elif text.startswith("", p) + if e < 0: + raise _XMLError("unclosed comment") + p = e + 3 + else: + return p + + def element(p): + # text[p] == "<" + p += 1 + q = p + while q < n and text[q] not in " \t\r\n/>": + q += 1 + if q == p: + raise _XMLError("no tag name") + el = _Element(text[p:q]) + p = q + while True: + while p < n and text[p] in " \t\r\n": + p += 1 + if p >= n: + raise _XMLError("unclosed tag") + if text.startswith("/>", p): + return el, p + 2 + if text[p] == ">": + p += 1 + break + q = p + while q < n and text[q] not in " \t\r\n=/>": + q += 1 + name = text[p:q] + p = q + while p < n and text[p] in " \t\r\n": + p += 1 + if not name or p >= n or text[p] != "=": + raise _XMLError("bad attribute") + p += 1 + while p < n and text[p] in " \t\r\n": + p += 1 + if p >= n or text[p] not in "\"'": + raise _XMLError("unquoted attribute") + e = text.find(text[p], p + 1) + if e < 0: + raise _XMLError("unclosed attribute") + if name not in el.attrib: + el.attrib[name] = _unescape(text[p + 1:e]) + p = e + 1 + while True: + if p >= n: + raise _XMLError("unclosed element") + if text.startswith("", p) + if e < 0: + raise _XMLError("unclosed end tag") + return el, e + 1 + if text.startswith("", p) + if e < 0: + raise _XMLError("unclosed comment") + p = e + 3 + elif text.startswith("", p) + if e < 0: + raise _XMLError("unclosed CDATA") + p = e + 3 + elif text.startswith("", p) + if e < 0: + raise _XMLError("unclosed PI") + p = e + 2 + elif text[p] == "<": + child, p = element(p) + el.children.append(child) + else: + e = text.find("<", p) + if e < 0: + raise _XMLError("unclosed element") + _unescape(text[p:e]) + p = e + + try: + pos = skip_misc(pos) + if pos >= n or text[pos] != "<": + return None + root, _ = element(pos) + return root + except _XMLError: + return None + + def parse_smartlist(xml_str, tags_by_id): """Parse a rekordbox djmdPlaylist.SmartList XML blob into a structured rule object. rekordbox encodes a smart playlist as: @@ -82,12 +260,14 @@ def parse_smartlist(xml_str, tags_by_id): `myTag` conditions store the tag ID as a SIGNED int32; djmdMyTag.ID is its unsigned form — resolve to the tag NAME here (only the DB knows the map). BPM values are ×100 (handled Go-side).""" - try: - root = ET.fromstring(xml_str) - except Exception: + root = parse_smartlist_xml(xml_str or "") + if root is None: return None + # Element names match in any case; a root other than NODE has no rules. + if root.tag.lower() != "node": + return {"logical": 1, "conditions": []} conds = [] - for cnd in root.findall("CONDITION"): + for cnd in (c for c in root.children if c.tag.lower() == "condition"): a = cnd.attrib prop = a.get("PropertyName", "") left = a.get("ValueLeft", "") @@ -99,13 +279,15 @@ def parse_smartlist(xml_str, tags_by_id): pass conds.append({ "property": prop, - "operator": int(a.get("Operator", "0") or 0), + "operator": juce_int(a.get("Operator", "")), "unit": a.get("ValueUnit", ""), "left": left, "right": a.get("ValueRight", ""), }) + # rekordbox treats any LogicalOperator but 2 (any-of) as all-of. + logical = 2 if juce_int(root.attrib.get("LogicalOperator", "")) == 2 else 1 return { - "logical": int(root.attrib.get("LogicalOperator", "1") or 1), + "logical": logical, "conditions": conds, } @@ -155,8 +337,12 @@ def main(): cols = [d[0] for d in cur.description] for row in cur.fetchall(): d = dict(zip(cols, row)) - # rekordbox stores BPM × 100; Length in ms? Check schema. - bpm = (d.get("BPM") or 0) / 100.0 if d.get("BPM") else 0 + # Numeric columns are read the way rekordbox's smart playlists read + # them (link-spec smart-numeric-boundaries, rekordbox 7.2.19), which + # leaves real library values unchanged: BPM (stored x100) as a signed + # 32-bit integer, Length (seconds), ReleaseYear and DJPlayCount as + # small unsigned integers, Rating as a signed byte. + bpm = db_int(d.get("BPM")) / 100.0 # File path: FolderPath is a "/Users/.../Music/..." style native path # (may include Windows drive letters). Use as-is — Go-side caller # may need to translate to USB layout. @@ -172,7 +358,7 @@ def main(): "file_name": d.get("FileNameL") or "", "file_size": d.get("FileSize") or 0, "bpm": bpm, - "duration_sec": d.get("Length") or 0, + "duration_sec": db_int(d.get("Length"), 16, False), "bitrate": d.get("BitRate") or 0, "sample_rate": d.get("SampleRate") or 0, # rekordbox FileType enum (1=mp3, 4=m4a, 5=flac, 11=wav, 12=aiff). @@ -181,12 +367,12 @@ def main(): "file_type_id": d.get("FileType") or 0, # Non-empty when the deck should auto-load this track's hot cues. "hot_cue_auto_load": d.get("HotCueAutoLoad") or "", - "rating": d.get("Rating") or 0, # rekordbox stores the 0-5 star count directly - "year": d.get("ReleaseYear") or 0, + "rating": db_int(d.get("Rating"), 8), # rekordbox stores the 0-5 star count directly + "year": db_int(d.get("ReleaseYear"), 16, False), "track_num": d.get("TrackNo") or 0, "disc_num": d.get("DiscNo") or 0, "comment": d.get("Commnt") or "", - "play_count": d.get("DJPlayCount") or 0, + "play_count": db_int(d.get("DJPlayCount"), 16, False), # StockDate is rekordbox's "Date Added" (TRACK INFO, the DATE # ADDED menu); DateCreated is often empty. "date_added": d.get("StockDate") or d.get("DateCreated") or "",