diff --git a/api/smart_import.go b/api/smart_import.go index 0b5a80c..9597c96 100644 --- a/api/smart_import.go +++ b/api/smart_import.go @@ -3,7 +3,7 @@ package api import ( - "strconv" + "math" "strings" "time" @@ -44,6 +44,20 @@ var rbFieldMap = map[string]string{ "dateCreated": "date_created", } +// rbField looks up a SmartList PropertyName, which rekordbox matches in any +// letter case (smart-xml-matrix, rekordbox 7.2.19: GENRE works as genre). +func rbField(property string) (string, bool) { + if field, ok := rbFieldMap[property]; ok { + return field, true + } + for name, field := range rbFieldMap { + if strings.EqualFold(name, property) { + return field, true + } + } + return "", false +} + func isNumericField(field string) bool { switch field { case "bpm", "year", "rating", "play_count", "duration_sec", "color": @@ -66,6 +80,29 @@ func rbDate(s string) string { return t.UTC().Format(time.RFC3339) } +// rbInt reads a numeric rule value as rekordbox does (link-spec +// smart-numeric-matrix and smart-numeric-boundaries, rekordbox 7.2.19): an +// optionally signed run of leading digits, so 0.5 is 0 and blank or other +// text is 0, saturating at the int32 limits. +func rbInt(s string) int64 { + s = strings.TrimLeft(s, " \t\r\n") + neg := false + if s != "" && (s[0] == '-' || s[0] == '+') { + neg = s[0] == '-' + s = s[1:] + } + var n int64 + for i := 0; i < len(s) && s[i] >= '0' && s[i] <= '9'; i++ { + if n = n*10 + int64(s[i]-'0'); n > math.MaxInt32+1 { + n = math.MaxInt32 + 1 + } + } + if neg { + return max(-n, math.MinInt32) + } + return min(n, math.MaxInt32) +} + // smartRulesFromRekordbox translates a parsed rekordbox SmartList into the // app's SmartRules. Returns the rules and the number of conditions that // could not be mapped (unsupported property/operator) so the caller can log @@ -99,7 +136,7 @@ func smartRulesFromRekordbox(rb *library.RBSmartList) (rules *SmartRules, mapped // 11=ends-with. BPM values are stored ×100; myTag values arrive already // resolved to the tag name by the dump helper. func rbCondToSmart(c library.RBSmartCond) (SmartCond, bool) { - field, ok := rbFieldMap[c.Property] + field, ok := rbField(c.Property) if !ok { return SmartCond{}, false } @@ -109,7 +146,7 @@ func rbCondToSmart(c library.RBSmartCond) (SmartCond, bool) { // num parses a rekordbox numeric value string, scaling BPM down by 100. num := func(s string) float64 { - f, _ := strconv.ParseFloat(strings.TrimSpace(s), 64) + f := float64(rbInt(s)) if field == "bpm" { f /= 100 } diff --git a/api/smart_import_numeric_test.go b/api/smart_import_numeric_test.go new file mode 100644 index 0000000..5b9135f --- /dev/null +++ b/api/smart_import_numeric_test.go @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package api + +import ( + "testing" + "time" + + "github.com/vynulldev/vynull/library" +) + +// Numeric rule values and stored numbers compare as rekordbox's do (link-spec +// smart-numeric-matrix and smart-numeric-boundaries, rekordbox 7.2.19). +func TestSmartImportNumericLikeRekordbox(t *testing.T) { + for in, want := range map[string]int64{ + "0.5": 0, "1.5": 1, "-1": -1, "": 0, "not-a-number": 0, " 7": 7, + "2147483647": 2147483647, "2147483648": 2147483647, "4294967296": 2147483647, + "-99999999999": -2147483648, + } { + if got := rbInt(in); got != want { + t.Errorf("rbInt(%q) = %d, want %d", in, got, want) + } + } + match := func(prop string, op int, left, right string, tr *library.Track) bool { + rules, _, _ := smartRulesFromRekordbox(&library.RBSmartList{Logical: 1, + Conditions: []library.RBSmartCond{{Property: prop, Operator: op, Left: left, Right: right}}}) + return rules.Match(tr, nil, time.Now()) + } + // A stored rating of -1 (255 in a byte) is below zero. + if !match("rating", 4, "0", "", &library.Track{Rating: 255}) { + t.Errorf("rating 255 (signed -1) should be less than 0") + } + // A negative stored BPM matches no rule, not even its complement. + neg := &library.Track{BPM: -0.01} + if match("bpm", 2, "100", "", neg) || match("bpm", 5, "-100", "100", neg) { + t.Errorf("a negative BPM should match no BPM rule") + } + // 0.5 truncates to 0. + if !match("bpm", 1, "0.5", "", &library.Track{BPM: 0}) { + t.Errorf("BPM equal 0.5 should match a zero BPM") + } +} diff --git a/api/smart_playlist.go b/api/smart_playlist.go index 9be9296..ecbb5f8 100644 --- a/api/smart_playlist.go +++ b/api/smart_playlist.go @@ -79,11 +79,14 @@ func (c SmartCond) match(t *library.Track, tags TagLookup, now time.Time) bool { func evalLeaf(field, op string, value any, t *library.Track, tags TagLookup, now time.Time) bool { switch field { case "bpm": - return numericMatch(op, value, t.BPM) + // rekordbox's smart playlists skip a track whose stored BPM reads + // as negative (link-spec smart-numeric-boundaries, rekordbox 7.2.19). + return t.BPM >= 0 && numericMatch(op, value, t.BPM) case "year": return numericMatch(op, value, float64(t.Year)) case "rating": - return numericMatch(op, value, float64(t.Rating)) + // rekordbox reads the rating as a signed byte. + return numericMatch(op, value, float64(int8(t.Rating))) case "play_count": return numericMatch(op, value, float64(t.PlayCount)) case "duration_sec": diff --git a/dbserver/menuitem.go b/dbserver/menuitem.go index f8fd597..548bd1a 100644 --- a/dbserver/menuitem.go +++ b/dbserver/menuitem.go @@ -4,6 +4,7 @@ package dbserver import ( "fmt" + "math" "sort" "strconv" "strings" @@ -287,9 +288,15 @@ func (h *Handler) applyTrackDetail(m *menuItem, t *library.Track) { m.Label2 += " - " + t.Key } case "key": - // rekordbox composes " - bpm" even for an empty key. + // rekordbox composes " - bpm" even for an empty key, but + // shows the key alone when the tempo is zero, and a stored negative + // tempo by its magnitude (link-spec smart-numeric-boundaries, + // rekordbox 7.2.19). m.ParentID = h.trackKeyID(t) - m.Label2 = fmt.Sprintf("%s - %.1f bpm", t.Key, t.BPM) + m.Label2 = t.Key + if raw := math.Round(t.BPM * 100); raw != 0 { + m.Label2 = fmt.Sprintf("%s - %.1f bpm", t.Key, math.Abs(raw)/100) + } case "artist": m.ParentID = h.catID("artist", t.Artist) m.Label2 = t.Artist diff --git a/dbserver/songinfo_display_test.go b/dbserver/songinfo_display_test.go index 9c9f5c8..854d78e 100644 --- a/dbserver/songinfo_display_test.go +++ b/dbserver/songinfo_display_test.go @@ -177,6 +177,23 @@ func TestKeyColumnEmptyKey(t *testing.T) { } } +// The KEY column shows the key alone for a zero tempo and a stored negative +// tempo by its magnitude (link-spec smart-numeric-boundaries). +func TestKeyColumnZeroAndNegativeTempo(t *testing.T) { + h := &Handler{menu: keyColumnMenu{}} + for _, tc := range []struct { + bpm float64 + want string + }{ + {0, "Am"}, {0.015, "Am - 0.0 bpm"}, {-0.01, "Am - 0.0 bpm"}, + {-21474836.48, "Am - 21474836.5 bpm"}, {128, "Am - 128.0 bpm"}, + } { + if got := h.trackToStdItem(&library.Track{ID: 1, Key: "Am", BPM: tc.bpm}).Label2; got != tc.want { + t.Errorf("BPM %v: %q, want %q", tc.bpm, got, tc.want) + } + } +} + // keyColumnMenu is a MenuSource whose track lists show the KEY column. type keyColumnMenu struct{} diff --git a/library/rekordbox_dump_parse_test.go b/library/rekordbox_dump_parse_test.go new file mode 100644 index 0000000..39e9bc6 --- /dev/null +++ b/library/rekordbox_dump_parse_test.go @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package library + +import ( + "encoding/json" + "os/exec" + "testing" +) + +// The dump helper reads SmartList XML and numeric columns as rekordbox does +// (link-spec smart-xml-matrix and smart-numeric-boundaries, rekordbox +// 7.2.19). Runs the helper's pure functions under python3 with sqlcipher3 +// stubbed out, so no database or key is needed. +func TestDumpHelperParsesLikeRekordbox(t *testing.T) { + if _, err := exec.LookPath("python3"); err != nil { + t.Skip("python3 not available") + } + const script = ` +import importlib.util, json, sys, types +sys.modules["sqlcipher3"] = types.ModuleType("sqlcipher3") +spec = importlib.util.spec_from_file_location("dump", "../tools/rekordbox_dump.py") +dump = importlib.util.module_from_spec(spec) +spec.loader.exec_module(dump) +house = '' +canonical = '' + house + '' +def rule(x): + r = dump.parse_smartlist(x, {}) + if r is None: + return None + return [r["logical"]] + [c["operator"] for c in r["conditions"]] + [c["left"] for c in r["conditions"]] +out = { + "canonical": rule(canonical), + "lowercase": rule(canonical.replace("NODE", "node").replace("CONDITION", "condition")), + "mismatched_close": rule(canonical.replace("", "")), + "two_roots": rule(canonical + canonical.replace("House", "Techno")), + "trailing": rule(canonical + "trailing"), + "nul_junk": rule(canonical + "\0junk"), + "duplicate_value": rule(canonical.replace('ValueLeft="Fixture House"', 'ValueLeft="Fixture House" ValueLeft="Fixture Techno"')), + "duplicate_logical": rule(canonical.replace('LogicalOperator="1"', 'LogicalOperator="2" LogicalOperator="1"')), + "invalid_logical": rule(canonical.replace('LogicalOperator="1"', 'LogicalOperator="wat"')), + "entity": rule(canonical.replace("Fixture House", "Fixture House")), + "bom": rule("\ufeff" + canonical), + "leading_text": rule("leading" + canonical), + "unclosed": rule(canonical[:-len("")]), + "unknown_entity": rule(canonical.replace("Fixture House", "Fixture &bogus; House")), + "wrapper_root": rule("" + canonical + ""), + "ints": [dump.db_int(v) for v in (None, 0.5, 1.5, -0.5, -1, 2**31, 2**32)], + "u16": [dump.db_int(v, 16, False) for v in (-1, 2**31 - 1, 2**31, 2**32 - 1, 2**32)], + "s8": [dump.db_int(v, 8) for v in (-1, 2**31 - 1, 2**31, 1.5)], +} +json.dump(out, sys.stdout) +` + raw, err := exec.Command("python3", "-c", script).CombinedOutput() + if err != nil { + t.Fatalf("python3: %v\n%s", err, raw) + } + var got map[string]any + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatalf("decode %s: %v", raw, err) + } + want := map[string]string{ + "canonical": `[1,1,"Fixture House"]`, + "lowercase": `[1,1,"Fixture House"]`, + "mismatched_close": `[1,1,"Fixture House"]`, + "two_roots": `[1,1,"Fixture House"]`, + "trailing": `[1,1,"Fixture House"]`, + "nul_junk": `[1,1,"Fixture House"]`, + "duplicate_value": `[1,1,"Fixture House"]`, + "duplicate_logical": `[2,1,"Fixture House"]`, + "invalid_logical": `[1,1,"Fixture House"]`, + "entity": `[1,1,"Fixture House"]`, + "bom": `null`, + "leading_text": `null`, + "unclosed": `null`, + "unknown_entity": `null`, + "wrapper_root": `[1]`, + "ints": `[0,0,1,0,-1,-2147483648,0]`, + "u16": `[65535,65535,0,65535,0]`, + "s8": `[-1,-1,0,1]`, + } + for k, w := range want { + g, _ := json.Marshal(got[k]) + if string(g) != w { + t.Errorf("%s = %s, want %s", k, g, w) + } + } +} diff --git a/tools/rekordbox_dump.py b/tools/rekordbox_dump.py index ebf03fa..0af0b7c 100644 --- a/tools/rekordbox_dump.py +++ b/tools/rekordbox_dump.py @@ -26,8 +26,8 @@ import json import os +import re import sys -import xml.etree.ElementTree as ET import sqlcipher3 @@ -67,6 +67,184 @@ def open_db(db_path, provided_key): sys.exit(1) +def juce_int(s): + """Read text as rekordbox does an integer attribute (JUCE getIntValue): + an optionally signed run of leading digits after any spaces, 0 if none, + so "1.5" is 1 and "wat" is 0.""" + m = re.match(r"\s*([+-]?\d+)", s or "") + return int(m.group(1)) if m else 0 + + +def db_int(v, bits=32, signed=True): + """A stored numeric column as rekordbox reads it into a fixed-width + integer: truncated toward zero (REAL 1.5 is 1, -0.5 is 0), NULL as 0, + keeping the low `bits` bits (link-spec smart-numeric-boundaries, + rekordbox 7.2.19).""" + try: + n = int(v or 0) + except (TypeError, ValueError): + return 0 + n &= (1 << bits) - 1 + if signed and n >= 1 << (bits - 1): + n -= 1 << bits + return n + + +class _XMLError(Exception): + pass + + +_ENTITIES = {"amp": "&", "lt": "<", "gt": ">", "quot": '"', "apos": "'"} + + +def _unescape(text): + out, i = [], 0 + while True: + j = text.find("&", i) + if j < 0: + out.append(text[i:]) + return "".join(out) + out.append(text[i:j]) + k = text.find(";", j) + if k < 0: + raise _XMLError("bare &") + name = text[j + 1:k] + if name.startswith("#x") or name.startswith("#X"): + try: + out.append(chr(int(name[2:], 16))) + except ValueError: + raise _XMLError("bad entity") + elif name.startswith("#"): + try: + out.append(chr(int(name[1:]))) + except ValueError: + raise _XMLError("bad entity") + elif name in _ENTITIES: + out.append(_ENTITIES[name]) + else: + raise _XMLError("unknown entity") + i = k + 1 + + +class _Element: + def __init__(self, tag): + self.tag, self.attrib, self.children = tag, {}, [] + + +def parse_smartlist_xml(text): + """Parse a SmartList blob the way rekordbox's JUCE XmlDocument does + (link-spec smart-xml-matrix, rekordbox 7.2.19), which is more lenient + than a conforming XML parser: a closing tag need not match its opening + tag, a repeated attribute keeps its first value, and anything after the + first root element (another root, text, a NUL) is ignored. A byte-order + mark or text before the root, an unclosed element, or a bad entity or + attribute makes it unreadable. Returns the root element, or None.""" + if text.startswith("\ufeff"): + return None + text = text.split("\0", 1)[0] + pos = 0 + n = len(text) + + def skip_misc(p): + while True: + while p < n and text[p] in " \t\r\n": + p += 1 + if text.startswith("", p) + if e < 0: + raise _XMLError("unclosed PI") + p = e + 2 + elif text.startswith("", p) + if e < 0: + raise _XMLError("unclosed comment") + p = e + 3 + else: + return p + + def element(p): + # text[p] == "<" + p += 1 + q = p + while q < n and text[q] not in " \t\r\n/>": + q += 1 + if q == p: + raise _XMLError("no tag name") + el = _Element(text[p:q]) + p = q + while True: + while p < n and text[p] in " \t\r\n": + p += 1 + if p >= n: + raise _XMLError("unclosed tag") + if text.startswith("/>", p): + return el, p + 2 + if text[p] == ">": + p += 1 + break + q = p + while q < n and text[q] not in " \t\r\n=/>": + q += 1 + name = text[p:q] + p = q + while p < n and text[p] in " \t\r\n": + p += 1 + if not name or p >= n or text[p] != "=": + raise _XMLError("bad attribute") + p += 1 + while p < n and text[p] in " \t\r\n": + p += 1 + if p >= n or text[p] not in "\"'": + raise _XMLError("unquoted attribute") + e = text.find(text[p], p + 1) + if e < 0: + raise _XMLError("unclosed attribute") + if name not in el.attrib: + el.attrib[name] = _unescape(text[p + 1:e]) + p = e + 1 + while True: + if p >= n: + raise _XMLError("unclosed element") + if text.startswith("", p) + if e < 0: + raise _XMLError("unclosed end tag") + return el, e + 1 + if text.startswith("", p) + if e < 0: + raise _XMLError("unclosed comment") + p = e + 3 + elif text.startswith("", p) + if e < 0: + raise _XMLError("unclosed CDATA") + p = e + 3 + elif text.startswith("", p) + if e < 0: + raise _XMLError("unclosed PI") + p = e + 2 + elif text[p] == "<": + child, p = element(p) + el.children.append(child) + else: + e = text.find("<", p) + if e < 0: + raise _XMLError("unclosed element") + _unescape(text[p:e]) + p = e + + try: + pos = skip_misc(pos) + if pos >= n or text[pos] != "<": + return None + root, _ = element(pos) + return root + except _XMLError: + return None + + def parse_smartlist(xml_str, tags_by_id): """Parse a rekordbox djmdPlaylist.SmartList XML blob into a structured rule object. rekordbox encodes a smart playlist as: @@ -82,12 +260,14 @@ def parse_smartlist(xml_str, tags_by_id): `myTag` conditions store the tag ID as a SIGNED int32; djmdMyTag.ID is its unsigned form — resolve to the tag NAME here (only the DB knows the map). BPM values are ×100 (handled Go-side).""" - try: - root = ET.fromstring(xml_str) - except Exception: + root = parse_smartlist_xml(xml_str or "") + if root is None: return None + # Element names match in any case; a root other than NODE has no rules. + if root.tag.lower() != "node": + return {"logical": 1, "conditions": []} conds = [] - for cnd in root.findall("CONDITION"): + for cnd in (c for c in root.children if c.tag.lower() == "condition"): a = cnd.attrib prop = a.get("PropertyName", "") left = a.get("ValueLeft", "") @@ -99,13 +279,15 @@ def parse_smartlist(xml_str, tags_by_id): pass conds.append({ "property": prop, - "operator": int(a.get("Operator", "0") or 0), + "operator": juce_int(a.get("Operator", "")), "unit": a.get("ValueUnit", ""), "left": left, "right": a.get("ValueRight", ""), }) + # rekordbox treats any LogicalOperator but 2 (any-of) as all-of. + logical = 2 if juce_int(root.attrib.get("LogicalOperator", "")) == 2 else 1 return { - "logical": int(root.attrib.get("LogicalOperator", "1") or 1), + "logical": logical, "conditions": conds, } @@ -155,8 +337,12 @@ def main(): cols = [d[0] for d in cur.description] for row in cur.fetchall(): d = dict(zip(cols, row)) - # rekordbox stores BPM × 100; Length in ms? Check schema. - bpm = (d.get("BPM") or 0) / 100.0 if d.get("BPM") else 0 + # Numeric columns are read the way rekordbox's smart playlists read + # them (link-spec smart-numeric-boundaries, rekordbox 7.2.19), which + # leaves real library values unchanged: BPM (stored x100) as a signed + # 32-bit integer, Length (seconds), ReleaseYear and DJPlayCount as + # small unsigned integers, Rating as a signed byte. + bpm = db_int(d.get("BPM")) / 100.0 # File path: FolderPath is a "/Users/.../Music/..." style native path # (may include Windows drive letters). Use as-is — Go-side caller # may need to translate to USB layout. @@ -172,7 +358,7 @@ def main(): "file_name": d.get("FileNameL") or "", "file_size": d.get("FileSize") or 0, "bpm": bpm, - "duration_sec": d.get("Length") or 0, + "duration_sec": db_int(d.get("Length"), 16, False), "bitrate": d.get("BitRate") or 0, "sample_rate": d.get("SampleRate") or 0, # rekordbox FileType enum (1=mp3, 4=m4a, 5=flac, 11=wav, 12=aiff). @@ -181,12 +367,12 @@ def main(): "file_type_id": d.get("FileType") or 0, # Non-empty when the deck should auto-load this track's hot cues. "hot_cue_auto_load": d.get("HotCueAutoLoad") or "", - "rating": d.get("Rating") or 0, # rekordbox stores the 0-5 star count directly - "year": d.get("ReleaseYear") or 0, + "rating": db_int(d.get("Rating"), 8), # rekordbox stores the 0-5 star count directly + "year": db_int(d.get("ReleaseYear"), 16, False), "track_num": d.get("TrackNo") or 0, "disc_num": d.get("DiscNo") or 0, "comment": d.get("Commnt") or "", - "play_count": d.get("DJPlayCount") or 0, + "play_count": db_int(d.get("DJPlayCount"), 16, False), # StockDate is rekordbox's "Date Added" (TRACK INFO, the DATE # ADDED menu); DateCreated is often empty. "date_added": d.get("StockDate") or d.get("DateCreated") or "",