improved speed, added local data lists, some cleaning

2025-09-10 22:50:41 +02:00
parent f35461cc50
commit 56b8c73297
9 changed files with 336 additions and 280 deletions
--- a/src/sh/local.sh
+++ b/src/sh/local.sh
@@ -1,3 +1,24 @@
+if [ ! "${LOCAL_LOADED:-}" ]; then
+  LOCALDATADIR="$HOME/.cache/$APP_NAME/local"
+  LOCALDATA_ARTISTS="$LOCALDATADIR/artists"
+  LOCALDATA_RELEASEGROUPS="$LOCALDATADIR/releasegroups"
+  LOCALDATA_RELEASES="$LOCALDATADIR/releases"
+  LOCALDATA_ARTISTS_VIEW="$LOCALDATADIR/artists_view"
+  LOCALDATA_RELEASEGROUPS_VIEW="$LOCALDATADIR/releasegroups_view"
+  LOCALDATA_RELEASES_VIEW="$LOCALDATADIR/releases_view"
+  LOCALDATA_ARTISTS_LIST="$LOCALDATADIR/artists_list"
+  LOCALDATA_RELEASEGROUPS_LIST="$LOCALDATADIR/releasegroups_list"
+  LOCALDATA_RELEASES_LIST="$LOCALDATADIR/releases_list"
+  DECORATION_FILENAME=${DECORATION_FILENAME:-"mbid.json"}
+
+  export LOCALDATADIR LOCALDATA_ARTISTS LOCALDATA_RELEASEGROUPS \
+    LOCALDATA_RELEASES LOCALDATA_ARTISTS_VIEW LOCALDATA_RELEASEGROUPS_VIEW \
+    LOCALDATA_RELEASES_VIEW LOCALDATA_ARTISTS_LIST LOCALDATA_RELEASEGROUPS_LIST \
+    LOCALDATA_RELEASES_LIST DECORATION_FILENAME
+
+  export LOCAL_LOADED=1
+fi
+
 gettags() {
  ffprobe -v error -show_entries format_tags -print_format json "$1" |
    $JQ '.format.tags | {
@@ -47,192 +68,173 @@ decorate() {
 }

 # Load missing cache entries (batch mode)
+#
 # argument $1: type
-# argument $2: File with one ID per line
+#
+# This method reads one MusicBrainz IDs of the specified type from stdin (one
+# per line), and fetches the missing items.
 __batch_load_missing() {
  tmpf=$(mktemp)
-  while IFS= read -r mbid; do
-    if ! in_cache "$1" "$mbid"; then
-      echo "$mbid" >>"$tmpf"
-    fi
-  done <"$2"
-  if [ -s "$tmpf" ]; then
-    lines=$(wc -l "$tmpf" | cut -d ' ' -f 1)
-    if [ "$lines" -gt 0 ]; then
+  cat |
+    cache_get_file_batch "$1" |
+    xargs \
+      sh -c 'for f; do [ -e "$f" ] || echo "$f"; done' _ |
+    cache_mbid_from_path_batch >"$tmpf"
+  lines=$(wc -l "$tmpf" | cut -d ' ' -f 1)
+  if [ "$lines" -gt 0 ]; then
+    case "$1" in
+    "$TYPE_ARTIST") tt="artists" ;;
+    "$TYPE_RELEASEGROUP") tt="release groups" ;;
+    "$TYPE_RELEASE") tt="releases" ;;
+    esac
+    info "Fetching missing $tt"
+    cnt=0
+    while IFS= read -r mbid; do
      case "$1" in
      "$TYPE_ARTIST")
-        tt="artists"
-        ;;
-      "$TYPE_RELEASEGROUP")
-        tt="release groups"
-        ;;
-      "$TYPE_RELEASE")
-        tt="releases"
+        name=$(mb_artist "$mbid" | $JQ '.name')
        ;;
+      "$TYPE_RELEASEGROUP") name=$(mb_releasegroup "$mbid" | $JQ '.title') ;;
+      "$TYPE_RELEASE") name=$(mb_release "$mbid" | $JQ '.title') ;;
      esac
-      info "Fetching $lines missing $tt"
-      cnt=0
-      while IFS= read -r mbid; do
-        case "$1" in
-        "$TYPE_ARTIST")
-          name=$(mb_artist "$mbid" | $JQ ".name")
-          ;;
-        "$TYPE_RELEASEGROUP")
-          name=$(mb_releasegroup "$mbid" | $JQ ".title")
-          ;;
-        "$TYPE_RELEASE")
-          name=$(mb_release "$mbid" | $JQ ".title")
-          ;;
-        esac
-        cnt=$((cnt + 1))
-        printf "\033[2K\r%d/%d (%s: %s)" "$cnt" "$lines" "$mbid" "$name"
-        sleep 1
-      done <"$tmpf"
-      printf "\n"
-    fi
+      cnt=$((cnt + 1))
+      info "$(printf "%d/%d (%s: %s)" "$cnt" "$lines" "$mbid" "$name")"
+      sleep 1
+    done <"$tmpf"
  fi
  rm -f "$tmpf"
 }

-LOCALDATADIR="$HOME/.cache/$APP_NAME/local"
-LOCALDATA_ARTISTS="$LOCALDATADIR/artists"
-LOCALDATA_RELEASEGROUPS="$LOCALDATADIR/releasegroups"
-LOCALDATA_RELEASES="$LOCALDATADIR/releases"
-LOCALDATA_ARTISTS_VIEW="$LOCALDATADIR/artists_view"
-LOCALDATA_RELEASEGROUPS_VIEW="$LOCALDATADIR/releasegroups_view"
-LOCALDATA_RELEASES_VIEW="$LOCALDATADIR/releases_view"
-DECORATION_FILENAME=${DECORATION_FILENAME:-"mbid.json"}
-
 # Precompute views
-precompute_view() {
-  info "Precomputing artist view"
-  while IFS= read -r aid; do
-    mb_artist "$aid" | $JQ '[
-    .id,
-    .type,
-    .name,
-    .disambiguation,
-    .["life-span"].begin,
-    .["life-span"].end
-    ] | join("\t")'
-  done <"$LOCALDATA_ARTISTS" |
-    awk \
-      -F "\t" \
-      -v file_local_artists="${LOCALDATA_ARTISTS:-}" \
-      -v format_person="$AV_PERSON" \
-      -v format_group="$AV_GROUP" \
-      -v format_disambiguation="$AV_DISAMBIGUATION" \
-      -v format_local="$FORMAT_LOCAL" \
-      "$AWK_ARTISTS" |
-    sort |
-    column -t -s "$(printf '\t')" -E 0 |
-    sed 's| \+\([0-9a-f-]\+\) \+\([0-9a-f-]\+\)$|\t\1\t\2|' >"$LOCALDATA_ARTISTS_VIEW"
-  info "Precomputing releasegroup view"
-  while IFS= read -r rgid; do
-    mb_releasegroup "$rgid" | $JQ '[
-    .id,
-    ."primary-type",
-    (."secondary-types" // []|join(";")),
-    ."first-release-date",
-    .title,
-    (."artist-credit" | map(([.name, .joinphrase]|join(""))) | join(""))
-    ] | join("\t")'
-  done <"$LOCALDATA_RELEASEGROUPS" |
-    awk \
-      -F "\t" \
-      -v file_local_releasegroups="${LOCALDATA_RELEASEGROUPS:-}" \
-      -v format_release="$RGV_RELEASE" \
-      -v format_release_w_artist="$RGV_RELEASE_W_ARTIST" \
-      -v format_year="$RGV_YEAR" \
-      -v format_album="$FORMAT_TYPE_ALBUM" \
-      -v format_single="$FORMAT_TYPE_SINGLE" \
-      -v format_ep="$FORMAT_TYPE_EP" \
-      -v format_broadcast="$FORMAT_TYPE_BROADCAST" \
-      -v format_other="$FORMAT_TYPE_OTHER" \
-      -v format_has_secondary="$FORMAT_TYPE_HAS_SECONDARY" \
-      -v format_secondary="$FORMAT_TYPE_SECONDARY" \
-      -v format_compilation="$FORMAT_TYPE_SECONDARY_COMPILATION" \
-      -v format_soundtrack="$FORMAT_TYPE_SECONDARY_SOUNDTRACK" \
-      -v format_spokenword="$FORMAT_TYPE_SECONDARY_SPOKENWORD" \
-      -v format_interview="$FORMAT_TYPE_SECONDARY_INTERVIEW" \
-      -v format_audiobook="$FORMAT_TYPE_SECONDARY_AUDIOBOOK" \
-      -v format_audiodrama="$FORMAT_TYPE_SECONDARY_AUDIODRAMA" \
-      -v format_live="$FORMAT_TYPE_SECONDARY_LIVE" \
-      -v format_remix="$FORMAT_TYPE_SECONDARY_REMIX" \
-      -v format_djmix="$FORMAT_TYPE_SECONDARY_DJMIX" \
-      -v format_mixtape="$FORMAT_TYPE_SECONDARY_MIXTAPE" \
-      -v format_demo="$FORMAT_TYPE_SECONDARY_DEMO" \
-      -v format_fieldrec="$FORMAT_TYPE_SECONDARY_FIELDREC" \
-      -v format_local="$FORMAT_LOCAL" \
-      "$AWK_RELEASEGROUPS" |
-    sort -n -r |
-    cut -d "$(printf '\t')" -f 2- |
-    column -t -s "$(printf '\t')" -E 0 |
-    sed 's| \+\([0-9a-f-]\+\) \+\([0-9a-f-]\+\)$|\t\1\t\2|' >"$LOCALDATA_RELEASEGROUPS_VIEW"
-  info "Precomputing release view"
-  cut -d "$(printf '\t')" -f 1 "$LOCALDATA_RELEASES" |
-    while IFS= read -r rid; do
-      mb_release "$rid" | $JQ '[
+__precompute_lists() {
+  cache_get_file_batch "$TYPE_ARTIST" <"$LOCALDATA_ARTISTS" | xargs \
+    $JQ '[
      .id,
-      .status,
-      .date,
-      ."cover-art-archive".count,
-      (."label-info" | map(.label.name) | unique | join(", ")),
-      (.media | map(."track-count") | add),
-      (.media | map(.format) | unique | join(", ")),
-      .country,
+      .type,
+      .name,
+      .disambiguation,
+      .["life-span"].begin,
+      .["life-span"].end
+      ] | join("\t")' >"$LOCALDATA_ARTISTS_LIST"
+  cache_get_file_batch "$TYPE_RELEASEGROUP" <"$LOCALDATA_RELEASEGROUPS" | xargs \
+    $JQ '[
+      .id,
+      ."primary-type",
+      (."secondary-types" // []|join(";")),
+      ."first-release-date",
      .title,
      (."artist-credit" | map(([.name, .joinphrase]|join(""))) | join(""))
-      ] | join("\t")'
-    done |
-    awk \
-      -F "\t" \
-      -v file_local_releases="${LOCALDATA_RELEASES:-}" \
-      -v release_official="$FORMAT_STATUS_OFFICIAL" \
-      -v release_promotion="$FORMAT_STATUS_PROMO" \
-      -v release_bootleg="$FORMAT_STATUS_BOOTLEG" \
-      -v release_pseudo="$FORMAT_STATUS_PSEUDO" \
-      -v release_withdrawn="$FORMAT_STATUS_WITHDRAWN" \
-      -v release_expunged="$FORMAT_STATUS_EXPUNGED" \
-      -v release_cancelled="$FORMAT_STATUS_CANCELLED" \
-      -v release_format="$RV_FORMAT" \
-      -v release_format_title_artist="$RV_TITLE_ARTIST" \
-      -v release_format_title="$RV_TITLE" \
-      -v release_format_artist="$RV_ARTIST" \
-      -v format_local="$FORMAT_LOCAL" \
-      "$AWK_RELEASES" |
-    sort -n -r |
+      ] | join("\t")' >"$LOCALDATA_RELEASEGROUPS_LIST"
+  #  cache_get_file_batch "$TYPE_RELEASE" <"$LOCALDATA_RELEASES" | xargs \
+  #    $JQ '[
+  #      .id,
+  #      .status,
+  #      .date,
+  #      ."cover-art-archive".count,
+  #      (."label-info" | map(.label.name) | unique | join(", ")),
+  #      (.media | map(."track-count") | add),
+  #      (.media | map(.format) | unique | join(", ")),
+  #      .country,
+  #      .title,
+  #      (."artist-credit" | map(([.name, .joinphrase]|join(""))) | join(""))
+  #      ] | join("\t")' >"$LOCALDATA_RELEASES_LIST" &
+}
+
+# Precompute views
+# TODO: The sed opperations take too long, improve!
+precompute_views() {
+  awk \
+    -F "\t" \
+    -v file_local_artists="${LOCALDATA_ARTISTS:-}" \
+    -v format_person="$AV_PERSON" \
+    -v format_group="$AV_GROUP" \
+    -v format_disambiguation="$AV_DISAMBIGUATION" \
+    -v format_local="$FORMAT_LOCAL" \
+    "$AWK_ARTISTS" "$LOCALDATA_ARTISTS_LIST" |
+    sort |
+    column -t -s "$(printf '\t')" -l 2 >"$LOCALDATA_ARTISTS_VIEW"
+  #column -t -s "$(printf '\t')" |
+  #sed 's| \+\([0-9a-f-]\+\) \+\([0-9a-f-]\+\)$|\t\1\t\2|' >"$LOCALDATA_ARTISTS_VIEW"
+  awk \
+    -F "\t" \
+    -v file_local_releasegroups="${LOCALDATA_RELEASEGROUPS:-}" \
+    -v format_release="$RGV_RELEASE" \
+    -v format_release_w_artist="$RGV_RELEASE_W_ARTIST" \
+    -v format_year="$RGV_YEAR" \
+    -v format_album="$FORMAT_TYPE_ALBUM" \
+    -v format_single="$FORMAT_TYPE_SINGLE" \
+    -v format_ep="$FORMAT_TYPE_EP" \
+    -v format_broadcast="$FORMAT_TYPE_BROADCAST" \
+    -v format_other="$FORMAT_TYPE_OTHER" \
+    -v format_has_secondary="$FORMAT_TYPE_HAS_SECONDARY" \
+    -v format_secondary="$FORMAT_TYPE_SECONDARY" \
+    -v format_compilation="$FORMAT_TYPE_SECONDARY_COMPILATION" \
+    -v format_soundtrack="$FORMAT_TYPE_SECONDARY_SOUNDTRACK" \
+    -v format_spokenword="$FORMAT_TYPE_SECONDARY_SPOKENWORD" \
+    -v format_interview="$FORMAT_TYPE_SECONDARY_INTERVIEW" \
+    -v format_audiobook="$FORMAT_TYPE_SECONDARY_AUDIOBOOK" \
+    -v format_audiodrama="$FORMAT_TYPE_SECONDARY_AUDIODRAMA" \
+    -v format_live="$FORMAT_TYPE_SECONDARY_LIVE" \
+    -v format_remix="$FORMAT_TYPE_SECONDARY_REMIX" \
+    -v format_djmix="$FORMAT_TYPE_SECONDARY_DJMIX" \
+    -v format_mixtape="$FORMAT_TYPE_SECONDARY_MIXTAPE" \
+    -v format_demo="$FORMAT_TYPE_SECONDARY_DEMO" \
+    -v format_fieldrec="$FORMAT_TYPE_SECONDARY_FIELDREC" \
+    -v format_local="$FORMAT_LOCAL" \
+    "$AWK_RELEASEGROUPS" "$LOCALDATA_RELEASEGROUPS_LIST" | sort -n -r |
    cut -d "$(printf '\t')" -f 2- |
-    column -t -s "$(printf '\t')" -E 0 |
-    sed 's| \+\([0-9a-f-]\+\) \+\([0-9a-f-]\+\):\(.*$\)$|\t\1\t\2\t\3|' >"$LOCALDATA_RELEASES_VIEW"
+    column -t -s "$(printf '\t')" -l 5 >"$LOCALDATA_RELEASEGROUPS_VIEW"
+  #column -t -s "$(printf '\t')" |
+  #sed 's| \+\([0-9a-f-]\+\) \+\([0-9a-f-]\+\)$|\t\1\t\2|' >"$LOCALDATA_RELEASEGROUPS_VIEW"
 }

 # Load local music
+#
 # argument $1: path to decorated music files
-load_local() {
-  [ -d "$LOCALDATADIR" ] || mkdir -p "$LOCALDATADIR"
-  tmpreleases=$(mktemp)
-  [ -f "$tmpreleases" ] || exit 1
-  info "Locating and parsing decoration files ($DECORATION_FILENAME)"
+#
+# This method parses all decorations and generates a line-by-line database of
+# locally available artists, releases, and release groups.
+reloaddb() {
+  rm -rf "$LOCALDATADIR"
+  mkdir -p "$LOCALDATADIR"
  find "$1" -type f -name "$DECORATION_FILENAME" -print0 |
-    xargs -0 -P 4 $JQ '.releaseid+"\t"+input_filename' |
-    tee "$LOCALDATA_RELEASES" |
-    cut -d "$(printf '\t')" -f 1 >"$tmpreleases"
-  __batch_load_missing "$TYPE_RELEASE" "$tmpreleases"
-  # Get release groups and album artists
-  while IFS= read -r rid; do
-    mb=$(mb_release "$rid")
-    echo "$mb" | $JQ '."release-group".id' >>"$LOCALDATA_RELEASEGROUPS"
-    echo "$mb" | $JQ '."release-group"."artist-credit" | map(.artist.id) | join("\n")' >>"$LOCALDATA_ARTISTS"
-  done <"$tmpreleases"
-  tf=$(mktemp)
-  sort "$LOCALDATA_RELEASEGROUPS" | uniq >"$tf" && mv "$tf" "$LOCALDATA_RELEASEGROUPS"
-  sort "$LOCALDATA_ARTISTS" | uniq >"$tf" && mv "$tf" "$LOCALDATA_ARTISTS"
-  # Populate cache with missing data
-  __batch_load_missing "$TYPE_RELEASEGROUP" "$LOCALDATA_RELEASEGROUPS"
-  __batch_load_missing "$TYPE_ARTIST" "$LOCALDATA_ARTISTS"
-  rm -f "$tmpreleases"
-  info "Resetting views"
-  rm -f "$LOCALDATA_ARTISTS_VIEW" "$LOCALDATA_RELEASEGROUPS_VIEW" "$LOCALDATA_RELEASES_VIEW"
-  precompute_view
+    xargs -0 $JQ '.releaseid+"\t"+input_filename' >"$LOCALDATA_RELEASES"
+  # Get necessary metadata and setup lists
+  tmpreleases=$(mktemp)
+  cut -d "$(printf '\t')" -f 1 "$LOCALDATA_RELEASES" |
+    tee "$tmpreleases" |
+    __batch_load_missing "$TYPE_RELEASE"
+  tmpreleasefiles=$(mktemp)
+  cache_get_file_batch "$TYPE_RELEASE" <"$tmpreleases" >"$tmpreleasefiles"
+  xargs \
+    $JQ '."release-group".id' \
+    <"$tmpreleasefiles" >"$LOCALDATA_RELEASEGROUPS"
+  xargs \
+    $JQ '."release-group"."artist-credit" | map(.artist.id) | join("\n")' \
+    <"$tmpreleasefiles" >"$LOCALDATA_ARTISTS"
+  rm -f "$tmpreleases" "$tmpreleasefiles"
+  tf1=$(mktemp)
+  tf2=$(mktemp)
+  sort "$LOCALDATA_RELEASEGROUPS" | uniq >"$tf1"
+  mv "$tf1" "$LOCALDATA_RELEASEGROUPS"
+  sort "$LOCALDATA_ARTISTS" | uniq >"$tf2"
+  mv "$tf2" "$LOCALDATA_ARTISTS"
+  __batch_load_missing "$TYPE_RELEASEGROUP" <"$LOCALDATA_RELEASEGROUPS"
+  __batch_load_missing "$TYPE_ARTIST" <"$LOCALDATA_ARTISTS"
+  __precompute_lists
+}
+
+# Check if necessary cache files are present or not
+local_files_present() {
+  cache_get_file_batch "$TYPE_ARTIST" <"$LOCALDATA_ARTISTS" | xargs ls >/dev/null 2>&1 || return 1
+  cache_get_file_batch "$TYPE_RELEASEGROUP" <"$LOCALDATA_RELEASEGROUPS" | xargs ls >/dev/null 2>&1 || return 1
+  cut -d "$(printf '\t')" -f 1 "$LOCALDATA_RELEASES" | cache_get_file_batch "$TYPE_RELEASEGROUP" | xargs ls >/dev/null 2>&1 || return 1
+  return 0
+}
+
+# Load missing files
+load_missing_files() {
+  __batch_load_missing "$TYPE_ARTIST" <"$LOCALDATA_ARTISTS"
+  __batch_load_missing "$TYPE_RELEASEGROUP" <"$LOCALDATA_RELEASEGROUPS"
+  cut -d "$(printf '\t')" -f 1 "$LOCALDATA_RELEASES" | __batch_load_missing "$TYPE_RELEASE"
 }