# Database functionality to support local music. # # All local data is stored in the directory `LOCALDATADIR`. In the future, we # will also use the methods here, and modifications thereof, to support # MusicBainz collections. if [ ! "${LOCAL_LOADED:-}" ]; then LOCALDATADIR="$HOME/.cache/$APP_NAME/local" LOCALDATA_ARTISTS="$LOCALDATADIR/artists" LOCALDATA_RELEASEGROUPS="$LOCALDATADIR/releasegroups" LOCALDATA_RELEASES="$LOCALDATADIR/releases" LOCALDATA_ARTISTS_VIEW="$LOCALDATADIR/artists_view" LOCALDATA_RELEASEGROUPS_VIEW="$LOCALDATADIR/releasegroups_view" LOCALDATA_RELEASES_VIEW="$LOCALDATADIR/releases_view" LOCALDATA_ARTISTS_LIST="$LOCALDATADIR/artists_list" LOCALDATA_RELEASEGROUPS_LIST="$LOCALDATADIR/releasegroups_list" LOCALDATA_RELEASES_LIST="$LOCALDATADIR/releases_list" DECORATION_FILENAME=${DECORATION_FILENAME:-"mbid.json"} # Create necessary files [ -d "$LOCALDATADIR" ] || mkdir -p "$LOCALDATADIR" [ -f "$LOCALDATA_ARTISTS" ] || touch "$LOCALDATA_ARTISTS" [ -f "$LOCALDATA_RELEASEGROUPS" ] || touch "$LOCALDATA_RELEASEGROUPS" [ -f "$LOCALDATA_RELEASES" ] || touch "$LOCALDATA_RELEASES" [ -f "$LOCALDATA_ARTISTS_LIST" ] || touch "$LOCALDATA_ARTISTS_LIST" [ -f "$LOCALDATA_RELEASEGROUPS_LIST" ] || touch "$LOCALDATA_RELEASEGROUPS_LIST" [ -f "$LOCALDATA_RELEASES_LIST" ] || touch "$LOCALDATA_RELEASES_LIST" export LOCALDATADIR LOCALDATA_ARTISTS LOCALDATA_RELEASEGROUPS \ LOCALDATA_RELEASES LOCALDATA_ARTISTS_VIEW LOCALDATA_RELEASEGROUPS_VIEW \ LOCALDATA_RELEASES_VIEW LOCALDATA_ARTISTS_LIST LOCALDATA_RELEASEGROUPS_LIST \ LOCALDATA_RELEASES_LIST DECORATION_FILENAME export LOCAL_LOADED=1 fi # Retrieve tags as json object from music file # # @argument $1: path to music file # # The tags retrieved are the MusicBrainz release ID and the MusicBrainz track # ID __gettags() { $FFPROBE -v error -show_entries format_tags -print_format json "$1" | $JQ '.format.tags | { trackid: (."MusicBrainz Release Track Id" // ."MUSICBRAINZ_RELEASETRACKID" // ."MusicBrainz/Release Track Id" // ""), releaseid: (."MusicBrainz Album Id" // ."MUSICBRAINZ_ALBUMID" // ."MusicBrainz/Album Id" // "") }' } # Decorate locally available music # # @input $1: Path to directory with album # # This methods reads the music files in the specified directory and writes a # json file that points to all relevant MusicBrainz IDs. If the directory # contains untagged files, or files of different releases, then the decoration # process will fail, and an error is printed. decorate() { if [ -f "$1/$DECORATION_FILENAME" ]; then info "Directory $1 has already been decorated (skipping)" return 0 fi decoration=$($JQ -n '.tracks = {}') tmpf=$(mktemp) (cd "$1" && find . -type f -iname '*.mp3' -o -iname '*.mp4' -o -iname '*.flac' -o -iname '*.m4a' -o -iname '*.ogg') >"$tmpf" while IFS= read -r f; do mbid=$(__gettags "$1/$f") rid=$(echo "$mbid" | $JQ '.releaseid') tid=$(echo "$mbid" | $JQ '.trackid') if [ ! "$rid" ] || [ ! "$tid" ]; then err "File $f: Seems not tagged" releaseid="" break fi if [ "${releaseid:-}" ]; then if [ "$releaseid" != "$rid" ]; then err "Directory $1 contains files of different releases" releaseid="" break fi else info "Decorating $1 as release $rid" releaseid="$rid" fi decoration=$(echo "$decoration" | $JQ ".tracks += {\"$tid\": \"$f\"}") done <"$tmpf" rm -f "$tmpf" if [ "$releaseid" ]; then echo "$decoration" | $JQ ".releaseid = \"$releaseid\"" >"$1/$DECORATION_FILENAME" else return 1 fi } # Decorate locally available music with specified MusicBrainz release # # @input $1: Path to directory with album # @input $2: MusicBrainz release ID # # Similar as `decorate`, but the MusicBrainz IDs are not inferred from the # tags, but passed as argument. decorate_as() { if [ -f "$1/$DECORATION_FILENAME" ]; then rid="$($JQ '.releaseid' "$1/$DECORATION_FILENAME")" title="$(mb_release "$rid" | $JQ '.title // ""')" artist="$(mb_release "$rid" | $JQ '."artist-credit" | map([.name, .joinphrase] | join("")) | join("")')" [ "$rid" = "$2" ] && info "Directory $1 has already been decorated as the release '$title' - '$artist' with the identical MusicBrainz release ID." || info "Directory $1 has already been decorated as the release '$title' - '$artist' with the MusicBrainz release ID $rid." while true; do infonn "Do you want to redecorate $1? (yes/no)" read -r yn case $yn in "yes") break ;; "no") return 0 ;; *) info "Please answer \"yes\" or \"no\"." ;; esac done fi # Print info title="$(mb_release "$2" | $JQ '.title // ""')" artist="$(mb_release "$2" | $JQ '."artist-credit" | map([.name, .joinphrase] | join("")) | join("")')" info "Decorating $1 as the release $title by $artist" # Start decoration decoration=$($JQ -n '.tracks = {}') tmpf=$(mktemp) (cd "$1" && find . -type f -iname '*.mp3' -o -iname '*.mp4' -o -iname '*.flac' -o -iname '*.m4a' -o -iname '*.ogg' | sort) >"$tmpf" # Compare number of tracks with release rcnt="$(mb_release "$2" | $JQ '.media | map(."track-count") | add')" dcnt="$(wc -l "$tmpf" | cut -d ' ' -f 1)" if [ ! "$rcnt" -eq "$dcnt" ]; then err "Number of tracks in directory ($dcnt) does not match number of tracks in release ($rcnt)." return 1 fi # tmpj=$(mktemp) mb_release "$2" | $JQ '.media[] | .position as $pos | .tracks | map({ $pos, "id": .id, "n": .number, "t": .title }) | map(if(.n | type == "string" and test("^[0-9]+$")) then .n |= tonumber else . end) | sort_by([.pos, .n])[] | [.t, .id] | join("\t")' >"$tmpj" assocfile=$(mktemp) awk -F '\t' ' BEGIN { OFS = "\t" } FNR == NR { title[FNR] = $1; id[FNR] = $2 } FNR != NR { fname[FNR] = $1 } END { for (i in id) print title[i], id[i], fname[i] } ' "$tmpj" "$tmpf" >"$assocfile" rm -f "$tmpj" "$tmpf" # Ask user if this is ok info "We discovered the following associatoin:" while IFS= read -r line; do t="$(echo "$line" | cut -d "$(printf '\t')" -f 1)" f="$(echo "$line" | cut -d "$(printf '\t')" -f 3)" printf "Track '%s'\tFile '%s'\n" "$t" "$f" done <"$assocfile" | column -t -s "$(printf '\t')" while true; do infonn "Are the track correctly associated to the audio files? (yes/no)" read -r yn case $yn in "yes") break ;; "no") return 0 ;; *) info "Please answer \"yes\" or \"no\"." ;; esac done # Construct decoration decoration=$($JQ -n '.tracks = {}') while IFS= read -r line; do i="$(echo "$line" | cut -d "$(printf '\t')" -f 2)" f="$(echo "$line" | cut -d "$(printf '\t')" -f 3)" decoration=$(echo "$decoration" | $JQ ".tracks += {\"$i\": \"$f\"}") done <"$assocfile" echo "$decoration" | $JQ ".releaseid = \"$2\"" >"$1/$DECORATION_FILENAME" return 0 } # Load missing cache entries (batch mode) # # argument $1: type # # This method reads one MusicBrainz IDs of the specified type from stdin (one # per line), and fetches the missing items. __batch_load_missing() { tmpf=$(mktemp) cat | cache_get_file_batch "$1" | xargs \ sh -c 'for f; do [ -e "$f" ] || echo "$f"; done' _ | cache_mbid_from_path_batch >"$tmpf" lines=$(wc -l "$tmpf" | cut -d ' ' -f 1) if [ "$lines" -gt 0 ]; then case "$1" in "$TYPE_ARTIST") tt="artists" ;; "$TYPE_RELEASEGROUP") tt="release groups" ;; "$TYPE_RELEASE") tt="releases" ;; esac info "Fetching missing $tt" cnt=0 while IFS= read -r mbid; do case "$1" in "$TYPE_ARTIST") name=$(mb_artist "$mbid" | $JQ '.name') ;; "$TYPE_RELEASEGROUP") name=$(mb_releasegroup "$mbid" | $JQ '.title') ;; "$TYPE_RELEASE") name=$(mb_release "$mbid" | $JQ '.title') ;; esac cnt=$((cnt + 1)) info "$(printf "%d/%d (%s: %s)" "$cnt" "$lines" "$mbid" "$name")" sleep 1 done <"$tmpf" fi rm -f "$tmpf" } # Precompute lists # # The main views (VIEW_ARTIST and TYPE_RELEASEGROUP) for locally available # music are theme dependent. These views are generated from the lists that are # produced with the present method. It contains all essential data, but in a # theme-independent fashion. The lists are stored in the files # `LOCALDATA_ARTISTS_LIST` and `LOCALDATA_RELEASEGROUPS_LIST`. __precompute_lists() { cache_get_file_batch "$TYPE_ARTIST" <"$LOCALDATA_ARTISTS" | xargs \ $JQ '[ .id, .type, .name, ."sort-name", .disambiguation, .["life-span"].begin, .["life-span"].end ] | join("\t")' >"$LOCALDATA_ARTISTS_LIST" cache_get_file_batch "$TYPE_RELEASEGROUP" <"$LOCALDATA_RELEASEGROUPS" | xargs \ $JQ '[ .id, ."primary-type", (."secondary-types" // []|join(";")), ."first-release-date", .title, (."artist-credit" | map(([.name, .joinphrase]|join(""))) | join("")) ] | join("\t")' >"$LOCALDATA_RELEASEGROUPS_LIST" } # Precompute views # # This method injects the theme elements to the lists from `precompute_lists`. # The resulting views are stored in the files `LOCALDATA_ARTISTS_VIEW` and # `LOCALDATA_RELEASEGROUPS_VIEW`. precompute_views() { awk_artists "$SORT_ARTIST_DEFAULT" <"$LOCALDATA_ARTISTS_LIST" >"$LOCALDATA_ARTISTS_VIEW" awk_releasegroups "$SORT_RG_DEFAULT" <"$LOCALDATA_RELEASEGROUPS_LIST" >"$LOCALDATA_RELEASEGROUPS_VIEW" #column -t -s "$(printf '\t')" | #sed 's| \+\([0-9a-f-]\+\) \+\([0-9a-f-]\+\)$|\t\1\t\2|' >"$LOCALDATA_RELEASEGROUPS_VIEW" } # Load local music # # argument $1: path to decorated music files # # This method parses all decorations and generates a line-by-line database of # locally available artists, releases, and release groups. This data is stored # in the files `LOCALDATA_ARTISTS`, `LOCALDATA_RELEASES`, and # `LOCALDATA_RELEASEGROUPS`. reloaddb() { rm -rf "$LOCALDATADIR" mkdir -p "$LOCALDATADIR" find "$1" -type f -name "$DECORATION_FILENAME" -print0 | xargs -0 $JQ '.releaseid+"\t"+input_filename' >"$LOCALDATA_RELEASES" # Get necessary metadata and setup lists tmpreleases=$(mktemp) cut -d "$(printf '\t')" -f 1 "$LOCALDATA_RELEASES" | tee "$tmpreleases" | __batch_load_missing "$TYPE_RELEASE" tmpreleasefiles=$(mktemp) cache_get_file_batch "$TYPE_RELEASE" <"$tmpreleases" >"$tmpreleasefiles" xargs \ $JQ '."release-group".id' \ <"$tmpreleasefiles" >"$LOCALDATA_RELEASEGROUPS" xargs \ $JQ '."release-group"."artist-credit" | map(.artist.id) | join("\n")' \ <"$tmpreleasefiles" >"$LOCALDATA_ARTISTS" rm -f "$tmpreleases" "$tmpreleasefiles" tf1=$(mktemp) tf2=$(mktemp) sort "$LOCALDATA_RELEASEGROUPS" | uniq >"$tf1" mv "$tf1" "$LOCALDATA_RELEASEGROUPS" sort "$LOCALDATA_ARTISTS" | uniq >"$tf2" mv "$tf2" "$LOCALDATA_ARTISTS" __batch_load_missing "$TYPE_RELEASEGROUP" <"$LOCALDATA_RELEASEGROUPS" __batch_load_missing "$TYPE_ARTIST" <"$LOCALDATA_ARTISTS" __precompute_lists } # Check if necessary cache files are present or not # # This method returns a non-zero value if some cached file is required to exist # for the computation of the lists (and views). This does not include the # derivation of the MusicBrainz artist IDs and MusicBrainz release-group IDs # from the MusicBrainz releases (see the `reloaddb` method above). local_files_present() { cache_get_file_batch "$TYPE_ARTIST" <"$LOCALDATA_ARTISTS" | xargs ls >/dev/null 2>&1 || return 1 cache_get_file_batch "$TYPE_RELEASEGROUP" <"$LOCALDATA_RELEASEGROUPS" | xargs ls >/dev/null 2>&1 || return 1 #cut -d "$(printf '\t')" -f 1 "$LOCALDATA_RELEASES" | cache_get_file_batch "$TYPE_RELEASE" | xargs ls >/dev/null 2>&1 || return 1 return 0 } # Load missing files # # If missing files were detected with `local_files_present`, then these missing # files may be cached using the present method. load_missing_files() { __batch_load_missing "$TYPE_ARTIST" <"$LOCALDATA_ARTISTS" __batch_load_missing "$TYPE_RELEASEGROUP" <"$LOCALDATA_RELEASEGROUPS" #cut -d "$(printf '\t')" -f 1 "$LOCALDATA_RELEASES" | __batch_load_missing "$TYPE_RELEASE" }