From bfee05b744465d27c4ab93cd0e273bbc460b57fe Mon Sep 17 00:00:00 2001 From: dtb Date: Sun, 6 Nov 2022 01:05:41 -0400 Subject: [PATCH] store feeds as JSON --- rss/youtube | 102 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 30 deletions(-) diff --git a/rss/youtube b/rss/youtube index 3f8c6ce..6c5e078 100755 --- a/rss/youtube +++ b/rss/youtube @@ -4,6 +4,8 @@ set -e argv0="$0" +PIGPEN="$(pig_pen)" + id(){ youtube-dl --skip-download --print-json --playlist-items 1 "$1" \ | jq '.channel_id' \ @@ -11,38 +13,75 @@ id(){ } feedurl(){ - printf "https://www.youtube.com/feeds/videos.xml?channel_id=%b\n" "$1" + printf "http://www.youtube.com/feeds/videos.xml?channel_id=%b\n" "$1" } list_channels(){ - for file in "$(pig_pen)"/* - do printf "%s\t%s\n" "$(xq '.feed.title' <"$file")" "$file" - done + pscat \ + [ sh -c "for file in \"$PIGPEN\"/*.xml; \ + do xq '{(.feed.link[0].\"@href\"): .feed.title}' \"\$file\"; \ + done 2>/dev/null" ] \ + [ sh -c "for file in \"$PIGPEN\"/*.json; \ + do jq '{(.feed.link[0].\"@href\"): .feed.title}' \"\$file\"; \ + done 2>/dev/null" ] \ + [ printf "\n" ] \ + | sed \ + -e '/{/d' \ + -e '/}/d' \ + -e 's/^ \"//g' \ + -e 's/\": \"/ /g' \ + -e 's/\,$//g' \ + -e 's/\"$//g' } list_videos(){ - list_channels \ - | grep "$1" \ - | cut -f 2 \ - | xargs xq \ + file="$( \ + ls "$PIGPEN" \ + | grep "$( \ + list_channels \ + | sed "/$(printf '%b' "$1")/q" \ + | tail -n 1 \ + | cut -f 1 \ + | xargs pig_name + )" \ + )" + case "$file" in + *json) + jq \ 'reduce .feed.entry[] as $item ( {}; . + { ($item.link."@href"): $item.title } - )' \ - | sed \ - -e '1d' \ - -e '$d' \ - -e 's/\": \"/ /g' \ - -e 's/^ \"//g' \ - -e 's/\"\,$//g' # this is really hacky + )' "$file" + ;; + *xml) + xq \ + 'reduce .feed.entry[] as $item ( + {}; + . + { ($item.link."@href"): $item.title } + )' "$file" + ;; + *) + printf "No file found!\n" 1>&2 + true + ;; + esac | sed \ + -e '1d' \ + -e '$d' \ + -e 's/\": \"/ /g' \ + -e 's/^ \"//g' \ + -e 's/\"\,$//g' # this is really hacky } latin(){ - xq '.feed.pig' <"$1" + case "$1" in + json) jq '.feed.pig' <"$1" ;; + xml) xq '.feed.pig' <"$1" ;; + esac } stow(){ - pig_fetch "$1" | xq -x "$( + name="$PIGPEN/$(pig_name "$1")" + pig_fetch "$1" | xq "$( date '+%Y-%m-%dT%T' \ | xargs printf ' . * { feed: { pig: { @@ -51,7 +90,8 @@ stow(){ stow_date: "%b", version: "0" } } }' "$1" \ - )" >"$(pig_pen)/$(pig_name "$1")" + )" >"$name.json" + rm -f "$name.xml" } watch(){ @@ -82,7 +122,7 @@ case "$1" in list_channels) ! str isvalue "$2" \ || usage - list_channels | sort + list_channels ;; list_videos) # do something with youtube-dl? str isvalue "$2" \ @@ -103,19 +143,21 @@ listen) ;; refresh) if ! str isvalue "$2" - then for file in "$(pig_pen)"/* - do latin "$file" \ - | jq '.feed_url' \ - | xargs stow & + then for file in "$PIGPEN"/* + do stow "$( \ + latin "$file" \ + | jq -r '.feed_url' \ + )" done else while str isvalue "$2" - do list_channels \ - | grep "$2" \ - | cut -f 2 \ - | xargs latin \ - | jq '.feed_url' \ - | xargs stow + do stow "$( \ + latin "$( \ + list_channels \ + | grep "$2" \ + | cut -f 2 \ + )" | jq '.feed_url' \ + )" shift done fi @@ -124,7 +166,7 @@ subscribe) str isvalue "$2" \ || usage while str isvalue "$2" - do stow "$(feedurl "$(id "$2")")" + do stow "http://www.youtube.com/feeds/videos.xml?channel_id=$(id "$2")" shift done ;;