#!/bin/bash # Searches the website for RSS feeds and adds them to newsboat url list. Can # also find hidden RSS feeds on various websites, namely Youtube, Reddit, # Vimeo, Github, Gitlab and Medium. Gets site url as $1 or (if not present) # from X clipboard. Gets tags as $2. If it finds more than one feed, calls # mew for the user to choose which one to add. I have bound it to a keyboard # shortcut so i copy a site link and easily add its feed to the reader. # Inspired by and based on the logic of this extension: # https://github.com/shevabam/get-rss-feed-url-extension # This script requires rssadd to add feeds to the list. getlink () { local url="$1" feeds="$(curl -s "$url" | rg -x '.*type=.*(rss|rdf|atom).*' | sed 's/ //g')" url="$(echo $url | sed 's|^\(https://[^/]*/\).*|\1|')" for rsspath in $feeds; do rsspath="$(echo $rsspath | sed -n "s|.*href=['\"]\([^'\"]*\)['\"].*|\1|p")" if echo "$rsspath" | rg "http" > /dev/null; then link="$rsspath" elif echo "$rsspath" | rg "^/" > /dev/null; then link="$url$(echo $rsspath | sed 's|^/||')" else link="$url$rsspath" fi echo $link done } getRedditRss() { echo "${1%/}.rss" } getYoutubeRss() { local url="$1" path=$(echo "$url" | sed -e 's|^http[s]*://||') case "$path" in *"/channel/"*) channel_id="$(echo $path | sed -r 's|.*channel/([^/]*).*|\1|')" && feed="https://www.youtube.com/feeds/videos.xml?channel_id=${channel_id}" ;; *"/c/"*|*"/user/"*) feed=$(wget -q "$url" -O tmp_rssget_yt \ && sed -n 's|.*\("rssUrl":"[^"]*\).*|\1|; p' tmp_rssget_yt \ | rg rssUrl \ | sed 's|"rssUrl":"||') ;; esac echo "$feed" } getVimeoRss() { local url="$1" if echo "$url" | rg -q "/videos$"; then feed_url=$(echo "$url" | sed 's/\/videos$//' | sed 's/\/$/\/rss/') else feed_url="${url}/videos/rss" fi echo "$feed_url" } getGithubRss () { local url="${1%/}" if echo $url | rg "github.com/[^/]*/[a-zA-Z0-9].*" >/dev/null ; then echo "${url}/commits.atom" echo "${url}/releases.atom" echo "${url}/tags.atom" elif echo $url | rg "github.com/[^/]*(/)" >/dev/null ; then echo "${url}.atom" fi } getGitlabRss () { local url="${1%/}" echo "${url}.atom" } getMediumRss () { echo $1 | sed 's|/tag/|/feed/|' } if [ -n "$1" ] ; then url="$1" else url="$(wl-paste)" [ -z "$url" ] && echo "usage: $0 url 'tag1 tag2 tag3'" && exit 1 fi declare -a list=() yt_regex="^(http(s)?://)?((w){3}\.)?(youtube\.com|invidio\.us|invidious\.flokinet\.to|invidious\.materialio\.us|iv\.datura\.network|invidious\.perennialte\.ch|invidious\.fdn\.fr|invidious\.private\.coffee|invidious\.protokolla\.fi|invidious\.privacyredirect\.com|yt\.artemislena\.eu|yt\.drgnz\.club|invidious\.incogniweb\.net|yewtu\.be|inv\.tux\.pizza|invidious\.reallyaweso\.me|iv\.melmac\.space|inv\.us\.projectsegfau\.lt|inv\.nadeko\.net|invidious\.darkness\.services|invidious\.jing\.rocks|invidious\.privacydev\.net|inv\.in\.projectsegfau\.lt|invidious\.drgns\.space)/(channel|user|c).+" reddit_regex="^(http(s)?://)?((w){3}\.)?reddit\.com.*" vimeo_regex="^(http(s)?://)?((w){3}.)?vimeo\.com.*" if echo $url | rg -x "$yt_regex" >/dev/null ; then list="$(getYoutubeRss "$url")" elif echo $url | rg -x "$reddit_regex" >/dev/null ; then list="$(getRedditRss "$url")" # vimeo actually works with getlink elif echo $url | rg "$vimeo_regex" >/dev/null ; then list="$(getVimeoRss "$url")" elif echo $url | rg "github.com" >/dev/null ; then list="$(getGithubRss "$url")" # gitlab also works with getlink elif echo $url | rg "gitlab.com/[a-zA-Z0-9].*" >/dev/null ; then list="$(getGitlabRss "$url")" elif echo $url | rg "medium.com/tag" >/dev/null ; then list="$(getMediumRss "$url")" else list="$(getlink "$url")" fi [ "$(echo "$list" | wc -l)" -eq 1 ] && chosen_link="$list" || chosen_link=$(printf '%s\n' "${list[@]}" | mew -p "Choose a feed:") tags="$2" ifinstalled rssadd && rssadd "$chosen_link" "$tags" echo "$chosen_link" "$tags"