class Rss
Constants
- DEFAULT_FEED_TITLE
- DEFAULT_FEED_URL
- PLUGIN
- PREVIEW_ITEMS_LIMIT
- RECENT_ITEMS_LIMIT
- REFRESH_INTERVAL_SEC
- STATE_KEY
- USER_AGENT
Public Class Methods
Source
# File plugins/rss.rb, line 227 def self.add_feed(state, url, fetch:, page_url: nil) normalized = normalize_feed_url(url) return { error: "Feed already subscribed" } if state["feeds"].any? { |feed| feed["url"] == normalized } body = fetch_feed_body(normalized, fetch: fetch) items = parse_rss_items(body) meta = parse_feed_meta(body) title = meta[:title] title = normalized if title.to_s.strip.empty? feed = { "id" => feed_id(normalized), "url" => normalized, "title" => title, "last_guid" => nil } if page_url normalized_filter = RssFilter.normalize_page_url(page_url) feed["link_filter"] = normalized_filter feed = bootstrap_feed_last_guid(feed, fetch: fetch) else feed["last_guid"] = items.empty? ? nil : item_key(items.first) end { state: state.merge("feeds" => state["feeds"] + [feed]), feed: feed } rescue StandardError { error: "Could not fetch feed" } end
Source
# File plugins/rss.rb, line 277 def self.bootstrap_feed_last_guid(feed, fetch:) items = feed_items(feed, fetch: fetch) items.empty? ? feed.merge("last_guid" => nil) : feed.merge("last_guid" => item_key(items.first)) end
Source
# File plugins/rss.rb, line 122 def self.clean_paragraph(html) text = html.to_s.gsub(/<[^>]+>/, " ") text.gsub(/\s+/, " ").strip end
Source
# File plugins/rss.rb, line 297 def self.clear_feed_link_filter(state, feed_url) feed = find_feed(state, feed_url) return { error: "Feed not found" } unless feed updated_feed = feed.dup updated_feed.delete("link_filter") feeds = state["feeds"].map { |entry| entry["url"] == feed["url"] ? updated_feed : entry } { state: state.merge("feeds" => feeds), feed: updated_feed } end
Source
# File plugins/rss.rb, line 32 def self.command_pattern /rss(?: (.+))?$/i end
Source
# File plugins/rss.rb, line 177 def self.default_state { "feeds" => [ { "id" => feed_id(DEFAULT_FEED_URL), "url" => DEFAULT_FEED_URL, "title" => DEFAULT_FEED_TITLE, "last_guid" => nil } ], "last_refresh_at" => nil } end
Source
# File plugins/rss.rb, line 311 def self.due_for_refresh?(state, now: Time.now, interval_sec: REFRESH_INTERVAL_SEC) raw = state["last_refresh_at"] return true if raw.nil? || raw.to_s.strip.empty? last = RabbotDb.parse_time(raw) return true unless last (now - last) >= interval_sec end
Source
# File plugins/rss.rb, line 114 def self.feed_id(url) Digest::SHA1.hexdigest(url.to_s)[0, 12] end
Source
# File plugins/rss.rb, line 223 def self.feed_items(feed, fetch:) fetch_feed_items(feed["url"], fetch: fetch, link_filter: feed["link_filter"]) end
Source
# File plugins/rss.rb, line 211 def self.fetch_feed_body(url, fetch:) fetch.call(url) end
Source
# File plugins/rss.rb, line 215 def self.fetch_feed_items(url, fetch:, link_filter: nil) body = fetch_feed_body(url, fetch: fetch) items = parse_rss_items(body) RssFilter.apply(items, link_filter: link_filter) rescue StandardError [] end
Source
# File plugins/rss.rb, line 270 def self.find_feed(state, url) normalized = normalize_feed_url(url) return nil unless normalized state["feeds"].find { |feed| feed["url"] == normalized } end
Source
# File plugins/rss.rb, line 353 def self.format_item_card(feed, item) RssReport.format_item_card(feed, item) end
Source
# File plugins/rss.rb, line 357 def self.format_list(state) interval_min = REFRESH_INTERVAL_SEC / 60 RssReport.format_list(state["feeds"], interval_min: interval_min) end
Source
# File plugins/rss.rb, line 436 def self.gather_latest(state, fetch:) show_headlines(state, fetch: fetch)[:text] end
Source
# File plugins/rss.rb, line 161 def self.item_key(item) value = item[:guid].to_s.strip value = item[:link].to_s.strip if value.empty? value = item[:title].to_s.strip if value.empty? value end
Source
# File plugins/rss.rb, line 191 def self.load_state(network:, channel:, store: RabbotDb) store.get_plugin_json( plugin: PLUGIN, key: STATE_KEY, network: network, channel: channel, default: nil ) || default_state end
Source
# File plugins/rss.rb, line 168 def self.new_items_since(feed, items) last = feed["last_guid"] return [] if last.nil? || last.to_s.empty? items.take_while do |item| item_key(item) != last end end
Source
# File plugins/rss.rb, line 51 def self.normalize_feed_url(url) value = url.to_s.strip return nil unless valid_feed_url?(value) value end
Source
# File plugins/rss.rb, line 58 def self.parse_command(args) text = args.to_s.strip return { action: :show } if text.empty? return { action: :list } if text.casecmp?("list") return { action: :preview } if text.casecmp?("show") if (match = text.match(/\Arecent(?:\s+(.+))?\z/i)) label = match[1].to_s.strip return { error: usage_message } if label.empty? return { action: :recent, label: label } end if (match = text.match(/\Aadd\s+(\S+)(?:\s+(\S+))?\z/i)) url = normalize_feed_url(match[1]) return { error: usage_message } unless url if match[2] page_url = RssFilter.normalize_page_url(match[2]) return { error: usage_message } unless page_url return { action: :add, url: url, page_url: page_url } end return { action: :add, url: url } end if (match = text.match(/\Arem(?:ove)?\s+(.+)\z/i)) url = normalize_feed_url(match[1]) return { error: usage_message } unless url return { action: :rem, url: url } end if text.match?(/\Afilter\s/i) if (match = text.match(/\Afilter\s+rem(?:ove)?\s+(\S+)\z/i)) url = normalize_feed_url(match[1]) return { error: usage_message } unless url return { action: :filter_rem, url: url } end if (match = text.match(/\Afilter\s+(\S+)\s+(\S+)\z/i)) feed_url = normalize_feed_url(match[1]) page_url = RssFilter.normalize_page_url(match[2]) return { error: usage_message } unless feed_url && page_url return { action: :filter, feed_url: feed_url, page_url: page_url } end return { error: usage_message } end { error: usage_message } end
Source
# File plugins/rss.rb, line 152 def self.parse_feed_meta(xml) doc = REXML::Document.new(xml.to_s) channel = REXML::XPath.first(doc, "//*[local-name()='channel' or local-name()='feed']") title = channel ? rss_element_text(channel, "title") : "" { title: title } rescue REXML::ParseException, StandardError { title: "" } end
Source
# File plugins/rss.rb, line 127 def self.parse_rss_items(xml) doc = REXML::Document.new(xml.to_s) REXML::XPath.match(doc, "//*[local-name()='item' or local-name()='entry']").filter_map do |node| title = rss_element_text(node, "title") link = rss_element_text(node, "link") link = node.attributes["href"].to_s.strip if link.empty? && node.name == "entry" guid = rss_element_text(node, "guid") guid = rss_element_text(node, "id") if guid.empty? description = rss_element_text(node, "description") description = rss_element_text(node, "summary") if description.empty? description = rss_element_text(node, "content") if description.empty? description = clean_paragraph(description) next if title.empty? { guid: guid, title: title, link: link, description: description } end rescue REXML::ParseException, StandardError [] end
Source
# File plugins/rss.rb, line 376 def self.preview_subscriptions(state, fetch:, limit: PREVIEW_ITEMS_LIMIT) sections = [] state["feeds"].each do |feed| items = feed_items(feed, fetch: fetch) next if items.empty? preview_items = items.first(limit) sections << RssReport.format_feed_preview(feed, preview_items) end text = sections.any? ? sections.join("\n") : "No RSS headlines found" { text: text } rescue StandardError { text: "Could not fetch RSS headlines" } end
Source
# File plugins/rss.rb, line 362 def self.recent_subscription(state, label:, fetch:, limit: RECENT_ITEMS_LIMIT) lookup = RssLookup.find_feed_by_label(state["feeds"], label) return { error: lookup[:error] } if lookup[:error] feed = lookup[:feed] items = feed_items(feed, fetch: fetch) return { text: "No RSS headlines found" } if items.empty? preview_items = items.first(limit) { text: RssReport.format_feed_preview(feed, preview_items) } rescue StandardError { text: "Could not fetch RSS headlines" } end
Source
# File plugins/rss.rb, line 321 def self.refresh_feed(feed, items) last_guid = feed["last_guid"] if last_guid.nil? || last_guid.to_s.empty? updated = items.empty? ? feed : feed.merge("last_guid" => item_key(items.first)) return { feed: updated, announcements: [] } end fresh = new_items_since(feed, items) updated = items.empty? ? feed : feed.merge("last_guid" => item_key(items.first)) announcements = fresh.empty? ? [] : [{ feed: feed, items: fresh.reverse }] { feed: updated, announcements: announcements } end
Source
# File plugins/rss.rb, line 335 def self.refresh_state(state, fetch:, now: Time.now) announcements = [] feeds = state["feeds"].map do |feed| items = feed_items(feed, fetch: fetch) result = refresh_feed(feed, items) announcements.concat(result[:announcements]) result[:feed] end { state: state.merge( "feeds" => feeds, "last_refresh_at" => RabbotDb.iso_time(now) ), announcements: announcements } end
Source
# File plugins/rss.rb, line 260 def self.remove_feed(state, url) normalized = normalize_feed_url(url) feeds = state["feeds"] return { error: "Feed not found" } unless feeds.any? { |feed| feed["url"] == normalized } { state: state.merge("feeds" => feeds.reject { |feed| feed["url"] == normalized }) } end
Source
# File plugins/rss.rb, line 118 def self.rss_element_text(node, name) REXML::XPath.first(node, ".//*[local-name()='#{name}']")&.text.to_s.strip end
Source
# File plugins/rss.rb, line 201 def self.save_state(network:, channel:, state:, store: RabbotDb) store.set_plugin_json( plugin: PLUGIN, key: STATE_KEY, network: network, channel: channel, value: state ) end
Source
# File plugins/rss.rb, line 282 def self.set_feed_link_filter(state, feed_url, page_url, fetch:) feed = find_feed(state, feed_url) return { error: "Feed not found" } unless feed normalized_filter = RssFilter.normalize_page_url(page_url) updated_feed = feed.merge("link_filter" => normalized_filter) updated_feed = bootstrap_feed_last_guid(updated_feed, fetch: fetch) feeds = state["feeds"].map { |entry| entry["url"] == feed["url"] ? updated_feed : entry } { state: state.merge("feeds" => feeds), feed: updated_feed } end
Source
# File plugins/rss.rb, line 394 def self.show_headlines(state, fetch:) sections = [] saw_items = false updated_feeds = state["feeds"].map do |feed| items = feed_items(feed, fetch: fetch) next feed if items.empty? saw_items = true last_guid = feed["last_guid"] if last_guid.nil? || last_guid.to_s.empty? sections << format_item_card(feed, items.first) feed.merge("last_guid" => item_key(items.first)) else fresh = new_items_since(feed, items) if fresh.empty? feed else fresh.reverse.each { |item| sections << format_item_card(feed, item) } feed.merge("last_guid" => item_key(items.first)) end end end text = if sections.any? sections.join("\n") elsif saw_items "No new RSS headlines" else "No RSS headlines found" end { text: text, state: state.merge("feeds" => updated_feeds) } rescue StandardError { text: "Could not fetch RSS headlines", state: state } end
Source
# File plugins/rss.rb, line 40 def self.usage_message "Usage: !rss [list|add <url> [filter-page-url]|rem <url>|filter <feed-url> <page-url>|filter rem <feed-url>|show|recent <feed>]" end
Source
# File plugins/rss.rb, line 44 def self.valid_feed_url?(url) uri = URI.parse(url.to_s.strip) uri.is_a?(URI::HTTP) && !uri.host.to_s.empty? rescue URI::InvalidURIError false end
Public Instance Methods
Source
# File plugins/rss.rb, line 465 def execute(m, args) return unless m.channel parsed = self.class.parse_command(args) if parsed[:error] m.reply parsed[:error] return end network = bot.config.server channel = m.channel.name state = self.class.load_state(network: network, channel: channel) case parsed[:action] when :list themed_flood_safe_reply(m, self.class.format_list(state)) when :add result = self.class.add_feed( state, parsed[:url], fetch: method(:fetch_feed), page_url: parsed[:page_url] ) if result[:error] m.reply result[:error] return end self.class.save_state(network: network, channel: channel, state: result[:state]) m.reply "Subscribed to #{result[:feed]['title']}" when :rem result = self.class.remove_feed(state, parsed[:url]) if result[:error] m.reply result[:error] return end self.class.save_state(network: network, channel: channel, state: result[:state]) m.reply "Feed removed" when :filter result = self.class.set_feed_link_filter( state, parsed[:feed_url], parsed[:page_url], fetch: method(:fetch_feed) ) if result[:error] m.reply result[:error] return end self.class.save_state(network: network, channel: channel, state: result[:state]) m.reply "Link filter set for #{result[:feed]['title']}" when :filter_rem result = self.class.clear_feed_link_filter(state, parsed[:url]) if result[:error] m.reply result[:error] return end self.class.save_state(network: network, channel: channel, state: result[:state]) m.reply "Link filter cleared" when :preview result = self.class.preview_subscriptions(state, fetch: method(:fetch_feed)) themed_flood_safe_reply(m, result[:text]) when :recent result = self.class.recent_subscription( state, label: parsed[:label], fetch: method(:fetch_feed) ) if result[:error] m.reply result[:error] return end themed_flood_safe_reply(m, result[:text]) else result = self.class.show_headlines(state, fetch: method(:fetch_feed)) self.class.save_state(network: network, channel: channel, state: result[:state]) themed_flood_safe_reply(m, result[:text]) end end
Source
# File plugins/rss.rb, line 440 def fetch_feed(url) RabbotHttp.fetch(url, user_agent: USER_AGENT) end
Source
# File plugins/rss.rb, line 444 def poll_feeds return unless bot network = bot.config.server bot.config.channels.each do |channel_name| channel = bot.channels.find { |entry| entry.name.casecmp?(channel_name.to_s) } next unless channel state = self.class.load_state(network: network, channel: channel_name) next unless self.class.due_for_refresh?(state) result = self.class.refresh_state(state, fetch: method(:fetch_feed)) self.class.save_state(network: network, channel: channel_name, state: result[:state]) result[:announcements].each do |entry| entry[:items].each do |item| channel.send(self.class.format_item_card(entry[:feed], item)) end end end end