module AnimalNews
AnimalNews format โ IRC banners, article lines, and user-facing messages. Public: animal_label, news_banner, report_footer, format_article, format_report, format_article_detail, format_lookup_report, format_location_name, no_news_message, fetch_error_message, lookup_first_message Depends: irc_format, text_util Tests: test/plugins/animal_test.rb
AnimalNews plugin โ Cinch command patterns and define_plugin factory for dedicated animal bots. Public: animal_token_pattern, command_pattern_for, lookup_pattern_for, location_pattern_for, dedicated_command_pattern_for, dedicated_lookup_pattern_for, dedicated_location_pattern_for, define_plugin Depends: flood_safe, animal_news/{parse,format,fetch} Tests: test/plugins/animal_test.rb, test/plugins/{ducks,spiders,snakes,crocodiles}_test.rb
Constants
- AREA_CATCHES_LIMIT
- ARTICLE_LIMIT
- DEFAULT_EMOJI
- MIN_ARTICLE_PARAGRAPH_LENGTH
- RECENT_ARTICLES_MUTEX
- REGION_FEEDS
- STYLE_BANNER
- USER_AGENT
Public Class Methods
Source
# File lib/animal_news/format.rb, line 11 def self.animal_label(animal) animal.to_s.strip.split(/\s+/).map(&:capitalize).join(" ").upcase end
Source
# File lib/animal_news/plugin.rb, line 11 def self.animal_token_pattern "[a-zA-Z][a-zA-Z0-9\\-_]*(?:[ \\-_][a-zA-Z][a-zA-Z0-9\\-_]*)*" end
Source
# File lib/animal_news/fetch.rb, line 90 def self.area_catches(animal, region, exclude_titles:, fetch:, limit: AREA_CATCHES_LIMIT) feed_url = feed_url_for_area(animal, region) return [] if feed_url.to_s.empty? excluded = exclude_titles.map { |title| normalize_title(title) }.to_h { |key| [key, true] } body = fetch.call(feed_url) items = parse_rss_items(body, source: "Google News").map do |item| item.merge(region: region.to_s) end result = [] items.each do |item| key = normalize_title(item[:title]) next if key.empty? || excluded[key] excluded[key] = true result << item break if result.length >= limit end result rescue StandardError [] end
Source
# File lib/animal_news/parse.rb, line 202 def self.article_detail_paragraphs(url, item:, fetch:, limit: 2) paragraphs = fetch_article_paragraphs(url, fetch: fetch, limit: limit) paragraphs = fallback_article_paragraphs(item, limit: limit) if paragraphs.empty? paragraphs end
Source
# File lib/animal_news/fetch.rb, line 35 def self.batchexecute_post(_uri, body) uri = URI("https://news.google.com/_/DotsSplashUi/data/batchexecute") request = Net::HTTP::Post.new(uri) request["Content-Type"] = "application/x-www-form-urlencoded;charset=UTF-8" request["User-Agent"] = USER_AGENT request.body = body response = Net::HTTP.start(uri.host, uri.port, use_ssl: true) { |http| http.request(request) } response.body.to_s end
Source
# File lib/animal_news/parse.rb, line 156 def self.build_google_news_batch_request(article_id, signature:, timestamp:) payload = [ [ [ "Fbv4je", %(["garturlreq",[["X","X",["X","X"],null,null,1,1,"US:en",null,1,null,null,null,null,null,0,1],"X","X",1,[1,1,1],1,1,null,0,0,null,0],"#{article_id}",#{timestamp},"#{signature}"]) ] ] ] "f.req=#{CGI.escape(JSON.generate(payload))}" end
Source
# File lib/animal_news/fetch.rb, line 30 def self.build_location_feed_url(animal, location) query = "#{encode_query(animal)}+#{encode_query(location)}" "https://news.google.com/rss/search?q=#{query}&hl=en-AU&gl=AU&ceid=AU:en" end
Source
# File lib/animal_news/parse.rb, line 29 def self.clean_google_title(title) title.to_s.sub(/\s+-\s+[^-]+\z/, "").strip end
Source
# File lib/animal_news/parse.rb, line 25 def self.clean_paragraph(html) TextUtil.squish(Nokogiri::HTML.fragment(html.to_s).text) end
Source
# File lib/animal_news/fetch.rb, line 50 def self.clear_recent_articles! RECENT_ARTICLES_MUTEX.synchronize do recent_articles_store.clear end end
Source
# File lib/animal_news/fetch.rb, line 161 def self.collect_news(animal, fetch:) collected = [] feeds = feeds_for(animal) feeds.each do |feed| body = fetch.call(feed[:url]) items = parse_rss_items(body, source: "Google News").map do |item| item.merge(region: feed[:region]) end collected.concat(items) collected = unique_items(collected, limit: ARTICLE_LIMIT) break if collected.length >= ARTICLE_LIMIT end collected end
Source
# File lib/animal_news/fetch.rb, line 178 def self.collect_news_for_location(animal, location, fetch:) area = format_location_name(location) collected = [] body = fetch.call(build_location_feed_url(animal, location)) items = parse_rss_items(body, source: "Google News").map do |item| item.merge(region: area) end collected.concat(items) collected = unique_items(collected, limit: ARTICLE_LIMIT) return collected if collected.length >= ARTICLE_LIMIT feeds_for(animal).each do |feed| body = fetch.call(feed[:url]) items = parse_rss_items(body, source: "Google News").map do |item| item.merge(region: feed[:region]) end collected.concat(items) collected = unique_items(collected, limit: ARTICLE_LIMIT) break if collected.length >= ARTICLE_LIMIT end collected end
Source
# File lib/animal_news/plugin.rb, line 54 def self.command_pattern AnimalNews.dedicated_command_pattern_for(command_name) end
Source
# File lib/animal_news/plugin.rb, line 15 def self.command_pattern_for(command) /#{command} (#{animal_token_pattern})$/ end
Source
# File lib/animal_news/parse.rb, line 144 def self.decode_google_news_batch_response(body) header = '["garturlres","' text = body.to_s.gsub('\\"', '"') return nil unless text.include?(header) start = text[text.index(header) + header.length..] footer = '",' return nil unless start&.include?(footer) start[0...start.index(footer)] end
Source
# File lib/animal_news/parse.rb, line 168 def self.decode_google_news_url(source_url, fetch:, post:) return source_url unless google_news_article_url?(source_url) article_id = google_news_article_id(source_url) return source_url if article_id.empty? params = extract_google_news_decode_params(fetch.call(source_url)) return source_url unless params body = post.call( "https://news.google.com/_/DotsSplashUi/data/batchexecute", build_google_news_batch_request( article_id, signature: params[:signature], timestamp: params[:timestamp] ) ) decoded = decode_google_news_batch_response(body) decoded.to_s.empty? ? source_url : decoded rescue StandardError source_url end
Source
# File lib/animal_news/plugin.rb, line 27 def self.dedicated_command_pattern_for(command) /#{command}$/ end
Source
# File lib/animal_news/plugin.rb, line 35 def self.dedicated_location_pattern_for(command) /#{command} ((?![1-5]$)[a-zA-Z0-9]+(?:[ \-_][a-zA-Z0-9]+)*)$/ end
Source
# File lib/animal_news/plugin.rb, line 31 def self.dedicated_lookup_pattern_for(command) /#{command} ([1-5])$/ end
Source
# File lib/animal_news/plugin.rb, line 39 def self.define_plugin(animal:, command:, emoji: DEFAULT_EMOJI) require_relative "../flood_safe" animal_term = animal.to_s command_name = command.to_s plugin_emoji = emoji Class.new do include Cinch::Plugin include FloodSafe define_singleton_method(:animal_term) { animal_term } define_singleton_method(:command_name) { command_name } define_singleton_method(:plugin_emoji) { plugin_emoji } def self.command_pattern AnimalNews.dedicated_command_pattern_for(command_name) end def self.lookup_pattern AnimalNews.dedicated_lookup_pattern_for(command_name) end def self.location_pattern AnimalNews.dedicated_location_pattern_for(command_name) end match command_pattern match lookup_pattern, method: :lookup_article_command match location_pattern, method: :execute_location define_method(:fetch_feed) do |url| RabbotHttp.fetch(url, user_agent: USER_AGENT) end define_method(:channel_key) do |message| target = message.channel || message.user target.respond_to?(:name) ? target.name : target.to_s end define_method(:batchexecute_post) do |uri, body| AnimalNews.batchexecute_post(uri, body) end define_method(:execute) do |m| fetch = method(:fetch_feed) items = AnimalNews.collect_news(animal_term, fetch: fetch) AnimalNews.store_recent_articles(channel_key(m), animal_term, items) text = items.empty? ? AnimalNews.no_news_message(animal_term) : AnimalNews.format_report(animal_term, items, emoji: plugin_emoji) flood_safe_reply(m, text) rescue StandardError flood_safe_reply(m, AnimalNews.fetch_error_message(animal_term)) end define_method(:execute_location) do |m, location_text| location = AnimalNews.parse_location(location_text) unless location flood_safe_reply(m, "Pick a story from 1 to 5") return end fetch = method(:fetch_feed) items = AnimalNews.collect_news_for_location(animal_term, location, fetch: fetch) AnimalNews.store_recent_articles(channel_key(m), animal_term, items) area = AnimalNews.format_location_name(location) text = items.empty? ? AnimalNews.no_news_message(animal_term) : AnimalNews.format_report(animal_term, items, center: area, emoji: plugin_emoji) flood_safe_reply(m, text) rescue StandardError flood_safe_reply(m, AnimalNews.fetch_error_message(animal_term)) end define_method(:lookup_article_command) do |m, index_text| index = AnimalNews.parse_article_index(index_text) unless index flood_safe_reply(m, "Pick a story from 1 to 5") return end articles = AnimalNews.recent_articles_snapshot(channel_key(m), animal_term) fetch = method(:fetch_feed) post = method(:batchexecute_post) text = AnimalNews.lookup_article(animal_term, index, articles: articles, command: command_name, fetch: fetch, post: post) flood_safe_reply(m, text) end end end
Source
# File lib/animal_news/fetch.rb, line 15 def self.encode_query(term) term.to_s.strip.downcase.gsub(/\s+/, "+") end
Source
# File lib/animal_news/parse.rb, line 110 def self.extract_article_paragraphs(html, limit: 2) doc = Nokogiri::HTML(html.to_s) doc.css("p").filter_map do |paragraph| text = TextUtil.squish(paragraph.text) next if text.length < MIN_ARTICLE_PARAGRAPH_LENGTH text end.uniq.first(limit) end
Source
# File lib/animal_news/parse.rb, line 133 def self.extract_google_news_decode_params(html) div = Nokogiri::HTML(html.to_s).at_css("c-wiz > div") return nil unless div signature = div["data-n-a-sg"].to_s.strip timestamp = div["data-n-a-ts"].to_s.strip return nil if signature.empty? || timestamp.empty? { signature: signature, timestamp: timestamp } end
Source
# File lib/animal_news/parse.rb, line 195 def self.fallback_article_paragraphs(item, limit: 2) paragraph = item[:paragraph].to_s.strip return [] if paragraph.empty? [paragraph].first(limit) end
Source
# File lib/animal_news/fetch.rb, line 76 def self.feed_for_region(animal, region) feeds_for(animal).find { |feed| feed[:region] == region.to_s } end
Source
# File lib/animal_news/fetch.rb, line 80 def self.feed_url_for_area(animal, area) area = area.to_s.strip return nil if area.empty? feed = feed_for_region(animal, area) return feed[:url] if feed build_location_feed_url(animal, area) end
Source
# File lib/animal_news/fetch.rb, line 19 def self.feeds_for(animal) term = encode_query(animal) REGION_FEEDS.map do |feed| query = feed[:suffix] ? "#{term}+#{feed[:suffix]}" : term { region: feed[:region], url: "https://news.google.com/rss/search?q=#{query}&hl=en-AU&gl=AU&ceid=AU:en" } end end
Source
# File lib/animal_news/parse.rb, line 208 def self.fetch_article_paragraphs(url, fetch:, limit: 2) html = fetch.call(url) extract_article_paragraphs(html, limit: limit) rescue StandardError [] end
Source
# File lib/animal_news/format.rb, line 79 def self.fetch_error_message(animal) "Could not fetch #{animal.to_s.downcase} news" end
Source
# File lib/animal_news/format.rb, line 26 def self.format_article(index, title:, paragraph:, source:) "#{index}. [#{source}] #{title} #{paragraph}" end
Source
# File lib/animal_news/format.rb, line 50 def self.format_article_detail(index, title:, paragraphs:, source:) body = paragraphs.map(&:to_s).map(&:strip).reject(&:empty?).join(" ") "#{index}. [#{source}] #{title} #{body}" end
Source
# File lib/animal_news/format.rb, line 71 def self.format_location_name(location) location.to_s.strip.split(/\s+/).map(&:capitalize).join(" ") end
Source
# File lib/animal_news/format.rb, line 55 def self.format_lookup_report(primary, region:, area_catches:) return primary if area_catches.empty? region_line = IrcFormat.divider("More in #{region}") area_lines = area_catches.each_with_index.map do |item, offset| format_article( offset + 1, title: item[:title], paragraph: item[:paragraph], source: item[:source] ) end [primary, region_line, *area_lines].join("\n") end
Source
# File lib/animal_news/format.rb, line 30 def self.format_report(animal, items, center: nil, emoji: DEFAULT_EMOJI) regions = items.map { |item| item[:region] }.uniq regions.unshift(center) if center && !regions.include?(center) meta = regions.join(" ยท ") lines = items.each_with_index.map do |item, index| format_article( index + 1, title: item[:title], paragraph: item[:paragraph], source: item[:source] ) end [ news_banner(animal, meta: meta), *lines, report_footer(animal) ].join("\n") end
Source
# File lib/animal_news/fetch.rb, line 203 def self.gather_news(animal, fetch:, emoji: DEFAULT_EMOJI) collected = collect_news(animal, fetch: fetch) return no_news_message(animal) if collected.empty? format_report(animal, collected, emoji: emoji) rescue StandardError fetch_error_message(animal) end
Source
# File lib/animal_news/fetch.rb, line 212 def self.gather_news_for_location(animal, location, fetch:, emoji: DEFAULT_EMOJI) area = format_location_name(location) collected = collect_news_for_location(animal, location, fetch: fetch) return no_news_message(animal) if collected.empty? format_report(animal, collected, center: area, emoji: emoji) rescue StandardError fetch_error_message(animal) end
Source
# File lib/animal_news/parse.rb, line 127 def self.google_news_article_id(url) URI.parse(url.to_s).path.split("/").reject(&:empty?).last.to_s.split("?").first rescue URI::InvalidURIError "" end
Source
# File lib/animal_news/parse.rb, line 120 def self.google_news_article_url?(url) uri = URI.parse(url.to_s) uri.host == "news.google.com" && uri.path.include?("/articles/") rescue URI::InvalidURIError false end
Source
# File lib/animal_news/plugin.rb, line 62 def self.location_pattern AnimalNews.dedicated_location_pattern_for(command_name) end
Source
# File lib/animal_news/plugin.rb, line 23 def self.location_pattern_for(command) /#{command} (#{animal_token_pattern}) ((?![1-5]$)[a-zA-Z][a-zA-Z0-9\-_]*(?:[ \-_][a-zA-Z][a-zA-Z0-9\-_]*)*)$/ end
Source
# File lib/animal_news/fetch.rb, line 115 def self.lookup_article(animal, index, articles:, command:, fetch:, post: method(:batchexecute_post)) return lookup_first_message(animal, command: command) if articles.empty? return "Pick a story from 1 to 5" unless valid_article_index?(index) item = articles[index - 1] return "Pick a story from 1 to 5" unless item url = item[:url].to_s.strip return "No article link available" if url.empty? resolved_url = resolve_article_url(url, fetch: fetch, post: post) paragraphs = article_detail_paragraphs(resolved_url, item: item, fetch: fetch) return "Could not fetch article" if paragraphs.empty? primary = format_article_detail( index, title: item[:title], paragraphs: paragraphs, source: item[:source] ) area_catches = area_catches( animal, item[:region], exclude_titles: articles.map { |article| article[:title] }, fetch: fetch ) format_lookup_report(primary, region: item[:region], area_catches: area_catches) rescue StandardError paragraphs = fallback_article_paragraphs(item, limit: 2) return "Could not fetch article" if paragraphs.empty? primary = format_article_detail( index, title: item[:title], paragraphs: paragraphs, source: item[:source] ) area_catches = area_catches( animal, item[:region], exclude_titles: articles.map { |article| article[:title] }, fetch: fetch ) format_lookup_report(primary, region: item[:region], area_catches: area_catches) end
Source
# File lib/animal_news/format.rb, line 83 def self.lookup_first_message(animal, command:) "Run !#{command} #{animal.to_s.downcase} first" end
Source
# File lib/animal_news/plugin.rb, line 58 def self.lookup_pattern AnimalNews.dedicated_lookup_pattern_for(command_name) end
Source
# File lib/animal_news/plugin.rb, line 19 def self.lookup_pattern_for(command) /#{command} (#{animal_token_pattern}) ([1-5])$/ end
Source
# File lib/animal_news/format.rb, line 75 def self.no_news_message(animal) "No #{animal.to_s.downcase} news found" end
Source
# File lib/animal_news/parse.rb, line 52 def self.normalize_item(node, source:) title = rss_element_text(node, "title") paragraph = rss_element_text(node, "description") paragraph = rss_element_text(node, "summary") if paragraph.empty? paragraph = rss_element_text(node, "content") if paragraph.empty? url = rss_item_link(node) title = clean_google_title(title) paragraph = clean_paragraph(paragraph) return nil if title.empty? || paragraph.empty? { title: title, paragraph: paragraph, source: source, url: url } end
Source
# File lib/animal_news/parse.rb, line 33 def self.normalize_title(title) TextUtil.squish(title.to_s.downcase) end
Source
# File lib/animal_news/parse.rb, line 17 def self.parse_animal(text) animal = TextUtil.squish(text) return nil if animal.empty? return nil unless animal.match?(/\A[a-zA-Z0-9 \-_]+\z/) animal end
Source
# File lib/animal_news/parse.rb, line 96 def self.parse_article_index(text) index = text.to_s.strip.to_i valid_article_index?(index) ? index : nil end
Source
# File lib/animal_news/parse.rb, line 101 def self.parse_location(text) location = text.to_s.strip.gsub(/\s+/, " ") return nil if location.empty? return nil if location.match?(/\A[1-5]\z/) return nil unless location.match?(/\A[a-zA-Z0-9 \-_]+\z/) location end
Source
# File lib/animal_news/parse.rb, line 67 def self.parse_rss_items(xml, source: "Google News") doc = REXML::Document.new(xml.to_s) REXML::XPath.match(doc, "//*[local-name()='item' or local-name()='entry']").filter_map do |node| normalize_item(node, source: source) end rescue REXML::ParseException, StandardError [] end
Source
# File lib/animal_news/fetch.rb, line 56 def self.recent_articles_key(channel, animal) "#{channel}:#{animal.to_s.downcase}" end
Source
# File lib/animal_news/fetch.rb, line 70 def self.recent_articles_snapshot(channel, animal) RECENT_ARTICLES_MUTEX.synchronize do (recent_articles_store[recent_articles_key(channel, animal)] || []).map(&:dup) end end
Source
# File lib/animal_news/fetch.rb, line 46 def self.recent_articles_store @recent_articles_store ||= {} end
Source
# File lib/animal_news/parse.rb, line 191 def self.resolve_article_url(url, fetch:, post:) google_news_article_url?(url) ? decode_google_news_url(url, fetch: fetch, post: post) : url end
Source
# File lib/animal_news/parse.rb, line 37 def self.rss_element_text(node, name) REXML::XPath.first(node, ".//*[local-name()='#{name}']")&.text.to_s.strip end
Source
# File lib/animal_news/parse.rb, line 41 def self.rss_item_link(node) link = rss_element_text(node, "link") return link unless link.empty? link_nodes = REXML::XPath.match(node, ".//*[local-name()='link']") alternate = link_nodes.find { |entry| entry.attributes["rel"].to_s == "alternate" } href = alternate&.attributes&.[]("href") href = link_nodes.first&.attributes&.[]("href") if href.to_s.empty? href.to_s.strip end
Source
# File lib/animal_news/fetch.rb, line 60 def self.store_recent_articles(channel, animal, items) channel = channel.to_s animal = animal.to_s return if channel.empty? || animal.empty? RECENT_ARTICLES_MUTEX.synchronize do recent_articles_store[recent_articles_key(channel, animal)] = items.first(ARTICLE_LIMIT).map(&:dup) end end
Source
# File lib/animal_news/parse.rb, line 76 def self.unique_items(items, limit: ARTICLE_LIMIT) seen = {} result = [] items.each do |item| key = normalize_title(item[:title]) next if key.empty? || seen[key] seen[key] = true result << item break if result.length >= limit end result end
Source
# File lib/animal_news/parse.rb, line 92 def self.valid_article_index?(index) index.is_a?(Integer) && index.between?(1, ARTICLE_LIMIT) end