module WhatsonSearch
Find upcoming events for a town via Google News RSS and DuckDuckGo HTML search. HTTP is injected via fetch lambdas — no live network in tests.
Constants
- DATE_PATTERN
- DDG_HTML_URL
- DEFAULT_LIMIT
- EVENT_PATTERN
- MAX_LIMIT
- MIN_LIMIT
- REJECT_PATTERN
Public Instance Methods
Source
# File lib/whatson_search.rb, line 70 def build_ddg_query(town) quoted = town.to_s.strip free_terms = WhatsonFreeEvents.search_clause %("#{quoted}" upcoming events OR calendar OR "what's on" OR #{free_terms} site:*.gov.au OR site:*.org.au) end
Source
# File lib/whatson_search.rb, line 76 def build_rss_url(town) free_terms = WhatsonFreeEvents.search_clause.gsub('"', "") query = CGI.escape("#{town} events upcoming OR calendar OR festival OR market OR #{free_terms}") "https://news.google.com/rss/search?q=#{query}&hl=en-AU&gl=AU&ceid=AU:en" end
Source
# File lib/whatson_search.rb, line 82 def build_search_url(town) "#{DDG_HTML_URL}?q=#{CGI.escape(build_ddg_query(town))}" end
Source
# File lib/whatson_search.rb, line 194 def clean_paragraph(html) TextUtil.squish(Nokogiri::HTML.fragment(html.to_s).text) end
Source
# File lib/whatson_search.rb, line 145 def clean_title(title) TextUtil.squish(title.to_s.sub(/\s+-\s+[^-]+\z/, "")) end
Source
# File lib/whatson_search.rb, line 86 def clean_url(href) return "" if href.nil? || href.to_s.strip.empty? href = href.to_s.strip href = "https:#{href}" if href.start_with?("//") if href.include?("uddg=") decoded = URI.decode_www_form(URI.parse(href).query.to_s).to_h["uddg"] return decoded if decoded && !decoded.empty? end href rescue StandardError href.to_s end
Source
# File lib/whatson_search.rb, line 102 def event_like?(title:, snippet:, url:) text = "#{title} #{snippet} #{url}" return false if text.match?(REJECT_PATTERN) && !text.match?(EVENT_PATTERN) text.match?(EVENT_PATTERN) || url.to_s.match?(%r{/events?/|/whats-on|/calendar|/whatson}i) end
Source
# File lib/whatson_search.rb, line 116 def extract_date(text) match = text.to_s.match(DATE_PATTERN) return nil unless match normalize_date(match[0]) end
Source
# File lib/whatson_search.rb, line 137 def extract_location(snippet, town) match = snippet.to_s.match(/\bat\s+(.+?)(?:\s+on\s+\d|\.\z)/i) location = match ? match[1].strip : nil return location unless location.to_s.empty? town.to_s.strip.empty? ? nil : TextUtil.title_words(town) end
Source
# File lib/whatson_search.rb, line 110 def mentions_town?(text, town) town_words = town.to_s.downcase.split(/\s+/) body = text.to_s.downcase town_words.all? { |word| body.include?(word) } end
Source
# File lib/whatson_search.rb, line 222 def merge_events(*lists, limit: DEFAULT_LIMIT) seen = {} merged = [] lists.flatten.each do |event| key = normalize_event_key(event) next if key.empty? || seen[key] seen[key] = true merged << event break if merged.length >= limit end merged end
Source
# File lib/whatson_search.rb, line 123 def normalize_date(raw) text = raw.to_s.strip text = text.sub(/\A(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)[a-z]*\s+/i, "") month_map = { "january" => "Jan", "february" => "Feb", "march" => "Mar", "april" => "Apr", "june" => "Jun", "july" => "Jul", "august" => "Aug", "september" => "Sep", "october" => "Oct", "november" => "Nov", "december" => "Dec" } month_map.each do |long, short| text = text.sub(/\b#{long}\b/i, short) end TextUtil.squish(text) end
Source
# File lib/whatson_search.rb, line 149 def normalize_event_key(event) [event[:title], event[:date], event[:url]].map { |part| part.to_s.downcase.strip }.join("|") end
Source
# File lib/whatson_search.rb, line 153 def parse_ddg_results(html, town:, parsed: nil) doc = Nokogiri::HTML(html.to_s) doc.css(".result").filter_map do |result| link_el = result.at_css(".result__a") next unless link_el title = clean_title(link_el.text) snippet = result.at_css(".result__snippet")&.text&.strip.to_s url = clean_url(link_el["href"]) next if title.empty? next unless event_like?(title: title, snippet: snippet, url: url) next unless result_matches_town?(parsed, town, title: title, snippet: snippet, url: url) { title: title, date: extract_date("#{title} #{snippet}"), location: extract_location(snippet, town), url: url, source: "web search" } end rescue StandardError [] end
Source
# File lib/whatson_search.rb, line 198 def parse_rss_events(xml, town:, parsed: nil) doc = REXML::Document.new(xml.to_s) REXML::XPath.match(doc, "//*[local-name()='item' or local-name()='entry']").filter_map do |node| title = clean_title(rss_element_text(node, "title")) paragraph = clean_paragraph(rss_element_text(node, "description")) paragraph = clean_paragraph(rss_element_text(node, "summary")) if paragraph.empty? url = rss_item_link(node) next if title.empty? next unless event_like?(title: title, snippet: paragraph, url: url) next unless result_matches_town?(parsed, town, title: title, snippet: paragraph, url: url) { title: title, date: extract_date("#{title} #{paragraph}"), location: extract_location(paragraph, town), url: url, source: "Google News" } end rescue REXML::ParseException, StandardError [] end
Source
# File lib/whatson_search.rb, line 66 def parse_town(text) WhatsonTowns.resolve(text) end
Source
# File lib/whatson_search.rb, line 236 def result_matches_town?(parsed, town, title:, snippet:, url:) search_town = parsed&.dig(:search_town) || town return false unless WhatsonTowns.accept_result?(parsed || {}, title: title, snippet: snippet, url: url) return true if mentions_town?("#{title} #{snippet} #{url}", search_town) mentions_town?("#{title} #{snippet} #{url}", town) || url.match?(/\.gov\.au/i) end
Source
# File lib/whatson_search.rb, line 179 def rss_element_text(node, name) REXML::XPath.first(node, ".//*[local-name()='#{name}']")&.text.to_s.strip end
Source
# File lib/whatson_search.rb, line 183 def rss_item_link(node) link = rss_element_text(node, "link") return link unless link.empty? link_nodes = REXML::XPath.match(node, ".//*[local-name()='link']") alternate = link_nodes.find { |entry| entry.attributes["rel"].to_s == "alternate" } href = alternate&.attributes&.[]("href") href = link_nodes.first&.attributes&.[]("href") if href.to_s.empty? href.to_s.strip end
Source
# File lib/whatson_search.rb, line 248 def search(town, fetch:, fetch_rss: nil, limit: DEFAULT_LIMIT, report: WhatsonReport.method(:build_report)) parsed = parse_town(town) return report.call([], parsed) if parsed[:error] return nil if WhatsonTowns.silent?(parsed) query_town = search_town_name(parsed) ddg_url = build_search_url(query_town) ddg_html = fetch.call(ddg_url) ddg_events = parse_ddg_results(ddg_html, town: parsed[:town], parsed: parsed) rss_events = [] if fetch_rss rss_url = build_rss_url(query_town) rss_xml = fetch_rss.call(rss_url) rss_events = parse_rss_events(rss_xml, town: parsed[:town], parsed: parsed) end events = merge_events(rss_events, ddg_events, limit: limit) report.call(events, parsed) rescue StandardError "Could not search for events right now." end
Source
# File lib/whatson_search.rb, line 244 def search_town_name(parsed) parsed[:search_town] || parsed[:town] end
Source
# File lib/whatson_search.rb, line 62 def usage_message "Usage: !whatson <town>" end