module WikiSearch
Constants
- API_URL
- DISAMBIGUATION_ITEM_PATTERN
- DISAMBIGUATION_LIMIT
- LISTICLE_MIN_MATCH_RATIO
- LISTICLE_TITLE_PATTERN
- MIN_RELEVANCE_SCORE
- SEARCH_LIMIT
- STOP_WORDS
Public Instance Methods
Source
# File lib/wiki_search.rb, line 119 def api_url(params) query_string = params.map { |key, value| "#{key}=#{CGI.escape(value.to_s)}" }.join("&") "#{API_URL}?#{query_string}" end
Source
# File lib/wiki_search.rb, line 228 def best_page(pages, query) pages .map { |page| [page, score_title(page["title"], query)] } .select { |_page, score| score >= MIN_RELEVANCE_SCORE } .max_by { |_page, score| score } &.first end
Source
# File lib/wiki_search.rb, line 124 def build_api_url(query, limit: SEARCH_LIMIT) api_url( action: "query", generator: "search", gsrsearch: query, gsrlimit: limit.to_s, prop: "extracts|info|pageprops", inprop: "url", exintro: "1", explaintext: "1", format: "json", utf8: "1" ) end
Source
# File lib/wiki_search.rb, line 153 def build_opensearch_url(query, limit: SEARCH_LIMIT) api_url( action: "opensearch", search: query, limit: limit.to_s, namespace: "0", format: "json" ) end
Source
# File lib/wiki_search.rb, line 163 def build_parse_url(title) api_url( action: "parse", page: title, prop: "wikitext", format: "json" ) end
Source
# File lib/wiki_search.rb, line 139 def build_title_url(title) api_url( action: "query", titles: title, redirects: "1", prop: "extracts|info|pageprops", inprop: "url", exintro: "1", explaintext: "1", format: "json", utf8: "1" ) end
Source
# File lib/wiki_search.rb, line 178 def disambiguation_intro_only?(extract) text = WikiReport.normalize_extract(extract) return false unless text.match?(/\bmay refer to:/i) text.match?(/\bmay refer to:\s*\z/i) end
Source
# File lib/wiki_search.rb, line 172 def disambiguation_page?(page) return true if page.dig("pageprops", "disambiguation") page["extract"].to_s.match?(/\bmay refer to:/i) end
Source
# File lib/wiki_search.rb, line 202 def fetch_disambiguation_items(title, fetch:) json = fetch.call(build_parse_url(title)) data = json.is_a?(String) ? JSON.parse(json) : json wikitext = data.dig("parse", "wikitext", "*") return [] if wikitext.to_s.strip.empty? parse_disambiguation_items(wikitext) rescue JSON::ParserError, StandardError [] end
Source
# File lib/wiki_search.rb, line 265 def fetch_pages_for_titles(titles, fetch:) return [] if titles.empty? json = fetch.call(build_title_url(titles.join("|"))) parse_pages(json) end
Source
# File lib/wiki_search.rb, line 46 def listicle_title?(title) title.to_s.match?(LISTICLE_TITLE_PATTERN) end
Source
# File lib/wiki_search.rb, line 282 def lookup_opensearch(query, fetch:) json = fetch.call(build_opensearch_url(query)) data = JSON.parse(json) titles = data[1].to_a return nil if titles.empty? pages = fetch_pages_for_titles(titles, fetch: fetch) page = best_page(pages, query) page ? page_result(page, fetch: fetch) : nil rescue JSON::ParserError, StandardError nil end
Source
# File lib/wiki_search.rb, line 295 def lookup_search(query, fetch:) search_query_variants(query).each do |variant| json = fetch.call(build_api_url(variant)) result = parse_response(json, query: query, fetch: fetch) return result if result end nil end
Source
# File lib/wiki_search.rb, line 272 def lookup_title_candidates(query, fetch:) title_candidates(query).each do |title| pages = fetch_pages_for_titles([title], fetch: fetch) page = best_page(pages, query) return page_result(page, fetch: fetch) if page end nil end
Source
# File lib/wiki_search.rb, line 33 def normalize_text(text) text.to_s .downcase .gsub(/[''']/, "") .gsub(/[^a-z0-9\s]/, " ") .squeeze(" ") .strip end
Source
# File lib/wiki_search.rb, line 236 def page_result(page, fetch: nil) if disambiguation_page?(page) && disambiguation_intro_only?(page["extract"]) && fetch items = fetch_disambiguation_items(page["title"], fetch: fetch) if items.any? limited = items.first(DISAMBIGUATION_LIMIT) more_count = items.size - limited.size return WikiReport.format_disambiguation( page["title"], page["extract"], page["fullurl"], limited, more_count: more_count ) end end WikiReport.format_result(page["title"], page["extract"], page["fullurl"]) end
Source
# File lib/wiki_search.rb, line 185 def parse_disambiguation_items(wikitext) wikitext.to_s.each_line.filter_map do |line| stripped = line.strip next unless stripped.start_with?("*") && !stripped.start_with?("**") content = stripped.sub(/\A\*\s*/, "") next unless (match = content.match(DISAMBIGUATION_ITEM_PATTERN)) display = match[2] || match[1] trailing = match[3].to_s.strip trailing = trailing.sub(/\A[,:\sāā-]+/, "").strip item = display.to_s.strip item += " ā #{trailing}" unless trailing.empty? item end end
Source
# File lib/wiki_search.rb, line 213 def parse_pages(json) data = json.is_a?(String) ? JSON.parse(json) : json pages = data.dig("query", "pages") return [] if pages.nil? || pages.empty? pages.values.filter_map do |page| next if page["missing"] next if page["extract"].to_s.strip.empty? page end rescue JSON::ParserError [] end
Source
# File lib/wiki_search.rb, line 255 def parse_response(json, query: nil, fetch: nil) pages = parse_pages(json) return nil if pages.empty? page = query ? best_page(pages, query) : pages.first return nil unless page page_result(page, fetch: fetch) end
Source
# File lib/wiki_search.rb, line 78 def possessive_title_candidates(query) text = query.to_s.strip words = text.split(/\s+/) variants = [] words.each_with_index do |word, index| next unless word.match?(/\A[a-z]+s\z/i) next if word.length < 4 stem = word[0..-2] next if stem.length < 2 variant_words = words.dup variant_words[index] = "#{stem.capitalize}'s" variants << variant_words.join(" ") variants << variant_words.map(&:capitalize).join(" ") end variants.uniq.reject(&:empty?) end
Source
# File lib/wiki_search.rb, line 42 def query_tokens(query) normalize_text(query).split.reject { |word| word.length < 3 || STOP_WORDS.include?(word) } end
Source
# File lib/wiki_search.rb, line 50 def score_title(title, query) title_norm = normalize_text(title) query_norm = normalize_text(query) tokens = query_tokens(query) return 0 if tokens.empty? matched = tokens.count { |token| title_norm.include?(token) } match_ratio = matched.to_f / tokens.size score = 0 score += 250 if title_norm == query_norm score += 180 if title_norm.include?(query_norm) || query_norm.include?(title_norm) score += matched * 45 score += 40 if match_ratio == 1.0 if listicle_title?(title) score -= 600 return score if match_ratio < LISTICLE_MIN_MATCH_RATIO end score -= 300 if title.match?(/disambiguation\z/i) score -= 200 if title.match?(/\A(?:Category|Index of):/i) return -1 if match_ratio < 0.5 && title_norm != query_norm score end
Source
# File lib/wiki_search.rb, line 305 def search(query, fetch:) cleaned = query.to_s.strip return "No article found" if cleaned.empty? lookup_title_candidates(cleaned, fetch: fetch) || lookup_opensearch(cleaned, fetch: fetch) || lookup_search(cleaned, fetch: fetch) || "No article found" rescue StandardError "No article found" end
Source
# File lib/wiki_search.rb, line 103 def search_query_variants(query) text = query.to_s.strip variants = [text, titleize_query(text)] + possessive_title_candidates(text) words = text.split(/\s+/) if words.length >= 3 quoted = %("#{words.first(2).join(" ")}") variants << [quoted, words.drop(2).join(" ")].join(" ").strip end variants.uniq.reject(&:empty?) end
Source
# File lib/wiki_search.rb, line 114 def title_candidates(query) text = query.to_s.strip ([text, titleize_query(text)] + possessive_title_candidates(text)).uniq.reject(&:empty?) end
Source
# File lib/wiki_search.rb, line 99 def titleize_query(query) query.to_s.strip.split(/\s+/).map(&:capitalize).join(" ") end