module DictionaryLookup
DictionaryLookup — WordNet query parsing, fuzzy spell-correct, and wn subprocess. Public: parse_query, lookup, run_lookup, fuzzy_suggest_word, wordnet_words Tests: test/lib/dictionary_lookup_test.rb
Constants
- ALLOWED_OPTIONS
- DEFAULT_FLAGS
- FUZZY_MATCH_MAX_DISTANCE
- FUZZY_MIN_WORD_LENGTH
- INDEX_FILES
- MODIFIER_OPTIONS
- SEARCH_OPTIONS
- SENSE_NUMBER_PATTERN
- WN_BIN
- WN_DICT_DIR
- WORD_TOKEN_PATTERN
Public Instance Methods
Source
# File lib/dictionary_lookup.rb, line 52 def allowed_token?(token) value = token.to_s return true if ALLOWED_OPTIONS.include?(value) return true if value.match?(SENSE_NUMBER_PATTERN) return true if value.match?(WORD_TOKEN_PATTERN) false end
Source
# File lib/dictionary_lookup.rb, line 229 def build_argv(word, flags) [WN_BIN, word, *flags] end
Source
# File lib/dictionary_lookup.rb, line 233 def default_run lambda do |argv| Open3.capture3(*argv) end end
Source
# File lib/dictionary_lookup.rb, line 155 def fuzzy_suggest_word(needle, words:, max_distance: FUZZY_MATCH_MAX_DISTANCE) query = needle.to_s.strip.downcase return nil if query.length < FUZZY_MIN_WORD_LENGTH return nil if query.include?("_") entries = words.is_a?(Hash) ? words : words.each_with_object({}) { |word, map| map[word.to_s.downcase] = word.to_s } return nil if entries.key?(query) best = nil best_distance = max_distance + 1 best_letter_distance = max_distance + 1 entries.each do |candidate_down, candidate| next if (query.length - candidate_down.length).abs > max_distance distance = levenshtein_distance(query, candidate_down) next if distance > max_distance letter_distance = letter_bag_distance(query, candidate_down) next if distance > best_distance next if distance == best_distance && letter_distance >= best_letter_distance best = candidate best_distance = distance best_letter_distance = letter_distance end best end
Source
# File lib/dictionary_lookup.rb, line 48 def invalid_query_message "Invalid WordNet query — use letters, numbers, underscores, and WordNet flags only." end
Source
# File lib/dictionary_lookup.rb, line 183 def letter_bag_distance(a, b) bag_a = a.to_s.chars.tally bag_b = b.to_s.chars.tally keys = (bag_a.keys + bag_b.keys).uniq keys.sum do |key| (bag_a[key].to_i - bag_b[key].to_i).abs end end
Source
# File lib/dictionary_lookup.rb, line 124 def levenshtein_distance(a, b) left = a.to_s right = b.to_s return right.length if left.empty? return left.length if right.empty? rows = left.length + 1 cols = right.length + 1 matrix = Array.new(rows) { Array.new(cols, 0) } (0...rows).each { |i| matrix[i][0] = i } (0...cols).each { |j| matrix[0][j] = j } (1...rows).each do |i| (1...cols).each do |j| cost = left[i - 1] == right[j - 1] ? 0 : 1 matrix[i][j] = [ matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + cost ].min next unless i > 1 && j > 1 && left[i - 1] == right[j - 2] && left[i - 2] == right[j - 1] matrix[i][j] = [matrix[i][j], matrix[i - 2][j - 2] + 1].min end end matrix[left.length][right.length] end
Source
# File lib/dictionary_lookup.rb, line 192 def load_words_from_dict(dict_dir) words = {} INDEX_FILES.each do |filename| path = File.join(dict_dir.to_s, filename) next unless File.file?(path) File.foreach(path) do |line| token = line.split(/\s+/, 2).first next unless token&.match?(WORD_TOKEN_PATTERN) words[token.downcase] = token end end words end
Source
# File lib/dictionary_lookup.rb, line 239 def lookup(query, run: default_run, words: nil) parsed = parse_query(query) return invalid_query_message if parsed[:invalid] return usage_message if parsed[:word].nil? word = parsed[:word] flags = parsed[:flags] label = search_label(flags) candidates = words || wordnet_words output, status = run_lookup(word, flags, run: run) corrected_word = nil if wn_miss?(output, status) suggestion = fuzzy_suggest_word(word, words: candidates) if suggestion retry_output, retry_status = run_lookup(suggestion, flags, run: run) unless wn_miss?(retry_output, retry_status) output = retry_output status = retry_status corrected_word = suggestion end end end return DictionaryReport.format_no_results(word) if wn_miss?(output, status) DictionaryReport.format_wn_output(output, word: word, search_label: label, corrected_word: corrected_word) rescue StandardError DictionaryReport.format_no_results(word || query.to_s.strip) end
Source
# File lib/dictionary_lookup.rb, line 93 def modifier_token?(token) MODIFIER_OPTIONS.include?(token) || token.match?(SENSE_NUMBER_PATTERN) end
Source
# File lib/dictionary_lookup.rb, line 87 def option_token?(token) ALLOWED_OPTIONS.include?(token) || token.match?(SENSE_NUMBER_PATTERN) || SEARCH_OPTIONS.include?(token) end
Source
# File lib/dictionary_lookup.rb, line 61 def parse_query(text) tokens = text.to_s.strip.split(/\s+/).reject(&:empty?) return { word: nil, flags: DEFAULT_FLAGS.dup, invalid: false } if tokens.empty? unless tokens.all? { |token| allowed_token?(token) } return { word: nil, flags: DEFAULT_FLAGS.dup, invalid: true } end flags = [] words = [] tokens.each do |token| if option_token?(token) flags << token else words << token end end word = words.map { |part| part.tr("-", "_") }.join("_") word = nil if word.to_s.empty? flags = DEFAULT_FLAGS.dup if flags.empty? || flags.all? { |flag| modifier_token?(flag) } { word: word, flags: flags, invalid: false } end
Source
# File lib/dictionary_lookup.rb, line 223 def run_lookup(word, flags, run:) stdout, stderr, status = run.call(build_argv(word, flags)) output = [stdout, stderr].join("\n").strip [output, status] end
Source
# File lib/dictionary_lookup.rb, line 97 def search_label(flags) modifiers = [] modifiers << "gloss" if flags.include?("-g") primary = flags.find { |flag| SEARCH_OPTIONS.include?(flag) } || "-over" label = case primary when "-over" then "overview" when "-synsn" then "synonyms (noun)" when "-synsv" then "synonyms (verb)" when "-synsa" then "synonyms (adjective)" when "-synsr" then "synonyms (adverb)" when "-antsn" then "antonyms (noun)" when "-antsv" then "antonyms (verb)" when "-antsa" then "antonyms (adjective)" when "-antsr" then "antonyms (adverb)" when "-hypen", "-hypev" then "hypernyms" when "-hypon", "-hypov" then "hyponyms" when "-treen", "-treev" then "hyponym tree" when "-grepn", "-grepv", "-grepa", "-grepr" then "compound grep" when "-framv" then "verb frames" when "-derin", "-deriv" then "derivations" else primary.delete_prefix("-") end modifiers.empty? ? label : "#{label} + #{modifiers.join(' + ')}" end
Source
# File lib/dictionary_lookup.rb, line 44 def usage_message "Usage: !dict <word> [-over|-synsn|-hypen|...]" end
Source
# File lib/dictionary_lookup.rb, line 213 def wn_miss?(output, status) value = output.to_s.strip return true if value.empty? return true if !status.success? && DictionaryReport.no_match_output?(value) return true if DictionaryReport.no_match_output?(value) body = DictionaryReport.extract_body_lines(value.lines.map(&:rstrip)) body.empty? end
Source
# File lib/dictionary_lookup.rb, line 208 def wordnet_words(dict_dir: WN_DICT_DIR, loader: method(:load_words_from_dict)) @wordnet_words ||= {} @wordnet_words[dict_dir] ||= loader.call(dict_dir) end