module CarSearch
Parse car search queries, fetch listings from DDG/Brave HTML, filter roadworthy results. HTTP is injected via fetch lambdas โ no live network in tests. Public: CarSearch.search and all methods in car_search/{query,listings,details,filter}.rb Depends: GeoLookup, CarReport (lazy, for default report callback) Tests: test/lib/car_search_test.rb, test/plugins/car_test.rb
CarSearch filter โ roadworthy and price filter pipeline for listing selection. Public: matches_price?, filter_listings Depends: CarSearch details and listings helpers Tests: test/lib/car_search_test.rb, test/plugins/car_test.rb
Constants
- AU_STATE_SLUGS
- BRAVE_SEARCH_URL
- CARSALES_BASE
- DDG_HTML_URL
- DEFAULT_LIMIT
- DETAILS_PATH
- FOR_SALE_STATUS_PATTERN
- MAX_LIMIT
- MIN_LIMIT
- NOT_ROADWORTHY_PATTERN
- SALE_STATUS_KEYS
- SOLD_AVAILABILITY_PATTERN
- SOLD_PATTERN
- SOLD_STATUS_PATTERN
Public Class Methods
Source
# File lib/car_search/details.rb, line 217 def apply_details_to_listing(listing, fetch:) html = fetch.call(listing[:url]) details = parse_details_listing(html) updated = listing.merge(for_sale_verified: true) updated[:for_sale] = details[:for_sale] unless details[:for_sale].nil? if details[:price] && listing_price(updated).nil? updated = merge_listing_price(updated, details[:price]) end updated rescue StandardError listing.merge(for_sale_verified: false) end
Source
# File lib/car_search/query.rb, line 17 def australian_location?(result) if result.respond_to?(:country_code) && result.country_code.to_s.strip.downcase == "au" return true end result.country.to_s.downcase.include?("australia") end
Source
# File lib/car_search/listings.rb, line 68 def brave_snippet_text(container) text = container&.at_css(".generic-snippet .content, .description, [class*='description']")&.text&.strip.to_s text.sub(/\A[A-Za-z]+ \d{1,2}, \d{4}\s*-\s*/o, "").strip end
Source
# File lib/car_search/query.rb, line 148 def build_ddg_query(make:, model:, min_price:, max_price:, location_label: nil) terms = ["site:carsales.com.au/cars/details"] unless location_label.to_s.strip.empty? terms << location_label.to_s.split(",").first.strip end terms << make.to_s.strip unless make.to_s.strip.empty? terms << model.to_s.strip unless model.to_s.strip.empty? terms << "for sale" terms << "roadworthy" if min_price && max_price terms << "$#{min_price} to $#{max_price}" elsif max_price terms << "under $#{max_price}" end terms.join(" ") end
Source
# File lib/car_search/query.rb, line 123 def build_search_url(make:, model:, min_price:, max_price:, location_state: nil) if min_price && max_price && make.to_s.strip.empty? query_parts = [] unless location_state.to_s.strip.empty? query_parts << "State.#{carsales_state_name(location_state)}" end query_parts << "Price.range(#{min_price}..#{max_price})" query = "(And.#{query_parts.join('._')}.)" return "#{CARSALES_BASE}/cars/?q=#{CGI.escape(query)}&sort=Price-asc" end parts = %w[cars used] parts << slugify(make) unless make.to_s.strip.empty? parts << slugify(model) unless model.to_s.strip.empty? parts << "#{location_state}-state" unless location_state.to_s.strip.empty? if min_price && max_price parts << "between-#{min_price}-#{max_price}" elsif max_price parts << "under-#{max_price}" end "#{CARSALES_BASE}/#{parts.join('/')}/?sort=Price-asc" end
Source
# File lib/car_search/query.rb, line 33 def carsales_state_name(slug) slug.to_s.split("-").map(&:capitalize).join(" ") end
Source
# File lib/car_search/listings.rb, line 13 def clean_url(href) return "" if href.nil? || href.to_s.strip.empty? href = href.to_s.strip href = "https:#{href}" if href.start_with?("//") if href.include?("uddg=") decoded = URI.decode_www_form(URI.parse(href).query.to_s).to_h["uddg"] return decoded if decoded && !decoded.empty? end href rescue StandardError href.to_s end
Source
# File lib/car_search/listings.rb, line 92 def collect_listings(html, fallback_html: nil) listings = parse_listings(html) return listings unless listings.empty? return listings if fallback_html.to_s.strip.empty? parse_brave_listings(fallback_html) end
Source
# File lib/car_search/listings.rb, line 29 def ddg_blocked?(html) body = html.to_s body.include?("challenge-form") || body.include?("anomaly-modal") end
Source
# File lib/car_search/query.rb, line 37 def detect_location(tokens, lookup_place: GeoLookup.method(:lookup_place)) return { location: nil, location_label: nil, tokens: tokens } if tokens.empty? tokens.length.downto(1) do |count| phrase = tokens.first(count).join(" ") result = lookup_place.call(phrase) next unless result next unless australian_location?(result) remaining = tokens[count..] || [] return { location: result, location_label: GeoLookup.location_label(result), tokens: remaining } end { location: nil, location_label: nil, tokens: tokens } end
Source
# File lib/car_search/details.rb, line 58 def dig_for_price(value, depth = 0) return nil if depth > 12 case value when Hash %w[price driveAwayPrice listingPrice egcPrice displayPrice].each do |key| parsed = numeric_price_value(value[key]) return parsed if parsed end value.each_value do |child| parsed = dig_for_price(child, depth + 1) return parsed if parsed end when Array value.each do |child| parsed = dig_for_price(child, depth + 1) return parsed if parsed end end nil end
Source
# File lib/car_search/details.rb, line 110 def dig_for_sale_status(value, depth = 0) return nil if depth > 12 case value when Hash SALE_STATUS_KEYS.each do |key| status = normalize_sale_status(value[key]) return status unless status.nil? end availability = value["availability"].to_s return false if availability.match?(SOLD_AVAILABILITY_PATTERN) value.each_value do |child| status = dig_for_sale_status(child, depth + 1) return status unless status.nil? end when Array value.each do |child| status = dig_for_sale_status(child, depth + 1) return status unless status.nil? end end nil end
Source
# File lib/car_search/details.rb, line 241 def enrich_listing_prices(listings, fetch:, max: nil) enrich_listings_from_details(listings, fetch: fetch, max: max) end
Source
# File lib/car_search/details.rb, line 230 def enrich_listings_from_details(listings, fetch:, max: nil) remaining = max || listings.length listings.map do |listing| next listing if remaining <= 0 next listing if listing[:for_sale_verified] remaining -= 1 apply_details_to_listing(listing, fetch: fetch) end end
Source
# File lib/car_search/details.rb, line 19 def extract_price(text) match = text.to_s.match(/\$\s*([\d,]+)/) return nil unless match match[1].delete(",").to_i end
Source
# File lib/car_search/filter.rb, line 19 def filter_listings(listings, parsed, fetch_detail: nil) limit = parsed[:limit] || DEFAULT_LIMIT pool = listings.select { |listing| roadworthy?(listing) && !obviously_sold?(listing) } selected = [] fetch_budget = [pool.length, limit * 4].max fetches = 0 pool.each do |listing| break if selected.length >= limit if fetch_detail && listing[:for_sale].nil? && fetches < fetch_budget listing = apply_details_to_listing(listing, fetch: fetch_detail) fetches += 1 end next unless for_sale?(listing, strict: !fetch_detail.nil?) next unless matches_price?(listing, min_price: parsed[:min_price], max_price: parsed[:max_price]) selected << listing end selected end
Source
# File lib/car_search/details.rb, line 87 def for_sale?(listing, strict: false) return false if listing[:for_sale] == false return true if listing[:for_sale] == true return false if obviously_sold?(listing) if strict return true if listing[:for_sale_verified] && listing_price(listing) return true if listing_price(listing) false else true end end
Source
# File lib/car_search/details.rb, line 245 def format_price(amount) value = amount.to_i "$#{value.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse}" end
Source
# File lib/car_search/listings.rb, line 34 def humanize_slug(slug) slug.to_s.split("-").map { |word| word.match?(/\A\d+\z/) ? word : word.capitalize }.join(" ") end
Source
# File lib/car_search/details.rb, line 33 def listing_network_id(url) match = url.to_s.match(/(SSE-AD-\d+)/i) match ? match[1].upcase : nil end
Source
# File lib/car_search/details.rb, line 26 def listing_price(listing) price = listing[:price] return price.to_i if price extract_price("#{listing[:title]} #{listing[:snippet]}") end
Source
# File lib/car_search/listings.rb, line 45 def listing_title(url, fallback_text: nil) title_from_details_url(url) || fallback_text.to_s.strip end
Source
# File lib/car_search/query.rb, line 25 def location_state_slug(result) return nil unless result return nil unless result.respond_to?(:state_code) code = result.state_code.to_s.strip.downcase AU_STATE_SLUGS[code] end
Source
# File lib/car_search/filter.rb, line 9 def matches_price?(listing, min_price:, max_price:) price = listing_price(listing) return true if price.nil? return false if min_price && price < min_price return false if max_price && price > max_price true end
Source
# File lib/car_search/details.rb, line 210 def merge_listing_price(listing, price) listing.merge( price: price, snippet: [listing[:snippet], format_price(price)].compact.reject(&:empty?).join(" ยท ") ) end
Source
# File lib/car_search/details.rb, line 102 def normalize_sale_status(value) text = value.to_s.strip.downcase.gsub(/[\s_-]+/, " ") return true if text.match?(FOR_SALE_STATUS_PATTERN) return false if text.match?(SOLD_STATUS_PATTERN) nil end
Source
# File lib/car_search/details.rb, line 38 def numeric_price_value(value) case value when Integer return value if value >= 100 when Float int_value = value.to_i return int_value if int_value >= 100 when String digits = value.delete(",").match(/\A\d+\z/) return digits[0].to_i if digits && digits[0].to_i >= 100 when Hash %w[amount value price].each do |key| parsed = numeric_price_value(value[key]) return parsed if parsed end end nil end
Source
# File lib/car_search/details.rb, line 82 def obviously_sold?(listing) text = "#{listing[:title]} #{listing[:snippet]}" text.match?(SOLD_PATTERN) end
Source
# File lib/car_search/listings.rb, line 73 def parse_brave_listings(html) doc = Nokogiri::HTML(html.to_s) seen = {} doc.css('a[href*="carsales.com.au/cars/details"]').filter_map do |link_el| url = clean_url(link_el["href"]) next unless url.match?(DETAILS_PATH) next if seen[url] seen[url] = true container = link_el.ancestors.find { |node| node["class"].to_s.include?("snippet") } snippet = brave_snippet_text(container) title = listing_title(url, fallback_text: link_el.text) { title: title, url: url, snippet: snippet } end rescue StandardError [] end
Source
# File lib/car_search/details.rb, line 137 def parse_details_for_sale(doc, html) doc.css('script[type="application/ld+json"]').each do |script| data = JSON.parse(script.text) items = data.is_a?(Array) ? data : [data] items.each do |item| offers = item["offers"] offers = [offers] if offers.is_a?(Hash) Array(offers).each do |offer| availability = offer["availability"].to_s return false if availability.match?(SOLD_AVAILABILITY_PATTERN) end status = dig_for_sale_status(item) return status unless status.nil? end rescue JSON::ParserError, StandardError next end next_data = doc.at_css("script#__NEXT_DATA__")&.text if next_data && !next_data.strip.empty? status = dig_for_sale_status(JSON.parse(next_data)) return status unless status.nil? end body = doc.text.to_s return false if body.match?(/this (?:car|vehicle) (?:has been )?sold|sold on|no longer available/i) return false if html.to_s.match?(/"saleStatus"\s*:\s*"sold"/i) nil rescue StandardError nil end
Source
# File lib/car_search/details.rb, line 198 def parse_details_listing(html) doc = Nokogiri::HTML(html.to_s) { for_sale: parse_details_for_sale(doc, html), price: parse_details_price_from_doc(doc, html) } end
Source
# File lib/car_search/details.rb, line 206 def parse_details_price(html) parse_details_listing(html)[:price] end
Source
# File lib/car_search/details.rb, line 171 def parse_details_price_from_doc(doc, html) doc.css('script[type="application/ld+json"]').each do |script| data = JSON.parse(script.text) items = data.is_a?(Array) ? data : [data] items.each do |item| offers = item["offers"] offers = [offers] if offers.is_a?(Hash) Array(offers).each do |offer| parsed = numeric_price_value(offer["price"]) return parsed if parsed end end rescue JSON::ParserError, StandardError next end next_data = doc.at_css("script#__NEXT_DATA__")&.text if next_data && !next_data.strip.empty? parsed = dig_for_price(JSON.parse(next_data)) return parsed if parsed end extract_price(html) rescue StandardError nil end
Source
# File lib/car_search/query.rb, line 57 def parse_limit_token(token) return nil unless token.to_s.match?(/\A\d+\z/) value = token.to_i return value if (MIN_LIMIT..MAX_LIMIT).cover?(value) nil end
Source
# File lib/car_search/listings.rb, line 49 def parse_listings(html) doc = Nokogiri::HTML(html.to_s) doc.css(".result").filter_map do |result| link_el = result.at_css(".result__a") next unless link_el url = clean_url(link_el["href"]) next unless url.match?(DETAILS_PATH) title = link_el.text.to_s.strip snippet = result.at_css(".result__snippet")&.text&.strip.to_s next if title.empty? { title: title, url: url, snippet: snippet } end rescue StandardError [] end
Source
# File lib/car_search/query.rb, line 66 def parse_query(text, lookup_place: GeoLookup.method(:lookup_place)) tokens = text.to_s.strip.split(/\s+/) return { error: usage_message } if tokens.empty? limit = DEFAULT_LIMIT limit_explicit = false if (parsed_limit = parse_limit_token(tokens.first)) limit = parsed_limit limit_explicit = true tokens.shift end prices = [] while !tokens.empty? && tokens.last.match?(/\A\d+\z/) prices.unshift(tokens.pop.to_i) end return { error: usage_message } if prices.length > 2 return { error: "Min price must be less than max price." } if prices.length == 2 && prices[0] >= prices[1] min_price = prices.length == 2 ? prices[0] : nil max_price = prices.last if prices.any? detected = detect_location(tokens, lookup_place: lookup_place) tokens = detected[:tokens] location = detected[:location] location_label = detected[:location_label] location_state = location_state_slug(location) make = tokens.empty? ? nil : tokens[0] model = tokens.length > 1 ? tokens[1..].join(" ") : nil if make.nil? && model.nil? && location.nil? && min_price.nil? && max_price.nil? && !limit_explicit return { error: usage_message } end { make: make, model: model, min_price: min_price, max_price: max_price, limit: limit, location: location, location_label: location_label, location_state: location_state, raw: text.to_s.strip } end
Source
# File lib/car_search/details.rb, line 14 def roadworthy?(listing) text = "#{listing[:title]} #{listing[:snippet]}" !text.match?(NOT_ROADWORTHY_PATTERN) end
Source
# File lib/car_search/query.rb, line 115 def slugify(term) term.to_s .downcase .gsub(/[^a-z0-9]+/, "-") .gsub(/-+/, "-") .gsub(/\A-|-\z/, "") end
Source
# File lib/car_search/listings.rb, line 38 def title_from_details_url(url) match = url.to_s.match(%r{/cars/details/([^/]+)/}i) return nil unless match humanize_slug(match[1]) end
Source
# File lib/car_search/query.rb, line 13 def usage_message "Usage: !car [<count>] [<location>] [<make> [<model>]] [<min>] [<max>]" end
Public Instance Methods
Source
# File lib/car_search.rb, line 62 def search(parsed, fetch:, fallback_fetch: nil, fetch_detail: nil, report: CarReport.method(:build_report)) query = build_ddg_query(**parsed.slice(:make, :model, :min_price, :max_price, :location_label)) html = fetch.call(query) fallback_html = nil if parse_listings(html).empty? && fallback_fetch fallback_html = fallback_fetch.call(query) end listings = filter_listings(collect_listings(html, fallback_html: fallback_html), parsed, fetch_detail: fetch_detail) report.call(listings, parsed) rescue StandardError "Could not search for cars right now." end