module UrlFetch

# File lib/url_fetch.rb, line 27
def bot_block_page?(html)
  text = html.to_s
  return false if text.strip.empty?
  return true if text.match?(CHALLENGE_BODY_PATTERN)

  title = text[%r{<title[^>]*>([^<]+)</title>}i, 1]
  return true if title&.match?(CHALLENGE_TITLE_PATTERN)

  false
end

# File lib/url_fetch.rb, line 96
def default_http_fetch(url, user_agent:, headers:)
  RabbotHttp.fetch(url, user_agent: user_agent, headers: headers)
end

# File lib/url_fetch.rb, line 100
def default_lookup_fetch(api_url)
  RabbotHttp.fetch(api_url, user_agent: USER_AGENT, headers: BROWSER_HEADERS)
end

# File lib/url_fetch.rb, line 63
def fetch(url, http_fetch: nil, wayback_lookup: nil, wayback_fetch: nil)
  http_fetch ||= method(:default_http_fetch)
  wayback_lookup ||= method(:default_lookup_fetch)
  wayback_fetch ||= method(:default_http_fetch)
  html = nil
  begin
    html = fetch_direct(url, http_fetch: http_fetch)
  rescue StandardError
    html = nil
  end

  return html if usable_html?(html)

  snapshot_url = wayback_snapshot_url(url, lookup_fetch: wayback_lookup)
  raise "No usable HTML" if snapshot_url.to_s.strip.empty?

  snap_html = wayback_fetch.call(snapshot_url, user_agent: USER_AGENT, headers: BROWSER_HEADERS)
  snap_html = normalize_body(snap_html)
  raise "No usable HTML" unless usable_html?(snap_html)

  snap_html
end

# File lib/url_fetch.rb, line 38
def fetch_direct(url, http_fetch: nil)
  http_fetch ||= method(:default_http_fetch)
  body = http_fetch.call(url, user_agent: USER_AGENT, headers: BROWSER_HEADERS)
  normalize_body(body)
rescue OpenURI::HTTPError => e
  body = e.io.read
  raise if body.nil? || body.to_s.strip.empty?

  normalize_body(body)
end

# File lib/url_fetch.rb, line 90
def normalize_body(body)
  text = body.to_s.dup
  text.force_encoding(Encoding::BINARY) unless text.encoding == Encoding::BINARY
  text
end

# File lib/url_fetch.rb, line 86
def usable_html?(html)
  !html.to_s.strip.empty? && !bot_block_page?(html)
end

# File lib/url_fetch.rb, line 49
def wayback_snapshot_url(url, lookup_fetch: nil)
  lookup_fetch ||= method(:default_lookup_fetch)
  api_url = "#{WAYBACK_API}?url=#{URI.encode_www_form_component(url.to_s)}"
  data = JSON.parse(lookup_fetch.call(api_url))
  snap = data.dig("archived_snapshots", "closest")
  return nil unless snap
  return nil unless snap["available"]
  return nil unless snap["status"].to_s == "200"

  snap["url"].to_s.strip
rescue StandardError
  nil
end