module EhcFetch
EhcFetch — passive HTTP GET with status and security header capture. Public: USER_AGENT, fetch, default_fetch, robots_url, extract_security_headers, extract_cookie_names, format_status_line Depends: (stdlib Net::HTTP) Tests: test/lib/ehc_fetch_test.rb
Constants
- SECURITY_HEADER_KEYS
- USER_AGENT
Public Instance Methods
Source
# File lib/ehc_fetch.rb, line 29 def default_fetch ->(url) { http_get(url) } end
Source
# File lib/ehc_fetch.rb, line 71 def extract_security_headers(headers) data = headers.is_a?(Hash) ? headers : {} lines = [] SECURITY_HEADER_KEYS.each do |key| value = data[key] next if value.to_s.strip.empty? label = key.tr("-", " ").split.map(&:capitalize).join(" ") lines << "#{label}: #{value}" end cookies = extract_cookie_names(data["set-cookie"]) lines << "Cookies: #{cookies.join(', ')}" if cookies.any? lines end
Source
# File lib/ehc_fetch.rb, line 25 def fetch(url, fetch: nil) (fetch || default_fetch).call(url) end
Source
# File lib/ehc_fetch.rb, line 95 def format_status_line(status:, content_type: nil) parts = ["HTTP #{status}"] type = content_type.to_s.split(";").first.to_s.strip parts << type unless type.empty? parts.join(" \u00b7 ") end
Source
# File lib/ehc_fetch.rb, line 40 def http_get(url, user_agent: USER_AGENT) uri = URI.parse(url.to_s) raise URI::InvalidURIError, "missing host" if uri.host.to_s.strip.empty? http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = (uri.scheme == "https") http.open_timeout = 10 http.read_timeout = 15 request = Net::HTTP::Get.new(uri) request["User-Agent"] = user_agent response = http.request(request) { status: response.code.to_i, headers: normalize_headers(response), body: response.body.to_s } rescue StandardError => e { error: e.message.to_s } end
Source
# File lib/ehc_fetch.rb, line 62 def normalize_headers(response) headers = {} response.each_header do |key, value| headers[key.downcase] = value end headers["set-cookie"] = response.get_fields("set-cookie") if response.get_fields("set-cookie") headers end
Source
# File lib/ehc_fetch.rb, line 33 def robots_url(page_url) uri = URI.parse(page_url.to_s) "#{uri.scheme}://#{uri.host}/robots.txt" rescue URI::InvalidURIError nil end