module EhcProbe
EhcProbe — extract CTF-relevant signals from HTML page bodies. Public: analyze, signal_lines, robots_signals Depends: UrlPreview, TextUtil Tests: test/lib/ehc_probe_test.rb
Constants
- COMMENT_LIMIT
- MAX_SIGNALS
- PATH_HINT_LIMIT
Public Instance Methods
Source
# File lib/ehc_probe.rb, line 21 def analyze(body, url:) html = body.to_s doc = Nokogiri::HTML(html) metadata = UrlPreview.parse_metadata(html) { url: url.to_s, title: metadata[:title].to_s.strip, description: metadata[:description].to_s.strip, forms: extract_forms(doc, base_url: url), comments: extract_comments(doc), path_hints: extract_path_hints(doc), external_hosts: extract_external_hosts(doc, base_url: url) } end
Source
# File lib/ehc_probe.rb, line 122 def comment_line(comment) return nil if comment.to_s.strip.empty? "HTML comment: #{comment}" end
Source
# File lib/ehc_probe.rb, line 132 def external_host_lines(hosts) Array(hosts).first(2).map { |host| "External host: #{host}" } end
Source
# File lib/ehc_probe.rb, line 85 def extract_comments(doc) doc.xpath("//comment()").map { |node| node.text.to_s.strip.gsub(/\s+/, " ") } .reject(&:empty?) .map { |text| TextUtil.truncate(text, COMMENT_LIMIT) } end
Source
# File lib/ehc_probe.rb, line 102 def extract_external_hosts(doc, base_url:) base_host = host_from_url(base_url) hosts = doc.css("script[src], link[href]").filter_map do |node| src = node["src"] || node["href"] next unless src.to_s.match?(%r{\Ahttps?://}i) host_from_url(src) end hosts.reject { |host| host.empty? || host.casecmp?(base_host) }.uniq end
Source
# File lib/ehc_probe.rb, line 70 def extract_forms(doc, base_url:) doc.css("form").filter_map do |form| method = form["method"].to_s.strip.upcase method = "GET" if method.empty? action = resolve_path(form["action"].to_s, base_url: base_url) inputs = form.css("input, textarea, select").filter_map do |node| name = node["name"].to_s.strip name unless name.empty? end.uniq next if inputs.empty? { method: method, action: action, inputs: inputs } end end
Source
# File lib/ehc_probe.rb, line 91 def extract_path_hints(doc) hints = [] doc.css("[href], [src]").each do |node| value = node["href"] || node["src"] next unless value.to_s.match?(%r{\A/[^/]}) hints << value.to_s.split(/[?#]/).first end hints.uniq.first(PATH_HINT_LIMIT) end
Source
# File lib/ehc_probe.rb, line 113 def form_line(form) return nil unless form inputs = Array(form[:inputs]).join(", ") hidden = Array(form[:inputs]).grep(/user|id|token|csrf/i) detail = hidden.any? ? " (#{hidden.join(', ')})" : "" "#{form[:method]} form #{form[:action]}#{detail}" end
Source
# File lib/ehc_probe.rb, line 150 def host_from_url(url) URI.parse(url.to_s).host.to_s.sub(/\Awww\./, "") rescue URI::InvalidURIError "" end
Source
# File lib/ehc_probe.rb, line 128 def path_hint_lines(hints) Array(hints).first(2).map { |path| "Path hint: #{path}" } end
Source
# File lib/ehc_probe.rb, line 136 def resolve_path(action, base_url:) action = action.to_s.strip return "/" if action.empty? if action.match?(%r{\Ahttps?://}i) path = URI.parse(action).path.to_s return path.empty? ? "/" : path end action.start_with?("/") ? action.split(/[?#]/).first : "/#{action.split(/[?#]/).first}" rescue URI::InvalidURIError action end
Source
# File lib/ehc_probe.rb, line 60 def robots_signals(body) text = body.to_s.strip return [] if text.empty? interesting = text.lines.map(&:strip).grep(/\A(?:Disallow|Allow):/i).first(3) return [] if interesting.empty? ["robots.txt: #{interesting.join(' \u00b7 ')}"] end
Source
# File lib/ehc_probe.rb, line 37 def signal_lines(probe, headers: [], robots_body: nil) header_list = Array(headers) lines = [] lines << header_list.first if header_list.any? title_line = title_signal_line(probe[:title]) lines << title_line if title_line form = form_line(probe[:forms].first) lines << form if form lines.concat(header_list.drop(1)) lines << comment_line(probe[:comments].first) if probe[:comments].any? lines.concat(path_hint_lines(probe[:path_hints])) lines.concat(robots_signals(robots_body)) lines.concat(external_host_lines(probe[:external_hosts])) lines.map(&:strip).reject(&:empty?).first(MAX_SIGNALS) end
Source
# File lib/ehc_probe.rb, line 53 def title_signal_line(title) text = title.to_s.strip return nil if text.empty? "Title: #{text}" end