module CharacterTranscripts
Fetches fandom.com character transcripts once a bot is identified by its fandom source link, splits large transcripts into sorted numbered files, and loads them back as plain context for bots. Pass a fetch: lambda in tests so no live HTTP runs.
Constants
- DEFAULT_CHUNK_BYTES
- DEFAULT_ROOT
- FANDOM_HOST_SUFFIX
- PERSONALITIES_DIR
- Result
- TRANSCRIPT_SUFFIX
- USER_AGENT
Public Instance Methods
Source
# File lib/character_transcripts.rb, line 71 def api_url_for(transcript_url) uri = URI.parse(transcript_url) title = page_title_from_url(transcript_url) uri.path = "/api.php" uri.query = URI.encode_www_form( action: "query", prop: "extracts", explaintext: "1", redirects: "1", titles: title, format: "json", utf8: "1" ) uri.to_s end
Source
# File lib/character_transcripts.rb, line 138 def character_dir(slug_name, root: DEFAULT_ROOT) File.join(root, slug_name) end
Source
# File lib/character_transcripts.rb, line 54 def character_name(url) page_title_from_url(url).sub(%r{/#{TRANSCRIPT_SUFFIX}\z}i, "").strip end
Source
# File lib/character_transcripts.rb, line 126 def chunk_filename(slug_name, index) format("%s_%03d.txt", slug_name, index) end
Source
# File lib/character_transcripts.rb, line 142 def chunk_files(slug_name, root: DEFAULT_ROOT) sort_chunks(Dir[File.join(character_dir(slug_name, root: root), "#{slug_name}_*.txt")]) end
Source
# File lib/character_transcripts.rb, line 130 def chunk_index(path) File.basename(path.to_s)[/_(\d+)\.txt\z/, 1].to_i end
Source
# File lib/character_transcripts.rb, line 87 def clean_text(text) text.to_s .gsub(/\r\n?/, "\n") .gsub(/[ \t]+/, " ") .gsub(/ *\n */, "\n") .gsub(/\n{3,}/, "\n\n") .strip end
Source
# File lib/character_transcripts.rb, line 212 def default_fetch(url) RabbotHttp.fetch(url, user_agent: USER_AGENT) end
Source
# File lib/character_transcripts.rb, line 188 def download(target, fetch: method(:default_fetch), root: DEFAULT_ROOT, max_bytes: DEFAULT_CHUNK_BYTES, profiles_dir: PERSONALITIES_DIR) source_url = resolve_source_url(target, profiles_dir: profiles_dir) return Result.new(ok: false, error: "No fandom.com link for #{target}") unless source_url transcript_url = transcript_url_for(source_url) text = fetch_extract(transcript_url, fetch: fetch) if text.strip.empty? return Result.new( ok: false, source_url: source_url, transcript_url: transcript_url, error: "No transcript found at #{transcript_url}" ) end character = character_name(source_url) chunks = split_transcript(text, max_bytes: max_bytes) paths = write_chunks(slug(character), chunks, root: root) Result.new( ok: true, character: character, source_url: source_url, transcript_url: transcript_url, chunk_paths: paths, bytes: text.bytesize ) end
Source
# File lib/character_transcripts.rb, line 37 def fandom_url?(url) uri = URI.parse(url.to_s.strip) return false unless uri.is_a?(URI::HTTP) host = uri.host.to_s.downcase host == "fandom.com" || host.end_with?(FANDOM_HOST_SUFFIX) rescue URI::InvalidURIError false end
Source
# File lib/character_transcripts.rb, line 96 def fetch_extract(transcript_url, fetch: method(:default_fetch)) body = fetch.call(api_url_for(transcript_url)) data = JSON.parse(body) page = data.dig("query", "pages")&.values&.find { |candidate| candidate["missing"].nil? } return "" unless page clean_text(page["extract"]) rescue JSON::ParserError, StandardError "" end
Source
# File lib/character_transcripts.rb, line 159 def load_context(slug_name, root: DEFAULT_ROOT, max_chunks: nil) files = chunk_files(slug_name, root: root) files = files.first(max_chunks) if max_chunks files.map { |path| File.read(path).strip }.reject(&:empty?).join("\n\n") end
Source
# File lib/character_transcripts.rb, line 47 def page_title_from_url(url) path = URI.parse(url.to_s.strip).path.to_s.sub(%r{\A/wiki/}, "").sub(%r{\A/+}, "") CGI.unescape(path).tr("_", " ").strip rescue URI::InvalidURIError "" end
Source
# File lib/character_transcripts.rb, line 165 def profile_source_url(name, profiles_dir: PERSONALITIES_DIR) slugged = slug(name) candidates = [ File.join(profiles_dir, "#{slugged}.yml"), File.join(profiles_dir, "#{name.to_s.strip.downcase}.yml") ] path = candidates.find { |candidate| File.file?(candidate) } return nil unless path data = YAML.safe_load_file(path, aliases: false) data.is_a?(Hash) ? data["source_url"].to_s.strip : nil rescue StandardError nil end
Source
# File lib/character_transcripts.rb, line 180 def resolve_source_url(target, profiles_dir: PERSONALITIES_DIR) text = target.to_s.strip return text if fandom_url?(text) url = profile_source_url(text, profiles_dir: profiles_dir) fandom_url?(url) ? url : nil end
Source
# File lib/character_transcripts.rb, line 67 def slug(value) value.to_s.strip.downcase.gsub(/[^a-z0-9]+/, "_").gsub(/\A_+|_+\z/, "") end
Source
# File lib/character_transcripts.rb, line 134 def sort_chunks(paths) Array(paths).sort_by { |path| [chunk_index(path), File.basename(path.to_s)] } end
Source
# File lib/character_transcripts.rb, line 107 def split_transcript(text, max_bytes: DEFAULT_CHUNK_BYTES) lines = clean_text(text).split("\n") chunks = [] current = +"" lines.each do |line| candidate = current.empty? ? line : "#{current}\n#{line}" if candidate.bytesize > max_bytes && !current.empty? chunks << current current = line.dup else current = candidate end end chunks << current unless current.empty? chunks end
Source
# File lib/character_transcripts.rb, line 58 def transcript_url_for(url) uri = URI.parse(url.to_s.strip) base = uri.path.to_s.sub(%r{/+\z}, "") return uri.to_s if base.downcase.end_with?("/#{TRANSCRIPT_SUFFIX.downcase}") uri.path = "#{base}/#{TRANSCRIPT_SUFFIX}" uri.to_s end
Source
# File lib/character_transcripts.rb, line 146 def write_chunks(slug_name, chunks, root: DEFAULT_ROOT) dir = character_dir(slug_name, root: root) FileUtils.mkdir_p(dir) Dir[File.join(dir, "#{slug_name}_*.txt")].each { |path| File.delete(path) } paths = Array(chunks).each_with_index.map do |chunk, index| path = File.join(dir, chunk_filename(slug_name, index + 1)) File.write(path, "#{chunk}\n") path end sort_chunks(paths) end