Integrate transcript captions into captions API

This commit is contained in:
syeopite 2023-07-23 05:02:02 -07:00
parent caac7e2166
commit e4942b188f
No known key found for this signature in database
GPG Key ID: A73C186DA3955A1A
5 changed files with 91 additions and 54 deletions

View File

@ -182,6 +182,19 @@ https_only: false
#force_resolve: #force_resolve:
##
## Use Innertube's transcripts API instead of timedtext for closed captions
##
## Useful for larger instances as InnerTube is **not ratelimited**. See https://github.com/iv-org/invidious/issues/2567
##
## Subtitle experience may differ slightly on Invidious.
##
## Accepted values: true, false
## Default: false
##
# use_innertube_for_captions: false
# ----------------------------- # -----------------------------
# Logging # Logging
# ----------------------------- # -----------------------------

View File

@ -129,6 +129,9 @@ class Config
# Use quic transport for youtube api # Use quic transport for youtube api
property use_quic : Bool = false property use_quic : Bool = false
# Use Innertube's transcripts API instead of timedtext for closed captions
property use_innertube_for_captions : Bool = false
# Saved cookies in "name1=value1; name2=value2..." format # Saved cookies in "name1=value1; name2=value2..." format
@[YAML::Field(converter: Preferences::StringToCookies)] @[YAML::Field(converter: Preferences::StringToCookies)]
property cookies : HTTP::Cookies = HTTP::Cookies.new property cookies : HTTP::Cookies = HTTP::Cookies.new

View File

@ -87,70 +87,78 @@ module Invidious::Routes::API::V1::Videos
caption = caption[0] caption = caption[0]
end end
url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target if CONFIG.use_innertube_for_captions
params = Invidious::Videos::Transcript.generate_param(id, caption.language_code, caption.auto_generated)
initial_data = YoutubeAPI.transcript(params.to_s)
# Auto-generated captions often have cues that aren't aligned properly with the video, webvtt = Invidious::Videos::Transcript.convert_transcripts_to_vtt(initial_data, caption.language_code)
# as well as some other markup that makes it cumbersome, so we try to fix that here else
if caption.name.includes? "auto-generated" # Timedtext API handling
caption_xml = YT_POOL.client &.get(url).body url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target
if caption_xml.starts_with?("<?xml") # Auto-generated captions often have cues that aren't aligned properly with the video,
webvtt = caption.timedtext_to_vtt(caption_xml, tlang) # as well as some other markup that makes it cumbersome, so we try to fix that here
else if caption.name.includes? "auto-generated"
caption_xml = XML.parse(caption_xml) caption_xml = YT_POOL.client &.get(url).body
webvtt = String.build do |str| if caption_xml.starts_with?("<?xml")
str << <<-END_VTT webvtt = caption.timedtext_to_vtt(caption_xml, tlang)
WEBVTT else
Kind: captions caption_xml = XML.parse(caption_xml)
Language: #{tlang || caption.language_code}
webvtt = String.build do |str|
str << <<-END_VTT
WEBVTT
Kind: captions
Language: #{tlang || caption.language_code}
END_VTT END_VTT
caption_nodes = caption_xml.xpath_nodes("//transcript/text") caption_nodes = caption_xml.xpath_nodes("//transcript/text")
caption_nodes.each_with_index do |node, i| caption_nodes.each_with_index do |node, i|
start_time = node["start"].to_f.seconds start_time = node["start"].to_f.seconds
duration = node["dur"]?.try &.to_f.seconds duration = node["dur"]?.try &.to_f.seconds
duration ||= start_time duration ||= start_time
if caption_nodes.size > i + 1 if caption_nodes.size > i + 1
end_time = caption_nodes[i + 1]["start"].to_f.seconds end_time = caption_nodes[i + 1]["start"].to_f.seconds
else else
end_time = start_time + duration end_time = start_time + duration
end
start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}"
end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}"
text = HTML.unescape(node.content)
text = text.gsub(/<font color="#[a-fA-F0-9]{6}">/, "")
text = text.gsub(/<\/font>/, "")
if md = text.match(/(?<name>.*) : (?<text>.*)/)
text = "<v #{md["name"]}>#{md["text"]}</v>"
end
str << <<-END_CUE
#{start_time} --> #{end_time}
#{text}
END_CUE
end end
start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}"
end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}"
text = HTML.unescape(node.content)
text = text.gsub(/<font color="#[a-fA-F0-9]{6}">/, "")
text = text.gsub(/<\/font>/, "")
if md = text.match(/(?<name>.*) : (?<text>.*)/)
text = "<v #{md["name"]}>#{md["text"]}</v>"
end
str << <<-END_CUE
#{start_time} --> #{end_time}
#{text}
END_CUE
end end
end end
end
else
# Some captions have "align:[start/end]" and "position:[num]%"
# attributes. Those are causing issues with VideoJS, which is unable
# to properly align the captions on the video, so we remove them.
#
# See: https://github.com/iv-org/invidious/issues/2391
webvtt = YT_POOL.client &.get("#{url}&format=vtt").body
if webvtt.starts_with?("<?xml")
webvtt = caption.timedtext_to_vtt(webvtt)
else else
# Some captions have "align:[start/end]" and "position:[num]%"
# attributes. Those are causing issues with VideoJS, which is unable
# to properly align the captions on the video, so we remove them.
#
# See: https://github.com/iv-org/invidious/issues/2391
webvtt = YT_POOL.client &.get("#{url}&format=vtt").body webvtt = YT_POOL.client &.get("#{url}&format=vtt").body
.gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1") if webvtt.starts_with?("<?xml")
webvtt = caption.timedtext_to_vtt(webvtt)
else
webvtt = YT_POOL.client &.get("#{url}&format=vtt").body
.gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1")
end
end end
end end

View File

@ -6,7 +6,9 @@ module Invidious::Videos
property language_code : String property language_code : String
property base_url : String property base_url : String
def initialize(@name, @language_code, @base_url) property auto_generated : Bool
def initialize(@name, @language_code, @base_url, @auto_generated)
end end
# Parse the JSON structure from Youtube # Parse the JSON structure from Youtube
@ -25,7 +27,12 @@ module Invidious::Videos
language_code = caption["languageCode"].to_s language_code = caption["languageCode"].to_s
base_url = caption["baseUrl"].to_s base_url = caption["baseUrl"].to_s
captions_list << CaptionMetadata.new(name, language_code, base_url) auto_generated = false
if caption["kind"]? && caption["kind"] == "asr"
auto_generated = true
end
captions_list << CaptionMetadata.new(name, language_code, base_url, auto_generated)
end end
return captions_list return captions_list

View File

@ -85,7 +85,13 @@ module Invidious::Videos
lines = [] of TranscriptLine lines = [] of TranscriptLine
body.each do |line| body.each do |line|
# Transcript section headers. They are not apart of the captions and as such we can safely skip them.
if line.as_h.has_key?("transcriptSectionHeaderRenderer")
next
end
line = line["transcriptSegmentRenderer"] line = line["transcriptSegmentRenderer"]
start_ms = line["startMs"].as_s.to_i.millisecond start_ms = line["startMs"].as_s.to_i.millisecond
end_ms = line["endMs"].as_s.to_i.millisecond end_ms = line["endMs"].as_s.to_i.millisecond