Commit 97acd69c authored by Kourser's avatar Kourser
Browse files

fix(transcripts): prefer timed formats, and read word-by-word JSON as sentences

An episode may list several transcripts; the parser kept the first. On
Buzzsprout that is the HTML one, shown as a single block that does not
follow the playback, while a timed JSON sits right after it. The parser
now keeps a format whose lines carry times (Podcasting 2.0 JSON, WebVTT,
SubRip, by type or extension), the first among equals; HTML and plain
text only when nothing else is offered. A feed refresh rewrites the
episode, so stored ones switch too.

That JSON is timed word by word, though: 31,729 segments for one Podnews
episode, which would have made a line per word. Such transcripts (ten
segments or more, mostly single words) are now regrouped into sentences:
a new line at the end of a sentence, when the speaker changes, after a
two-second pause, or past about 280 characters. The Podnews episode reads
as 1,139 timed lines. Transcripts already cut into sentences are kept as
they are.

Co-Authored-By: default avatarClaude Opus 5.5 <noreply@anthropic.com>
parent 1fa68be3
Loading
Loading
Loading
Loading
+20 −2
Changes for Packages/Core/Sources/FeedKit/FeedParser.swift: 20 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -84,9 +84,15 @@ private final class Builder: NSObject, XMLParserDelegate {
        case "itunes:category" where !inItem:
            if let label = attrs["text"], !label.isEmpty { channelCategories.append(label) }
        case "podcast:transcript":
            if current?.transcriptURL == nil,
               let urlString = attrs["url"], let url = URL(string: urlString) {
            // An episode may offer several formats (Buzzsprout lists HTML
            // first): keep one whose lines carry times, so it follows the
            // playback; among equals, the first.
            if let item = current, let urlString = attrs["url"], let url = URL(string: urlString) {
                let rank = ItemAccumulator.transcriptRank(type: attrs["type"], url: url)
                if item.transcriptURL == nil || rank > item.transcriptRank {
                    current?.transcriptURL = url
                    current?.transcriptRank = rank
                }
            }
        case "podcast:chapters":
            if let urlString = attrs["url"], let url = URL(string: urlString) {
@@ -242,4 +248,16 @@ private struct ItemAccumulator {
    var seasonNumber: Int?
    var chaptersURL: URL?
    var transcriptURL: URL?
    var transcriptRank = 0

    /// 2 for timed lines (Podcasting 2.0 JSON, WebVTT, SubRip), 1 for a block
    /// of text (HTML, plain text, unknown).
    static func transcriptRank(type: String?, url: URL) -> Int {
        let type = type?.lowercased() ?? ""
        let timedTypes = ["json", "vtt", "srt", "subrip"]
        if timedTypes.contains(where: type.contains) || ["json", "vtt", "srt"].contains(url.pathExtension.lowercased()) {
            return 2
        }
        return 1
    }
}
+75
Changes for Packages/Core/Sources/FeedKit/Transcript.swift: 75 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -25,10 +25,12 @@ public enum TranscriptParser {
        let startTime: Double?
        let endTime: Double?
        let body: String?
        let speaker: String?
    }

    private static func parseJSON(_ data: Data) -> [TranscriptCue]? {
        guard let doc = try? JSONDecoder().decode(JSONDoc.self, from: data) else { return nil }
        if isWordByWord(doc.segments) { return sentences(from: doc.segments) }
        var cues: [TranscriptCue] = []
        for segment in doc.segments {
            let body = (segment.body ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
@@ -40,6 +42,79 @@ public enum TranscriptParser {
        return cues
    }

    // MARK: Word-by-word JSON

    /// Some hosts (Buzzsprout) time every word: a segment per word, and a
    /// line per word is unreadable. Such a transcript has thousands of
    /// segments; a handful of one-word lines is a short transcript, kept as is.
    private static func isWordByWord(_ segments: [Segment]) -> Bool {
        let words = segments.compactMap { $0.body?.trimmingCharacters(in: .whitespacesAndNewlines) }
            .filter { !$0.isEmpty }
        guard words.count >= 10 else { return false }
        let single = words.filter { !$0.contains(where: \.isWhitespace) }.count
        return Double(single) >= Double(words.count) * 0.8
    }

    /// Regroups timed words into sentences: a new line at the end of a
    /// sentence, when the speaker changes, after a pause, or past ~280
    /// characters.
    private static func sentences(from segments: [Segment]) -> [TranscriptCue] {
        // Spaces may come as segments of their own; otherwise words are joined.
        let spacedInline = segments.contains { segment in
            guard let body = segment.body, !body.isEmpty else { return false }
            return body.allSatisfy(\.isWhitespace)
        }
        var cues: [TranscriptCue] = []
        var text = ""
        var start: TimeInterval = 0
        var end: TimeInterval?
        var speaker: String?

        func flush() {
            let clean = text.trimmingCharacters(in: .whitespacesAndNewlines)
            if !clean.isEmpty {
                cues.append(TranscriptCue(id: cues.count, start: start, end: end, text: clean))
            }
            text = ""
        }

        for segment in segments {
            guard let body = segment.body, !body.isEmpty else { continue }
            let isBlank = body.allSatisfy(\.isWhitespace)
            if !text.isEmpty, !isBlank {
                let newSpeaker = segment.speaker != nil && segment.speaker != speaker
                let pause = (segment.startTime ?? 0) - (end ?? 0) > 2
                if newSpeaker || pause { flush() }
            }
            if text.isEmpty {
                if isBlank { continue }
                start = segment.startTime ?? end ?? 0
            }
            text = spacedInline ? text + body : joined(text, body)
            if let time = segment.endTime ?? segment.startTime { end = time }
            speaker = segment.speaker ?? speaker

            let line = text.trimmingCharacters(in: .whitespaces)
            // A short "Mr." or "Dr." does not end a line.
            if let last = line.last, ".?!…".contains(last), line.count >= 12 {
                flush()
            } else if line.count >= 280 {
                flush()
            }
        }
        flush()
        return cues
    }

    /// Appends a word, with a space unless punctuation follows.
    private static func joined(_ text: String, _ word: String) -> String {
        guard let last = text.last, let first = word.first else { return text + word }
        if last.isWhitespace || first.isWhitespace || ",.;:!?…)%".contains(first) {
            return text + word
        }
        return text + " " + word
    }

    // MARK: SRT / WebVTT

    private static func parseCueBlocks(_ text: String) -> [TranscriptCue] {
+35
Changes for Packages/Core/Tests/FeedKitTests/FeedParserTests.swift: 35 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -61,6 +61,41 @@ private func utcDate(_ y: Int, _ mo: Int, _ d: Int, _ h: Int, _ mi: Int, _ s: In
        #expect(ep.publicationDate == utcDate(2002, 10, 2, 13, 0, 0))
    }

    @Test func prefersATranscriptWithTimes() throws {
        let xml = """
        <?xml version="1.0" encoding="UTF-8"?>
        <rss version="2.0" xmlns:podcast="https://podcastindex.org/namespace/1.0">
        <channel><title>T</title>
        <item><title>A</title><guid>a</guid>
          <enclosure url="https://example.com/a.mp3" type="audio/mpeg"/>
          <podcast:transcript url="https://example.com/a/transcript" type="text/html"/>
          <podcast:transcript url="https://example.com/a/transcript.json" type="application/json"/>
        </item>
        <item><title>B</title><guid>b</guid>
          <enclosure url="https://example.com/b.mp3" type="audio/mpeg"/>
          <podcast:transcript url="https://example.com/b.html" type="text/html"/>
        </item>
        <item><title>C</title><guid>c</guid>
          <enclosure url="https://example.com/c.mp3" type="audio/mpeg"/>
          <podcast:transcript url="https://example.com/c.srt" type="application/srt"/>
          <podcast:transcript url="https://example.com/c.json" type="application/json"/>
        </item>
        <item><title>D</title><guid>d</guid>
          <enclosure url="https://example.com/d.mp3" type="audio/mpeg"/>
          <podcast:transcript url="https://example.com/d.txt"/>
          <podcast:transcript url="https://example.com/d.vtt"/>
        </item>
        </channel></rss>
        """
        let feed = try FeedParser.parse(data: Data(xml.utf8), feedURL: URL(string: "https://example.com/feed.xml")!)
        #expect(feed.episodes.map(\.transcriptURL) == [
            URL(string: "https://example.com/a/transcript.json"), // timed beats HTML listed first
            URL(string: "https://example.com/b.html"),            // HTML when it is all there is
            URL(string: "https://example.com/c.srt"),             // among timed formats, the first
            URL(string: "https://example.com/d.vtt"),             // no type: the extension tells
        ])
    }

    @Test func fallsBackToEnclosureURLForMissingGUID() throws {
        let feed = try loadSampleFeed()
        let ep = feed.episodes[1]
+64
Changes for Packages/Core/Tests/FeedKitTests/TranscriptParserTests.swift: 64 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -8,6 +8,70 @@ import PodcastModel
        TranscriptParser.parse(data: Data(string.utf8))
    }

    @Test func regroupsWordByWordJSONIntoSentences() {
        // Buzzsprout's shape: a segment per word, spaces as segments of their own.
        let json = """
        {"version": "1.0.0", "segments": [
          {"startTime": 0.1, "endTime": 0.4, "body": "Hello"},
          {"startTime": 0.4, "endTime": 0.4, "body": " "},
          {"startTime": 0.4, "endTime": 0.7, "body": "there,"},
          {"startTime": 0.7, "endTime": 0.7, "body": " "},
          {"startTime": 0.7, "endTime": 1.1, "body": "everyone."},
          {"startTime": 1.1, "endTime": 1.1, "body": " "},
          {"startTime": 1.2, "endTime": 1.5, "body": "Welcome"},
          {"startTime": 1.5, "endTime": 1.5, "body": " "},
          {"startTime": 1.5, "endTime": 1.8, "body": "back"},
          {"startTime": 1.8, "endTime": 1.8, "body": " "},
          {"startTime": 1.8, "endTime": 2.0, "body": "to"},
          {"startTime": 2.0, "endTime": 2.0, "body": " "},
          {"startTime": 2.0, "endTime": 2.2, "body": "the"},
          {"startTime": 2.2, "endTime": 2.2, "body": " "},
          {"startTime": 2.2, "endTime": 2.6, "body": "show"},
          {"startTime": 5.0, "endTime": 5.3, "body": "After"},
          {"startTime": 5.3, "endTime": 5.3, "body": " "},
          {"startTime": 5.3, "endTime": 5.6, "body": "a"},
          {"startTime": 5.6, "endTime": 5.6, "body": " "},
          {"startTime": 5.6, "endTime": 6.0, "body": "pause"}
        ]}
        """
        let cues = parse(json)
        #expect(cues.map(\.text) == ["Hello there, everyone.", "Welcome back to the show", "After a pause"])
        #expect(cues.map(\.start) == [0.1, 1.2, 5.0])
        #expect(cues[0].end == 1.1)
    }

    @Test func joinsWordsWithoutSpaceSegments() {
        let json = """
        {"segments": [
          {"startTime": 0, "body": "Bonjour", "speaker": "A"},
          {"startTime": 0.5, "body": "à", "speaker": "A"},
          {"startTime": 0.8, "body": "toutes", "speaker": "A"},
          {"startTime": 1.1, "body": "et", "speaker": "A"},
          {"startTime": 1.2, "body": "à", "speaker": "A"},
          {"startTime": 1.3, "body": "tous", "speaker": "A"},
          {"startTime": 1.6, "body": ",", "speaker": "A"},
          {"startTime": 1.8, "body": "merci", "speaker": "B"},
          {"startTime": 2.0, "body": "de", "speaker": "B"},
          {"startTime": 2.1, "body": "votre", "speaker": "B"},
          {"startTime": 2.4, "body": "écoute", "speaker": "B"},
          {"startTime": 2.8, "body": "!", "speaker": "B"}
        ]}
        """
        let cues = parse(json)
        // A new speaker starts a new line, even mid-sentence.
        #expect(cues.map(\.text) == ["Bonjour à toutes et à tous,", "merci de votre écoute!"])
    }

    @Test func keepsSentenceLevelJSONAsItIs() {
        let json = """
        {"segments": [
          {"startTime": 0, "body": "Première phrase complète."},
          {"startTime": 3, "body": "Une seconde, sans point final"}
        ]}
        """
        #expect(parse(json).map(\.text) == ["Première phrase complète.", "Une seconde, sans point final"])
    }

    @Test func parsesSRT() {
        let srt = """
        1