From bb7dda3f1ae8f865a3d36ec697f57a300d63158a Mon Sep 17 00:00:00 2001 From: Isaac <> Date: Mon, 14 Oct 2024 18:44:21 +0400 Subject: [PATCH 1/3] Temp API (cherry picked from commit 7a2629b8fe04645b8d4ce855bd6afe6d3ce989eb) --- submodules/TelegramApi/Sources/Api0.swift | 2 +- submodules/TelegramApi/Sources/Api23.swift | 22 +++++++++++++------ .../Sources/State/Serialization.swift | 2 +- .../TelegramEngine/Payments/StarGifts.swift | 2 +- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/submodules/TelegramApi/Sources/Api0.swift b/submodules/TelegramApi/Sources/Api0.swift index aa22fb967be..32e60d1664b 100644 --- a/submodules/TelegramApi/Sources/Api0.swift +++ b/submodules/TelegramApi/Sources/Api0.swift @@ -894,7 +894,7 @@ fileprivate let parsers: [Int32 : (BufferReader) -> Any?] = { dict[-425595208] = { return Api.SmsJob.parse_smsJob($0) } dict[1301522832] = { return Api.SponsoredMessage.parse_sponsoredMessage($0) } dict[1124938064] = { return Api.SponsoredMessageReportOption.parse_sponsoredMessageReportOption($0) } - dict[-1365150482] = { return Api.StarGift.parse_starGift($0) } + dict[1237678029] = { return Api.StarGift.parse_starGift($0) } dict[1577421297] = { return Api.StarsGiftOption.parse_starsGiftOption($0) } dict[-1798404822] = { return Api.StarsGiveawayOption.parse_starsGiveawayOption($0) } dict[1411605001] = { return Api.StarsGiveawayWinnersOption.parse_starsGiveawayWinnersOption($0) } diff --git a/submodules/TelegramApi/Sources/Api23.swift b/submodules/TelegramApi/Sources/Api23.swift index a89148bf05f..00f070a3eff 100644 --- a/submodules/TelegramApi/Sources/Api23.swift +++ b/submodules/TelegramApi/Sources/Api23.swift @@ -574,13 +574,13 @@ public extension Api { } public extension Api { enum StarGift: TypeConstructorDescription { - case starGift(flags: Int32, id: Int64, sticker: Api.Document, stars: Int64, availabilityRemains: Int32?, availabilityTotal: Int32?, convertStars: Int64) + case starGift(flags: Int32, id: Int64, sticker: Api.Document, stars: Int64, availabilityRemains: Int32?, availabilityTotal: Int32?, convertStars: Int64, firstSaleDate: Int32?, lastSaleDate: Int32?) public func serialize(_ buffer: Buffer, _ boxed: Swift.Bool) { switch self { - case .starGift(let flags, let id, let sticker, let stars, let availabilityRemains, let availabilityTotal, let convertStars): + case .starGift(let flags, let id, let sticker, let stars, let availabilityRemains, let availabilityTotal, let convertStars, let firstSaleDate, let lastSaleDate): if boxed { - buffer.appendInt32(-1365150482) + buffer.appendInt32(1237678029) } serializeInt32(flags, buffer: buffer, boxed: false) serializeInt64(id, buffer: buffer, boxed: false) @@ -589,14 +589,16 @@ public extension Api { if Int(flags) & Int(1 << 0) != 0 {serializeInt32(availabilityRemains!, buffer: buffer, boxed: false)} if Int(flags) & Int(1 << 0) != 0 {serializeInt32(availabilityTotal!, buffer: buffer, boxed: false)} serializeInt64(convertStars, buffer: buffer, boxed: false) + if Int(flags) & Int(1 << 1) != 0 {serializeInt32(firstSaleDate!, buffer: buffer, boxed: false)} + if Int(flags) & Int(1 << 1) != 0 {serializeInt32(lastSaleDate!, buffer: buffer, boxed: false)} break } } public func descriptionFields() -> (String, [(String, Any)]) { switch self { - case .starGift(let flags, let id, let sticker, let stars, let availabilityRemains, let availabilityTotal, let convertStars): - return ("starGift", [("flags", flags as Any), ("id", id as Any), ("sticker", sticker as Any), ("stars", stars as Any), ("availabilityRemains", availabilityRemains as Any), ("availabilityTotal", availabilityTotal as Any), ("convertStars", convertStars as Any)]) + case .starGift(let flags, let id, let sticker, let stars, let availabilityRemains, let availabilityTotal, let convertStars, let firstSaleDate, let lastSaleDate): + return ("starGift", [("flags", flags as Any), ("id", id as Any), ("sticker", sticker as Any), ("stars", stars as Any), ("availabilityRemains", availabilityRemains as Any), ("availabilityTotal", availabilityTotal as Any), ("convertStars", convertStars as Any), ("firstSaleDate", firstSaleDate as Any), ("lastSaleDate", lastSaleDate as Any)]) } } @@ -617,6 +619,10 @@ public extension Api { if Int(_1!) & Int(1 << 0) != 0 {_6 = reader.readInt32() } var _7: Int64? _7 = reader.readInt64() + var _8: Int32? + if Int(_1!) & Int(1 << 1) != 0 {_8 = reader.readInt32() } + var _9: Int32? + if Int(_1!) & Int(1 << 1) != 0 {_9 = reader.readInt32() } let _c1 = _1 != nil let _c2 = _2 != nil let _c3 = _3 != nil @@ -624,8 +630,10 @@ public extension Api { let _c5 = (Int(_1!) & Int(1 << 0) == 0) || _5 != nil let _c6 = (Int(_1!) & Int(1 << 0) == 0) || _6 != nil let _c7 = _7 != nil - if _c1 && _c2 && _c3 && _c4 && _c5 && _c6 && _c7 { - return Api.StarGift.starGift(flags: _1!, id: _2!, sticker: _3!, stars: _4!, availabilityRemains: _5, availabilityTotal: _6, convertStars: _7!) + let _c8 = (Int(_1!) & Int(1 << 1) == 0) || _8 != nil + let _c9 = (Int(_1!) & Int(1 << 1) == 0) || _9 != nil + if _c1 && _c2 && _c3 && _c4 && _c5 && _c6 && _c7 && _c8 && _c9 { + return Api.StarGift.starGift(flags: _1!, id: _2!, sticker: _3!, stars: _4!, availabilityRemains: _5, availabilityTotal: _6, convertStars: _7!, firstSaleDate: _8, lastSaleDate: _9) } else { return nil diff --git a/submodules/TelegramCore/Sources/State/Serialization.swift b/submodules/TelegramCore/Sources/State/Serialization.swift index 12c50ddd302..dd47bffef63 100644 --- a/submodules/TelegramCore/Sources/State/Serialization.swift +++ b/submodules/TelegramCore/Sources/State/Serialization.swift @@ -210,7 +210,7 @@ public class BoxedMessage: NSObject { public class Serialization: NSObject, MTSerialization { public func currentLayer() -> UInt { - return 189 + return 191 } public func parseMessage(_ data: Data!) -> Any! { diff --git a/submodules/TelegramCore/Sources/TelegramEngine/Payments/StarGifts.swift b/submodules/TelegramCore/Sources/TelegramEngine/Payments/StarGifts.swift index 1aaffb2ea51..5445368f7c6 100644 --- a/submodules/TelegramCore/Sources/TelegramEngine/Payments/StarGifts.swift +++ b/submodules/TelegramCore/Sources/TelegramEngine/Payments/StarGifts.swift @@ -132,7 +132,7 @@ public struct StarGift: Equatable, Codable, PostboxCoding { extension StarGift { init?(apiStarGift: Api.StarGift) { switch apiStarGift { - case let .starGift(_, id, sticker, stars, availabilityRemains, availabilityTotal, convertStars): + case let .starGift(_, id, sticker, stars, availabilityRemains, availabilityTotal, convertStars, _, _): var availability: Availability? if let availabilityRemains, let availabilityTotal { availability = Availability(remains: availabilityRemains, total: availabilityTotal) From d1028fdcc7c7d09e4f93941cb8c50678df133b00 Mon Sep 17 00:00:00 2001 From: Denis Barinov Date: Tue, 15 Oct 2024 00:40:51 +0500 Subject: [PATCH 2/3] Build --- .../PublicHeaders/AsyncDisplayKit/ASRecursiveUnfairLock.h | 2 +- submodules/Telegram | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 120000 submodules/Telegram diff --git a/submodules/AsyncDisplayKit/Source/PublicHeaders/AsyncDisplayKit/ASRecursiveUnfairLock.h b/submodules/AsyncDisplayKit/Source/PublicHeaders/AsyncDisplayKit/ASRecursiveUnfairLock.h index cc1b9100d24..57646909c2e 100644 --- a/submodules/AsyncDisplayKit/Source/PublicHeaders/AsyncDisplayKit/ASRecursiveUnfairLock.h +++ b/submodules/AsyncDisplayKit/Source/PublicHeaders/AsyncDisplayKit/ASRecursiveUnfairLock.h @@ -11,7 +11,7 @@ #import #import -#if defined(__aarch64__) +#if defined(__aarch64__) || __IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_10_0 #define AS_USE_OS_LOCK true #else #define AS_USE_OS_LOCK false diff --git a/submodules/Telegram b/submodules/Telegram new file mode 120000 index 00000000000..a3126780182 --- /dev/null +++ b/submodules/Telegram @@ -0,0 +1 @@ +/Users/denis/Desktop/Telegram-iOS/Telegram \ No newline at end of file From 2e474667724c8ce18ba04729b15aac43950659d0 Mon Sep 17 00:00:00 2001 From: Denis Barinov Date: Mon, 28 Oct 2024 00:29:34 +0500 Subject: [PATCH 3/3] Create HLS Player --- submodules/HLSPlayer/BUILD | 18 ++ submodules/HLSPlayer/Sources/HLSPlayer.swift | 284 ++++++++++++++++++ .../Sources/HLSPlayerAudioOutput.swift | 135 +++++++++ .../HLSPlayer/Sources/HLSPlayerItem.swift | 191 ++++++++++++ .../HLSPlayer/Sources/HLSPlayerLayer.swift | 190 ++++++++++++ .../Sources/HLSPlayerPlaylistEntity.swift | 16 + .../Sources/HLSPlayerStreamEntity.swift | 9 + .../Sources/HLSPlayerVideoOutput.swift | 18 ++ .../TelegramUniversalVideoContent/BUILD | 1 + .../Sources/HLSVideoContent.swift | 73 ++--- 10 files changed, 882 insertions(+), 53 deletions(-) create mode 100644 submodules/HLSPlayer/BUILD create mode 100644 submodules/HLSPlayer/Sources/HLSPlayer.swift create mode 100644 submodules/HLSPlayer/Sources/HLSPlayerAudioOutput.swift create mode 100644 submodules/HLSPlayer/Sources/HLSPlayerItem.swift create mode 100644 submodules/HLSPlayer/Sources/HLSPlayerLayer.swift create mode 100644 submodules/HLSPlayer/Sources/HLSPlayerPlaylistEntity.swift create mode 100644 submodules/HLSPlayer/Sources/HLSPlayerStreamEntity.swift create mode 100644 submodules/HLSPlayer/Sources/HLSPlayerVideoOutput.swift diff --git a/submodules/HLSPlayer/BUILD b/submodules/HLSPlayer/BUILD new file mode 100644 index 00000000000..2cd5417be1a --- /dev/null +++ b/submodules/HLSPlayer/BUILD @@ -0,0 +1,18 @@ +load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") + +swift_library( + name = "HLSPlayer", + module_name = "HLSPlayer", + srcs = glob([ + "Sources/**/*.swift", + ]), + copts = [ + "-warnings-as-errors", + ], + deps = [ + "//submodules/TelegramCore:TelegramCore", + ], + visibility = [ + "//visibility:public", + ], +) diff --git a/submodules/HLSPlayer/Sources/HLSPlayer.swift b/submodules/HLSPlayer/Sources/HLSPlayer.swift new file mode 100644 index 00000000000..279e54594ff --- /dev/null +++ b/submodules/HLSPlayer/Sources/HLSPlayer.swift @@ -0,0 +1,284 @@ +import AVFoundation +import TelegramCore + +public final class HLSPlayer: NSObject { + + protocol LayerDelegate { + func play() + func pause() + func stop() + func rendering(at: CMSampleBuffer) -> Bool + } + + public enum ActionAtItemEnd { + case pause + case replay + } + + public var volume: Double { + get { + audioOutput.volume + } + set { + audioOutput.volume(at: newValue) + } + } + + public var rate: Float { + get { + currentItem?.rate ?? 0.0 + } + set { + currentItem?.rate = newValue + } + } + + public var actionAtItemEnd: ActionAtItemEnd = .pause + + var layerDelegate: LayerDelegate? + + public private(set) var currentItem: HLSPlayerItem? + + private let queue: DispatchQueue + private let queueAudio: DispatchQueue + private let queueVideo: DispatchQueue + private let fileManager: FileManager + private var tempFileURL: URL? + private var reader: AVAssetReader? + private var audioOutput: HLSPlayerAudioOutput + private var videoOutput: HLSPlayerVideoOutput + + private var currentSegmentNumber = -1 + private var currenTimeSeconds = 0.0 + private var currentBandwidth = 0.0 + private var mastersData: [Int: Data] = [:] + + public override init() { + queue = DispatchQueue( + label: "HLSPlayerQueue", + qos: .userInitiated, + attributes: .concurrent) + queueAudio = DispatchQueue( + label: "HLSPlayerAudioQueue", + qos: .userInitiated) + queueVideo = DispatchQueue( + label: "HLSPlayerVideoQueue", + qos: .userInitiated) + fileManager = FileManager.default + audioOutput = HLSPlayerAudioOutput() + videoOutput = HLSPlayerVideoOutput() + } + + deinit { + if let url = tempFileURL { + try? fileManager.removeItem(at: url) + } + } + + public func play() { + DispatchQueue.main.async { + self.audioOutput.play() + self.layerDelegate?.play() + } + } + + public func pause() { + DispatchQueue.main.async { + self.audioOutput.pause() + self.layerDelegate?.pause() + } + } + + public func stiop() { + DispatchQueue.main.async { + self.audioOutput.stop() + self.layerDelegate?.stop() + } + } + + public func replaceCurrent(item: HLSPlayerItem?) { + self.currentItem = item + if item != nil { + currentItem?.fetch { [weak self] in + self?.loadSegment() + } + } + } + + public func seek(to: CMTime) { + + } + + public func currentTime() -> CMTime { + CMTime(seconds: currenTimeSeconds, preferredTimescale: 30) + } +} + +// MARK: - Private + +private extension HLSPlayer { + + func loadSegment() { + guard let item = currentItem else { + Logger.shared.log("HLSPlayer", "Error item for load segment") + return + } + let bitRate = currentBitRate(at: item) + guard let stream = item.streams[bitRate] else { + Logger.shared.log("HLSPlayer", "Error item for load segment") + return + } + + if currentSegmentNumber < 0 { + currentSegmentNumber = stream.playlist.sequence + } + + let segmentNumber = mastersData[bitRate] == nil ? 0 : currentSegmentNumber + let isEndItem = (segmentNumber + 1) == stream.playlist.segments.count + guard segmentNumber < stream.playlist.segments.count else { + Logger.shared.log("HLSPlayer", "Error the number exceeds of segments") + return + } + + let segment = stream.playlist.segments[segmentNumber] + guard let segmentURL = URL(string: "\(item.baseURL)\(segment.uri)") else { + Logger.shared.log("HLSPlayer", "Error segment URL") + return + } + + var request = URLRequest(url: segmentURL, cachePolicy: .reloadIgnoringLocalCacheData) + request.setValue("bytes=\(segment.byteRange.offset)-\(segment.byteRange.offset + segment.byteRange.length - 1)", forHTTPHeaderField: "Range") + + queue.async { + let startLoad = CACurrentMediaTime() + let task = URLSession.shared.dataTask(with: request) { [weak self] data, responce, error in + if let data = data { + let endLoad = CACurrentMediaTime() + self?.currentBandwidth = stream.bandWidth / (endLoad - startLoad) * 8 + if let masterData = self?.mastersData[bitRate] { + let fileURLWithPath = NSTemporaryDirectory().appending(segment.uri) + let tempFileURL = URL(fileURLWithPath: fileURLWithPath) + self?.tempFileURL = tempFileURL + try? (masterData + data).write(to: tempFileURL) + self?.currentItem?.presentationSize = stream.resolution + self?.readingData(at: tempFileURL, isEndItem: isEndItem) + } else { + self?.mastersData[bitRate] = data + self?.loadSegment() + } + } else if let error = error { + Logger.shared.log("HLSPlayer", "Error loading segment \(error)") + } + } + task.resume() + } + } + + func currentBitRate(at item: HLSPlayerItem) -> Int { + if item.preferredPeakBitRate == 0.0 { + var bitRate = 0 + if currentBandwidth == 0 { + if bitRate == 0 { + item.streams.forEach { (key, value) in + if key > bitRate { + bitRate = key + } + } + } + } else { + item.streams.forEach { (key, value) in + if Double(key) < currentBandwidth && key > bitRate { + bitRate = key + } + } + if bitRate == 0 { + item.streams.forEach { (key, value) in + if bitRate > key || bitRate == 0 { + bitRate = key + } + } + } + } + return bitRate + } else { + return Int(item.preferredPeakBitRate) + } + } + + func getReader(at asset: AVURLAsset) -> AVAssetReader? { + if let reader = reader { + return reader + } + guard let reader = try? AVAssetReader(asset: asset) else { + return nil + } + return reader + } + + func readingData(at tempURL: URL, isEndItem: Bool) { + currentSegmentNumber += 1 + + let asset = AVURLAsset(url: tempURL) + guard let reader = getReader(at: asset) else { + try? fileManager.removeItem(at: tempURL) + Logger.shared.log("HLSPlayer", "Error create AVassetReader") + return + } + + if let audioOutput = audioOutput.track(at: asset) { + if reader.canAdd(audioOutput) { + reader.add(audioOutput) + } + } + + if let videoOutput = videoOutput.track(at: asset) { + if reader.canAdd(videoOutput) { + reader.add(videoOutput) + } + } + + reader.startReading() + + let group = DispatchGroup() + reader.outputs.forEach { output in + switch output.mediaType { + case .audio: +// queueAudio.async { [weak self] in + group.enter() +// while let sampleBuffer = output.copyNextSampleBuffer() { +// if self?.audioOutput.rendering(at: sampleBuffer) != true { +// break +// } +// } + group.leave() +// } + case .video: + queueVideo.async { [weak self] in + group.enter() + while let sampleBuffer = output.copyNextSampleBuffer() { + if self?.layerDelegate?.rendering(at: sampleBuffer) != true { + break + } + } + group.leave() + } + default: + return + } + } + + group.wait() + try? fileManager.removeItem(at: tempURL) + if isEndItem { + currentSegmentNumber = -1 + switch actionAtItemEnd { + case .pause: + pause() + case .replay: + play() + } + NotificationCenter.default.post(name: Notification.Name("HLSPlayeItemDidPlayToEndTime"), object: nil) + loadSegment() + } + } +} diff --git a/submodules/HLSPlayer/Sources/HLSPlayerAudioOutput.swift b/submodules/HLSPlayer/Sources/HLSPlayerAudioOutput.swift new file mode 100644 index 00000000000..d2e110e7ca3 --- /dev/null +++ b/submodules/HLSPlayer/Sources/HLSPlayerAudioOutput.swift @@ -0,0 +1,135 @@ +import AudioToolbox +import AVFAudio +import AVFoundation +import CoreMedia +import TelegramCore + +final class HLSPlayerAudioOutput { + + private(set) var volume: Double = 1.0 + + private var audioQueue: AudioQueueRef? + private var audioQueueBuffer: AudioQueueBufferRef? + private var streamFormat: AudioStreamBasicDescription + + init() { + streamFormat = AudioStreamBasicDescription() + + congigure() + } + + deinit { + if let audioQueue = audioQueue { + AudioQueueStop(audioQueue, false) + } + audioQueueBuffer?.deallocate() + } + + func play() { + guard let audioQueue = audioQueue else { return } + AudioQueueStart(audioQueue, nil) + } + + func pause() { + guard let audioQueue = audioQueue else { return } + AudioQueueStop(audioQueue, true) + } + + func stop() { + if let audioQueue = audioQueue { + AudioQueueStop(audioQueue, false) + } + audioQueueBuffer?.deallocate() + } + + func volume(at newValue: Double) { + guard let audioQueue = audioQueue else { return } + let errorSetVolume = AudioQueueSetParameter(audioQueue, kAudioQueueParam_Volume, Float32(newValue)) + if errorSetVolume != noErr { + Logger.shared.log("HLSPlayer", "Error AudioQueueSetParameterVolume \(errorSetVolume)") + return + } + self.volume = newValue + } + + func rendering(at sampleBuffer: CMSampleBuffer) -> Bool { + var bufferListSize = 0 + let errorBlockBufferInit = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sampleBuffer, bufferListSizeNeededOut: &bufferListSize, bufferListOut: nil, bufferListSize: 0, blockBufferAllocator: nil, blockBufferMemoryAllocator: nil, flags: kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment, blockBufferOut: nil) + if errorBlockBufferInit != noErr { + Logger.shared.log("HLSPlayer", "Error AudioBufferListWithRetainedBlockBuffer \(errorBlockBufferInit)") + return false + } + + var blockBuffer: CMBlockBuffer? + let mData = UnsafeMutablePointer.allocate(capacity: 1024) + var bufferList = AudioBufferList(mNumberBuffers: UInt32(bufferListSize), mBuffers: AudioBuffer(mNumberChannels: 2, mDataByteSize: 1024, mData: mData)) + let errorBlockBuffer = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sampleBuffer, bufferListSizeNeededOut: &bufferListSize, bufferListOut: &bufferList, bufferListSize: bufferListSize, blockBufferAllocator: kCFAllocatorDefault, blockBufferMemoryAllocator: kCFAllocatorDefault, flags: kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment, blockBufferOut: &blockBuffer) + if errorBlockBuffer != noErr { + Logger.shared.log("HLSPlayer", "Error AudioBufferListWithRetainedBlockBuffer \(errorBlockBuffer)") + return false + } + + guard let audioQueueBuffer = audioQueueBuffer, let audioQueue = audioQueue else { return false } + + if nil == memcpy(self.audioQueueBuffer?.pointee.mAudioData, bufferList.mBuffers.mData, Int(bufferList.mBuffers.mDataByteSize)) { + Logger.shared.log("HLSPlayer", "Error memcpy audioQueueBuffer") + } + + let errorEnqueue = AudioQueueEnqueueBuffer(audioQueue, audioQueueBuffer, 0, nil) + if errorEnqueue != noErr { + Logger.shared.log("HLSPlayer", "Error AudioQueueEnqueueBuffer \(errorEnqueue)") + return false + } + return true + } + + func track(at asset: AVAsset) -> AVAssetReaderTrackOutput? { + guard let track = asset.tracks(withMediaType: .audio).first else { + return nil + } + let outputSettings: [String: Any] = [ + AVFormatIDKey: kAudioFormatLinearPCM, + AVSampleRateKey: 48000.0, + AVNumberOfChannelsKey: 2, + AVLinearPCMBitDepthKey: 16, + AVLinearPCMIsFloatKey: false, + AVLinearPCMIsBigEndianKey: false, + ] + let output = AVAssetReaderTrackOutput(track: track, outputSettings: outputSettings) + output.alwaysCopiesSampleData = false + return output + } +} + +// MARK: - Configuration + +private extension HLSPlayerAudioOutput { + + func congigure() { + streamFormat.mSampleRate = 48000.0 + streamFormat.mFormatID = kAudioFormatLinearPCM + streamFormat.mFormatFlags = kAudioFormatFlagIsNonInterleaved | kAudioFormatFlagIsSignedInteger + streamFormat.mBytesPerPacket = 4 + streamFormat.mFramesPerPacket = 1 + streamFormat.mBytesPerFrame = 4 + streamFormat.mChannelsPerFrame = 2 + streamFormat.mBitsPerChannel = 16 + streamFormat.mReserved = 0 + + congigureAudioQueue() + } + + func congigureAudioQueue() { + let errorNewOutput = AudioQueueNewOutput(&streamFormat, { _, _, _ in }, nil, nil, nil, 0, &audioQueue) + if errorNewOutput != noErr { + Logger.shared.log("HLSPlayer", "Error AudioQueueNewOutput \(errorNewOutput)") + } + + guard let audioQueue = audioQueue else { return } + + let errorAllocate = AudioQueueAllocateBuffer(audioQueue, 4096, &self.audioQueueBuffer) + if errorAllocate != noErr { + Logger.shared.log("HLSPlayer", "Error AudioQueueAllocateBuffer \(errorAllocate)") + } + } +} diff --git a/submodules/HLSPlayer/Sources/HLSPlayerItem.swift b/submodules/HLSPlayer/Sources/HLSPlayerItem.swift new file mode 100644 index 00000000000..813536f63d0 --- /dev/null +++ b/submodules/HLSPlayer/Sources/HLSPlayerItem.swift @@ -0,0 +1,191 @@ +import Foundation +import TelegramCore + +public final class HLSPlayerItem: NSObject { + + public var preferredPeakBitRate = 0.0 + public var presentationSize: CGSize = .zero + + var rate: Float = 0.0 + + private(set) var streams: [Int: HLSPlayerStreamEntity] = [:] + private(set) var baseURL: String + + private let masterURL: URL + private let queue: DispatchQueue + + public init(url: URL) { + baseURL = url.deletingLastPathComponent().absoluteString + masterURL = url + queue = DispatchQueue( + label: "HLSPlayerItemQueue", + qos: .userInitiated, + attributes: .concurrent) + } + + public func fetch(completion: @escaping () -> Void) { + queue.async { [weak self] in + self?.configure(completion: completion) + } + } +} + +// MARK: - Private Configuration + +private extension HLSPlayerItem { + + func configure(completion: @escaping () -> Void) { + guard let masterPlaylistData = try? Data(contentsOf: masterURL), + let masterPlaylistString = String(data: masterPlaylistData, encoding: .utf8) else { + Logger.shared.log("HLSPlayer", "Error receiving master playlist data") + return + } + + handler(masterPlaylist: masterPlaylistString, completion: completion) + } + + func handler(masterPlaylist: String, completion: @escaping () -> Void) { + let group = DispatchGroup() + let masterPlaylistLines = masterPlaylist.components(separatedBy: "\n") + masterPlaylistLines.forEach { line in + autoreleasepool { + if line.hasPrefix("#EXT-X-STREAM-INF:") { + group.enter() + + let parts = line.components(separatedBy: ":") + let attributes = parts.last?.components(separatedBy: ",") ?? [] + + var bandWidth: Double? + var resolution: CGSize? + var codecs: String? + var frameRate: Double? + + attributes.forEach { attribute in + autoreleasepool { + let keyValue = attribute.trimmingCharacters(in: .whitespacesAndNewlines).components(separatedBy: "=") + let key = keyValue.first ?? "" + let value = keyValue.last ?? "" + + switch key { + case "BANDWIDTH": + bandWidth = Double(value) + case "RESOLUTION": + let resolutionValues = value.components(separatedBy: "x") + if let width = Int(resolutionValues.first ?? ""), + let height = Int(resolutionValues.last ?? "") { + resolution = CGSize(width: width, height: height) + } + case "CODECS": + codecs = value + case "FRAME-RATE": + frameRate = Double(value) + default: + Logger.shared.log("HLSPlayer", "#EXT-X-STREAM-INF:\(key) key is not supported") + } + } + } + + guard let index = masterPlaylistLines.firstIndex(of: line), + masterPlaylistLines.count > index + 1, + let mediaPlaylistURL = URL(string: "\(baseURL)\("\(masterPlaylistLines[index + 1])")"), + let mediaPlaylistData = try? Data(contentsOf: mediaPlaylistURL), + let mediaPlaylist = String(data: mediaPlaylistData, encoding: .utf8) else { + Logger.shared.log("HLSPlayer", "Error receiving playlist data") + group.leave() + return + } + + handler(mediaPlaylist: mediaPlaylist, bandWidth: bandWidth, resolution: resolution, codecs: codecs, frameRate: frameRate) + group.leave() + } + } + } + + group.wait() + DispatchQueue.main.async { + completion() + } + } + + func handler(mediaPlaylist: String, bandWidth: Double?, resolution: CGSize?, codecs: String?, frameRate: Double?) { + var duration: Double? + var sequence: Int? + var segments: [HLSPlayerPlaylistEntity.Segment] = [] + + let mediaPlaylistLines = mediaPlaylist.components(separatedBy: "\n") + mediaPlaylistLines.forEach { line in + autoreleasepool { + if line.hasPrefix("#EXT-X-TARGETDURATION:") { + if let targetDurationString = line.components(separatedBy: ":").last, + let targetDuration = Double(targetDurationString) { + duration = targetDuration + } + } else if line.hasPrefix("#EXT-X-MEDIA-SEQUENCE:") { + if let mediaSequenceString = line.components(separatedBy: ":").last, + let mediaSequence = Int(mediaSequenceString) { + sequence = mediaSequence + } + } else if line.hasPrefix("#EXT-X-INDEPENDENT-SEGMENTS") { + // TODO: Add segment dependency processing + } else if line.hasPrefix("#EXT-X-MAP:") { + if let parts = line.components(separatedBy: ":").last?.replacingOccurrences(of: "\"", with: "").components(separatedBy: ",") { + var uri: String? + var byteRange: HLSPlayerPlaylistEntity.ByteRange? + parts.forEach { part in + autoreleasepool { + let keyValue = part.trimmingCharacters(in: .whitespacesAndNewlines).components(separatedBy: "=") + let key = keyValue.first ?? "" + let value = keyValue.last ?? "" + switch key { + case "URI": + uri = value + case "BYTERANGE": + let byteRangeParts = value.components(separatedBy: "@") + if let length = Int(byteRangeParts.first ?? ""), + let offset = Int(byteRangeParts.last ?? "") { + byteRange = HLSPlayerPlaylistEntity.ByteRange(length: length, offset: offset) + } + default: + Logger.shared.log("HLSPlayer", "#EXT-X-MAP:\(key) key is not supported") + } + } + } + if let duration = duration, + let byteRange = byteRange, + let uri = uri { + let playlistMap = HLSPlayerPlaylistEntity.Segment(duration: duration, byteRange: byteRange, uri: uri) + segments.append(playlistMap) + } + } + } else if line.hasPrefix("#EXTINF:") { + if let durationString = line.components(separatedBy: ":").last, + let duration = Double(durationString), + let index = mediaPlaylistLines.firstIndex(of: line), + mediaPlaylistLines.count > index + 2, + let byteRange = mediaPlaylistLines[index + 1].components(separatedBy: ":").last { + let uri = mediaPlaylistLines[index + 2] + let byteRangeParts = byteRange.components(separatedBy: "@") + if let length = Int(byteRangeParts.first ?? ""), + let offset = Int(byteRangeParts.last ?? "") { + let byteRange = HLSPlayerPlaylistEntity.ByteRange(length: length, offset: offset) + let playlist = HLSPlayerPlaylistEntity.Segment(duration: duration, byteRange: byteRange, uri: uri) + segments.append(playlist) + } + } + } + } + } + + if let duration = duration, let sequence = sequence { + let playlist = HLSPlayerPlaylistEntity(duration: duration, sequence: sequence, segments: segments) + if let bandWidth = bandWidth, let resolution = resolution { + let stream = HLSPlayerStreamEntity(bandWidth: bandWidth, resolution: resolution, codecs: codecs, frameRate: frameRate, playlist: playlist) + streams[Int(bandWidth)] = stream + } else { + Logger.shared.log("HLSPlayer", "Error create stream") + } + } else { + Logger.shared.log("HLSPlayer", "Error create playlist") + } + } +} diff --git a/submodules/HLSPlayer/Sources/HLSPlayerLayer.swift b/submodules/HLSPlayer/Sources/HLSPlayerLayer.swift new file mode 100644 index 00000000000..e14346838a9 --- /dev/null +++ b/submodules/HLSPlayer/Sources/HLSPlayerLayer.swift @@ -0,0 +1,190 @@ +import AVFoundation +import TelegramCore +import MetalKit +import MetalPerformanceShaders + +public final class HLSPlayerLayer: CALayer { + + private weak var player: HLSPlayer? + + private var mtlDevice: MTLDevice? + private var mtlCommandQueue: MTLCommandQueue? + private var mtlLibrary: MTLLibrary? + private var mtlTexture: MTLTexture? + private var mtlRenderPipelineState: MTLRenderPipelineState? + private var mtkView: MTKView? + + private var width = 0 + private var height = 0 + + public init(player: HLSPlayer?) { + self.player = player + super.init() + configure() + } + + public required init?(coder: NSCoder) { + super.init(coder: coder) + configure() + } + + public func setPlayer(at player: HLSPlayer?) { + self.player = player + self.player?.layerDelegate = self + } +} + +// MARK: - LayerDelegate + +extension HLSPlayerLayer: HLSPlayer.LayerDelegate { + + func play() { + mtkView?.isPaused = false + } + + func pause() { + mtkView?.isPaused = true + } + + func stop() { + mtkView = nil + } + + func rendering(at sampleBuffer: CMSampleBuffer) -> Bool { + guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer), + texture(at: pixelBuffer) else { + // TODO: Modify without using Metal if necessary + Logger.shared.log("HLSPlayer", "Error Metal not available") + return false + } + + replaceTexture(from: pixelBuffer) + + DispatchQueue.main.async { + self.mtkView?.setNeedsDisplay() + } + + return true + } +} + +// MARK: - MTKViewDelegate + +extension HLSPlayerLayer: MTKViewDelegate { + + public func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) { + + } + + public func draw(in view: MTKView) { + guard let commandBuffer = mtlCommandQueue?.makeCommandBuffer(), + let renderPipelineState = mtlRenderPipelineState, + let texture = mtlTexture else { + return + } + + if let currentDrawable = view.currentDrawable { + let passDescriptor = MTLRenderPassDescriptor() + let colorAttachment = MTLRenderPassColorAttachmentDescriptor() + colorAttachment.texture = texture + colorAttachment.loadAction = .clear + colorAttachment.storeAction = .store + passDescriptor.colorAttachments[0] = colorAttachment + let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: passDescriptor) + renderEncoder?.setRenderPipelineState(renderPipelineState) + renderEncoder?.setFragmentTexture(texture, index: 0) + renderEncoder?.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4, instanceCount: 1) + renderEncoder?.endEncoding() + commandBuffer.present(currentDrawable) + } else { + DispatchQueue.main.async { + /* + let mpsImage = MPSImage(texture: texture, featureChannels: 4) + self.contents = CIImage(mtlTexture: mpsImage.texture, options: nil) + */ + let ciContext = CIContext(mtlDevice: texture.device) + if let ciImage = CIImage(mtlTexture: texture, options: nil) { + self.contents = ciContext.createCGImage(ciImage, from: ciImage.extent) + } + } + } + + commandBuffer.commit() + commandBuffer.waitUntilCompleted() + } +} + +// MARK: - Configuration + +private extension HLSPlayerLayer { + + func configure() { + player?.layerDelegate = self + + configureMetal() + + let mtkView = MTKView() + mtkView.device = mtlDevice + mtkView.frame = frame + mtkView.delegate = self + mtkView.enableSetNeedsDisplay = true + mtkView.contentScaleFactor = 1.0 + mtkView.preferredFramesPerSecond = 60 + mtkView.isPaused = true + addSublayer(mtkView.layer) + self.mtkView = mtkView + + shouldRasterize = true + drawsAsynchronously = true + } + + func configureMetal() { + mtlDevice = MTLCreateSystemDefaultDevice() + mtlCommandQueue = mtlDevice?.makeCommandQueue() + let vertexShaderSource = """ + vertex float4 vertexShader(uint vertexID [[vertex_id]]) { + return float4(vertexID, 0.0, 1.0, 1.0); + } + """ + mtlLibrary = try? mtlDevice?.makeLibrary(source: vertexShaderSource, options: nil) + let pipelineDescriptor = MTLRenderPipelineDescriptor() + pipelineDescriptor.vertexFunction = mtlLibrary?.makeFunction(name: "vertexShader") + pipelineDescriptor.fragmentFunction = mtlLibrary?.makeFunction(name: "fragmentShader") + pipelineDescriptor.colorAttachments[0].pixelFormat = mtkView?.colorPixelFormat ?? .bgra8Unorm + mtlRenderPipelineState = try? mtlDevice?.makeRenderPipelineState(descriptor: pipelineDescriptor) + } + + func texture(at pixelBuffer: CVPixelBuffer) -> Bool { + let width = CVPixelBufferGetWidth(pixelBuffer) + guard mtlTexture != nil, self.width == width else { + let height = CVPixelBufferGetHeight(pixelBuffer) + self.width = width + self.height = height + guard width > 0, height > 0 else { return false } + let textureDescriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .bgra8Unorm, width: width, height: height, mipmapped: false) + textureDescriptor.usage = [.shaderRead, .shaderWrite, .renderTarget] + let texture = mtlDevice?.makeTexture(descriptor: textureDescriptor) + mtlTexture = texture + return texture != nil + } + return true + } + + func replaceTexture(from pixelBuffer: CVPixelBuffer) { + guard let commandBuffer = mtlCommandQueue?.makeCommandBuffer() else { return } + + let region = MTLRegion(origin: MTLOrigin(x: 0, y: 0, z: 0), size: MTLSize(width: width, height: height, depth: 1)) + CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly) + guard let pixelBufferAdress = CVPixelBufferGetBaseAddress(pixelBuffer) else { + CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly) + return + } + CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly) + let bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer) + let bytesPerImage = CVPixelBufferGetDataSize(pixelBuffer) + mtlTexture?.replace(region: region, mipmapLevel: 0, slice: 0, withBytes: pixelBufferAdress, bytesPerRow: bytesPerRow, bytesPerImage: bytesPerImage) + + commandBuffer.commit() + commandBuffer.waitUntilCompleted() + } +} diff --git a/submodules/HLSPlayer/Sources/HLSPlayerPlaylistEntity.swift b/submodules/HLSPlayer/Sources/HLSPlayerPlaylistEntity.swift new file mode 100644 index 00000000000..819e39c3432 --- /dev/null +++ b/submodules/HLSPlayer/Sources/HLSPlayerPlaylistEntity.swift @@ -0,0 +1,16 @@ +struct HLSPlayerPlaylistEntity { + let duration: Double + let sequence: Int + let segments: [Segment] + + struct Segment { + let duration: Double + let byteRange: ByteRange + let uri: String + } + + struct ByteRange { + let length: Int + let offset: Int + } +} diff --git a/submodules/HLSPlayer/Sources/HLSPlayerStreamEntity.swift b/submodules/HLSPlayer/Sources/HLSPlayerStreamEntity.swift new file mode 100644 index 00000000000..ebd30f363bb --- /dev/null +++ b/submodules/HLSPlayer/Sources/HLSPlayerStreamEntity.swift @@ -0,0 +1,9 @@ +import Foundation + +struct HLSPlayerStreamEntity { + let bandWidth: Double + let resolution: CGSize + let codecs: String? + let frameRate: Double? + let playlist: HLSPlayerPlaylistEntity +} diff --git a/submodules/HLSPlayer/Sources/HLSPlayerVideoOutput.swift b/submodules/HLSPlayer/Sources/HLSPlayerVideoOutput.swift new file mode 100644 index 00000000000..30a4c77a629 --- /dev/null +++ b/submodules/HLSPlayer/Sources/HLSPlayerVideoOutput.swift @@ -0,0 +1,18 @@ +import AVFoundation + +final class HLSPlayerVideoOutput { + + func track(at asset: AVAsset) -> AVAssetReaderTrackOutput? { + guard let track = asset.tracks(withMediaType: .video).first else { + return nil + } + let outputSettings: [String : Any] = [ + kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, + kCVPixelBufferWidthKey as String: track.naturalSize.width, + kCVPixelBufferHeightKey as String: track.naturalSize.height, + ] + let output = AVAssetReaderTrackOutput(track: track, outputSettings: outputSettings) + output.alwaysCopiesSampleData = false + return output + } +} diff --git a/submodules/TelegramUniversalVideoContent/BUILD b/submodules/TelegramUniversalVideoContent/BUILD index b705ae876bd..8d08dee013d 100644 --- a/submodules/TelegramUniversalVideoContent/BUILD +++ b/submodules/TelegramUniversalVideoContent/BUILD @@ -25,6 +25,7 @@ swift_library( "//submodules/Utils/RangeSet:RangeSet", "//submodules/TelegramVoip", "//submodules/ManagedFile", + "//submodules/HLSPlayer", ], visibility = [ "//visibility:public", diff --git a/submodules/TelegramUniversalVideoContent/Sources/HLSVideoContent.swift b/submodules/TelegramUniversalVideoContent/Sources/HLSVideoContent.swift index 3cd8c3b4e3c..5edd26cc917 100644 --- a/submodules/TelegramUniversalVideoContent/Sources/HLSVideoContent.swift +++ b/submodules/TelegramUniversalVideoContent/Sources/HLSVideoContent.swift @@ -13,6 +13,7 @@ import PhotoResources import RangeSet import TelegramVoip import ManagedFile +import HLSPlayer public final class HLSVideoContent: UniversalVideoContent { public let id: AnyHashable @@ -277,8 +278,8 @@ private final class HLSVideoContentNode: ASDisplayNode, UniversalVideoContentNod private let imageNode: TransformImageNode - private var playerItem: AVPlayerItem? - private var player: AVPlayer? + private var playerItem: HLSPlayerItem? + private var player: HLSPlayer? private let playerNode: ASDisplayNode private var loadProgressDisposable: Disposable? @@ -325,19 +326,16 @@ private final class HLSVideoContentNode: ASDisplayNode, UniversalVideoContentNod self.imageNode = TransformImageNode() - var player: AVPlayer? - player = AVPlayer(playerItem: nil) + var player: HLSPlayer? + player = HLSPlayer() self.player = player - if #available(iOS 16.0, *) { - player?.defaultRate = Float(baseRate) - } if !enableSound { player?.volume = 0.0 } - + self.playerNode = ASDisplayNode() self.playerNode.setLayerBlock({ - return AVPlayerLayer(player: player) + HLSPlayerLayer(player: player) }) self.intrinsicDimensions = fileReference.media.dimensions?.cgSize ?? CGSize(width: 480.0, height: 320.0) @@ -418,37 +416,32 @@ private final class HLSVideoContentNode: ASDisplayNode, UniversalVideoContentNod if let playerSource = self.playerSource { self.serverDisposable = SharedHLSServer.shared.registerPlayer(source: playerSource, completion: { [weak self] in Queue.mainQueue().async { - guard let self else { + guard let self, + let assetUrl = URL(string: "http://127.0.0.1:\(SharedHLSServer.shared.port)/\(playerSource.id)/master.m3u8") else { return } - - let playerItem: AVPlayerItem - let assetUrl = "http://127.0.0.1:\(SharedHLSServer.shared.port)/\(playerSource.id)/master.m3u8" + #if DEBUG print("HLSVideoContentNode: playing \(assetUrl)") #endif - playerItem = AVPlayerItem(url: URL(string: assetUrl)!) - - if #available(iOS 14.0, *) { - playerItem.startsOnFirstEligibleVariant = true - } - + + let playerItem = HLSPlayerItem(url: assetUrl) self.setPlayerItem(playerItem) } }) } self.didBecomeActiveObserver = NotificationCenter.default.addObserver(forName: UIApplication.willEnterForegroundNotification, object: nil, queue: nil, using: { [weak self] _ in - guard let strongSelf = self, let layer = strongSelf.playerNode.layer as? AVPlayerLayer else { + guard let strongSelf = self, let layer = strongSelf.playerNode.layer as? HLSPlayerLayer else { return } - layer.player = strongSelf.player + layer.setPlayer(at: strongSelf.player) }) self.willResignActiveObserver = NotificationCenter.default.addObserver(forName: UIApplication.didEnterBackgroundNotification, object: nil, queue: nil, using: { [weak self] _ in - guard let strongSelf = self, let layer = strongSelf.playerNode.layer as? AVPlayerLayer else { + guard let strongSelf = self, let layer = strongSelf.playerNode.layer as? HLSPlayerLayer else { return } - layer.player = nil + layer.setPlayer(at: nil) }) } @@ -484,7 +477,7 @@ private final class HLSVideoContentNode: ASDisplayNode, UniversalVideoContentNod self.statusTimer?.invalidate() } - private func setPlayerItem(_ item: AVPlayerItem?) { + private func setPlayerItem(_ item: HLSPlayerItem?) { if let playerItem = self.playerItem { playerItem.removeObserver(self, forKeyPath: "playbackBufferEmpty") playerItem.removeObserver(self, forKeyPath: "playbackLikelyToKeepUp") @@ -514,26 +507,9 @@ private final class HLSVideoContentNode: ASDisplayNode, UniversalVideoContentNod self.playerItem = item if let item { - self.didPlayToEndTimeObserver = NotificationCenter.default.addObserver(forName: NSNotification.Name.AVPlayerItemDidPlayToEndTime, object: item, queue: nil, using: { [weak self] notification in + self.didPlayToEndTimeObserver = NotificationCenter.default.addObserver(forName: NSNotification.Name("HLSPlayeItemDidPlayToEndTime"), object: nil, queue: OperationQueue.current) { [weak self] notification in self?.performActionAtEnd() - }) - - self.failureObserverId = NotificationCenter.default.addObserver(forName: AVPlayerItem.failedToPlayToEndTimeNotification, object: item, queue: .main, using: { notification in -#if DEBUG - print("Player Error: \(notification.description)") -#endif - }) - self.errorObserverId = NotificationCenter.default.addObserver(forName: AVPlayerItem.newErrorLogEntryNotification, object: item, queue: .main, using: { [weak item] notification in - if let item { - let event = item.errorLog()?.events.last - if let event { - let _ = event -#if DEBUG - print("Player Error: \(event.errorComment ?? "")") -#endif - } - } - }) + } item.addObserver(self, forKeyPath: "presentationSize", options: [], context: nil) } @@ -542,15 +518,9 @@ private final class HLSVideoContentNode: ASDisplayNode, UniversalVideoContentNod playerItem.addObserver(self, forKeyPath: "playbackLikelyToKeepUp", options: .new, context: nil) playerItem.addObserver(self, forKeyPath: "playbackBufferFull", options: .new, context: nil) playerItem.addObserver(self, forKeyPath: "status", options: .new, context: nil) - self.playerItemFailedToPlayToEndTimeObserver = NotificationCenter.default.addObserver(forName: NSNotification.Name.AVPlayerItemFailedToPlayToEndTime, object: playerItem, queue: OperationQueue.main, using: { [weak self] _ in - guard let self else { - return - } - let _ = self - }) } - self.player?.replaceCurrentItem(with: self.playerItem) + self.player?.replaceCurrent(item: self.playerItem) } private func updateStatus() { @@ -733,9 +703,6 @@ private final class HLSVideoContentNode: ASDisplayNode, UniversalVideoContentNod return } self.baseRate = baseRate - if #available(iOS 16.0, *) { - player.defaultRate = Float(baseRate) - } if player.rate != 0.0 { player.rate = Float(baseRate) }