From 05f1a879300a4b1fb259fa49531d673567652e98 Mon Sep 17 00:00:00 2001 From: zx5656 <56567244@qq.com> Date: Tue, 10 Nov 2020 13:47:54 +0800 Subject: [PATCH 1/2] add function to transmit audio data of AudioStream in time division --- SmartDeviceLink-iOS.xcodeproj/project.pbxproj | 8 + .../public/SDLAudioStreamManager.h | 3 +- .../public/SDLAudioStreamManager.m | 154 ++++++++++++++---- SmartDeviceLink/public/dispatch_timer.h | 18 ++ SmartDeviceLink/public/dispatch_timer.m | 38 +++++ 5 files changed, 186 insertions(+), 35 deletions(-) mode change 100755 => 100644 SmartDeviceLink/public/SDLAudioStreamManager.h mode change 100755 => 100644 SmartDeviceLink/public/SDLAudioStreamManager.m create mode 100644 SmartDeviceLink/public/dispatch_timer.h create mode 100644 SmartDeviceLink/public/dispatch_timer.m diff --git a/SmartDeviceLink-iOS.xcodeproj/project.pbxproj b/SmartDeviceLink-iOS.xcodeproj/project.pbxproj index 3515205d63..6793899b85 100644 --- a/SmartDeviceLink-iOS.xcodeproj/project.pbxproj +++ b/SmartDeviceLink-iOS.xcodeproj/project.pbxproj @@ -15,6 +15,8 @@ 000DD57622EF0971005AB7A7 /* SDLReleaseInteriorVehicleDataModuleResponseSpec.m in Sources */ = {isa = PBXBuildFile; fileRef = 000DD57522EF0971005AB7A7 /* SDLReleaseInteriorVehicleDataModuleResponseSpec.m */; }; 00EADD3322DFE54B0088B608 /* SDLEncryptionLifecycleManagerSpec.m in Sources */ = {isa = PBXBuildFile; fileRef = 00EADD3222DFE54B0088B608 /* SDLEncryptionLifecycleManagerSpec.m */; }; 00EADD3522DFE5670088B608 /* SDLEncryptionConfigurationSpec.m in Sources */ = {isa = PBXBuildFile; fileRef = 00EADD3422DFE5670088B608 /* SDLEncryptionConfigurationSpec.m */; }; + 040C7934255A589300AAB8EB /* dispatch_timer.h in Headers */ = {isa = PBXBuildFile; fileRef = 040C7932255A589300AAB8EB /* dispatch_timer.h */; }; + 040C7935255A589300AAB8EB /* dispatch_timer.m in Sources */ = {isa = PBXBuildFile; fileRef = 040C7933255A589300AAB8EB /* dispatch_timer.m */; }; 106187B924AA75540045C4EC /* SDLRPCPermissionStatusSpec.m in Sources */ = {isa = PBXBuildFile; fileRef = 106187B824AA75540045C4EC /* SDLRPCPermissionStatusSpec.m */; }; 109566F6242986F300E24F66 /* SDLCacheFileManagerSpec.m in Sources */ = {isa = PBXBuildFile; fileRef = 109566F5242986F300E24F66 /* SDLCacheFileManagerSpec.m */; }; 1098F03824A39699004F53CC /* SDLPermissionElementSpec.m in Sources */ = {isa = PBXBuildFile; fileRef = 1098F03724A39699004F53CC /* SDLPermissionElementSpec.m */; }; @@ -1789,6 +1791,8 @@ 000DD57522EF0971005AB7A7 /* SDLReleaseInteriorVehicleDataModuleResponseSpec.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SDLReleaseInteriorVehicleDataModuleResponseSpec.m; sourceTree = ""; }; 00EADD3222DFE54B0088B608 /* SDLEncryptionLifecycleManagerSpec.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SDLEncryptionLifecycleManagerSpec.m; sourceTree = ""; }; 00EADD3422DFE5670088B608 /* SDLEncryptionConfigurationSpec.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SDLEncryptionConfigurationSpec.m; sourceTree = ""; }; + 040C7932255A589300AAB8EB /* dispatch_timer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = dispatch_timer.h; path = SmartDeviceLink/public/dispatch_timer.h; sourceTree = SOURCE_ROOT; }; + 040C7933255A589300AAB8EB /* dispatch_timer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = dispatch_timer.m; path = SmartDeviceLink/public/dispatch_timer.m; sourceTree = SOURCE_ROOT; }; 106187B824AA75540045C4EC /* SDLRPCPermissionStatusSpec.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SDLRPCPermissionStatusSpec.m; sourceTree = ""; }; 109566F5242986F300E24F66 /* SDLCacheFileManagerSpec.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SDLCacheFileManagerSpec.m; sourceTree = ""; }; 1098F03724A39699004F53CC /* SDLPermissionElementSpec.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SDLPermissionElementSpec.m; sourceTree = ""; }; @@ -4371,6 +4375,8 @@ 5D23C9471FCF59F400002CA5 /* Utilities */ = { isa = PBXGroup; children = ( + 040C7932255A589300AAB8EB /* dispatch_timer.h */, + 040C7933255A589300AAB8EB /* dispatch_timer.m */, 4ABB260F24F7F3520061BF55 /* SDLPCMAudioConverter.h */, 4ABB260E24F7F3520061BF55 /* SDLPCMAudioConverter.m */, 4ABB260B24F7F33F0061BF55 /* SDLAudioFile.h */, @@ -7360,6 +7366,7 @@ 4ABB276A24F7FE480061BF55 /* SDLHybridAppPreference.h in Headers */, 4A8BD37A24F9468B000945E3 /* SDLProtocolConstants.h in Headers */, 4ABB2B0F24F84D950061BF55 /* SDLAppInfo.h in Headers */, + 040C7934255A589300AAB8EB /* dispatch_timer.h in Headers */, 4ABB2AB124F847F40061BF55 /* SDLSetDisplayLayoutResponse.h in Headers */, 4ABB2A5E24F847B10061BF55 /* SDLGetDTCsResponse.h in Headers */, 4A8BD39A24F94741000945E3 /* SDLIAPControlSessionDelegate.h in Headers */, @@ -7768,6 +7775,7 @@ 4ABB291724F842160061BF55 /* SDLCancelInteraction.m in Sources */, 4ABB279424F7FF0B0061BF55 /* SDLKeypressMode.m in Sources */, 4A8BD34324F945CC000945E3 /* SDLControlFramePayloadEndService.m in Sources */, + 040C7935255A589300AAB8EB /* dispatch_timer.m in Sources */, 4A8BD26A24F933C7000945E3 /* SDLParameterPermissions.m in Sources */, 4ABB25EB24F7E7C20061BF55 /* SDLCarWindowViewController.m in Sources */, 4ABB284D24F828630061BF55 /* SDLTransmissionType.m in Sources */, diff --git a/SmartDeviceLink/public/SDLAudioStreamManager.h b/SmartDeviceLink/public/SDLAudioStreamManager.h old mode 100755 new mode 100644 index 7ec8a5918c..44c64e0a6b --- a/SmartDeviceLink/public/SDLAudioStreamManager.h +++ b/SmartDeviceLink/public/SDLAudioStreamManager.h @@ -59,8 +59,9 @@ NS_ASSUME_NONNULL_BEGIN @note This happens on a serial background thread and will provide an error callback using the delegate if the conversion fails. @param fileURL File URL to convert + @param forceInterrupt force Audio Stream Interrupt */ -- (void)pushWithFileURL:(NSURL *)fileURL; +- (void)pushWithFileURL:(NSURL *)fileURL forceInterrupt:(BOOL)forceInterrupt; /** Push a new audio buffer onto the queue. Call `playNextWhenReady` to start playing the pushed audio buffer. diff --git a/SmartDeviceLink/public/SDLAudioStreamManager.m b/SmartDeviceLink/public/SDLAudioStreamManager.m old mode 100755 new mode 100644 index 35b492cce5..eac8dffce6 --- a/SmartDeviceLink/public/SDLAudioStreamManager.m +++ b/SmartDeviceLink/public/SDLAudioStreamManager.m @@ -17,6 +17,7 @@ #import "SDLManager.h" #import "SDLPCMAudioConverter.h" #import "SDLStreamingAudioManagerType.h" +#import "dispatch_timer.h" NS_ASSUME_NONNULL_BEGIN @@ -28,11 +29,19 @@ @interface SDLAudioStreamManager () @property (assign, nonatomic, readwrite, getter=isPlaying) BOOL playing; @property (assign, nonatomic) BOOL shouldPlayWhenReady; +@property (nonatomic, strong, nullable) dispatch_source_t audioStreamTimer; +@property (nonatomic) NSTimeInterval streamingEndTimeOfHU; @end @implementation SDLAudioStreamManager +// Byte length of voice data per second +static const NSInteger PerSecondVoiceData = 32000; + +// How many seconds the handset can precede the head unit +static const NSTimeInterval ThresholdPrecedeSec = 3.0f; + - (instancetype)initWithManager:(id)streamManager { self = [super init]; if (!self) { return nil; } @@ -47,11 +56,27 @@ - (instancetype)initWithManager:(id)streamManager - (void)stop { dispatch_async(_audioQueue, ^{ - self.shouldPlayWhenReady = NO; - [self.mutableQueue removeAllObjects]; + [self sdl_stop]; }); } +- (void)sdl_stop { + NSError *error = nil; + for (SDLAudioFile *file in self.mutableQueue) { + if (file.outputFileURL != nil) { + [[NSFileManager defaultManager] removeItemAtURL:file.outputFileURL error:&error]; + } + } + [self.mutableQueue removeAllObjects]; + + if (self.audioStreamTimer != nil) { + dispatch_stop_timer(self.audioStreamTimer); + self.audioStreamTimer = nil; + } + self.shouldPlayWhenReady = NO; + self.playing = NO; +} + #pragma mark - Getters - (NSArray *)queue { @@ -61,13 +86,12 @@ - (void)stop { #pragma mark - Pushing to the Queue #pragma mark Files -- (void)pushWithFileURL:(NSURL *)fileURL { - dispatch_async(_audioQueue, ^{ - [self sdl_pushWithContentsOfURL:fileURL]; +- (void)pushWithFileURL:(NSURL *)fileURL forceInterrupt:(BOOL)forceInterrupt { dispatch_async(_audioQueue, ^{ + [self sdl_pushWithContentsOfURL:fileURL forceInterrupt:forceInterrupt]; }); } -- (void)sdl_pushWithContentsOfURL:(NSURL *)fileURL { +- (void)sdl_pushWithContentsOfURL:(NSURL *)fileURL forceInterrupt:(BOOL)forceInterrupt { // Convert and store in the queue NSError *error = nil; SDLPCMAudioConverter *converter = [[SDLPCMAudioConverter alloc] initWithFileURL:fileURL]; @@ -82,6 +106,19 @@ - (void)sdl_pushWithContentsOfURL:(NSURL *)fileURL { return; } + if (self.mutableQueue.count == 0) { + NSTimeInterval precedeTime = self.streamingEndTimeOfHU - [[NSDate date] timeIntervalSince1970]; + if (precedeTime > 0.0f) { + SDLLogD(@"Time when handset is ahead of head unit: %f", precedeTime); + [NSThread sleepForTimeInterval:precedeTime]; + } + self.streamingEndTimeOfHU = [[NSDate date] timeIntervalSince1970]; + } + + if (forceInterrupt) { + [self sdl_stop]; + } + SDLAudioFile *audioFile = [[SDLAudioFile alloc] initWithInputFileURL:fileURL outputFileURL:outputFileURL estimatedDuration:estimatedDuration]; [self.mutableQueue addObject:audioFile]; @@ -126,50 +163,99 @@ - (void)sdl_playNextWhenReady { } self.shouldPlayWhenReady = NO; - __block SDLAudioFile *file = self.mutableQueue.firstObject; - [self.mutableQueue removeObjectAtIndex:0]; - - // Strip the first bunch of bytes (because of how Apple outputs the data) and send to the audio stream, if we don't do this, it will make a weird click sound - NSData *audioData = nil; - if (file.inputFileURL != nil) { - audioData = [file.data subdataWithRange:NSMakeRange(5760, (file.data.length - 5760))]; - } else { - audioData = file.data; - } + if (self.playing == NO) { + self.playing = YES; + SDLAudioFile *file = self.mutableQueue.firstObject; + + // Strip the first bunch of bytes (because of how Apple outputs the data) and send to the audio stream, if we don't do this, it will make a weird click sound + __block NSData *audioData = nil; + if (file.inputFileURL != nil) { + audioData = [file.data subdataWithRange:NSMakeRange(5760, (file.data.length - 5760))]; + } else { + audioData = file.data; + } - // Send the audio file, which starts it playing immediately - SDLLogD(@"Playing audio file: %@", file); - __block BOOL success = [self.streamManager sendAudioData:audioData]; - self.playing = YES; + NSTimeInterval precedeTime = self.streamingEndTimeOfHU - [[NSDate date] timeIntervalSince1970]; + if(precedeTime > ThresholdPrecedeSec){ + SDLLogD(@"The time during which the handset precedes the head unit exceeds the threshold: %f", precedeTime); + [NSThread sleepForTimeInterval:ThresholdPrecedeSec]; + } - // Determine the length of the audio PCM data and perform a few items once the audio has finished playing - float audioLengthSecs = (float)audioData.length / (float)32000.0; - __weak typeof(self) weakself = self; - dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(audioLengthSecs * NSEC_PER_SEC)), [SDLGlobals sharedGlobals].sdlProcessingQueue, ^{ - __strong typeof(weakself) strongSelf = weakself; + // Send the audio file, which starts it playing immediately + SDLLogD(@"Playing audio file: %@", file); + BOOL success = [self sendAudioData:&audioData of:PerSecondVoiceData * 2]; + if ((success) && (audioData.length > 0)) { + __weak typeof(self) weakSelf = self; + self.audioStreamTimer = dispatch_create_timer(1.0f, YES, ^{ + BOOL success = [weakSelf sendAudioData:&audioData of:PerSecondVoiceData]; + if ((success) && (audioData.length > 0)) { + SDLLogD(@"sendAudioData continue: %lu", (unsigned long)audioData.length); + } else { + SDLLogD(@"sendAudioData end"); + dispatch_stop_timer(weakSelf.audioStreamTimer); + weakSelf.audioStreamTimer = nil; + + [weakSelf sdl_finishAudioStreaming:file success:success]; + } + }); + } else { + [self sdl_finishAudioStreaming:file success:success]; + } + } +} - strongSelf.playing = NO; +- (void)sdl_finishAudioStreaming:(SDLAudioFile *)file success:(BOOL)success { + __weak typeof(self) weakself = self; + dispatch_async(_audioQueue, ^{ + weakself.playing = NO; + if (weakself.mutableQueue.count > 0) { + [weakself.mutableQueue removeObjectAtIndex:0]; + [weakself sdl_playNextWhenReady]; + } + }); + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(1 * NSEC_PER_MSEC)), dispatch_get_main_queue(), ^{ NSError *error = nil; - if (strongSelf.delegate != nil) { + if (weakself.delegate != nil) { if (file.inputFileURL != nil) { - [strongSelf.delegate audioStreamManager:strongSelf fileDidFinishPlaying:file.inputFileURL successfully:success]; - } else if ([strongSelf.delegate respondsToSelector:@selector(audioStreamManager:dataBufferDidFinishPlayingSuccessfully:)]) { - [strongSelf.delegate audioStreamManager:strongSelf dataBufferDidFinishPlayingSuccessfully:success]; + [weakself.delegate audioStreamManager:weakself fileDidFinishPlaying:file.inputFileURL successfully:success]; + } else if ([weakself.delegate respondsToSelector:@selector(audioStreamManager:dataBufferDidFinishPlayingSuccessfully:)]) { + [weakself.delegate audioStreamManager:weakself dataBufferDidFinishPlayingSuccessfully:success]; } } SDLLogD(@"Ending Audio file: %@", file); [[NSFileManager defaultManager] removeItemAtURL:file.outputFileURL error:&error]; - if (strongSelf.delegate != nil && error != nil) { + if (weakself.delegate != nil && error != nil) { if (file.inputFileURL != nil) { - [strongSelf.delegate audioStreamManager:strongSelf errorDidOccurForFile:file.inputFileURL error:error]; - } else if ([strongSelf.delegate respondsToSelector:@selector(audioStreamManager:errorDidOccurForDataBuffer:)]) { - [strongSelf.delegate audioStreamManager:strongSelf errorDidOccurForDataBuffer:error]; + [weakself.delegate audioStreamManager:weakself errorDidOccurForFile:file.inputFileURL error:error]; + } else if ([weakself.delegate respondsToSelector:@selector(audioStreamManager:errorDidOccurForDataBuffer:)]) { + [weakself.delegate audioStreamManager:weakself errorDidOccurForDataBuffer:error]; } } }); } +- (BOOL)sendAudioData:(NSData **)data of:(NSUInteger)byteLength{ + if (self.streamManager.isAudioConnected == NO) { + return NO; + } + + NSUInteger sByte = byteLength; + if ((*data).length < byteLength) { + sByte = (*data).length; + } + + if ([self.streamManager sendAudioData:[*data subdataWithRange:NSMakeRange(0, sByte)]]) { + // Set remaining voice data + *data = [(*data) subdataWithRange:NSMakeRange(sByte, (*data).length - sByte)]; + + // Calculate the AudioStreaming end time on the HU side. + self.streamingEndTimeOfHU += (double)sByte / (double)PerSecondVoiceData; + return YES; + } + return NO; +} + @end NS_ASSUME_NONNULL_END diff --git a/SmartDeviceLink/public/dispatch_timer.h b/SmartDeviceLink/public/dispatch_timer.h new file mode 100644 index 0000000000..1dd8bb9f28 --- /dev/null +++ b/SmartDeviceLink/public/dispatch_timer.h @@ -0,0 +1,18 @@ +// +// dispatch_timer.h +// MobileNav +// +// Created by Muller, Alexander (A.) on 5/12/16. +// Copyright © 2016 Alex Muller. All rights reserved. +// + +#ifndef dispatch_timer_h +#define dispatch_timer_h + +#include +#include + +dispatch_source_t dispatch_create_timer(double afterInterval, bool repeating, dispatch_block_t block); +void dispatch_stop_timer(dispatch_source_t timer); + +#endif /* dispatch_timer_h */ diff --git a/SmartDeviceLink/public/dispatch_timer.m b/SmartDeviceLink/public/dispatch_timer.m new file mode 100644 index 0000000000..445095eb66 --- /dev/null +++ b/SmartDeviceLink/public/dispatch_timer.m @@ -0,0 +1,38 @@ +// +// dispatch_timer.c +// MobileNav +// +// Created by Muller, Alexander (A.) on 5/12/16. +// Copyright © 2016 Alex Muller. All rights reserved. +// + +#include "dispatch_timer.h" + +dispatch_source_t dispatch_create_timer(double afterInterval, bool repeating, dispatch_block_t block) { + dispatch_queue_t queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, + 0); + dispatch_source_t timer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, + 0, + 0, + queue); + dispatch_source_set_timer(timer, + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(afterInterval * NSEC_PER_SEC)), + (uint64_t)(afterInterval * NSEC_PER_SEC), + (1ull * NSEC_PER_SEC) / 10); + dispatch_source_set_event_handler(timer, ^{ + if (!repeating) { + dispatch_stop_timer(timer); + } + if (block) { + block(); + } + }); + dispatch_resume(timer); + + return timer; +} + +void dispatch_stop_timer(dispatch_source_t timer) { + dispatch_source_set_event_handler(timer, NULL); + dispatch_source_cancel(timer); +} From 1da9990bfb6501618f6f6aba18983c69e17a383c Mon Sep 17 00:00:00 2001 From: zx5656 <56567244@qq.com> Date: Wed, 11 Nov 2020 12:58:57 +0800 Subject: [PATCH 2/2] fix CI check error for pushWithFileURL --- SmartDeviceLinkTests/DevAPISpecs/SDLAudioStreamManagerSpec.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SmartDeviceLinkTests/DevAPISpecs/SDLAudioStreamManagerSpec.m b/SmartDeviceLinkTests/DevAPISpecs/SDLAudioStreamManagerSpec.m index 373003311f..44eac1861d 100644 --- a/SmartDeviceLinkTests/DevAPISpecs/SDLAudioStreamManagerSpec.m +++ b/SmartDeviceLinkTests/DevAPISpecs/SDLAudioStreamManagerSpec.m @@ -32,7 +32,7 @@ context(@"with a file URL", ^{ beforeEach(^{ mockAudioManager.audioConnected = NO; - [testManager pushWithFileURL:testAudioFileURL]; + [testManager pushWithFileURL:testAudioFileURL forceInterrupt:true]; }); describe(@"after attempting to play the file", ^{ @@ -73,7 +73,7 @@ describe(@"after adding an audio file to the queue", ^{ beforeEach(^{ mockAudioManager.audioConnected = YES; - [testManager pushWithFileURL:testAudioFileURL]; + [testManager pushWithFileURL:testAudioFileURL forceInterrupt:true]; }); it(@"should have a file in the queue", ^{