Skip to content

fix: MJPEG video streaming and xcode-select detection on modern macOS #898

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
amilich wants to merge 1 commit into from
Closed

fix: MJPEG video streaming and xcode-select detection on modern macOS #898

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion FBControlCore/Utility/FBArchiveOperations.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,6 @@ typedef NS_ENUM(NSUInteger, FBCompressionFormat) {
To confirm that the stream has been correctly written, the caller should check the exit code of the returned task upon completion.

@param path the path to archive.
@param queue the queue to do work on
@param logger the logger to log to.
@return a Future containing a task with an NSInputStream attached to stdout.
*/
Expand Down
6 changes: 6 additions & 0 deletions FBControlCore/Utility/FBXcodeConfiguration.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,13 @@ + (nullable NSString *)findXcodeDeveloperDirectory:(NSError **)error
static NSError *savedError;
dispatch_once(&onceToken, ^{
NSError *innerError = nil;
// First try the symlink (older macOS)
directory = [FBXcodeDirectory symlinkedDeveloperDirectoryWithError:&innerError];
// If that fails, fall back to running xcode-select -p
if (!directory) {
FBFuture<NSString *> *future = [FBXcodeDirectory xcodeSelectDeveloperDirectory];
directory = [future block:&innerError];
}
savedError = innerError;
});
if (error) {
Expand Down
4 changes: 2 additions & 2 deletions FBSimulatorControl.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@
AA0CA38720643CCF00347424 /* FBCrashLogCommands.h in Headers */ = {isa = PBXBuildFile; fileRef = AA0CA38620643C6800347424 /* FBCrashLogCommands.h */; settings = {ATTRIBUTES = (Public, ); }; };
AA111CCE1BBE7C5A0054AFDD /* CoreSimulatorDoubles.m in Sources */ = {isa = PBXBuildFile; fileRef = AA111CCD1BBE7C5A0054AFDD /* CoreSimulatorDoubles.m */; };
AA1174B21CEA17DB00EB699E /* FBApplicationCommands.h in Headers */ = {isa = PBXBuildFile; fileRef = AA1174B11CEA17DB00EB699E /* FBApplicationCommands.h */; settings = {ATTRIBUTES = (Public, ); }; };
AA1174B61CEA183F00EB699E /* FBSimulatorApplicationCommands.h in Headers */ = {isa = PBXBuildFile; fileRef = AA1174B41CEA183F00EB699E /* FBSimulatorApplicationCommands.h */; };
AA1174B61CEA183F00EB699E /* FBSimulatorApplicationCommands.h in Headers */ = {isa = PBXBuildFile; fileRef = AA1174B41CEA183F00EB699E /* FBSimulatorApplicationCommands.h */; settings = {ATTRIBUTES = (Public, ); }; };
AA1174B71CEA183F00EB699E /* FBSimulatorApplicationCommands.m in Sources */ = {isa = PBXBuildFile; fileRef = AA1174B51CEA183F00EB699E /* FBSimulatorApplicationCommands.m */; };
AA11EDF9272C44410055D6D1 /* FBFileContainerTests.m in Sources */ = {isa = PBXBuildFile; fileRef = AA11EDF8272C44410055D6D1 /* FBFileContainerTests.m */; };
AA12D5441CE1086300CCD944 /* FBTestReporterAdapter.h in Headers */ = {isa = PBXBuildFile; fileRef = AA12D5421CE1086300CCD944 /* FBTestReporterAdapter.h */; settings = {ATTRIBUTES = (Private, ); }; };
Expand Down Expand Up @@ -128,7 +128,7 @@
AA3C18441D5DE47D00419EAA /* CoreImage.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = AA3C18431D5DE47D00419EAA /* CoreImage.framework */; };
AA3E44401F14AE2C00F333D2 /* FBDeviceApplicationCommands.h in Headers */ = {isa = PBXBuildFile; fileRef = AA3E443E1F14AE2C00F333D2 /* FBDeviceApplicationCommands.h */; };
AA3E44411F14AE2C00F333D2 /* FBDeviceApplicationCommands.m in Sources */ = {isa = PBXBuildFile; fileRef = AA3E443F1F14AE2C00F333D2 /* FBDeviceApplicationCommands.m */; };
AA3EA8531F31B20D003FBDC1 /* FBSimulatorFileCommands.h in Headers */ = {isa = PBXBuildFile; fileRef = AA3EA8511F31B20D003FBDC1 /* FBSimulatorFileCommands.h */; };
AA3EA8531F31B20D003FBDC1 /* FBSimulatorFileCommands.h in Headers */ = {isa = PBXBuildFile; fileRef = AA3EA8511F31B20D003FBDC1 /* FBSimulatorFileCommands.h */; settings = {ATTRIBUTES = (Public, ); }; };
AA3EA8541F31B20D003FBDC1 /* FBSimulatorFileCommands.m in Sources */ = {isa = PBXBuildFile; fileRef = AA3EA8521F31B20D003FBDC1 /* FBSimulatorFileCommands.m */; };
AA3EA8561F31B494003FBDC1 /* FBSimulatorApplicationDataTests.m in Sources */ = {isa = PBXBuildFile; fileRef = AA3EA8551F31B494003FBDC1 /* FBSimulatorApplicationDataTests.m */; };
AA3FD04D1C876E4F001093CA /* FBSimulatorLaunchTests.m in Sources */ = {isa = PBXBuildFile; fileRef = AA3FD03D1C876E4F001093CA /* FBSimulatorLaunchTests.m */; };
Expand Down
101 changes: 69 additions & 32 deletions FBSimulatorControl/Framebuffer/FBSimulatorVideoStream.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,25 +66,25 @@ - (instancetype)initWithConfiguration:(FBVideoStreamConfiguration *)configuratio
static CVPixelBufferPoolRef createScaledPixelBufferPool(CVPixelBufferRef sourceBuffer, NSNumber *scaleFactor) {
size_t sourceWidth = CVPixelBufferGetWidth(sourceBuffer);
size_t sourceHeight = CVPixelBufferGetHeight(sourceBuffer);

size_t destinationWidth = (size_t) floor(scaleFactor.doubleValue * (double)sourceWidth);
size_t destinationHeight = (size_t) floor(scaleFactor.doubleValue * (double) sourceHeight);

NSDictionary<NSString *, id> *pixelBufferAttributes = @{
(NSString *) kCVPixelBufferWidthKey: @(destinationWidth),
(NSString *) kCVPixelBufferHeightKey: @(destinationHeight),
(NSString *) kCVPixelBufferPixelFormatTypeKey: @(CVPixelBufferGetPixelFormatType(sourceBuffer)),
};

NSDictionary<NSString *, id> *pixelBufferPoolAttributes = @{
(NSString *) kCVPixelBufferPoolMinimumBufferCountKey: @(100), // we will have at least 100 pixel buffers in the pool
(NSString *) kCVPixelBufferPoolAllocationThresholdKey: @(250), // to guard from OOM only 250 pixel buffers are allowed
};


CVPixelBufferPoolRef scaledPixelBufferPool;
CVPixelBufferPoolCreate(nil, (__bridge CFDictionaryRef) pixelBufferPoolAttributes, (__bridge CFDictionaryRef) pixelBufferAttributes, &scaledPixelBufferPool);

return scaledPixelBufferPool;
}

Expand All @@ -96,12 +96,12 @@ static CVPixelBufferPoolRef createScaledPixelBufferPool(CVPixelBufferRef sourceB
size_t rowSize = CVPixelBufferGetBytesPerRow(pixelBuffer);
OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer);
NSString *pixelFormatString = (__bridge_transfer NSString *) UTCreateStringForOSType(pixelFormat);

size_t columnLeft;
size_t columnRight;
size_t rowsTop;
size_t rowsBottom;

CVPixelBufferGetExtendedPixels(pixelBuffer, &columnLeft, &columnRight, &rowsTop, &rowsBottom);
return @{
@"width" : @(width),
Expand All @@ -119,21 +119,21 @@ static CVPixelBufferPoolRef createScaledPixelBufferPool(CVPixelBufferRef sourceB
static void scaleFromSourceToDestinationBuffer(CVPixelBufferRef sourceBuffer, CVPixelBufferRef destinationBuffer) {
CVPixelBufferLockBaseAddress(sourceBuffer, kCVPixelBufferLock_ReadOnly);
CVPixelBufferLockBaseAddress(destinationBuffer, kCVPixelBufferLock_ReadOnly);

vImage_Buffer scaleInput;
scaleInput.width = CVPixelBufferGetWidth(sourceBuffer);
scaleInput.height = CVPixelBufferGetHeight(sourceBuffer);
scaleInput.rowBytes = CVPixelBufferGetBytesPerRow(sourceBuffer);
scaleInput.data = CVPixelBufferGetBaseAddress(sourceBuffer);

vImage_Buffer scaleOutput;
scaleOutput.width = CVPixelBufferGetWidth(destinationBuffer);
scaleOutput.height = CVPixelBufferGetHeight((destinationBuffer));
scaleOutput.rowBytes = CVPixelBufferGetBytesPerRow(destinationBuffer);
scaleOutput.data = CVPixelBufferGetBaseAddress(destinationBuffer);

vImageScale_ARGB8888(&scaleInput, &scaleOutput, NULL, 0); // implicitly assumes a 4-channel image, like BGRA/RGBA

CVPixelBufferUnlockBaseAddress(sourceBuffer, kCVPixelBufferLock_ReadOnly);
CVPixelBufferUnlockBaseAddress(destinationBuffer, kCVPixelBufferLock_ReadOnly);
}
Expand All @@ -149,8 +149,20 @@ static void MJPEGCompressorCallback(void *outputCallbackRefCon, void *sourceFram
{
(void)(__bridge_transfer FBVideoCompressorCallbackSourceFrame *)(sourceFrameRefCon);
FBSimulatorVideoStreamFramePusher_VideoToolbox *pusher = (__bridge FBSimulatorVideoStreamFramePusher_VideoToolbox *)(outputCallbackRefCon);
[pusher.logger logFormat:@"MJPEGCompressorCallback: encodeStats=%d infoFlags=%u sampleBuffer=%p", (int)encodeStats, (unsigned int)infoFlags, sampleBuffer];
if (!sampleBuffer) {
[pusher.logger log:@"MJPEGCompressorCallback: sampleBuffer is NULL!"];
return;
}
CMBlockBufferRef blockBufffer = CMSampleBufferGetDataBuffer(sampleBuffer);
WriteJPEGDataToMJPEGStream(blockBufffer, pusher.consumer, pusher.logger, nil);
if (!blockBufffer) {
[pusher.logger log:@"MJPEGCompressorCallback: blockBuffer is NULL!"];
return;
}
size_t dataLen = CMBlockBufferGetDataLength(blockBufffer);
[pusher.logger logFormat:@"MJPEGCompressorCallback: writing %zu bytes to consumer", dataLen];
BOOL success = WriteJPEGDataToMJPEGStream(blockBufffer, pusher.consumer, pusher.logger, nil);
[pusher.logger logFormat:@"MJPEGCompressorCallback: write success=%d", success];
}

static void MinicapCompressorCallback(void *outputCallbackRefCon, void *sourceFrameRefCon, OSStatus encodeStats, VTEncodeInfoFlags infoFlags, CMSampleBufferRef sampleBuffer)
Expand Down Expand Up @@ -178,7 +190,7 @@ - (instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer frameNumber:(N

_pixelBuffer = pixelBuffer;
_frameNumber = frameNumber;

return self;
}

Expand All @@ -201,7 +213,7 @@ - (instancetype)initWithConsumer:(id<FBDataConsumer>)consumer scaleFactor:(NSNum

_consumer = consumer;
_scaleFactor = scaleFactor;

return self;
}

Expand Down Expand Up @@ -234,10 +246,10 @@ - (BOOL)writeEncodedFrame:(CVPixelBufferRef)pixelBuffer frameNumber:(NSUInteger)
}

CVPixelBufferLockBaseAddress(bufferToWrite, kCVPixelBufferLock_ReadOnly);

void *baseAddress = CVPixelBufferGetBaseAddress(bufferToWrite);
size_t size = CVPixelBufferGetDataSize(bufferToWrite);

if ([self.consumer conformsToProtocol:@protocol(FBDataConsumerSync)]) {
NSData *data = [NSData dataWithBytesNoCopy:baseAddress length:size freeWhenDone:NO];
[self.consumer consumeData:data];
Expand Down Expand Up @@ -275,15 +287,27 @@ - (instancetype)initWithConfiguration:(FBVideoStreamConfiguration *)configuratio

- (BOOL)setupWithPixelBuffer:(CVPixelBufferRef)pixelBuffer error:(NSError **)error
{
// Only require hardware acceleration for H.264, not JPEG
// JPEG may not have hardware encoder support on all systems
BOOL isH264 = (self.videoCodec == kCMVideoCodecType_H264);

NSDictionary<NSString *, id> * encoderSpecification = @{
(NSString *) kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder: @YES,
};

if (@available(macOS 12.1, *)) {
encoderSpecification = @{
(NSString *) kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder: @YES,
(NSString *) kVTVideoEncoderSpecification_EnableLowLatencyRateControl: @YES,
};
if (isH264) {
// H.264 has good hardware encoder support, require it for best performance
encoderSpecification = @{
(NSString *) kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder: @YES,
(NSString *) kVTVideoEncoderSpecification_EnableLowLatencyRateControl: @YES,
};
} else {
// For other codecs like JPEG, prefer hardware but don't require it
encoderSpecification = @{
(NSString *) kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder: @YES,
};
}
}
size_t sourceWidth = CVPixelBufferGetWidth(pixelBuffer);
size_t sourceHeight = CVPixelBufferGetHeight(pixelBuffer);
Expand Down Expand Up @@ -319,7 +343,7 @@ - (BOOL)setupWithPixelBuffer:(CVPixelBufferRef)pixelBuffer error:(NSError **)err
describeFormat:@"Failed to start Compression Session %d", status]
failBool:error];
}

status = VTSessionSetProperties(
compressionSession,
(__bridge CFDictionaryRef) self.compressionSessionProperties
Expand Down Expand Up @@ -359,7 +383,7 @@ - (BOOL)writeEncodedFrame:(CVPixelBufferRef)pixelBuffer frameNumber:(NSUInteger)
describeFormat:@"No compression session"]
failBool:error];
}

CVPixelBufferRef bufferToWrite = pixelBuffer;
FBVideoCompressorCallbackSourceFrame *sourceFrameRef = [[FBVideoCompressorCallbackSourceFrame alloc] initWithPixelBuffer:nil frameNumber:frameNumber];
CVPixelBufferPoolRef bufferPool = self.scaledPixelBufferPoolRef;
Expand All @@ -374,7 +398,7 @@ - (BOOL)writeEncodedFrame:(CVPixelBufferRef)pixelBuffer frameNumber:(NSUInteger)
[self.logger logFormat:@"Failed to get a pixel buffer from the pool: %d", returnStatus];
}
}

VTEncodeInfoFlags flags;
CMTime time = CMTimeMakeWithSeconds(CFAbsoluteTimeGetCurrent() - timeAtFirstFrame, NSEC_PER_SEC);
OSStatus status = VTCompressionSessionEncodeFrame(
Expand Down Expand Up @@ -580,22 +604,28 @@ - (BOOL)mountSurface:(IOSurface *)surface error:(NSError **)error
describe:@"Cannot mount surface when there is no consumer"]
failBool:error];
}

// Get the Attributes
NSDictionary<NSString *, id> *attributes = FBBitmapStreamPixelBufferAttributesFromPixelBuffer(buffer);
[self.logger logFormat:@"Mounting Surface with Attributes: %@", attributes];

// Swap the pixel buffers.
self.pixelBuffer = buffer;
self.pixelBufferAttributes = attributes;

id<FBSimulatorVideoStreamFramePusher> framePusher = [self.class framePusherForConfiguration:self.configuration compressionSessionProperties:self.compressionSessionProperties consumer:consumer logger:self.logger error:nil];
NSError *framePusherError = nil;
id<FBSimulatorVideoStreamFramePusher> framePusher = [self.class framePusherForConfiguration:self.configuration compressionSessionProperties:self.compressionSessionProperties consumer:consumer logger:self.logger error:&framePusherError];
[self.logger logFormat:@"framePusherForConfiguration returned: %@ error: %@", framePusher, framePusherError];
if (!framePusher) {
[self.logger logFormat:@"Failed to create frame pusher: %@", framePusherError];
return NO;
}
if (![framePusher setupWithPixelBuffer:buffer error:error]) {
NSError *setupError = nil;
if (![framePusher setupWithPixelBuffer:buffer error:&setupError]) {
[self.logger logFormat:@"Failed to setup frame pusher: %@", setupError];
return NO;
}
[self.logger log:@"Frame pusher setup succeeded"];
self.framePusher = framePusher;

// Signal that we've started
Expand All @@ -611,20 +641,27 @@ - (void)pushFrame
id<FBDataConsumer> consumer = self.consumer;
id<FBSimulatorVideoStreamFramePusher> framePusher = self.framePusher;
if (!pixelBufer || !consumer || !framePusher) {
[self.logger logFormat:@"pushFrame: missing preconditions - pixelBuffer=%p consumer=%p framePusher=%p", pixelBufer, consumer, framePusher];
return;
}
if (!checkConsumerBufferLimit(consumer, self.logger)) {
[self.logger log:@"pushFrame: consumer buffer limit exceeded, dropping frame"];
return;
}

NSUInteger frameNumber = self.frameNumber;
if (frameNumber == 0) {
self.timeAtFirstFrame = CFAbsoluteTimeGetCurrent();
[self.logger log:@"pushFrame: first frame!"];
}
CFTimeInterval timeAtFirstFrame = self.timeAtFirstFrame;

// Push the Frame
[framePusher writeEncodedFrame:pixelBufer frameNumber:frameNumber timeAtFirstFrame:timeAtFirstFrame error:nil];
NSError *error = nil;
BOOL success = [framePusher writeEncodedFrame:pixelBufer frameNumber:frameNumber timeAtFirstFrame:timeAtFirstFrame error:&error];
if (frameNumber % 30 == 0) {
[self.logger logFormat:@"pushFrame: frame %lu success=%d error=%@", (unsigned long)frameNumber, success, error];
}

// Increment frame counter
self.frameNumber = frameNumber + 1;
Expand Down
2 changes: 1 addition & 1 deletion Shims/Shimulator/TestCrashShim/TestCrashShim.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ static void PrintProcessInfo(void)
FBDebugLog(@"Environment %@", processInfo.environment);
}

__attribute__((constructor)) static void EntryPoint()
__attribute__((constructor)) static void EntryPoint(void)
{
NSLog(@"Start of Shimulator");

Expand Down
8 changes: 4 additions & 4 deletions Shims/Shimulator/TestLoadingShim/FBXCTestMain.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
#import <AppKit/AppKit.h>
#endif

__attribute__((constructor)) static void XCTestMainEntryPoint()
__attribute__((constructor)) static void XCTestMainEntryPoint(void)
{
FBDebugLog(@"[XCTestMainEntryPoint] Running inside: %@", [[NSBundle mainBundle] bundleIdentifier]);
if (!NSProcessInfo.processInfo.environment[kEnv_ShimStartXCTest]) {
Expand All @@ -45,7 +45,7 @@
FBDebugLog(@"[XCTestMainEntryPoint] End of XCTestMainEntryPoint");
}

BOOL FBLoadXCTestIfNeeded()
BOOL FBLoadXCTestIfNeeded(void)
{
FBDebugLog(@"Env: %@", [NSProcessInfo processInfo].environment);

Expand Down Expand Up @@ -84,7 +84,7 @@ BOOL FBLoadXCTestIfNeeded()
return NO;
}

void FBDeployBlockWhenAppLoads(void(^mainBlock)()) {
void FBDeployBlockWhenAppLoads(void(^mainBlock)(void)) {
#if TARGET_OS_IPHONE
NSString *notification = UIApplicationDidFinishLaunchingNotification;
#elif TARGET_OS_MAC
Expand Down Expand Up @@ -113,7 +113,7 @@ static id XCTestCase__xctTestIdentifier(id self, SEL sel)
return [[XCTTestIdentifier_class alloc] initWithStringRepresentation:[NSString stringWithFormat:@"%@/%@", classNameOut, methodNameOut] preserveModulePrefix:YES];
}

BOOL FBXCTestMain()
BOOL FBXCTestMain(void)
{
if (!FBLoadXCTestIfNeeded()) {
exit(TestShimExitCodeXCTestFailedLoading);
Expand Down
Loading