-
Notifications
You must be signed in to change notification settings - Fork 1.7k
[Crashlytics] Fix flaky tests #15551
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Closed
Closed
Changes from 11 commits
Commits
Show all changes
25 commits
Select commit
Hold shift + click to select a range
09d61fb
[Crashlytics] Fix flaky tests
paulb777 c6c90ad
Second try
paulb777 18d6d48
build fixes
paulb777 21e20ad
fixes
paulb777 3e59f61
timeout increases
paulb777 8c1c5bb
fix another flake
paulb777 7a6d04f
locks
paulb777 240a561
increase timeouts
paulb777 dbba8ce
review
paulb777 b3be77b
review
paulb777 d86bcf9
fixes
paulb777 db32311
wrap unresolved flake in ifdef
paulb777 7783cf6
Disable two more flakes
paulb777 9f5ffeb
More flake fixes
paulb777 cae9e43
better
paulb777 17fdac8
Mark another flaky test
paulb777 6f16835
Fix flakes in FIRCrashlyticsReportTests.m
paulb777 33a701d
address another flake
paulb777 91aa4ad
restore library file
paulb777 3883034
remove operationQueue changes
paulb777 32d465b
Restore FIRCLSSettingsTests.m
paulb777 62556dc
Skip flaky FIRCrashlyticsReportTests.m in nightlies
paulb777 9f25a5f
another try
paulb777 581dcf4
fix
paulb777 9dee833
fix
paulb777 File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Some comments aren't visible on the classic Files Changed page.
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -127,27 +127,33 @@ | |
| - (void)cacheSettingsWithGoogleAppID:(NSString *)googleAppID | ||
| currentTimestamp:(NSTimeInterval)currentTimestamp | ||
| expectedRemoveCount:(NSInteger)expectedRemoveCount { | ||
| self.fileManager.removeExpectation = [[XCTestExpectation alloc] | ||
| initWithDescription:@"FIRCLSMockFileManager.removeExpectation.cache"]; | ||
| self.fileManager.removeCount = 0; | ||
| self.fileManager.expectedRemoveCount = expectedRemoveCount; | ||
|
|
||
| XCTestExpectation *expectation = | ||
| [self expectationForNotification:FIRCLSMockFileManagerDidRemoveItemNotification | ||
| object:nil | ||
| handler:nil]; | ||
| expectation.expectedFulfillmentCount = expectedRemoveCount; | ||
|
|
||
| [self.settings cacheSettingsWithGoogleAppID:googleAppID currentTimestamp:currentTimestamp]; | ||
|
|
||
| [self waitForExpectations:@[ self.fileManager.removeExpectation ] timeout:1]; | ||
| [self waitForExpectations:@[ expectation ] timeout:16.0]; | ||
| } | ||
|
|
||
| - (void)reloadFromCacheWithGoogleAppID:(NSString *)googleAppID | ||
| currentTimestamp:(NSTimeInterval)currentTimestamp | ||
| expectedRemoveCount:(NSInteger)expectedRemoveCount { | ||
| self.fileManager.removeExpectation = [[XCTestExpectation alloc] | ||
| initWithDescription:@"FIRCLSMockFileManager.removeExpectation.reload"]; | ||
| self.fileManager.removeCount = 0; | ||
| self.fileManager.expectedRemoveCount = expectedRemoveCount; | ||
|
|
||
| XCTestExpectation *expectation = | ||
| [self expectationForNotification:FIRCLSMockFileManagerDidRemoveItemNotification | ||
| object:nil | ||
| handler:nil]; | ||
| expectation.expectedFulfillmentCount = expectedRemoveCount; | ||
|
|
||
| [self.settings reloadFromCacheWithGoogleAppID:googleAppID currentTimestamp:currentTimestamp]; | ||
|
|
||
| [self waitForExpectations:@[ self.fileManager.removeExpectation ] timeout:5.0]; | ||
| [self waitForExpectations:@[ expectation ] timeout:14.0]; | ||
|
Check failure on line 156 in Crashlytics/UnitTests/FIRCLSSettingsTests.m
|
||
| } | ||
|
|
||
| - (void)testActivatedSettingsCached { | ||
|
|
@@ -205,10 +211,6 @@ | |
| [self writeSettings:FIRCLSTestSettingsActivated error:&error]; | ||
| XCTAssertNil(error, "%@", error); | ||
|
|
||
| // 1 delete for clearing the cache key, plus 2 for the deletes from reloading and clearing the | ||
| // cache and cache key | ||
| self.fileManager.expectedRemoveCount = 3; | ||
|
|
||
| NSTimeInterval currentTimestamp = [NSDate timeIntervalSinceReferenceDate]; | ||
| [self.settings cacheSettingsWithGoogleAppID:TestGoogleAppID currentTimestamp:currentTimestamp]; | ||
|
|
||
|
|
@@ -238,10 +240,6 @@ | |
| [self writeSettings:FIRCLSTestSettingsActivated error:&error]; | ||
| XCTAssertNil(error, "%@", error); | ||
|
|
||
| // 1 delete for clearing the cache key, plus 2 for the deletes from reloading and clearing the | ||
| // cache and cache key | ||
| self.fileManager.expectedRemoveCount = 3; | ||
|
|
||
| NSTimeInterval currentTimestamp = [NSDate timeIntervalSinceReferenceDate]; | ||
| [self.settings cacheSettingsWithGoogleAppID:TestGoogleAppID currentTimestamp:currentTimestamp]; | ||
|
|
||
|
|
@@ -272,10 +270,6 @@ | |
| [self writeSettings:FIRCLSTestSettingsActivated error:&error]; | ||
| XCTAssertNil(error, "%@", error); | ||
|
|
||
| // 1 delete for clearing the cache key, plus 2 for the deletes from reloading and clearing the | ||
| // cache and cache key | ||
| self.fileManager.expectedRemoveCount = 3; | ||
|
|
||
| NSTimeInterval currentTimestamp = [NSDate timeIntervalSinceReferenceDate]; | ||
| [self.settings cacheSettingsWithGoogleAppID:TestGoogleAppID currentTimestamp:currentTimestamp]; | ||
|
|
||
|
|
@@ -387,7 +381,7 @@ | |
| // Cache them, and reload. Since it's corrupted we should delete it all | ||
| [self cacheSettingsWithGoogleAppID:TestGoogleAppID | ||
| currentTimestamp:currentTimestamp | ||
| expectedRemoveCount:2]; | ||
| expectedRemoveCount:3]; | ||
|
||
|
|
||
| // Should have default values because we deleted the cache and settingsDictionary | ||
| XCTAssertEqual(self.settings.isCacheExpired, YES); | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The timeout for waiting on this expectation has been increased significantly to 16.0 seconds (and similarly to 14.0 seconds in
reloadFromCacheWithGoogleAppID). While this is effective at preventing timeouts on slower CI environments and fixing flakiness, it's worth noting that it can also increase the overall test suite execution time. This is a reasonable and pragmatic approach to improve test stability, but if these operations aren't expected to take this long, it might be worth investigating if there's an underlying performance issue that could be addressed in the future.