From 69ec180c84710a28043a0f4a7cf280ff5000aaef Mon Sep 17 00:00:00 2001 From: Rachel McR Date: Wed, 26 Jun 2019 16:59:42 +0100 Subject: [PATCH] Improve UI test output (logs and screenshots) (#11996) * Add assertions that the editor was successfully closed * Group test steps into named activities for clarity in test logs * Take screenshot at the end of failed tests * Add wait before checking that editor is closed --- .../WordPressUITests/Flows/LoginFlow.swift | 32 ++++++++++------- .../WordPressUITests/Screens/BaseScreen.swift | 36 ++++++++++--------- .../Screens/Editor/AztecEditorScreen.swift | 34 +++++++++++------- .../Screens/Editor/BlockEditorScreen.swift | 35 +++++++++++------- .../Tests/EditorAztecTests.swift | 1 + .../Tests/EditorGutenbergTests.swift | 1 + .../WordPressUITests/Tests/LoginTests.swift | 1 + .../Tests/MainNavigationTests.swift | 6 +++- .../WordPressUITests/Tests/SignupTests.swift | 1 + .../Utils/XCTest+Extensions.swift | 8 +++++ 10 files changed, 99 insertions(+), 56 deletions(-) diff --git a/WordPress/WordPressUITests/Flows/LoginFlow.swift b/WordPress/WordPressUITests/Flows/LoginFlow.swift index fa043bcdddc5..9edd13e720ff 100644 --- a/WordPress/WordPressUITests/Flows/LoginFlow.swift +++ b/WordPress/WordPressUITests/Flows/LoginFlow.swift @@ -21,23 +21,29 @@ class LoginFlow { } static func logoutIfNeeded() { - if TabNavComponent.isLoaded() { - Logger.log(message: "Logging out...", event: .i) - let meScreen = TabNavComponent().gotoMeScreen() - if meScreen.isLoggedInToWpcom() { - _ = meScreen.logout() - } else { - _ = TabNavComponent().gotoMySiteScreen() - .removeSelfHostedSite() + XCTContext.runActivity(named: "Log out of app if currently logged in") { (activity) in + if TabNavComponent.isLoaded() { + Logger.log(message: "Logging out...", event: .i) + let meScreen = TabNavComponent().gotoMeScreen() + if meScreen.isLoggedInToWpcom() { + _ = meScreen.logout() + } else { + _ = TabNavComponent().gotoMySiteScreen() + .removeSelfHostedSite() + } + return } - return } - while LoginPasswordScreen.isLoaded() || LoginEmailScreen.isLoaded() || LinkOrPasswordScreen.isLoaded() || LoginSiteAddressScreen.isLoaded() || LoginUsernamePasswordScreen.isLoaded() || LoginCheckMagicLinkScreen.isLoaded() { - if LoginEmailScreen.isLoaded() && LoginEmailScreen.isEmailEntered() { - LoginEmailScreen().emailTextField.clearTextIfNeeded() + XCTContext.runActivity(named: "Return to app prologue screen if needed") { (activity) in + if !WelcomeScreen.isLoaded() { + while LoginPasswordScreen.isLoaded() || LoginEmailScreen.isLoaded() || LinkOrPasswordScreen.isLoaded() || LoginSiteAddressScreen.isLoaded() || LoginUsernamePasswordScreen.isLoaded() || LoginCheckMagicLinkScreen.isLoaded() { + if LoginEmailScreen.isLoaded() && LoginEmailScreen.isEmailEntered() { + LoginEmailScreen().emailTextField.clearTextIfNeeded() + } + navBackButton.tap() + } } - navBackButton.tap() } } } diff --git a/WordPress/WordPressUITests/Screens/BaseScreen.swift b/WordPress/WordPressUITests/Screens/BaseScreen.swift index 3f94d3d47f02..59e05429f0fa 100644 --- a/WordPress/WordPressUITests/Screens/BaseScreen.swift +++ b/WordPress/WordPressUITests/Screens/BaseScreen.swift @@ -19,9 +19,11 @@ class BaseScreen { @discardableResult func waitForPage() throws -> BaseScreen { - let result = waitFor(element: expectedElement, predicate: "isEnabled == true", timeout: 20) - XCTAssert(result, "Page \(self) is not loaded.") - Logger.log(message: "Page \(self) is loaded", event: .i) + XCTContext.runActivity(named: "Confirm page \(self) is loaded") { (activity) in + let result = waitFor(element: expectedElement, predicate: "isEnabled == true", timeout: 20) + XCTAssert(result, "Page \(self) is not loaded.") + Logger.log(message: "Page \(self) is loaded", event: .i) + } return self } @@ -46,23 +48,25 @@ class BaseScreen { } func openMagicLink() { - let safari = XCUIApplication(bundleIdentifier: "com.apple.mobilesafari") - safari.launch() + XCTContext.runActivity(named: "Open magic link in Safari") { (activity) in + let safari = XCUIApplication(bundleIdentifier: "com.apple.mobilesafari") + safari.launch() - // Select the URL bar when Safari opens - let urlBar = safari.otherElements["URL"] - waitFor(element: urlBar, predicate: "exists == true") - urlBar.tap() + // Select the URL bar when Safari opens + let urlBar = safari.otherElements["URL"] + waitFor(element: urlBar, predicate: "exists == true") + urlBar.tap() - // Follow the magic link - var magicLinkComponents = URLComponents(url: WireMock.URL(), resolvingAgainstBaseURL: false)! - magicLinkComponents.path = "/magic-link" - magicLinkComponents.queryItems = [URLQueryItem(name: "scheme", value: "wpdebug")] + // Follow the magic link + var magicLinkComponents = URLComponents(url: WireMock.URL(), resolvingAgainstBaseURL: false)! + magicLinkComponents.path = "/magic-link" + magicLinkComponents.queryItems = [URLQueryItem(name: "scheme", value: "wpdebug")] - safari.textFields["URL"].typeText("\(magicLinkComponents.url!.absoluteString)\n") + safari.textFields["URL"].typeText("\(magicLinkComponents.url!.absoluteString)\n") - // Accept the prompt to open the deep link - safari.scrollViews.element(boundBy: 0).buttons.element(boundBy: 1).tap() + // Accept the prompt to open the deep link + safari.scrollViews.element(boundBy: 0).buttons.element(boundBy: 1).tap() + } } } diff --git a/WordPress/WordPressUITests/Screens/Editor/AztecEditorScreen.swift b/WordPress/WordPressUITests/Screens/Editor/AztecEditorScreen.swift index dbbea06f8a34..f85401fe83b9 100644 --- a/WordPress/WordPressUITests/Screens/Editor/AztecEditorScreen.swift +++ b/WordPress/WordPressUITests/Screens/Editor/AztecEditorScreen.swift @@ -232,21 +232,29 @@ class AztecEditorScreen: BaseScreen { // returns void since return screen depends on from which screen it loaded func closeEditor() { - // Close the More menu if needed - if actionSheet.exists { - if isIpad { - app.otherElements["PopoverDismissRegion"].tap() - } else { - keepEditingButton.tap() + XCTContext.runActivity(named: "Close the Aztec editor") { (activity) in + XCTContext.runActivity(named: "Close the More menu if needed") { (activity) in + if actionSheet.exists { + if isIpad { + app.otherElements["PopoverDismissRegion"].tap() + } else { + keepEditingButton.tap() + } + } + } + + editorCloseButton.tap() + + XCTContext.runActivity(named: "Discard any local changes") { (activity) in + let notSavedState = app.staticTexts["You have unsaved changes."] + if notSavedState.exists { + Logger.log(message: "Discarding unsaved changes", event: .v) + discardButton.tap() + } } - } - // Close the editor and discard any local changes - editorCloseButton.tap() - let notSavedState = app.staticTexts["You have unsaved changes."] - if notSavedState.exists { - Logger.log(message: "Discarding unsaved changes", event: .v) - discardButton.tap() + let editorClosed = waitFor(element: editorCloseButton, predicate: "isEnabled == false") + XCTAssert(editorClosed, "Aztec editor should be closed but is still loaded.") } } diff --git a/WordPress/WordPressUITests/Screens/Editor/BlockEditorScreen.swift b/WordPress/WordPressUITests/Screens/Editor/BlockEditorScreen.swift index 08e3928f6d3a..e40be705a051 100644 --- a/WordPress/WordPressUITests/Screens/Editor/BlockEditorScreen.swift +++ b/WordPress/WordPressUITests/Screens/Editor/BlockEditorScreen.swift @@ -65,20 +65,29 @@ class BlockEditorScreen: BaseScreen { // returns void since return screen depends on from which screen it loaded func closeEditor() { - // Close the More menu if needed - if actionSheet.exists { - if isIpad { - app.otherElements["PopoverDismissRegion"].tap() - } else { - keepEditingButton.tap() + XCTContext.runActivity(named: "Close the block editor") { (activity) in + XCTContext.runActivity(named: "Close the More menu if needed") { (activity) in + if actionSheet.exists { + if isIpad { + app.otherElements["PopoverDismissRegion"].tap() + } else { + keepEditingButton.tap() + } + } } - } - // Close the editor and discard any local changes - editorCloseButton.tap() - let notSavedState = app.staticTexts["You have unsaved changes."] - if notSavedState.exists { - Logger.log(message: "Discarding unsaved changes", event: .v) - discardButton.tap() + + editorCloseButton.tap() + + XCTContext.runActivity(named: "Discard any local changes") { (activity) in + let notSavedState = app.staticTexts["You have unsaved changes."] + if notSavedState.exists { + Logger.log(message: "Discarding unsaved changes", event: .v) + discardButton.tap() + } + } + + let editorClosed = waitFor(element: editorNavBar, predicate: "isEnabled == false") + XCTAssert(editorClosed, "Block editor should be closed but is still loaded.") } } diff --git a/WordPress/WordPressUITests/Tests/EditorAztecTests.swift b/WordPress/WordPressUITests/Tests/EditorAztecTests.swift index 2e6625d2986a..980006ec38f4 100644 --- a/WordPress/WordPressUITests/Tests/EditorAztecTests.swift +++ b/WordPress/WordPressUITests/Tests/EditorAztecTests.swift @@ -13,6 +13,7 @@ class EditorAztecTests: XCTestCase { } override func tearDown() { + takeScreenshotOfFailedTest() if editorScreen != nil && !TabNavComponent.isVisible() { EditorFlow.returnToMainEditorScreen() editorScreen.closeEditor() diff --git a/WordPress/WordPressUITests/Tests/EditorGutenbergTests.swift b/WordPress/WordPressUITests/Tests/EditorGutenbergTests.swift index 7452bb41a588..c00da7ee11a9 100644 --- a/WordPress/WordPressUITests/Tests/EditorGutenbergTests.swift +++ b/WordPress/WordPressUITests/Tests/EditorGutenbergTests.swift @@ -13,6 +13,7 @@ class EditorGutenbergTests: XCTestCase { } override func tearDown() { + takeScreenshotOfFailedTest() if editorScreen != nil && !TabNavComponent.isVisible() { EditorFlow.returnToMainEditorScreen() editorScreen.closeEditor() diff --git a/WordPress/WordPressUITests/Tests/LoginTests.swift b/WordPress/WordPressUITests/Tests/LoginTests.swift index d73fe3295e0e..09099d034fdc 100644 --- a/WordPress/WordPressUITests/Tests/LoginTests.swift +++ b/WordPress/WordPressUITests/Tests/LoginTests.swift @@ -9,6 +9,7 @@ class LoginTests: XCTestCase { } override func tearDown() { + takeScreenshotOfFailedTest() LoginFlow.logoutIfNeeded() super.tearDown() } diff --git a/WordPress/WordPressUITests/Tests/MainNavigationTests.swift b/WordPress/WordPressUITests/Tests/MainNavigationTests.swift index f2df4d3268e4..28549e26efad 100644 --- a/WordPress/WordPressUITests/Tests/MainNavigationTests.swift +++ b/WordPress/WordPressUITests/Tests/MainNavigationTests.swift @@ -14,6 +14,7 @@ class MainNavigationTests: XCTestCase { } override func tearDown() { + takeScreenshotOfFailedTest() LoginFlow.logoutIfNeeded() super.tearDown() } @@ -27,6 +28,9 @@ class MainNavigationTests: XCTestCase { .tabBar.gotoBlockEditorScreen() .closeEditor() - XCTAssert(NotificationsScreen.isLoaded()) + XCTContext.runActivity(named: "Confirm Notifications screen and main navigation bar are loaded.") { (activity) in + XCTAssert(NotificationsScreen.isLoaded(), "Notifications screen isn't loaded.") + XCTAssert(TabNavComponent.isVisible(), "Main navigation bar isn't visible.") + } } } diff --git a/WordPress/WordPressUITests/Tests/SignupTests.swift b/WordPress/WordPressUITests/Tests/SignupTests.swift index 2c419424abeb..dbd57b023f54 100644 --- a/WordPress/WordPressUITests/Tests/SignupTests.swift +++ b/WordPress/WordPressUITests/Tests/SignupTests.swift @@ -9,6 +9,7 @@ class SignupTests: XCTestCase { } override func tearDown() { + takeScreenshotOfFailedTest() LoginFlow.logoutIfNeeded() super.tearDown() } diff --git a/WordPress/WordPressUITests/Utils/XCTest+Extensions.swift b/WordPress/WordPressUITests/Utils/XCTest+Extensions.swift index d7203d07bdff..ac357172a6d5 100644 --- a/WordPress/WordPressUITests/Utils/XCTest+Extensions.swift +++ b/WordPress/WordPressUITests/Utils/XCTest+Extensions.swift @@ -52,6 +52,14 @@ extension XCTestCase { systemAlertHandler(alertTitle: "“WordPress” Would Like to Access Your Photos", alertButton: "OK") } + public func takeScreenshotOfFailedTest() { + if let failureCount = testRun?.failureCount, failureCount > 0 { + XCTContext.runActivity(named: "Take a screenshot at the end of a failed test") { (activity) in + add(XCTAttachment(screenshot: XCUIApplication().windows.firstMatch.screenshot())) + } + } + } + public func systemAlertHandler(alertTitle: String, alertButton: String) { addUIInterruptionMonitor(withDescription: alertTitle) { (alert) -> Bool in let alertButtonElement = alert.buttons[alertButton]