Improve UI test output (logs and screenshots) (#11996)

* Add assertions that the editor was successfully closed * Group test steps into named activities for clarity in test logs * Take screenshot at the end of failed tests * Add wait before checking that editor is closed
wordpress-mobile · Jun 26, 2019 · 69ec180 · 69ec180
1 parent ff94001
commit 69ec180
Show file tree

Hide file tree

Showing 10 changed files with 99 additions and 56 deletions.
diff --git a/WordPress/WordPressUITests/Flows/LoginFlow.swift b/WordPress/WordPressUITests/Flows/LoginFlow.swift
@@ -21,23 +21,29 @@ class LoginFlow {
     }
 
     static func logoutIfNeeded() {
-        if TabNavComponent.isLoaded() {
-            Logger.log(message: "Logging out...", event: .i)
-            let meScreen = TabNavComponent().gotoMeScreen()
-            if meScreen.isLoggedInToWpcom() {
-                _ = meScreen.logout()
-            } else {
-                _ = TabNavComponent().gotoMySiteScreen()
-                .removeSelfHostedSite()
+        XCTContext.runActivity(named: "Log out of app if currently logged in") { (activity) in
+            if TabNavComponent.isLoaded() {
+                Logger.log(message: "Logging out...", event: .i)
+                let meScreen = TabNavComponent().gotoMeScreen()
+                if meScreen.isLoggedInToWpcom() {
+                    _ = meScreen.logout()
+                } else {
+                    _ = TabNavComponent().gotoMySiteScreen()
+                        .removeSelfHostedSite()
+                }
+                return
             }
-            return
         }
 
-        while LoginPasswordScreen.isLoaded() || LoginEmailScreen.isLoaded() || LinkOrPasswordScreen.isLoaded() || LoginSiteAddressScreen.isLoaded() || LoginUsernamePasswordScreen.isLoaded() || LoginCheckMagicLinkScreen.isLoaded() {
-            if LoginEmailScreen.isLoaded() && LoginEmailScreen.isEmailEntered() {
-                LoginEmailScreen().emailTextField.clearTextIfNeeded()
+        XCTContext.runActivity(named: "Return to app prologue screen if needed") { (activity) in
+            if !WelcomeScreen.isLoaded() {
+                while LoginPasswordScreen.isLoaded() || LoginEmailScreen.isLoaded() || LinkOrPasswordScreen.isLoaded() || LoginSiteAddressScreen.isLoaded() || LoginUsernamePasswordScreen.isLoaded() || LoginCheckMagicLinkScreen.isLoaded() {
+                    if LoginEmailScreen.isLoaded() && LoginEmailScreen.isEmailEntered() {
+                        LoginEmailScreen().emailTextField.clearTextIfNeeded()
+                    }
+                    navBackButton.tap()
+                }
             }
-            navBackButton.tap()
         }
     }
 }
diff --git a/WordPress/WordPressUITests/Screens/BaseScreen.swift b/WordPress/WordPressUITests/Screens/BaseScreen.swift
@@ -19,9 +19,11 @@ class BaseScreen {
 
     @discardableResult
     func waitForPage() throws -> BaseScreen {
-        let result = waitFor(element: expectedElement, predicate: "isEnabled == true", timeout: 20)
-        XCTAssert(result, "Page \(self) is not loaded.")
-        Logger.log(message: "Page \(self) is loaded", event: .i)
+        XCTContext.runActivity(named: "Confirm page \(self) is loaded") { (activity) in
+            let result = waitFor(element: expectedElement, predicate: "isEnabled == true", timeout: 20)
+            XCTAssert(result, "Page \(self) is not loaded.")
+            Logger.log(message: "Page \(self) is loaded", event: .i)
+        }
         return self
     }
 
@@ -46,23 +48,25 @@ class BaseScreen {
     }
 
     func openMagicLink() {
-        let safari = XCUIApplication(bundleIdentifier: "com.apple.mobilesafari")
-        safari.launch()
+        XCTContext.runActivity(named: "Open magic link in Safari") { (activity) in
+            let safari = XCUIApplication(bundleIdentifier: "com.apple.mobilesafari")
+            safari.launch()
 
-        // Select the URL bar when Safari opens
-        let urlBar = safari.otherElements["URL"]
-        waitFor(element: urlBar, predicate: "exists == true")
-        urlBar.tap()
+            // Select the URL bar when Safari opens
+            let urlBar = safari.otherElements["URL"]
+            waitFor(element: urlBar, predicate: "exists == true")
+            urlBar.tap()
 
-        // Follow the magic link
-        var magicLinkComponents = URLComponents(url: WireMock.URL(), resolvingAgainstBaseURL: false)!
-        magicLinkComponents.path = "/magic-link"
-        magicLinkComponents.queryItems = [URLQueryItem(name: "scheme", value: "wpdebug")]
+            // Follow the magic link
+            var magicLinkComponents = URLComponents(url: WireMock.URL(), resolvingAgainstBaseURL: false)!
+            magicLinkComponents.path = "/magic-link"
+            magicLinkComponents.queryItems = [URLQueryItem(name: "scheme", value: "wpdebug")]
 
-        safari.textFields["URL"].typeText("\(magicLinkComponents.url!.absoluteString)\n")
+            safari.textFields["URL"].typeText("\(magicLinkComponents.url!.absoluteString)\n")
 
-        // Accept the prompt to open the deep link
-        safari.scrollViews.element(boundBy: 0).buttons.element(boundBy: 1).tap()
+            // Accept the prompt to open the deep link
+            safari.scrollViews.element(boundBy: 0).buttons.element(boundBy: 1).tap()
+        }
     }
 }
 

diff --git a/WordPress/WordPressUITests/Screens/Editor/AztecEditorScreen.swift b/WordPress/WordPressUITests/Screens/Editor/AztecEditorScreen.swift
@@ -232,21 +232,29 @@ class AztecEditorScreen: BaseScreen {
 
     // returns void since return screen depends on from which screen it loaded
     func closeEditor() {
-        // Close the More menu if needed
-        if actionSheet.exists {
-            if isIpad {
-                app.otherElements["PopoverDismissRegion"].tap()
-            } else {
-                keepEditingButton.tap()
+        XCTContext.runActivity(named: "Close the Aztec editor") { (activity) in
+            XCTContext.runActivity(named: "Close the More menu if needed") { (activity) in
+                if actionSheet.exists {
+                    if isIpad {
+                        app.otherElements["PopoverDismissRegion"].tap()
+                    } else {
+                        keepEditingButton.tap()
+                    }
+                }
+            }
+
+            editorCloseButton.tap()
+
+            XCTContext.runActivity(named: "Discard any local changes") { (activity) in
+                let notSavedState = app.staticTexts["You have unsaved changes."]
+                if notSavedState.exists {
+                    Logger.log(message: "Discarding unsaved changes", event: .v)
+                    discardButton.tap()
+                }
             }
-        }
 
-        // Close the editor and discard any local changes
-        editorCloseButton.tap()
-        let notSavedState = app.staticTexts["You have unsaved changes."]
-        if notSavedState.exists {
-            Logger.log(message: "Discarding unsaved changes", event: .v)
-            discardButton.tap()
+            let editorClosed = waitFor(element: editorCloseButton, predicate: "isEnabled == false")
+            XCTAssert(editorClosed, "Aztec editor should be closed but is still loaded.")
         }
     }
 

diff --git a/WordPress/WordPressUITests/Screens/Editor/BlockEditorScreen.swift b/WordPress/WordPressUITests/Screens/Editor/BlockEditorScreen.swift
@@ -65,20 +65,29 @@ class BlockEditorScreen: BaseScreen {
 
     // returns void since return screen depends on from which screen it loaded
     func closeEditor() {
-        // Close the More menu if needed
-        if actionSheet.exists {
-            if isIpad {
-                app.otherElements["PopoverDismissRegion"].tap()
-            } else {
-                keepEditingButton.tap()
+        XCTContext.runActivity(named: "Close the block editor") { (activity) in
+            XCTContext.runActivity(named: "Close the More menu if needed") { (activity) in
+                if actionSheet.exists {
+                    if isIpad {
+                        app.otherElements["PopoverDismissRegion"].tap()
+                    } else {
+                        keepEditingButton.tap()
+                    }
+                }
             }
-        }
-        // Close the editor and discard any local changes
-        editorCloseButton.tap()
-        let notSavedState = app.staticTexts["You have unsaved changes."]
-        if notSavedState.exists {
-            Logger.log(message: "Discarding unsaved changes", event: .v)
-            discardButton.tap()
+
+            editorCloseButton.tap()
+
+            XCTContext.runActivity(named: "Discard any local changes") { (activity) in
+                let notSavedState = app.staticTexts["You have unsaved changes."]
+                if notSavedState.exists {
+                    Logger.log(message: "Discarding unsaved changes", event: .v)
+                    discardButton.tap()
+                }
+            }
+
+            let editorClosed = waitFor(element: editorNavBar, predicate: "isEnabled == false")
+            XCTAssert(editorClosed, "Block editor should be closed but is still loaded.")
         }
     }
 

diff --git a/WordPress/WordPressUITests/Tests/EditorAztecTests.swift b/WordPress/WordPressUITests/Tests/EditorAztecTests.swift
@@ -13,6 +13,7 @@ class EditorAztecTests: XCTestCase {
     }
 
     override func tearDown() {
+        takeScreenshotOfFailedTest()
         if editorScreen != nil && !TabNavComponent.isVisible() {
             EditorFlow.returnToMainEditorScreen()
             editorScreen.closeEditor()

diff --git a/WordPress/WordPressUITests/Tests/EditorGutenbergTests.swift b/WordPress/WordPressUITests/Tests/EditorGutenbergTests.swift
@@ -13,6 +13,7 @@ class EditorGutenbergTests: XCTestCase {
     }
 
     override func tearDown() {
+        takeScreenshotOfFailedTest()
         if editorScreen != nil && !TabNavComponent.isVisible() {
             EditorFlow.returnToMainEditorScreen()
             editorScreen.closeEditor()

diff --git a/WordPress/WordPressUITests/Tests/LoginTests.swift b/WordPress/WordPressUITests/Tests/LoginTests.swift
@@ -9,6 +9,7 @@ class LoginTests: XCTestCase {
     }
 
     override func tearDown() {
+        takeScreenshotOfFailedTest()
         LoginFlow.logoutIfNeeded()
         super.tearDown()
     }

diff --git a/WordPress/WordPressUITests/Tests/MainNavigationTests.swift b/WordPress/WordPressUITests/Tests/MainNavigationTests.swift
@@ -14,6 +14,7 @@ class MainNavigationTests: XCTestCase {
     }
 
     override func tearDown() {
+        takeScreenshotOfFailedTest()
         LoginFlow.logoutIfNeeded()
         super.tearDown()
     }
@@ -27,6 +28,9 @@ class MainNavigationTests: XCTestCase {
             .tabBar.gotoBlockEditorScreen()
             .closeEditor()
 
-        XCTAssert(NotificationsScreen.isLoaded())
+        XCTContext.runActivity(named: "Confirm Notifications screen and main navigation bar are loaded.") { (activity) in
+            XCTAssert(NotificationsScreen.isLoaded(), "Notifications screen isn't loaded.")
+            XCTAssert(TabNavComponent.isVisible(), "Main navigation bar isn't visible.")
+        }
     }
 }
diff --git a/WordPress/WordPressUITests/Tests/SignupTests.swift b/WordPress/WordPressUITests/Tests/SignupTests.swift
@@ -9,6 +9,7 @@ class SignupTests: XCTestCase {
     }
 
     override func tearDown() {
+        takeScreenshotOfFailedTest()
         LoginFlow.logoutIfNeeded()
         super.tearDown()
     }

diff --git a/WordPress/WordPressUITests/Utils/XCTest+Extensions.swift b/WordPress/WordPressUITests/Utils/XCTest+Extensions.swift
@@ -52,6 +52,14 @@ extension XCTestCase {
         systemAlertHandler(alertTitle: "“WordPress” Would Like to Access Your Photos", alertButton: "OK")
     }
 
+    public func takeScreenshotOfFailedTest() {
+        if let failureCount = testRun?.failureCount, failureCount > 0 {
+            XCTContext.runActivity(named: "Take a screenshot at the end of a failed test") { (activity) in
+                add(XCTAttachment(screenshot: XCUIApplication().windows.firstMatch.screenshot()))
+            }
+        }
+    }
+
     public func systemAlertHandler(alertTitle: String, alertButton: String) {
         addUIInterruptionMonitor(withDescription: alertTitle) { (alert) -> Bool in
             let alertButtonElement = alert.buttons[alertButton]