Make panel action tests more generic (#22174) (#22196)

* Make panel action tests more generic

* Remove test file

* Add simple app tests

* Shorten/fix import

* Remove unneeded task

* Add debug output

* Allow duplicate --plugin-path params

* Remove debugging options

* Add README

packages/kbn-test/src/functional_tests/lib/run_kibana_server.js

@@ -77,7 +77,10 @@ function filterCliArgs(args) {
 
     // Check if original argv has a later setting that overrides
     // the current val. If so, skip this val.
-    if (findIndexFrom(args, ++ind, opt => opt.split('=')[0] === val.split('=')[0]) > -1) {
+    if (
+      !allowsDuplicate(val) &&
+      findIndexFrom(args, ++ind, opt => opt.split('=')[0] === val.split('=')[0]) > -1
+    ) {
       return acc;
     }
 
@@ -96,6 +99,14 @@ function pipe(arr, ...fns) {
   }, arr);
 }
 
+/**
+ * Checks whether a specific parameter is allowed to appear multiple
+ * times in the Kibana parameters.
+ */
+function allowsDuplicate(val) {
+  return ['--plugin-path'].includes(val.split('=')[0]);
+}
+
 function isBasePathSettingOverridden(args, val, ind) {
   const key = val.split('=')[0];
   const basePathKeys = ['--no-base-path', '--server.basePath'];

scripts/functional_tests.js

@@ -21,5 +21,5 @@ require('../src/setup_node_env');
 require('@kbn/test').runTestsCli([
   require.resolve('../test/functional/config.js'),
   require.resolve('../test/api_integration/config.js'),
-  require.resolve('../test/panel_actions/config.js'),
+  require.resolve('../test/plugin_functional/config.js'),
 ]);

tasks/config/run.js

@@ -178,14 +178,17 @@ module.exports = function (grunt) {
       ],
     },
 
-    panelActionTests: {
+    pluginFunctionalTestsRelease: {
       cmd: process.execPath,
       args: [
         'scripts/functional_tests',
-        '--config', 'test/panel_actions/config.js',
+        '--config', 'test/plugin_functional/config.js',
         '--esFrom', 'source',
         '--bail',
         '--debug',
+        '--kibana-install-dir', `./build/oss/kibana-${PKG_VERSION}-${process.platform}-x86_64`,
+        '--',
+        '--server.maxPayloadBytes=1648576',
       ],
     },
 

tasks/jenkins.js

@@ -42,5 +42,6 @@ module.exports = function (grunt) {
   grunt.registerTask('jenkins:selenium', [
     'checkPlugins',
     'run:functionalTestsRelease',
+    'run:pluginFunctionalTestsRelease',
   ]);
 };

test/functional/page_objects/header_page.js

@@ -268,6 +268,16 @@ export function HeaderPageProvider({ getService, getPageObjects }) {
       await testSubjects.find('globalLoadingIndicator-hidden', defaultFindTimeout * 10);
     }
 
+    async getGlobalNavigationLink(linkText) {
+      const nav = await testSubjects.find('globalNav');
+      return await nav.findByPartialLinkText(linkText);
+    }
+
+    async clickGlobalNavigationLink(appTitle) {
+      const link = await this.getGlobalNavigationLink(appTitle);
+      await link.click();
+    }
+
     async getPrettyDuration() {
       return await testSubjects.getVisibleText('globalTimepickerRange');
     }

test/plugin_functional/README.md

@@ -0,0 +1,23 @@
+# Plugin Functional Tests
+
+This folder contains plugin functional tests, i.e. functional tests that should be executed
+against a Kibana instance with specific test plugins available.
+
+To add a plugin to the instance, just place the plugin folder in the `plugins`
+directory.
+
+Add new test suites into the `test_suites` folder and reference them from the
+`config.js` file. These test suites work the same as regular functional test
+except that they are executed against a Kibana with all plugins (from the
+`plugins` directory) installed.
+
+## Run the test
+
+To run these tests during development you can use the following commands:
+
+```
+# Start the test server (can continue running)
+node scripts/functional_tests_server.js --config test/plugin_functional/config.js
+# Start a test run
+node scripts/functional_test_runner.js --config test/plugin_functional/config.js
+```

test/panel_actions/config.js → test/plugin_functional/config.js

@@ -18,32 +18,37 @@
  */
 
 import path from 'path';
+import fs from 'fs';
 
 export default async function ({ readConfigFile }) {
   const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
 
+  // Find all folders in ./plugins since we treat all them as plugin folder
+  const allFiles = fs.readdirSync(path.resolve(__dirname, 'plugins'));
+  const plugins = allFiles.filter(file => fs.statSync(path.resolve(__dirname, 'plugins', file)).isDirectory());
+
   return {
     testFiles: [
-      require.resolve('./index'),
+      require.resolve('./test_suites/app_plugins'),
+      require.resolve('./test_suites/panel_actions'),
     ],
     services: functionalConfig.get('services'),
     pageObjects: functionalConfig.get('pageObjects'),
     servers: functionalConfig.get('servers'),
-    env: functionalConfig.get('env'),
     esTestCluster: functionalConfig.get('esTestCluster'),
     apps: functionalConfig.get('apps'),
     esArchiver: {
       directory: path.resolve(__dirname, '../es_archives')
     },
     screenshots: functionalConfig.get('screenshots'),
     junit: {
-      reportName: 'Panel Actions Functional Tests',
+      reportName: 'Plugin Functional Tests',
     },
     kbnTestServer: {
       ...functionalConfig.get('kbnTestServer'),
       serverArgs: [
         ...functionalConfig.get('kbnTestServer.serverArgs'),
-        `--plugin-path=${path.resolve(__dirname, './sample_panel_action')}`,
+        ...plugins.map(pluginDir => `--plugin-path=${path.resolve(__dirname, 'plugins', pluginDir)}`),
       ],
     },
   };

test/plugin_functional/plugins/sample_app_plugin/index.js

@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export default function (kibana) {
+  return new kibana.Plugin({
+    uiExports: {
+      app: {
+        title: 'Test Plugin App',
+        description: 'This is a sample plugin for the functional tests.',
+        main: 'plugins/sample_app_plugin/app',
+      }
+    }
+  });
+}

test/plugin_functional/plugins/sample_app_plugin/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "sample_app_plugin",
+  "version": "kibana"
+}

test/plugin_functional/plugins/sample_app_plugin/public/app.js

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import 'ui/autoload/all';
+
+import chrome from 'ui/chrome';
+
+chrome.setRootTemplate('<div data-test-subj="pluginContent">Super simple app plugin</div>');

test/panel_actions/sample_panel_action/package.json → test/plugin_functional/plugins/sample_panel_action/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sample_panel_action",
-  "version": "7.0.0-alpha1",
+  "version": "kibana",
   "dependencies": {
     "@elastic/eui": "0.0.55",
     "react": "^16.4.1"

test/panel_actions/sample_panel_action/public/sample_panel_action.js → test/plugin_functional/plugins/sample_panel_action/public/sample_panel_action.js

@@ -18,12 +18,12 @@
  */
 import { EuiFlyoutBody, EuiFlyoutHeader, EuiTitle } from '@elastic/eui';
 import React from 'react';
-import { openFlyout } from '../../../../src/ui/public/flyout';
+import { openFlyout } from 'ui/flyout';
 
 import {
   DashboardPanelAction,
   DashboardPanelActionsRegistryProvider,
-} from '../../../../src/ui/public/dashboard_panel_actions';
+} from 'ui/dashboard_panel_actions';
 
 class SamplePanelAction extends DashboardPanelAction {
   constructor() {

test/plugin_functional/test_suites/app_plugins/app_navigation.js

@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import expect from 'expect.js';
+
+export default function ({ getService, getPageObjects }) {
+  const testSubjects = getService('testSubjects');
+  const PageObjects = getPageObjects(['common', 'header', 'home']);
+
+  describe('app navigation', function describeIndexTests() {
+
+    before(async () => {
+      await PageObjects.common.navigateToApp('settings');
+    });
+
+    it('should show in navigation', async () => {
+      const link = await PageObjects.header.getGlobalNavigationLink('Test Plugin App');
+      expect(link).not.to.be(undefined);
+    });
+
+    it('should navigate to the app', async () => {
+      await PageObjects.header.clickGlobalNavigationLink('Test Plugin App');
+      const pluginContent = await testSubjects.find('pluginContent');
+      expect(await pluginContent.getVisibleText()).to.be('Super simple app plugin');
+    });
+  });
+
+}

test/plugin_functional/test_suites/app_plugins/index.js

@@ -0,0 +1,24 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export default function ({ loadTestFile }) {
+  describe('app plugins', () => {
+    loadTestFile(require.resolve('./app_navigation'));
+  });
+}

test/panel_actions/index.js → test/plugin_functional/test_suites/panel_actions/index.js

@@ -19,8 +19,8 @@
 
 import path from 'path';
 
-export const KIBANA_ARCHIVE_PATH = path.resolve(__dirname, '../functional/fixtures/es_archiver/dashboard/current/kibana');
-export const DATA_ARCHIVE_PATH = path.resolve(__dirname, '../functional/fixtures/es_archiver/dashboard/current/data');
+export const KIBANA_ARCHIVE_PATH = path.resolve(__dirname, '../../../functional/fixtures/es_archiver/dashboard/current/kibana');
+export const DATA_ARCHIVE_PATH = path.resolve(__dirname, '../../../functional/fixtures/es_archiver/dashboard/current/data');
 
 
 export default function ({ getService, getPageObjects, loadTestFile }) {