From fa86308669f42a2b2a1a8d498f650a68a5b00d2f Mon Sep 17 00:00:00 2001 From: Rabi Panda Date: Thu, 4 Feb 2021 11:04:56 -0800 Subject: [PATCH] Remove x-pack from build, distribution and packaging. (#43) This PR removes references to x-pack from buildSrc, distribution and qa modules. --- .../gradle/plugin/PluginBuildPlugin.groovy | 5 +- .../testclusters/ElasticsearchNode.java | 15 +--- .../plugins/InstallPluginCommand.java | 22 ----- .../plugins/InstallPluginCommandTests.java | 36 -------- gradle/formatting.gradle | 25 +----- .../packaging/test/DockerTests.java | 15 ---- .../packaging/util/Archives.java | 4 +- .../elasticsearch/packaging/util/Docker.java | 4 +- .../packaging/util/Packages.java | 4 +- .../packaging/util/ServerUtils.java | 22 ----- .../cluster/remote/test/RemoteClustersIT.java | 36 -------- .../org/elasticsearch/upgrades/XPackIT.java | 88 ------------------- server/build.gradle | 1 - settings.gradle | 1 - 14 files changed, 7 insertions(+), 271 deletions(-) delete mode 100644 qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/XPackIT.java diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy index 897717810ef27..f26ad552204ed 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy @@ -58,8 +58,7 @@ class PluginBuildPlugin implements Plugin { PluginPropertiesExtension extension = project.extensions.create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension, project) configureDependencies(project) - boolean isXPackModule = project.path.startsWith(':x-pack:plugin') - boolean isModule = project.path.startsWith(':modules:') || isXPackModule + boolean isModule = project.path.startsWith(':modules:') createBundleTasks(project, extension) @@ -103,7 +102,7 @@ class PluginBuildPlugin implements Plugin { expand(properties) inputs.properties(properties) } - if (isModule == false || isXPackModule) { + if (isModule == false) { addNoticeGeneration(project, extension1) } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index 821eff11b22a0..bfd4986b4db81 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -489,11 +489,6 @@ public synchronized void start() { pluginsToInstall.addAll(plugins.stream().map(Provider::get).map(p -> p.toURI().toString()).collect(Collectors.toList())); } - if (requiresAddingXPack()) { - logToProcessStdout("emulating the " + testDistribution + " flavor for " + getVersion() + " by installing x-pack"); - pluginsToInstall.add("x-pack"); - } - if (pluginsToInstall.isEmpty() == false) { if (getVersion().onOrAfter("7.6.0")) { logToProcessStdout("installing " + pluginsToInstall.size() + " plugins in a single transaction"); @@ -554,18 +549,10 @@ public synchronized void start() { startElasticsearchProcess(); } - private boolean requiresAddingXPack() { - return getVersion().before("6.3.0") && testDistribution == TestDistribution.DEFAULT; - } - private boolean canUseSharedDistribution() { // using original location can be too long due to MAX_PATH restrictions on windows CI // TODO revisit when moving to shorter paths on CI by using Teamcity - return OS.current() != OS.WINDOWS - && extraJarFiles.size() == 0 - && modules.size() == 0 - && plugins.size() == 0 - && requiresAddingXPack() == false; + return OS.current() != OS.WINDOWS && extraJarFiles.size() == 0 && modules.size() == 0 && plugins.size() == 0; } private void logToProcessStdout(String message) { diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java index 1f09cafb55e3b..e51b427db4efb 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java @@ -241,10 +241,6 @@ void execute(Terminal terminal, List pluginIds, boolean isBatch, Environ for (final String pluginId : pluginIds) { terminal.println("-> Installing " + pluginId); try { - if ("x-pack".equals(pluginId)) { - handleInstallXPack(buildFlavor()); - } - final List deleteOnFailure = new ArrayList<>(); deleteOnFailures.put(pluginId, deleteOnFailure); @@ -281,24 +277,6 @@ void execute(Terminal terminal, List pluginIds, boolean isBatch, Environ } } - Build.Flavor buildFlavor() { - return Build.CURRENT.flavor(); - } - - private static void handleInstallXPack(final Build.Flavor flavor) throws UserException { - switch (flavor) { - case DEFAULT: - throw new UserException(ExitCodes.CONFIG, "this distribution of Elasticsearch contains X-Pack by default"); - case OSS: - throw new UserException( - ExitCodes.CONFIG, - "X-Pack is not available with the oss distribution; to use X-Pack features use the default distribution" - ); - case UNKNOWN: - throw new IllegalStateException("your distribution is broken"); - } - } - /** Downloads the plugin and returns the file it was downloaded to. */ private Path download(Terminal terminal, String pluginId, Path tmpDir, boolean isBatch) throws Exception { if (OFFICIAL_PLUGINS.contains(pluginId)) { diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java index 933c1e512b675..9533c811db6da 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java @@ -525,15 +525,6 @@ public void testBuiltinModule() throws Exception { assertInstallCleaned(env.v2()); } - public void testBuiltinXpackModule() throws Exception { - Tuple env = createEnv(fs, temp); - Path pluginDir = createPluginDir(temp); - String pluginZip = createPluginUrl("x-pack", pluginDir); - UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1())); - assertTrue(e.getMessage(), e.getMessage().contains("is a system module")); - assertInstallCleaned(env.v2()); - } - public void testJarHell() throws Exception { // jar hell test needs a real filesystem assumeTrue("real filesystem", isReal); @@ -813,33 +804,6 @@ protected boolean addShutdownHook() { } } - public void testInstallXPack() throws IOException { - runInstallXPackTest(Build.Flavor.DEFAULT, UserException.class, "this distribution of Elasticsearch contains X-Pack by default"); - runInstallXPackTest( - Build.Flavor.OSS, - UserException.class, - "X-Pack is not available with the oss distribution; to use X-Pack features use the default distribution" - ); - runInstallXPackTest(Build.Flavor.UNKNOWN, IllegalStateException.class, "your distribution is broken"); - } - - private void runInstallXPackTest(final Build.Flavor flavor, final Class clazz, final String expectedMessage) - throws IOException { - final InstallPluginCommand flavorCommand = new InstallPluginCommand() { - @Override - Build.Flavor buildFlavor() { - return flavor; - } - }; - - final Environment environment = createEnv(fs, temp).v2(); - final T exception = expectThrows( - clazz, - () -> flavorCommand.execute(terminal, Collections.singletonList("x-pack"), false, environment) - ); - assertThat(exception, hasToString(containsString(expectedMessage))); - } - public void testInstallMisspelledOfficialPlugins() throws Exception { Tuple env = createEnv(fs, temp); diff --git a/gradle/formatting.gradle b/gradle/formatting.gradle index 5fc65042657c1..2b47c4915e2da 100644 --- a/gradle/formatting.gradle +++ b/gradle/formatting.gradle @@ -119,30 +119,7 @@ def projectPathsToExclude = [ ':test:fixtures:old-elasticsearch', ':test:fixtures:s3-fixture', ':test:framework', - ':test:logger-usage', - ':x-pack:license-tools', - ':x-pack:plugin:analytics', - ':x-pack:plugin:core', - ':x-pack:plugin:deprecation', - ':x-pack:plugin:frozen-indices', - ':x-pack:plugin:identity-provider', - ':x-pack:plugin:mapper-constant-keyword', - ':x-pack:plugin:mapper-flattened', - ':x-pack:plugin:ql', - ':x-pack:plugin:search-business-rules', - ':x-pack:plugin:spatial', - ':x-pack:plugin:vectors', - ':x-pack:plugin:voting-only-node', - ':x-pack:plugin:wildcard', - ':x-pack:qa', - ':x-pack:qa:security-example-spi-extension', - ':x-pack:snapshot-tool', - ':x-pack:snapshot-tool:qa:google-cloud-storage', - ':x-pack:snapshot-tool:qa:s3', - ':x-pack:test:feature-aware', - ':x-pack:test:idp-fixture', - ':x-pack:test:smb-fixture', - ':x-pack:transport-client' + ':test:logger-usage' ] subprojects { diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index 4101371f0f9d4..abcd0f521dd9c 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -67,7 +67,6 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.emptyString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; @@ -109,20 +108,6 @@ public void test010Install() { verifyContainerInstallation(installation, distribution()); } - /** - * Check that the /_xpack API endpoint's presence is correct for the type of distribution being tested. - */ - public void test011PresenceOfXpack() throws Exception { - waitForElasticsearch(installation); - final int statusCode = Request.Get("http://localhost:9200/_xpack").execute().returnResponse().getStatusLine().getStatusCode(); - - if (distribution.isOSS()) { - assertThat(statusCode, greaterThanOrEqualTo(400)); - } else { - assertThat(statusCode, equalTo(200)); - } - } - /** * Checks that no plugins are initially active. */ diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java index 72dab5a406c89..f7dba382557f1 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java @@ -210,9 +210,7 @@ private static void verifyDefaultInstallation(Installation es, Distribution dist "elasticsearch-saml-metadata", "elasticsearch-setup-passwords", "elasticsearch-syskeygen", - "elasticsearch-users", - "x-pack-env", - "x-pack-security-env" + "elasticsearch-users" ).forEach(executable -> { assertThat(es.bin(executable), file(File, owner, owner, p755)); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java index 7165c92a53488..0e6a4541f12c0 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java @@ -537,9 +537,7 @@ private static void verifyDefaultInstallation(Installation es) { "elasticsearch-saml-metadata", "elasticsearch-setup-passwords", "elasticsearch-syskeygen", - "elasticsearch-users", - "x-pack-env", - "x-pack-security-env" + "elasticsearch-users" ).forEach(executable -> assertPermissionsAndOwnership(es.bin(executable), p755)); Stream.of("role_mapping.yml", "roles.yml", "users", "users_roles") diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java index 8d286d2b80ec3..369ad170430ce 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java @@ -238,9 +238,7 @@ private static void verifyDefaultInstallation(Installation es, Distribution dist "elasticsearch-saml-metadata", "elasticsearch-setup-passwords", "elasticsearch-syskeygen", - "elasticsearch-users", - "x-pack-env", - "x-pack-security-env" + "elasticsearch-users" ).forEach(executable -> assertThat(es.bin(executable), file(File, "root", "root", p755))); Stream.of("users", "users_roles", "roles.yml", "role_mapping.yml", "log4j2.properties") diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java index 65c7c544c95c8..fa47baed058c4 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java @@ -41,8 +41,6 @@ import javax.net.ssl.TrustManagerFactory; import java.io.IOException; import java.io.InputStream; -import java.net.InetAddress; -import java.net.Socket; import java.nio.file.Files; import java.nio.file.Path; import java.security.KeyStore; @@ -114,26 +112,6 @@ private static HttpResponse execute(Request request, String username, String pas return executor.execute(request).returnResponse(); } - // polls every second for Elasticsearch to be running on 9200 - private static void waitForXpack() { - int retries = 60; - while (retries > 0) { - retries -= 1; - try (Socket s = new Socket(InetAddress.getLoopbackAddress(), 9200)) { - return; - } catch (IOException e) { - // ignore, only want to establish a connection - } - try { - Thread.sleep(1000); - } catch (InterruptedException interrupted) { - Thread.currentThread().interrupt(); - return; - } - } - throw new RuntimeException("Elasticsearch (with x-pack) did not start"); - } - public static void waitForElasticsearch(String status, String index, Installation installation, String username, String password) throws Exception { diff --git a/qa/remote-clusters/src/test/java/org/elasticsearch/cluster/remote/test/RemoteClustersIT.java b/qa/remote-clusters/src/test/java/org/elasticsearch/cluster/remote/test/RemoteClustersIT.java index 50ffe806dd424..586c2e6af66bf 100644 --- a/qa/remote-clusters/src/test/java/org/elasticsearch/cluster/remote/test/RemoteClustersIT.java +++ b/qa/remote-clusters/src/test/java/org/elasticsearch/cluster/remote/test/RemoteClustersIT.java @@ -116,40 +116,4 @@ public void testHAProxyModeConnectionWorks() throws IOException { assertEquals(2L, cluster1Client().search( new SearchRequest("haproxynosn:test2"), RequestOptions.DEFAULT).getHits().getTotalHits().value); } - - public void testHAProxyModeConnectionWithSNIToCluster1Works() throws IOException { - assumeThat("test is only supported if the distribution contains xpack", getDistribution(), equalTo("default")); - - ClusterUpdateSettingsRequest request = new ClusterUpdateSettingsRequest().persistentSettings(Settings.builder() - .put("cluster.remote.haproxysni1.mode", "proxy") - .put("cluster.remote.haproxysni1.proxy_address", "haproxy:9600") - .put("cluster.remote.haproxysni1.server_name", "application1.example.com") - .build()); - assertTrue(cluster2Client().cluster().putSettings(request, RequestOptions.DEFAULT).isAcknowledged()); - - RemoteConnectionInfo rci = cluster2Client().cluster().remoteInfo(new RemoteInfoRequest(), RequestOptions.DEFAULT).getInfos().get(0); - logger.info("Connection info: {}", rci); - assertTrue(rci.isConnected()); - - assertEquals(1L, cluster2Client().search( - new SearchRequest("haproxysni1:test1"), RequestOptions.DEFAULT).getHits().getTotalHits().value); - } - - public void testHAProxyModeConnectionWithSNIToCluster2Works() throws IOException { - assumeThat("test is only supported if the distribution contains xpack", getDistribution(), equalTo("default")); - - ClusterUpdateSettingsRequest request = new ClusterUpdateSettingsRequest().persistentSettings(Settings.builder() - .put("cluster.remote.haproxysni2.mode", "proxy") - .put("cluster.remote.haproxysni2.proxy_address", "haproxy:9600") - .put("cluster.remote.haproxysni2.server_name", "application2.example.com") - .build()); - assertTrue(cluster1Client().cluster().putSettings(request, RequestOptions.DEFAULT).isAcknowledged()); - - RemoteConnectionInfo rci = cluster1Client().cluster().remoteInfo(new RemoteInfoRequest(), RequestOptions.DEFAULT).getInfos().get(0); - logger.info("Connection info: {}", rci); - assertTrue(rci.isConnected()); - - assertEquals(2L, cluster1Client().search( - new SearchRequest("haproxysni2:test2"), RequestOptions.DEFAULT).getHits().getTotalHits().value); - } } diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/XPackIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/XPackIT.java deleted file mode 100644 index cc71b77a0af10..0000000000000 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/XPackIT.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.upgrades; - -import org.apache.http.util.EntityUtils; -import org.junit.Before; -import org.elasticsearch.client.Request; -import org.elasticsearch.rest.action.document.RestBulkAction; - -import java.io.IOException; - -import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assume.assumeThat; - -/** - * Basic tests for simple xpack functionality that are only run if the - * cluster is the on the default distribution. - */ -public class XPackIT extends AbstractRollingTestCase { - @Before - public void skipIfNotXPack() { - assumeThat("test is only supported if the distribution contains xpack", - System.getProperty("tests.distribution"), equalTo("default")); - assumeThat("running this on the unupgraded cluster would change its state and it wouldn't work prior to 6.3 anyway", - CLUSTER_TYPE, equalTo(ClusterType.UPGRADED)); - /* - * *Mostly* we want this for when we're upgrading from pre-6.3's - * zip distribution which doesn't contain xpack to post 6.3's zip - * distribution which *does* contain xpack. But we'll also run it - * on all upgrades for completeness's sake. - */ - } - - /** - * Test creating a trial license and using it. This is interesting because - * our other tests test cover starting a new cluster with the default - * distribution and enabling the trial license but this test is the only - * one that can upgrade from the oss distribution to the default - * distribution with xpack and the create a trial license. We don't - * do a lot with the trial license because for the most - * part those things are tested elsewhere, off in xpack. But we do use the - * trial license a little bit to make sure that it works. - */ - public void testTrialLicense() throws IOException { - Request startTrial = new Request("POST", "/_license/start_trial"); - startTrial.addParameter("acknowledge", "true"); - client().performRequest(startTrial); - - String noJobs = EntityUtils.toString( - client().performRequest(new Request("GET", "/_ml/anomaly_detectors")).getEntity()); - assertEquals("{\"count\":0,\"jobs\":[]}", noJobs); - - Request createJob = new Request("PUT", "/_ml/anomaly_detectors/test_job"); - createJob.setJsonEntity( - "{\n" - + " \"analysis_config\" : {\n" - + " \"bucket_span\": \"10m\",\n" - + " \"detectors\": [\n" - + " {\n" - + " \"function\": \"sum\",\n" - + " \"field_name\": \"total\"\n" - + " }\n" - + " ]\n" - + " },\n" - + " \"data_description\": {\n" - + " \"time_field\": \"timestamp\",\n" - + " \"time_format\": \"epoch_ms\"\n" - + " }\n" - + "}\n"); - client().performRequest(createJob); - } -} diff --git a/server/build.gradle b/server/build.gradle index 48ff7814b9ea2..e4a02bc64abf2 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -175,7 +175,6 @@ tasks.named("testingConventions").configure { def generateModulesList = tasks.register("generateModulesList") { List modules = project(':modules').subprojects.collect { it.name } - modules.add('x-pack') File modulesFile = new File(buildDir, 'generated-resources/modules.txt') processResources.from(modulesFile) inputs.property('modules', modules) diff --git a/settings.gradle b/settings.gradle index 98d7aa3cb63bc..e02d8f499ba5f 100644 --- a/settings.gradle +++ b/settings.gradle @@ -114,7 +114,6 @@ addSubProjects('', new File(rootProject.projectDir, 'modules')) addSubProjects('', new File(rootProject.projectDir, 'plugins')) addSubProjects('', new File(rootProject.projectDir, 'qa')) addSubProjects('test', new File(rootProject.projectDir, 'test/external-modules')) -addSubProjects('', new File(rootProject.projectDir, 'x-pack')) List startTasks = gradle.startParameter.taskNames