Add the common module of EngineConn of Linkis1.0 architecture

linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/pom.xml

@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright 2019 WeBank
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~ http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>linkis</artifactId>
+        <groupId>com.webank.wedatasphere.linkis</groupId>
+        <version>dev-1.0.0</version>
+        <relativePath>../../../pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>linkis-engineconn-common</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.webank.wedatasphere.linkis</groupId>
+            <artifactId>linkis-common</artifactId>
+            <version>${linkis.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.webank.wedatasphere.linkis</groupId>
+            <artifactId>linkis-protocol</artifactId>
+            <version>${linkis.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.webank.wedatasphere.linkis</groupId>
+            <artifactId>linkis-label-common</artifactId>
+            <version>${linkis.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.webank.wedatasphere.linkis</groupId>
+            <artifactId>linkis-manager-common</artifactId>
+            <version>${linkis.version}</version>
+        </dependency>
+
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-deploy-plugin</artifactId>
+            </plugin>
+
+            <plugin>
+                <groupId>net.alchim31.maven</groupId>
+                <artifactId>scala-maven-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <configuration>
+                    <!--<excludes>-->
+                    <!--<exclude>**/*.yml</exclude>-->
+                    <!--<exclude>**/*.properties</exclude>-->
+                    <!--<exclude>**/*.sh</exclude>-->
+                    <!--</excludes>-->
+                </configuration>
+            </plugin>
+        </plugins>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+            </resource>
+        </resources>
+        <finalName>${project.artifactId}-${project.version}</finalName>
+    </build>
+
+
+</project>

linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/conf/EngineConnConf.scala

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.common.conf
+
+import com.webank.wedatasphere.linkis.common.conf.CommonVars
+
+
+object EngineConnConf {
+
+  val ENGINE_EXECUTIONS = CommonVars("wds.linkis.engine.connector.executions", "com.webank.wedatasphere.linkis.engineconn.computation.executor.execute.ComputationEngineConnExecution")
+
+  val ENGINE_CONN_HOOKS = CommonVars("wds.linkis.engine.connector.hooks", "com.webank.wedatasphere.linkis.engineconn.computation.executor.hook.ComputationEngineConnHook")
+
+
+  val ENGINE_LAUNCH_CMD_PARAMS_USER_KEY = CommonVars("wds.linkis.engine.launch.cmd.params.user.key", "user")
+
+  val ENGINE_SUPPORT_PARALLELISM = CommonVars("wds.linkis.engine.parallelism.support.enabled", false)
+
+  val ENGINE_PUSH_LOG_TO_ENTRANCE = CommonVars("wds.linkis.engine.push.log.enable", true)
+
+  val ENGINECONN_PLUGIN_CLAZZ = CommonVars("wds.linkis.engineconn.plugin.default.clazz", "com.webank.wedatasphere.linkis.engineplugin.hive.HiveEngineConnPlugin")
+
+
+  val ENGINE_TASK_EXPIRE_TIME = CommonVars("wds.linkis.engine.task.expire.time", 1000 * 3600 * 24)
+
+  val ENGINE_LOCK_REFRESH_TIME = CommonVars("wds.linkis.engine.lock.refresh.time", 1000 * 60 * 3)
+
+  val ENGINE_CONN_LOCALPATH_PWD_KEY = CommonVars("wds.linkis.engine.localpath.pwd.key", "PWD")
+
+  val ENGINE_CONN_LOCAL_LOG_DIRS_KEY = CommonVars("wds.linkis.engine.logs.dir.key", "LOG_DIRS")
+}

linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/conf/EngineConnConstant.scala

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.common.conf
+
+
+object EngineConnConstant {
+
+  val MAX_TASK_NUM = 10000
+
+  val SPRING_CONF_MAP_NAME = "SpringConfMap"
+
+  val MAX_EXECUTOR_ID_NAME = "MaxExecutorId"
+}

linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/creation/EngineCreationContext.scala

@@ -0,0 +1,103 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.common.creation
+
+import java.util
+
+import com.webank.wedatasphere.linkis.common.ServiceInstance
+import com.webank.wedatasphere.linkis.manager.label.entity.Label
+
+
+trait EngineCreationContext {
+
+  def getUser: String
+
+  def setUser(user: String): Unit
+
+  def getTicketId: String
+
+  def setTicketId(ticketId: String): Unit
+
+  def getLabels(): util.List[Label[_]]
+
+  def setLabels(labels: util.List[Label[_]]): Unit
+
+  def getEMInstance: ServiceInstance
+
+  def setEMInstance(instance: ServiceInstance): Unit
+
+  def getOptions: util.Map[String, String]
+
+  def setOptions(options: util.Map[String, String])
+
+  def getExecutorId: Int
+
+  def setExecutorId(id: Int)
+
+  def setArgs(args: Array[String])
+
+  def getArgs: Array[String]
+}
+
+class DefaultEngineCreationContext extends EngineCreationContext {
+
+  private var user: String = _
+
+  private var ticketId: String = _
+
+  private var labels: util.List[Label[_]] = _
+
+  private var options: util.Map[String, String] = _
+
+  private var emInstance: ServiceInstance = _
+
+  private var executorId: Int = 0
+
+  private var args: Array[String] = null
+
+  override def getTicketId: String = this.ticketId
+
+  override def setTicketId(ticketId: String): Unit = this.ticketId = ticketId
+
+  override def getLabels(): util.List[Label[_]] = this.labels
+
+  override def setLabels(labels: util.List[Label[_]]): Unit = this.labels = labels
+
+  override def getEMInstance: ServiceInstance = this.emInstance
+
+  override def setEMInstance(instance: ServiceInstance): Unit = this.emInstance = instance
+
+  override def getOptions: util.Map[String, String] = this.options
+
+  override def setOptions(options: util.Map[String, String]): Unit = this.options = options
+
+  override def getUser: String = user
+
+  override def setUser(user: String): Unit = this.user = user
+
+  override def getExecutorId: Int = executorId
+
+  override def setExecutorId(id: Int): Unit = executorId = id
+
+  override def setArgs(args: Array[String]): Unit = this.args = args
+
+  override def getArgs: Array[String] = args
+
+
+  override def toString = s"DefaultEngineCreationContext(user-$user, ticketID-$ticketId, emInstance-$emInstance)"
+}
+

linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/engineconn/EngineConn.scala

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.common.engineconn
+
+import com.webank.wedatasphere.linkis.engineconn.common.creation.EngineCreationContext
+
+
+trait EngineConn {
+
+  /**
+    * 底层engine的类型比如：Spark、hive
+    *
+    * @return
+    */
+  def getEngineType(): String
+
+  def setEngineType(engineType: String): Unit
+
+  /**
+    * 底层计算存储Engine的具体连接信息，比如SparkSession,hive的sessionState
+    *
+    * @return
+    */
+  def getEngine(): Any
+
+  def setEngine(engine: Any): Unit
+
+  def getEngineCreationContext: EngineCreationContext
+
+
+}
+
+class DefaultEngineConn(engineCreationContext: EngineCreationContext) extends EngineConn {
+
+  var engineType: String = "spark"
+
+  var engine: Any = null
+
+
+  /**
+    * 底层engine的类型比如：Spark、hive
+    *
+    * @return
+    */
+  override def getEngineType(): String = engineType
+
+  override def setEngineType(engineType: String): Unit = this.engineType = engineType
+
+  /**
+    * 底层计算存储Engine的具体连接信息，比如SparkSession,hive的sessionState
+    *
+    * @return
+    */
+  override def getEngine(): Any = engine
+
+  override def setEngine(engine: Any): Unit = this.engine = engine
+
+  override def getEngineCreationContext: EngineCreationContext = engineCreationContext
+
+}

linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/com/webank/wedatasphere/linkis/engineconn/common/exception/EngineConnException.scala

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engineconn.common.exception
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException
+
+
+case class EngineTypeVersionNotMatchException(msg: String) extends ErrorException(40010, msg)
+
+