From 8d2179123065204b8ba4d85a92821d22ed016ce0 Mon Sep 17 00:00:00 2001 From: Casion Date: Tue, 28 Mar 2023 21:35:24 +0800 Subject: [PATCH] appcon compatibility (#4420) * engine bml resource compatible prefix v format --- ...linkis-engineconn-plugin-core-errorcode.md | 2 +- .../server/security/SecurityFilter.scala | 6 +++- .../EngineconnCoreErrorCodeSummary.java | 2 +- .../conf/EngineConnPluginConfiguration.scala | 4 +++ ...stractEngineConnBmlResourceGenerator.scala | 35 ++++++++++++++++--- .../DefaultEngineConnResourceService.scala | 13 +++++-- .../service/impl/DefaultResourceManager.scala | 2 +- .../src/apps/linkis/i18n/common/en.json | 2 +- 8 files changed, 53 insertions(+), 13 deletions(-) diff --git a/docs/errorcode/linkis-engineconn-plugin-core-errorcode.md b/docs/errorcode/linkis-engineconn-plugin-core-errorcode.md index a929d062d2..0fc2521a76 100644 --- a/docs/errorcode/linkis-engineconn-plugin-core-errorcode.md +++ b/docs/errorcode/linkis-engineconn-plugin-core-errorcode.md @@ -3,7 +3,7 @@ | module name(模块名) | error code(错误码) | describe(描述) |enumeration name(枚举)| Exception Class(类名)| | -------- | -------- | ----- |-----|-----| |linkis-engineconn-plugin-core |10001|Failed to createEngineConnLaunchRequest(创建 EngineConnLaunchRequest失败)|FAILED_CREATE_ELR|EngineconnCoreErrorCodeSummary| -|linkis-engineconn-plugin-core |10001|The engine plug-in material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)|EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION|EngineconnCoreErrorCodeSummary| +|linkis-engineconn-plugin-core |10001|The engine plugin material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)|EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION|EngineconnCoreErrorCodeSummary| |linkis-engineconn-plugin-core |10001|EngineTypeLabel are requested(需要参数 EngineTypeLabel)|ETL_REQUESTED|EngineconnCoreErrorCodeSummary| |linkis-engineconn-plugin-core |20000|Cannot instance EngineConnExecution(无法实例化 EngineConnExecution)|CANNOT_INSTANCE_ECE|EngineconnCoreErrorCodeSummary| |linkis-engineconn-plugin-core |20000|Cannot find default ExecutorFactory(找不到默认的 ExecutorFactory)|CANNOT_DEFAULT_EF|EngineconnCoreErrorCodeSummary| diff --git a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala index 411bda4820..5cc796d23e 100644 --- a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala +++ b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala @@ -199,7 +199,11 @@ object SecurityFilter { def getLoginUsername(req: HttpServletRequest): String = { if (Configuration.IS_TEST_MODE.getValue) { - ServerConfiguration.BDP_TEST_USER.getValue; + val testUser = ServerConfiguration.BDP_TEST_USER.getValue + if (StringUtils.isBlank(testUser)) { + throw new IllegalUserTicketException("Need to set test user when enable test module") + } + testUser } else { getLoginUser(req).getOrElse( throw new IllegalUserTicketException(ILLEGAL_USER_TOKEN.getErrorDesc) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java index 1685f4b652..f29b2a6ed8 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java @@ -23,7 +23,7 @@ public enum EngineconnCoreErrorCodeSummary implements LinkisErrorCode { FAILED_CREATE_ELR(10001, "Failed to createEngineConnLaunchRequest(创建 EngineConnLaunchRequest失败)"), EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION( 10001, - "The engine plug-in material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)"), + "The engine plugin material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)"), ETL_REQUESTED(10001, "EngineTypeLabel are requested(需要参数 EngineTypeLabel)"), CANNOT_INSTANCE_ECE(20000, "Cannot instance EngineConnExecution(无法实例化 EngineConnExecution)"), diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.scala index de1add173a..9a16d9b9e6 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.scala @@ -35,4 +35,8 @@ object EngineConnPluginConfiguration { val ENABLED_BML_UPLOAD_FAILED_EXIT: CommonVars[Boolean] = CommonVars("wds.linkis.engineconn.bml.upload.failed.enable", true) + // for third party eg appconn/datax, if all update, can set to false then to remove + val EC_BML_VERSION_MAY_WITH_PREFIX_V: CommonVars[Boolean] = + CommonVars("linkis.engineconn.bml.version.may.with.prefix", true) + } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala index fce3508c31..c21ee924e7 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala @@ -17,6 +17,7 @@ package org.apache.linkis.engineplugin.server.localize +import org.apache.linkis.engineplugin.server.conf.EngineConnPluginConfiguration import org.apache.linkis.engineplugin.server.conf.EngineConnPluginConfiguration.ENGINE_CONN_HOME import org.apache.linkis.engineplugin.server.localize.EngineConnBmlResourceGenerator.NO_VERSION_MARK import org.apache.linkis.manager.engineplugin.common.exception.EngineConnPluginErrorException @@ -53,13 +54,37 @@ abstract class AbstractEngineConnBmlResourceGenerator extends EngineConnBmlResou val engineConnPackageHome = Paths.get(engineConnDistHome, version).toFile.getPath logger.info("getEngineConnDistHome, engineConnPackageHome path:" + engineConnPackageHome) val engineConnPackageHomeFile = new File(engineConnPackageHome) + if (!engineConnPackageHomeFile.exists()) { - throw new EngineConnPluginErrorException( - ENGINE_VERSION_NOT_FOUND.getErrorCode, - MessageFormat.format(ENGINE_VERSION_NOT_FOUND.getErrorDesc, version, engineConnType) - ) + if ( + !version.startsWith( + "v" + ) && EngineConnPluginConfiguration.EC_BML_VERSION_MAY_WITH_PREFIX_V.getValue + ) { + val versionOld = "v" + version + val engineConnPackageHomeOld = Paths.get(engineConnDistHome, versionOld).toFile.getPath + logger.info( + "try to getEngineConnDistHome with prefix v, engineConnPackageHome path:" + engineConnPackageHomeOld + ) + val engineConnPackageHomeFileOld = new File(engineConnPackageHomeOld) + if (!engineConnPackageHomeFileOld.exists()) { + throw new EngineConnPluginErrorException( + ENGINE_VERSION_NOT_FOUND.getErrorCode, + MessageFormat.format(ENGINE_VERSION_NOT_FOUND.getErrorDesc, version, engineConnType) + ) + } else { + engineConnPackageHomeOld + } + } else { + throw new EngineConnPluginErrorException( + ENGINE_VERSION_NOT_FOUND.getErrorCode, + MessageFormat.format(ENGINE_VERSION_NOT_FOUND.getErrorDesc, version, engineConnType) + ) + } + } else { + engineConnPackageHome } - engineConnPackageHome + } private def checkEngineConnDistHome(engineConnPackageHomePath: String): Unit = { diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala index 3ddfdc7bbe..55f3b2568a 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala @@ -201,8 +201,6 @@ class DefaultEngineConnResourceService extends EngineConnResourceService with Lo engineConnBmlResource.setFileName(localizeResource.fileName) engineConnBmlResource.setFileSize(localizeResource.fileSize) engineConnBmlResource.setLastModified(localizeResource.lastModified) - if (version.startsWith("v")) engineConnBmlResource.setVersion(version.substring(1)) - else engineConnBmlResource.setVersion(version) engineConnBmlResource.setVersion(version) engineConnBmlResourceDao.save(engineConnBmlResource) } else { @@ -241,9 +239,18 @@ class DefaultEngineConnResourceService extends EngineConnResourceService with Lo ): EngineConnResource = { val engineConnType = engineConnBMLResourceRequest.getEngineConnType val version = engineConnBMLResourceRequest.getVersion - val engineConnBmlResources = asScalaBufferConverter( + var engineConnBmlResources = asScalaBufferConverter( engineConnBmlResourceDao.getAllEngineConnBmlResource(engineConnType, version) ) + if ( + engineConnBmlResources.asScala.size == 0 && EngineConnPluginConfiguration.EC_BML_VERSION_MAY_WITH_PREFIX_V.getValue + ) { + logger.info("Try to get engine conn bml resource with prefex v") + engineConnBmlResources = asScalaBufferConverter( + engineConnBmlResourceDao.getAllEngineConnBmlResource(engineConnType, "v" + version) + ) + } + val confBmlResourceMap = engineConnBmlResources.asScala .find(_.getFileName == LaunchConstants.ENGINE_CONN_CONF_DIR_NAME + ".zip") .map(parseToBmlResource) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala index f1284268b3..6145f8bc0b 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala @@ -710,7 +710,7 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ var heartbeatMsgMetrics = "" Utils.tryAndWarn { val oldMetrics = nodeMetricManagerPersistence.getNodeMetrics(ecNode) - if (StringUtils.isNotBlank(oldMetrics.getHeartBeatMsg)) { + if (oldMetrics != null && StringUtils.isNotBlank(oldMetrics.getHeartBeatMsg)) { heartbeatMsgMetrics = oldMetrics.getHeartBeatMsg } } diff --git a/linkis-web/src/apps/linkis/i18n/common/en.json b/linkis-web/src/apps/linkis/i18n/common/en.json index c6230555fb..845b6fe03a 100644 --- a/linkis-web/src/apps/linkis/i18n/common/en.json +++ b/linkis-web/src/apps/linkis/i18n/common/en.json @@ -629,7 +629,7 @@ "versionList": "Version List", "rollback": "Rollback", "checkEngineConnTypeAndVersion": "Please select the engine type and version", - "upload": "Please click the button to upload the engine plug-in" + "upload": "Please click the button to upload the engine plugin" } } }