From 4362f3e8d8147db5676a0a111a3c0a2509517b1e Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 17 Oct 2022 10:57:23 +0800 Subject: [PATCH 001/240] Support the method to get engine conn resource information. --- pom.xml | 2 +- .../url/LinkisURLStreamHandlerFactory.java | 1 - .../linkis/core/FlinkLogIterator.scala | 6 ++ .../linkis/entity/LogRequestPayload.scala | 5 ++ .../launcher/linkis/job/FlinkJobClient.scala | 67 +++++++++++++++++-- .../launcher/linkis/job/FlinkJobInfo.scala | 30 +++++++-- .../launcher/linkis/job/LinkisJobInfo.scala | 6 ++ .../manager/SimpleFlinkJobLaunchManager.scala | 9 +-- .../job/operator/FlinkClientLogOperator.scala | 34 ++++++++++ .../job/operator/FlinkYarnLogOperator.scala | 7 +- .../service/DefaultStreamTaskService.scala | 15 ++++- 11 files changed, 160 insertions(+), 22 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/operator/FlinkClientLogOperator.scala diff --git a/pom.xml b/pom.xml index eabaaba66..bd535bf6f 100644 --- a/pom.xml +++ b/pom.xml @@ -45,7 +45,7 @@ - 1.1.1 + 1.1.3 1.1.0 0.2.0 2.11.12 diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/url/LinkisURLStreamHandlerFactory.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/url/LinkisURLStreamHandlerFactory.java index 78701f646..1352c8b28 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/url/LinkisURLStreamHandlerFactory.java +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/url/LinkisURLStreamHandlerFactory.java @@ -15,7 +15,6 @@ package com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.url; -import org.apache.commons.lang.StringUtils; import java.net.URLStreamHandler; import java.net.URLStreamHandlerFactory; diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/core/FlinkLogIterator.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/core/FlinkLogIterator.scala index 2fdb12b7b..d6fb5c661 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/core/FlinkLogIterator.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/core/FlinkLogIterator.scala @@ -18,6 +18,7 @@ trait FlinkLogIterator extends Iterator[String] with Closeable { val engineConnLogOperator: EngineConnLogOperator def init(): Unit def getLogPath: String + def getLogDirSuffix: String def getLogs: util.ArrayList[String] def getEndLine: Long } @@ -28,6 +29,7 @@ class SimpleFlinkJobLogIterator(override val requestPayload: LogRequestPayload, private var logs: util.ArrayList[String] = _ private var index = 0 private var logPath: String = _ + private var logDirSuffix: String = _ private var isClosed = true private var endLine = 0 @@ -69,4 +71,8 @@ class SimpleFlinkJobLogIterator(override val requestPayload: LogRequestPayload, override def getLogs: util.ArrayList[String] = logs override def getEndLine: Long = endLine + + def setLogDirSuffix(logDirSuffix: String) : Unit = this.logDirSuffix = logDirSuffix + + override def getLogDirSuffix: String = logDirSuffix } diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/entity/LogRequestPayload.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/entity/LogRequestPayload.scala index 29f90d325..20cb4d081 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/entity/LogRequestPayload.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/entity/LogRequestPayload.scala @@ -14,6 +14,7 @@ class LogRequestPayload { private var onlyKeywords: String = _ private var lastRows = 0 private var logType: String = _ + private var logHistory: Boolean = false def getPageSize: Int = pageSize def setPageSize(pageSize: Int): Unit = this.pageSize = pageSize @@ -32,4 +33,8 @@ class LogRequestPayload { def getLogType: String = logType def setLogType(logType: String): Unit = this.logType = logType + + def isLogHistory: Boolean = logHistory + + def setLogHistory(logHistory: Boolean): Unit = this.logHistory = logHistory } diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/FlinkJobClient.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/FlinkJobClient.scala index 951145b94..b5c64a523 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/FlinkJobClient.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/FlinkJobClient.scala @@ -23,13 +23,17 @@ import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.core.{FlinkLo import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.entity.LogRequestPayload import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.exception.{FlinkJobLaunchErrorException, FlinkJobStateFetchException, FlinkSavePointException} import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.manager.FlinkJobLaunchManager -import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.operator.{FlinkTriggerSavepointOperator, FlinkYarnLogOperator} +import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.operator.{FlinkClientLogOperator, FlinkTriggerSavepointOperator, FlinkYarnLogOperator} import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.state.{Checkpoint, Savepoint} +import org.apache.commons.lang3.StringUtils import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.computation.client.once.OnceJob -import org.apache.linkis.computation.client.once.simple.SimpleOnceJob +import org.apache.linkis.computation.client.once.action.ECResourceInfoAction +import org.apache.linkis.computation.client.once.result.ECResourceInfoResult +import org.apache.linkis.computation.client.once.{LinkisManagerClient, LinkisManagerClientImpl, OnceJob} +import org.apache.linkis.computation.client.once.simple.{SimpleOnceJob, SimpleOnceJobBuilder} import org.apache.linkis.computation.client.operator.impl.EngineConnLogOperator - +import org.apache.linkis.httpclient.dws.DWSHttpClient +import java.util import java.net.URI class FlinkJobClient(onceJob: OnceJob, var jobInfo: FlinkJobInfo, stateManager: JobStateManager) @@ -39,9 +43,13 @@ class FlinkJobClient(onceJob: OnceJob, var jobInfo: FlinkJobInfo, stateManager: * Log operator */ private var logOperatorMap = Map( - "client" -> EngineConnLogOperator.OPERATOR_NAME, + "client" -> FlinkClientLogOperator.OPERATOR_NAME, "yarn" -> FlinkYarnLogOperator.OPERATOR_NAME ) + /** + * The linkis client in onceJob + */ + private var linkisClient: DWSHttpClient = _ override def getJobInfo: FlinkJobInfo = { getJobInfo(false) @@ -99,13 +107,38 @@ class FlinkJobClient(onceJob: OnceJob, var jobInfo: FlinkJobInfo, stateManager: case Some(operator) => onceJob.getOperator(operator) match { case engineConnLogOperator: EngineConnLogOperator => + val logIterator = new SimpleFlinkJobLogIterator(requestPayload, engineConnLogOperator) + engineConnLogOperator match { + case clientLogOperator: FlinkClientLogOperator => + var logDirSuffix = this.jobInfo.getLogDirSuffix + if (StringUtils.isBlank(logDirSuffix) && requestPayload.isLogHistory){ + // If want to fetch the history log, must get the log directory suffix first + getLinkisClient match { + case client: DWSHttpClient => + Option(Utils.tryCatch{ + client.execute(ECResourceInfoAction.newBuilder().setUser(jobInfo.getUser) + .setTicketid(clientLogOperator.getTicketId).build()).asInstanceOf[ECResourceInfoResult] + }{ + case e: Exception => + warn("Fail to query the engine conn resource info from linkis", e) + null + }) match { + case Some(result) => logDirSuffix = Utils.tryAndWarn{result.getData.getOrDefault("ecResourceInfoRecord", new util.HashMap[String, Any]).asInstanceOf[util.Map[String, Any]] + .getOrDefault("logDirSuffix", "").asInstanceOf[String]} + case _ => + } + } + } + clientLogOperator.setLogDirSuffix(logDirSuffix) + logIterator.setLogDirSuffix(logDirSuffix) + case _ => + } engineConnLogOperator match { case yarnLogOperator: FlinkYarnLogOperator => yarnLogOperator.setApplicationId(jobInfo.getApplicationId) case _ => } engineConnLogOperator.setECMServiceInstance(jobInfo.getECMInstance) engineConnLogOperator.setEngineConnType(FlinkJobLaunchManager.FLINK_ENGINE_CONN_TYPE) - val logIterator = new SimpleFlinkJobLogIterator(requestPayload, engineConnLogOperator) logIterator.init() jobInfo match { case jobInfo: FlinkJobInfo => jobInfo.setLogPath(logIterator.getLogPath) @@ -161,5 +194,27 @@ class FlinkJobClient(onceJob: OnceJob, var jobInfo: FlinkJobInfo, stateManager: triggerSavepoint(savepointURI.toString, JobLauncherConfiguration.FLINK_TRIGGER_SAVEPOINT_MODE.getValue) } + /** + * Get linkis client + * @return + */ + def getLinkisClient: DWSHttpClient = { + Utils.tryAndWarn{ + if (null == this.linkisClient){ + this.synchronized{ + if (null == this.linkisClient){ + this.linkisClient = SimpleOnceJobBuilder.getLinkisManagerClient match { + case client: LinkisManagerClient => + val dwsClientField = classOf[LinkisManagerClientImpl].getDeclaredField("dwsHttpClient") + dwsClientField.setAccessible(true) + dwsClientField.get(client).asInstanceOf[DWSHttpClient] + case _ => null + } + } + } + } + this.linkisClient + } + } } diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/FlinkJobInfo.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/FlinkJobInfo.scala index 5c2986609..0fd1f1783 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/FlinkJobInfo.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/FlinkJobInfo.scala @@ -34,41 +34,53 @@ class FlinkJobInfo extends YarnJobInfo { private var applicationUrl: String = _ private var status: String = _ private var logPath: String = _ + private var logDirSuffix: String = _ private var resources: java.util.Map[String, Object] = _ private var completedMsg: String = _ private var jobStates: Array[JobStateInfo] = _ + override def getApplicationId: String = applicationId + def setApplicationId(applicationId: String): Unit = this.applicationId = applicationId override def getApplicationUrl: String = applicationUrl + def setApplicationUrl(applicationUrl: String): Unit = this.applicationUrl = applicationUrl override def getId: String = id - def setId(id: String): Unit = this.id = id + def setId(id: String): Unit = this.id = id override def getECMInstance: ServiceInstance = ecmInstance + def setECMInstance(ecmInstance: ServiceInstance): Unit = this.ecmInstance = ecmInstance override def getUser: String = user + def setUser(user: String): Unit = this.user = user override def getStatus: String = status + override def setStatus(status: String): Unit = this.status = status override def getLogPath: String = logPath + def setLogPath(logPath: String): Unit = this.logPath = logPath override def getResources: util.Map[String, Object] = resources + def setResources(resources: java.util.Map[String, Object]): Unit = this.resources = resources def getSavepoint: String = savepoint + def setSavepoint(savepoint: String): Unit = this.savepoint = savepoint def getCheckpoint: String = checkpoint + def setCheckpoint(checkpoint: String): Unit = this.checkpoint = checkpoint override def getCompletedMsg: String = completedMsg + def setCompletedMsg(completedMsg: String): Unit = this.completedMsg = completedMsg override def toString: String = s"FlinkJobInfo(id: $id, status: $status, applicationId: $applicationId, applicationUrl: $applicationUrl, logPath: $logPath)" @@ -85,6 +97,7 @@ class FlinkJobInfo extends YarnJobInfo { def setJobStates(jobStates: Array[JobStateInfo]): Unit = { this.jobStates = jobStates } + /** * Job name * @@ -95,11 +108,16 @@ class FlinkJobInfo extends YarnJobInfo { def setName(name: String): Unit = { this.name = name } -} -object FlinkJobInfo{ - def main(args: Array[String]): Unit = { - val jobInfo = "{\"jobStates:\":{\"location\":\"xx\"}" - DWSHttpClient.jacksonJson.readValue(jobInfo, classOf[FlinkJobInfo]) + /** + * Job log directory suffix + * + * @return + */ + override def getLogDirSuffix: String = this.logDirSuffix + + override def setLogDirSuffix(logDirSuffix: String): Unit = { + this.logDirSuffix = logDirSuffix } } + diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/LinkisJobInfo.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/LinkisJobInfo.scala index da02fda50..4183f5025 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/LinkisJobInfo.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/LinkisJobInfo.scala @@ -11,5 +11,11 @@ trait LinkisJobInfo extends JobInfo { */ def getECMInstance: ServiceInstance + /** + * Job log directory suffix + * @return + */ + def getLogDirSuffix: String + def setLogDirSuffix(logDirSuffix: String): Unit } diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/SimpleFlinkJobLaunchManager.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/SimpleFlinkJobLaunchManager.scala index 6ad2e4f88..5e69766fe 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/SimpleFlinkJobLaunchManager.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/SimpleFlinkJobLaunchManager.scala @@ -15,16 +15,16 @@ package com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.manager -import com.webank.wedatasphere.streamis.jobmanager.launcher.job.state.{JobState, JobStateInfo} +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.state.JobState import com.webank.wedatasphere.streamis.jobmanager.launcher.job.{JobClient, LaunchJob} import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.conf.JobLauncherConfiguration -import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.{FlinkJobClient, FlinkJobInfo, LinkisJobInfo} import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.manager.SimpleFlinkJobLaunchManager.INSTANCE_NAME -import org.apache.commons.lang.StringEscapeUtils +import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.{FlinkJobClient, FlinkJobInfo, LinkisJobInfo} +import org.apache.commons.lang3.StringEscapeUtils import org.apache.linkis.common.utils.{RetryHandler, Utils} import org.apache.linkis.computation.client.once.simple.{SimpleOnceJob, SubmittableSimpleOnceJob} import org.apache.linkis.computation.client.once.{OnceJob, SubmittableOnceJob} -import org.apache.linkis.computation.client.operator.impl.{EngineConnApplicationInfoOperator, EngineConnLogOperator} +import org.apache.linkis.computation.client.operator.impl.EngineConnApplicationInfoOperator import org.apache.linkis.httpclient.dws.DWSHttpClient import org.apache.linkis.ujes.client.exception.UJESJobException @@ -69,6 +69,7 @@ class SimpleFlinkJobLaunchManager extends FlinkJobLaunchManager { jobInfo.setCompletedMsg(message) } jobInfo.setResources(nodeInfo.get("nodeResource").asInstanceOf[util.Map[String, Object]]) + jobInfo.setStatus("failed") // Set job state info into // Option(jobState).foreach(state => { // val stateInfo = new JobStateInfo diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/operator/FlinkClientLogOperator.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/operator/FlinkClientLogOperator.scala new file mode 100644 index 000000000..a41018a74 --- /dev/null +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/operator/FlinkClientLogOperator.scala @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.operator + +import org.apache.commons.lang3.StringUtils +import org.apache.linkis.computation.client.once.action.EngineConnOperateAction +import org.apache.linkis.computation.client.operator.impl.{EngineConnLogOperator, EngineConnLogs} + +/** + * Append "logDirSuffix" parameter + */ +class FlinkClientLogOperator extends EngineConnLogOperator{ + + private var logDirSuffix: String = _ + + def setLogDirSuffix(logDirSuffix: String): Unit = { + this.logDirSuffix = logDirSuffix + } + + protected override def addParameters(builder: EngineConnOperateAction.Builder): Unit = { + builder.operatorName(EngineConnLogOperator.OPERATOR_NAME) + if (StringUtils.isNotBlank(this.logDirSuffix)) { + builder.addParameter("logDirSuffix", logDirSuffix) + } + super.addParameters(builder) + } + + + override def getTicketId: String = super.getTicketId + + override def getName: String = FlinkClientLogOperator.OPERATOR_NAME +} + +object FlinkClientLogOperator { + val OPERATOR_NAME = "engineConnLog_flink" +} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/operator/FlinkYarnLogOperator.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/operator/FlinkYarnLogOperator.scala index a24e12580..975b23405 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/operator/FlinkYarnLogOperator.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/operator/FlinkYarnLogOperator.scala @@ -19,9 +19,9 @@ import org.apache.linkis.computation.client.once.action.EngineConnOperateAction import org.apache.linkis.computation.client.operator.impl.EngineConnLogOperator /** - * Extend the engine conn log operator + * Extend the flink client log operator */ -class FlinkYarnLogOperator extends EngineConnLogOperator{ +class FlinkYarnLogOperator extends FlinkClientLogOperator { private var applicationId: String = _ @@ -30,8 +30,9 @@ class FlinkYarnLogOperator extends EngineConnLogOperator{ } protected override def addParameters(builder: EngineConnOperateAction.Builder): Unit = { - builder.addParameter("yarnApplicationId", this.applicationId) super.addParameters(builder) + builder.operatorName(getName) + builder.addParameter("yarnApplicationId", this.applicationId) } override def getName: String = FlinkYarnLogOperator.OPERATOR_NAME diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala index 1913fc0c9..5e7d065f7 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala @@ -22,7 +22,7 @@ import com.webank.wedatasphere.streamis.jobmanager.launcher.job.state.JobState import com.webank.wedatasphere.streamis.jobmanager.launcher.job.{JobInfo, LaunchJob} import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.entity.LogRequestPayload import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.state.{Checkpoint, Savepoint} -import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.{FlinkJobClient, FlinkJobInfo} +import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.{FlinkJobClient, FlinkJobInfo, LinkisJobInfo} import com.webank.wedatasphere.streamis.jobmanager.manager.SpringContextHolder import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf.FLINK_JOB_STATUS_FAILED @@ -398,11 +398,24 @@ class DefaultStreamTaskService extends StreamTaskService with Logging{ val jobClient = jobLaunchManager.connect(streamTask.getLinkisJobId, streamTask.getLinkisJobInfo) jobClient match { case client: FlinkJobClient => + requestPayload.setLogHistory(JobConf.isCompleted(streamTask.getStatus)) val logIterator = client.fetchLogs(requestPayload) returnMap.put("logPath", logIterator.getLogPath) returnMap.put("logs", logIterator.getLogs) returnMap.put("endLine", logIterator.getEndLine) logIterator.close() + jobClient.getJobInfo match { + case linkisInfo: LinkisJobInfo => + if (StringUtils.isBlank(linkisInfo.getLogDirSuffix) && StringUtils.isNotBlank(logIterator.getLogDirSuffix)){ + Utils.tryAndWarn { + // Update the linkis job info and store into database + linkisInfo.setLogDirSuffix(logIterator.getLogDirSuffix) + streamTask.setLinkisJobInfo(DWSHttpClient.jacksonJson.writeValueAsString(linkisInfo)); + streamTaskMapper.updateTask(streamTask) + } + } + case _ => + } } }{ case e: Exception => // Just warn the exception From a2ecf89c48be994f4c4400df5e9ad705d28cedc5 Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 17 Oct 2022 10:58:10 +0800 Subject: [PATCH 002/240] Init the structure of log module. --- streamis-jobmanager/pom.xml | 1 + .../flink-streamis-log-collector/pom.xml | 28 +++++++++ streamis-jobmanager/streamis-job-log/pom.xml | 26 +++++++++ .../streamis-job-log-collector/pom.xml | 19 ++++++ .../streamis-job-log-common/pom.xml | 19 ++++++ .../jobmanager/log/entities/LogElement.java | 29 ++++++++++ .../log/entities/StreamisLogEvent.java | 4 ++ .../streamis-job-log-server/pom.xml | 26 +++++++++ .../StreamisJobLogAutoConfiguration.java | 4 ++ .../log/server/storage/JobLogStorage.java | 18 ++++++ .../server/storage/StreamisJobLogStorage.java | 4 ++ .../server/storage/bucket/JobLogBucket.java | 24 ++++++++ .../storage/bucket/JobLogBucketConfig.java | 58 +++++++++++++++++++ .../storage/bucket/JobLogBucketFactory.java | 15 +++++ .../storage/bucket/JobLogBucketState.java | 19 ++++++ .../storage/bucket/JobLogStorageWriter.java | 27 +++++++++ .../storage/bucket/Log4j2JobLogBucket.java | 4 ++ .../storage/bucket/StreamisJobLogBucket.java | 7 +++ 18 files changed, 332 insertions(+) create mode 100644 streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/pom.xml create mode 100644 streamis-jobmanager/streamis-job-log/pom.xml create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/StreamisJobLogAutoConfiguration.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucket.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketFactory.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketState.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogStorageWriter.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/StreamisJobLogBucket.java diff --git a/streamis-jobmanager/pom.xml b/streamis-jobmanager/pom.xml index b1a644e09..e88713dc2 100644 --- a/streamis-jobmanager/pom.xml +++ b/streamis-jobmanager/pom.xml @@ -32,6 +32,7 @@ streamis-job-manager streamis-jobmanager-server streamis-projectmanager-server + streamis-job-log diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/pom.xml new file mode 100644 index 000000000..5207e035a --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/pom.xml @@ -0,0 +1,28 @@ + + + + streamis-job-log + com.webank.wedatasphere.streamis + 0.2.0 + + 4.0.0 + + flink-streamis-log-collector + + + 8 + 8 + + 1.12.2 + + + + + org.apache.flink + flink-java + ${flink.version} + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/pom.xml b/streamis-jobmanager/streamis-job-log/pom.xml new file mode 100644 index 000000000..3446a972b --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/pom.xml @@ -0,0 +1,26 @@ + + + + streamis-jobmanager + com.webank.wedatasphere.streamis + 0.2.0 + + 4.0.0 + + streamis-job-log + pom + + streamis-job-log-collector + flink-streamis-log-collector + streamis-job-log-server + streamis-job-log-common + + + + 8 + 8 + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml new file mode 100644 index 000000000..aa20062f4 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml @@ -0,0 +1,19 @@ + + + + streamis-job-log + com.webank.wedatasphere.streamis + 0.2.0 + + 4.0.0 + + streamis-job-log-collector + + + 8 + 8 + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml new file mode 100644 index 000000000..51d283f53 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml @@ -0,0 +1,19 @@ + + + + streamis-job-log + com.webank.wedatasphere.streamis + 0.2.0 + + 4.0.0 + + streamis-job-log-common + + + 8 + 8 + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java new file mode 100644 index 000000000..a2b0a7807 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.entities; + +import java.util.concurrent.TimeUnit; + +/** + * Element defined of log + */ +public interface LogElement { + + /** + * Sequence id + * @return seq id + */ + int getSequenceId(); + + /** + * Log time + * @param unit unit + * @return log time + */ + long getLogTime(TimeUnit unit); + + /** + * Get content + * @return content array + */ + String[] getContents(); + +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java new file mode 100644 index 000000000..0c73d83d2 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.entities; + +public class StreamisLogEvent { +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml new file mode 100644 index 000000000..69936a7b4 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml @@ -0,0 +1,26 @@ + + + + streamis-job-log + com.webank.wedatasphere.streamis + 0.2.0 + + 4.0.0 + + streamis-job-log-service + + + 8 + 8 + + + + + com.webank.wedatasphere.streamis + streamis-job-log-common + 0.2.0 + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/StreamisJobLogAutoConfiguration.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/StreamisJobLogAutoConfiguration.java new file mode 100644 index 000000000..7b839893a --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/StreamisJobLogAutoConfiguration.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server; + +public class StreamisJobLogAutoConfiguration { +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java new file mode 100644 index 000000000..e951a1e7a --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.storage; + +import com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket.JobLogBucket; +import com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket.JobLogBucketConfig; + +/** + * Storage of job log + */ +public interface JobLogStorage { + + /** + * Create buckets + * @param jobName job name + * @param bucketConfig bucket config + * @return config + */ + JobLogBucket getOrCreateBucket(String jobName, JobLogBucketConfig bucketConfig); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java new file mode 100644 index 000000000..b357c30c6 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.storage; + +public class StreamisJobLogStorage { +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucket.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucket.java new file mode 100644 index 000000000..c20c08944 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucket.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket; + +/** + * Job log bucket for streamis + */ +public interface JobLogBucket { + + /** + * Bucket state + * @return state + */ + JobLogBucketState getBucketState(); + + /** + * Storage writer + * @return storage writer + */ + JobLogStorageWriter getBucketStorageWriter(); + + /** + * Close the bucket + */ + void close(); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java new file mode 100644 index 000000000..85a5c2d2c --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java @@ -0,0 +1,58 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket; + +import java.util.HashMap; +import java.util.Map; + +/** + * Configuration for job log bucket + */ +public class JobLogBucketConfig { + + /** + * Bucket class + */ + private Class bucketClass; + + /** + * Attribute + */ + protected Map attributes = new HashMap<>(); + + /** + * Max size of bucket part + */ + private int maxBucketPartSize; + + /** + * Layout pattern + */ + private String LayOutPattern; + + public Map getAttributes() { + return attributes; + } + + public void setAttributes(Map attributes) { + this.attributes = attributes; + } + + public int getMaxBucketPartSize() { + return maxBucketPartSize; + } + + public void setMaxBucketPartSize(int maxBucketPartSize) { + this.maxBucketPartSize = maxBucketPartSize; + } + + public String getLayOutPattern() { + return LayOutPattern; + } + + public void setLayOutPattern(String layOutPattern) { + LayOutPattern = layOutPattern; + } + + public static final class Define{ + + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketFactory.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketFactory.java new file mode 100644 index 000000000..d4b9b6b2a --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketFactory.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket; + +/** + * Factory of creating job log bucket + */ +public interface JobLogBucketFactory { + + /** + * Create bucket + * @param jobName job name + * @param config bucket config + * @return + */ + JobLogBucket createBucket(String jobName, JobLogBucketConfig config); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketState.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketState.java new file mode 100644 index 000000000..bdd64f265 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketState.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket; + +/** + * State of log bucket + */ +public interface JobLogBucketState { + + /** + * Bucket path + * @return path + */ + String getBucketPath(); + + double getBucketWriteRate(); + + int getBucketParts(); + + int getBucketWriteTime(); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogStorageWriter.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogStorageWriter.java new file mode 100644 index 000000000..772040374 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogStorageWriter.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket; + +import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; + +/** + * Storage writer for job log + */ +public interface JobLogStorageWriter { + + /** + * Write log element + * @param logEl elements + * @param + */ + void write(LogElement logEl); + + /** + * Write log line + * @param logLine log line + */ + void write(String logLine); + + /** + * Close log storage + */ + void close(); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java new file mode 100644 index 000000000..21c3303f5 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket; + +public class Log4j2JobLogBucket { +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/StreamisJobLogBucket.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/StreamisJobLogBucket.java new file mode 100644 index 000000000..ba9c002d6 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/StreamisJobLogBucket.java @@ -0,0 +1,7 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket; + +/** + * Use the appender and strategy of log4j (version 1.x) to implement the bucket + */ +public class StreamisJobLogBucket { +} From fd564d036a6955df47a5403d0716e21886b3b16a Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 18 Oct 2022 18:23:21 +0800 Subject: [PATCH 003/240] Add the new feature for adding task and stop task for existed jobs. --- .../manager/dao/StreamJobMapper.java | 1 + .../manager/dao/impl/StreamJobMapper.xml | 5 + .../service/DefaultStreamJobService.scala | 10 +- .../service/DefaultStreamTaskService.scala | 7 +- .../manager/service/StreamJobService.scala | 3 + .../manager/service/StreamTaskService.scala | 7 +- .../jobmanager/restful/api/JobRestfulApi.java | 110 +++++++++++++++++- 7 files changed, 131 insertions(+), 12 deletions(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java index c380ca954..3e002c45d 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java @@ -30,6 +30,7 @@ List getJobLists(@Param("projectName") String projectName, @Para StreamJob getJobById(@Param("jobId") Long jobId); + List getJobByName(@Param("jobName") String jobName); List getJobVersions(@Param("jobId") Long jobId); diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml index 71e2b38a6..875da3e4f 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml @@ -86,6 +86,11 @@ + + diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala index c3f467118..f12878433 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala @@ -17,26 +17,26 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.service import java.util import java.util.Date + import com.github.pagehelper.PageInfo import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants import com.webank.wedatasphere.streamis.jobmanager.launcher.service.StreamJobConfService import com.webank.wedatasphere.streamis.jobmanager.manager.alert.AlertLevel -import org.apache.linkis.common.exception.ErrorException -import org.apache.linkis.common.utils.Logging import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamAlertMapper, StreamJobMapper, StreamTaskMapper} import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{QueryJobListVo, TaskCoreNumVo, VersionDetailVo} -import com.webank.wedatasphere.streamis.jobmanager.manager.entity.{MetaJsonInfo, StreamAlertRecord, StreamJob, StreamJobVersion, StreamJobVersionFiles} +import com.webank.wedatasphere.streamis.jobmanager.manager.entity._ import com.webank.wedatasphere.streamis.jobmanager.manager.exception.{JobCreateErrorException, JobFetchErrorException} import com.webank.wedatasphere.streamis.jobmanager.manager.transform.JobContentParser import com.webank.wedatasphere.streamis.jobmanager.manager.transform.entity.StreamisTransformJobContent import com.webank.wedatasphere.streamis.jobmanager.manager.util.{ReaderUtils, ZipHelper} import org.apache.commons.lang.StringUtils +import org.apache.linkis.common.exception.ErrorException +import org.apache.linkis.common.utils.Logging import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional -import javax.annotation.Resource import scala.collection.JavaConverters._ @@ -60,6 +60,8 @@ class DefaultStreamJobService extends StreamJobService with Logging { this.streamJobMapper.getJobById(jobId) } + override def getJobByName(jobName: String): util.List[StreamJob] = streamJobMapper.getJobByName(jobName) + override def getByProList(projectName: String, userName: String, jobName: String, jobStatus: Integer, jobCreator: String): PageInfo[QueryJobListVo] = { val streamJobList = streamJobMapper.getJobLists(projectName, userName, jobName, jobStatus, jobCreator) if (streamJobList != null && !streamJobList.isEmpty) { diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala index 1913fc0c9..099866004 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala @@ -470,7 +470,7 @@ class DefaultStreamTaskService extends StreamTaskService with Logging{ }).asJava } - def getTask(jobId:Long, version: String): FlinkJobInfo ={ + def getTaskJobInfo(jobId:Long, version: String): FlinkJobInfo ={ val str = streamTaskMapper.getTask(jobId, version) if (StringUtils.isBlank(str)) { return new FlinkJobInfo @@ -511,6 +511,9 @@ class DefaultStreamTaskService extends StreamTaskService with Logging{ } } + + override def getLatestTaskByJobId(jobId: Long): StreamTask = streamTaskMapper.getLatestByJobId(jobId) + /** * Create new task use the latest job version * @@ -545,6 +548,8 @@ class DefaultStreamTaskService extends StreamTaskService with Logging{ } } + override def updateTask(streamTask: StreamTask): Unit = streamTaskMapper.updateTask(streamTask) + /** * Just launch task by task id * diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamJobService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamJobService.scala index 5f27f0864..5ac798177 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamJobService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamJobService.scala @@ -15,6 +15,9 @@ trait StreamJobService { def getJobById(jobId: Long): StreamJob + + def getJobByName(jobName: String): util.List[StreamJob] + /** * Page list query * @param projectName project name diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamTaskService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamTaskService.scala index 079a8d8f8..65c584804 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamTaskService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamTaskService.scala @@ -20,7 +20,6 @@ import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.entity.LogReq import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.FlinkJobInfo import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamTask import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{ExecResultVo, JobProgressVo, JobStatusVo, PauseResultVo, StreamTaskListVo} - import java.util import java.util.concurrent.Future /** @@ -87,6 +86,8 @@ trait StreamTaskService { */ def launch(taskId: Long, execUser: String): Unit + def getLatestTaskByJobId(jobId: Long): StreamTask + /** * Create new task use the latest job version * @param jobId job id @@ -95,6 +96,8 @@ trait StreamTaskService { */ def createTask(jobId: Long, status: Int, creator: String): StreamTask + def updateTask(streamTask: StreamTask): Unit + /** * Update the task status * @param jobId job id @@ -146,7 +149,7 @@ trait StreamTaskService { * @param version version * @return */ - def getTask(jobId: Long, version: String): FlinkJobInfo + def getTaskJobInfo(jobId: Long, version: String): FlinkJobInfo def getStateInfo(taskId: Long): JobState diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 6d6591d74..c6c9b5a97 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -15,6 +15,7 @@ package com.webank.wedatasphere.streamis.jobmanager.restful.api; +import com.fasterxml.jackson.core.JsonProcessingException; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.webank.wedatasphere.streamis.jobmanager.exception.JobException; @@ -22,16 +23,22 @@ import com.webank.wedatasphere.streamis.jobmanager.launcher.job.JobInfo; import com.webank.wedatasphere.streamis.jobmanager.launcher.job.manager.JobLaunchManager; import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.entity.LogRequestPayload; +import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.FlinkJobInfo; +import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.MetaJsonInfo; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJob; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJobVersion; +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamTask; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.*; import com.webank.wedatasphere.streamis.jobmanager.manager.project.service.ProjectPrivilegeService; import com.webank.wedatasphere.streamis.jobmanager.manager.service.StreamJobService; import com.webank.wedatasphere.streamis.jobmanager.manager.service.StreamTaskService; import com.webank.wedatasphere.streamis.jobmanager.manager.transform.entity.StreamisTransformJobContent; +import com.webank.wedatasphere.streamis.jobmanager.manager.utils.StreamTaskUtils; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.linkis.httpclient.dws.DWSHttpClient; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; import org.slf4j.Logger; @@ -43,10 +50,7 @@ import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; @RequestMapping(path = "/streamis/streamJobManager/job") @RestController @@ -210,7 +214,7 @@ public Message detailsJob(HttpServletRequest req, @RequestParam(value = "jobId", realTimeTrafficDTOS.add(realTimeTrafficDTO); - jobDetailsVO.setLinkisJobInfo(streamTaskService.getTask(jobId,version)); + jobDetailsVO.setLinkisJobInfo(streamTaskService.getTaskJobInfo(jobId,version)); jobDetailsVO.setDataNumber(dataNumberDTOS); jobDetailsVO.setLoadCondition(loadConditionDTOs); jobDetailsVO.setRealTimeTraffic(realTimeTrafficDTOS); @@ -238,6 +242,102 @@ public Message executeHistoryJob(HttpServletRequest req, return Message.ok().data("details", details); } + @RequestMapping(path = "/addTask", method = RequestMethod.GET) + public Message addTask(HttpServletRequest req, + @RequestParam(value = "jobName") String jobName, + @RequestParam(value = "appId") String appId, + @RequestParam(value = "appUrl") String appUrl) { + String username = SecurityFilter.getLoginUsername(req); + LOG.info("User {} try to add a new task for Streamis job {} with appId: {}, appUrl: {}.", username, jobName, appId, appUrl); + List streamJobs = streamJobService.getJobByName(jobName); + if(CollectionUtils.isEmpty(streamJobs)) { + return Message.error("Not exits Streamis job " + jobName); + } else if(streamJobs.size() > 1) { + return Message.error("Too many Streamis Job named " + jobName + ", we cannot distinguish between them."); + } else if(!"spark.jar".equals(streamJobs.get(0).getJobType())) { + return Message.error("Only spark.jar Job support to add new tasks."); + } + if (!streamJobService.hasPermission(streamJobs.get(0), username) && + !this.privilegeService.hasEditPrivilege(req, streamJobs.get(0).getProjectName())) { + return Message.error("Have no permission to add new task for StreamJob [" + jobName + "]."); + } + // 如果存在正在运行的，先将其停止掉 + StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJobs.get(0).getId()); + if(streamTask != null && JobConf.isRunning(streamTask.getStatus())) { + LOG.warn("Streamis Job {} exists running task, update its status to stopped at first.", jobName); + streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); + streamTaskService.updateTask(streamTask); + } else { + // 这里取个巧，从该工程该用户有权限的Job中找到一个Flink的历史作业，作为这个Spark Streaming作业的jobId和jobInfo + // 替换掉JobInfo中的 yarn 信息，这样我们前端就可以在不修改任何逻辑的情况下正常展示Spark Streaming作业了 + PageInfo jobList = streamJobService.getByProList(streamJobs.get(0).getProjectName(), username, null, 0, null); + Optional copyJob = jobList.getList().stream().min((job1, job2) -> { + if (job1.getStatus() > 0) { + return 0; + } else { + return 1; + } + }); + if(!copyJob.isPresent()) { + return Message.error("If no Flink Job has submitted, the register to Streamis cannot be succeeded."); + } + StreamTask copyTask = streamTaskService.getLatestTaskByJobId(copyJob.get().getId()); + LOG.warn("Streamis Job {} will bind the linkisJobInfo from history Flink Job {} with linkisJobId: {}, linkisJobInfo: {}.", + jobName, copyJob.get().getName(), copyTask.getLinkisJobId(), copyTask.getLinkisJobInfo()); + streamTask = streamTaskService.createTask(streamJobs.get(0).getId(), (Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue(), username); + streamTask.setLinkisJobId(copyTask.getLinkisJobId()); + streamTask.setLinkisJobInfo(copyTask.getLinkisJobInfo()); + } + streamTask.setStartTime(new Date()); + streamTask.setLastUpdateTime(new Date()); + FlinkJobInfo flinkJobInfo; + try { + flinkJobInfo = DWSHttpClient.jacksonJson().readValue(streamTask.getLinkisJobInfo(), FlinkJobInfo.class); + } catch (JsonProcessingException e) { + LOG.error("Job {} deserialize the jobInfo from history Job failed!", jobName, e); + return Message.error("Deserialize the jobInfo from history Job failed!"); + } + flinkJobInfo.setApplicationId(appId); + flinkJobInfo.setApplicationUrl(appUrl); + flinkJobInfo.setName(jobName); + flinkJobInfo.setStatus(JobConf.getStatusString((Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue())); + StreamTaskUtils.refreshInfo(streamTask, flinkJobInfo); + streamTaskService.updateTask(streamTask); + LOG.info("Streamis Job {} has added a new task successfully.", jobName); + return Message.ok(); + } + + @RequestMapping(path = "/stopTask", method = RequestMethod.GET) + public Message stopTask(HttpServletRequest req, + @RequestParam(value = "jobName") String jobName, + @RequestParam(value = "appId") String appId, + @RequestParam(value = "appUrl") String appUrl) { + String username = SecurityFilter.getLoginUsername(req); + LOG.info("User {} try to stop task for Streamis job {} with appId: {}, appUrl: {}.", username, jobName, appId, appUrl); + List streamJobs = streamJobService.getJobByName(jobName); + if(CollectionUtils.isEmpty(streamJobs)) { + return Message.error("Not exits Streamis job " + jobName); + } else if(streamJobs.size() > 1) { + return Message.error("Too many Streamis Job named " + jobName + ", we cannot distinguish between them."); + } else if(!"spark.jar".equals(streamJobs.get(0).getJobType())) { + return Message.error("Only spark.jar Job support to add new tasks."); + } + if (!streamJobService.hasPermission(streamJobs.get(0), username) && + !this.privilegeService.hasEditPrivilege(req, streamJobs.get(0).getProjectName())) { + return Message.error("Have no permission to add new task for StreamJob [" + jobName + "]."); + } + // 如果存在正在运行的，将其停止掉 + StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJobs.get(0).getId()); + if(streamTask != null && JobConf.isRunning(streamTask.getStatus())) { + LOG.warn("Streamis Job {} is exists running task, update its status to stopped.", jobName); + streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); + streamTaskService.updateTask(streamTask); + } else { + LOG.warn("Streamis Job {} is not exists running task, ignore to stop it.", jobName); + } + return Message.ok(); + } + @RequestMapping(path = "/progress", method = RequestMethod.GET) public Message progressJob(HttpServletRequest req, @RequestParam(value = "jobId", required = false) Long jobId, @RequestParam(value = "version", required = false) String version) throws IOException, JobException { From dcf6af8a043a7e7cddea9cee764609ee8977e70e Mon Sep 17 00:00:00 2001 From: davidhua Date: Wed, 19 Oct 2022 02:38:31 +0800 Subject: [PATCH 004/240] Use the "\u0001" instead of "\0x001" to act as blank placeholder. --- .../linkis/job/manager/SimpleFlinkJobLaunchManager.scala | 1 - .../streamis/jobmanager/manager/conf/JobConf.scala | 2 ++ .../streamis/jobmanager/manager/utils/JobUtils.scala | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/SimpleFlinkJobLaunchManager.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/SimpleFlinkJobLaunchManager.scala index 5e69766fe..8b4308a2b 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/SimpleFlinkJobLaunchManager.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/SimpleFlinkJobLaunchManager.scala @@ -69,7 +69,6 @@ class SimpleFlinkJobLaunchManager extends FlinkJobLaunchManager { jobInfo.setCompletedMsg(message) } jobInfo.setResources(nodeInfo.get("nodeResource").asInstanceOf[util.Map[String, Object]]) - jobInfo.setStatus("failed") // Set job state info into // Option(jobState).foreach(state => { // val stateInfo = new JobStateInfo diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala index 89cab6f92..7da10b5f5 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala @@ -27,6 +27,8 @@ object JobConf { val STREAMIS_JOB_MONITOR_ENABLE: CommonVars[Boolean] = CommonVars("wds.streamis.job.monitor.enable", true) + val STREAMIS_JOB_PARAM_BLANK_PLACEHOLDER: CommonVars[String] = CommonVars("wds.streamis.job.param.blank.placeholder", "\u0001") + val FLINK_JOB_STATUS_NOT_STARTED: CommonVars[Int] = CommonVars("wds.streamis.job.status.not-started", 0,"Not Started") val FLINK_JOB_STATUS_COMPLETED: CommonVars[Int] = CommonVars("wds.streamis.job.status.completed", 1,"Completed") diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/utils/JobUtils.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/utils/JobUtils.scala index 485f9c8ce..75a62b53d 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/utils/JobUtils.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/utils/JobUtils.scala @@ -1,5 +1,7 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.utils +import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf + import java.util import scala.collection.JavaConverters.{asScalaSetConverter, mapAsScalaMapConverter} @@ -12,7 +14,7 @@ object JobUtils { for (paramEntry <- params.entrySet().asScala){ val value = paramEntry.getValue value match { - case str: String => paramEntry.setValue(str.replace(" ", "\\0x001")) + case str: String => paramEntry.setValue(str.replace(" ", JobConf.STREAMIS_JOB_PARAM_BLANK_PLACEHOLDER.getValue)) case _ => } } From 4303fb13c71ded5e1f4348eeed887b686a922dfa Mon Sep 17 00:00:00 2001 From: davidhua Date: Wed, 19 Oct 2022 02:42:57 +0800 Subject: [PATCH 005/240] SendBuffer and BucketConfig. --- .../collector/sender/AbstractSendBuffer.java | 107 ++++++++++++++++++ .../collector/sender/ImmutableSendBuffer.java | 71 ++++++++++++ .../log/collector/sender/SendBuffer.java | 77 +++++++++++++ .../streamis-job-log-server/pom.xml | 4 + .../exception/StreamJobLogException.java | 29 +++++ .../storage/bucket/JobLogBucketConfig.java | 103 +++++++++++++++-- 6 files changed, 379 insertions(+), 12 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractSendBuffer.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/ImmutableSendBuffer.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendBuffer.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/exception/StreamJobLogException.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractSendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractSendBuffer.java new file mode 100644 index 000000000..2f778b231 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractSendBuffer.java @@ -0,0 +1,107 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; + +/** + * Abstract sender buffer; + * non-blocking and reduces out-of-bounds exceptions + */ +public abstract class AbstractSendBuffer implements SendBuffer{ + + protected enum Flag{ + WRITE_MODE, READ_MODE + } + + /** + * Access flag + */ + private Flag accessFlag = Flag.WRITE_MODE; + + private int position = 0; + private int limit; + /** + * The capacity is mutable + */ + protected int capacity; + + + public AbstractSendBuffer(int capacity){ + this.capacity = capacity; + limit(this.capacity); + } + + public AbstractSendBuffer(){ + this(Integer.MAX_VALUE); + } + @Override + public int capacity() { + return this.capacity; + } + + @Override + public int remaining() { + int rem = this.limit - this.position; + return Math.max(rem, 0); + } + + @Override + public void flip() { + this.limit = this.position; + this.position = 0; + this.accessFlag = Flag.READ_MODE; + } + + @Override + public void clear() { + limit(this.capacity); + this.position = 0; + this.accessFlag = Flag.WRITE_MODE; + clearBuf(); + } + + /** + * Change the limit value + * @param newLimit new limit + */ + final void limit(int newLimit){ + if (newLimit > this.capacity || (newLimit < 0)){ + throw new IllegalArgumentException("Set the illegal limit value: " + newLimit + " in send buffer, [capacity: " + this.capacity + "]"); + } + this.limit = newLimit; + if (this.position > newLimit){ + this.position = newLimit; + } + } + + /** + * Inc the position with offset + * @param offset offset value + * @param accessFlag access flag + * @return the current position value + */ + final int nextPosition(int offset, Flag accessFlag){ + if (this.accessFlag != accessFlag){ + throw new IllegalStateException("Illegal access flag [" + accessFlag + "] for send buffer"); + } + int p = position; + // Reach the limit, return -1 value + if (p >= limit){ + return -1; + } + if (p + offset > limit){ + this.position = limit; + } + return p; + } + + /** + * + * @return the current position + */ + final int position(){ + return this.position; + } + + /** + * Do the actual clear + */ + protected abstract void clearBuf(); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/ImmutableSendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/ImmutableSendBuffer.java new file mode 100644 index 000000000..3587079fa --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/ImmutableSendBuffer.java @@ -0,0 +1,71 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; + +import java.util.Arrays; +import java.util.function.Function; + +/** + * Immutable send buffer (use array) + */ +public class ImmutableSendBuffer extends AbstractSendBuffer{ + + /** + * Buffer object array + */ + private final Object[] buf; + + public ImmutableSendBuffer(int capacity) { + super(capacity); + buf = new Object[capacity]; + } + + @Override + protected void clearBuf() { + // Release the memory occupied + Arrays.fill(buf, null); + } + + @Override + public void capacity(String newCapacity) { + throw new IllegalArgumentException("Unsupported to scale-in/scale-up the send buffer"); + } + + @Override + public int writeBuf(E[] elements, int srcIndex, int length) { + if (srcIndex < elements.length){ + int startPos = nextPosition(Math.min(elements.length - srcIndex, length), Flag.WRITE_MODE); + if (startPos >= 0){ + int writes = position() - startPos; + for (int i = srcIndex; i < writes; i ++){ + buf[startPos++] = elements[i]; + } + return writes; + } + } + return -1; + } + + @Override + public int readBuf(E[] elements, int srcIndex, int length) { + return 0; + } + + @Override + public int writeBuf(E element) { + int startPos = nextPosition(1, Flag.WRITE_MODE); + if (startPos >= 0){ + buf[startPos] = element; + return 1; + } + return -1; + } + + @Override + public E readBuf() { + return null; + } + + @Override + public SendBuffer compact(Function dropAble) { + return null; + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendBuffer.java new file mode 100644 index 000000000..e8bee928b --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendBuffer.java @@ -0,0 +1,77 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; + +import java.util.function.Function; + +/** + * Buffer for Rpc sender + * @param buffer element + */ +public interface SendBuffer { + + /** + * Capacity + * @return int + */ + int capacity(); + + /** + * Scale-up or scale-in + * @param newCapacity new capacity + */ + void capacity(String newCapacity); + /** + * Remain size + * (remain space for writing or remain elements for reading) + * @return int + */ + int remaining(); + + /** + * transient between write-mode and read-mode + */ + void flip(); + + /** + * Clear to reuse the buffer + */ + void clear(); + /** + * Write buffer element + * @param element element + * @return if succeed + */ + int writeBuf(E element); + + /** + * Write buffer element array + * @param elements elements + * @param srcIndex the src index in elements + * @param length the length to read + * @return write num + */ + int writeBuf(E[] elements, int srcIndex, int length); + + /** + * Read buffer element + * @return element + */ + E readBuf(); + + /** + * Read buffer element array + * @param elements elements + * @param srcIndex the src index in elements + * @param length the length to write + * @return read num + */ + int readBuf(E[] elements, int srcIndex, int length); + + /** + * Compact the buffer, avoid the useless elements + * @param dropAble drop function + * @return send buffer + */ + SendBuffer compact(Function dropAble); + + +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml index 69936a7b4..7f2a5bc10 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml @@ -22,5 +22,9 @@ streamis-job-log-common 0.2.0 + + org.apache.linkis + linkis-module + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/exception/StreamJobLogException.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/exception/StreamJobLogException.java new file mode 100644 index 000000000..56edc2dd3 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/exception/StreamJobLogException.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.exception; + +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.common.exception.ExceptionLevel; +import org.apache.linkis.common.exception.LinkisRuntimeException; + +/** + * Stream job log exception + */ +public class StreamJobLogException extends ErrorException { + public StreamJobLogException(int errCode, String desc) { + super(errCode, desc); + } + public StreamJobLogException(int errCode, String desc, Throwable t){ + super(errCode, desc); + + } + public static class Runtime extends LinkisRuntimeException{ + + public Runtime(int errCode, String desc) { + super(errCode, desc); + } + + @Override + public ExceptionLevel getLevel() { + return ExceptionLevel.ERROR; + } + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java index 85a5c2d2c..2dc82e40f 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java @@ -1,5 +1,8 @@ package com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket; +import com.webank.wedatasphere.streamis.jobmanager.log.server.exception.StreamJobLogException; +import org.apache.linkis.common.conf.CommonVars; + import java.util.HashMap; import java.util.Map; @@ -8,25 +11,73 @@ */ public class JobLogBucketConfig { + @SuppressWarnings("unchecked") + public JobLogBucketConfig(){ + try { + Class defaultBucketClass = Class.forName(Define.JOB_LOG_BUCKET_CLASS.getValue()); + if (JobLogBucket.class.isAssignableFrom(defaultBucketClass)){ + this.bucketClass = (Class) defaultBucketClass; + } + } catch (ClassNotFoundException e) { +// throw new StreamJobLogException.Runtime(-1, "", e); + } + } + /** * Bucket class */ private Class bucketClass; + /** + * Root path for bucket + */ + private String bucketRootPath; + /** * Attribute */ protected Map attributes = new HashMap<>(); /** - * Max size of bucket part + * Max size of bucket active part */ - private int maxBucketPartSize; + private int maxBucketActivePartSize; + + /** + * Max number of bucket part + */ + private int maxBucketPartNum; + + /** + * The compress format used for bucket parts + */ + private String bucketPartCompress; + + /** + * Max hold time in minutes for bucket part + */ + private long bucketPartHoldTimeInMin; /** * Layout pattern */ - private String LayOutPattern; + private String LogLayOutPattern = "%msg%n"; + + public Class getBucketClass() { + return bucketClass; + } + + public void setBucketClass(Class bucketClass) { + this.bucketClass = bucketClass; + } + + public String getBucketRootPath() { + return bucketRootPath; + } + + public void setBucketRootPath(String bucketRootPath) { + this.bucketRootPath = bucketRootPath; + } public Map getAttributes() { return attributes; @@ -36,23 +87,51 @@ public void setAttributes(Map attributes) { this.attributes = attributes; } - public int getMaxBucketPartSize() { - return maxBucketPartSize; + public int getMaxBucketActivePartSize() { + return maxBucketActivePartSize; } - public void setMaxBucketPartSize(int maxBucketPartSize) { - this.maxBucketPartSize = maxBucketPartSize; + public void setMaxBucketActivePartSize(int maxBucketActivePartSize) { + this.maxBucketActivePartSize = maxBucketActivePartSize; } - public String getLayOutPattern() { - return LayOutPattern; + public int getMaxBucketPartNum() { + return maxBucketPartNum; } - public void setLayOutPattern(String layOutPattern) { - LayOutPattern = layOutPattern; + public void setMaxBucketPartNum(int maxBucketPartNum) { + this.maxBucketPartNum = maxBucketPartNum; } - public static final class Define{ + public String getBucketPartCompress() { + return bucketPartCompress; + } + + public void setBucketPartCompress(String bucketPartCompress) { + this.bucketPartCompress = bucketPartCompress; + } + + public long getBucketPartHoldTimeInMin() { + return bucketPartHoldTimeInMin; + } + + public void setBucketPartHoldTimeInMin(long bucketPartHoldTimeInMin) { + this.bucketPartHoldTimeInMin = bucketPartHoldTimeInMin; + } + + public String getLogLayOutPattern() { + return LogLayOutPattern; + } + public void setLogLayOutPattern(String logLayOutPattern) { + LogLayOutPattern = logLayOutPattern; + } + + + public static final class Define{ + /** + * Default bucket class + */ + public static final CommonVars JOB_LOG_BUCKET_CLASS = CommonVars.apply("wds.streamis.job.log.bucket.class", "com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket.Log4j2JobLogBucket"); } } From c1c76bb9fd4444bdbec08e2e1318c50a99dbcc7f Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Wed, 19 Oct 2022 21:35:34 +0800 Subject: [PATCH 006/240] Add the new feature for adding task and stop task for existed jobs. --- .../jobmanager/manager/conf/JobConf.scala | 3 + .../service/DefaultStreamJobService.scala | 5 +- .../parser/SparkJarJobContentParser.scala | 78 +++++++++++++++++++ .../jobmanager/restful/api/JobRestfulApi.java | 71 +++++++++++------ 4 files changed, 133 insertions(+), 24 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkJarJobContentParser.scala diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala index 89cab6f92..5f33fff2f 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala @@ -82,4 +82,7 @@ object JobConf { val TASK_SUBMIT_TIME_MAX: CommonVars[TimeType] = CommonVars("wds.streamis.task.submit.time.max", new TimeType("5m")) + val SUPPORTED_JOB_TYPES: CommonVars[String] = CommonVars("wds.streamis.supported.job.types", "flink.jar,flink.sql,spark.jar") + + val SUPPORTED_MANAGEMENT_JOB_TYPES: CommonVars[String] = CommonVars("wds.streamis.management.supported.job.types", "flink.jar,flink.sql") } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala index f12878433..e0c270d99 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala @@ -24,8 +24,8 @@ import com.webank.wedatasphere.streamis.jobmanager.launcher.service.StreamJobCon import com.webank.wedatasphere.streamis.jobmanager.manager.alert.AlertLevel import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamAlertMapper, StreamJobMapper, StreamTaskMapper} -import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{QueryJobListVo, TaskCoreNumVo, VersionDetailVo} import com.webank.wedatasphere.streamis.jobmanager.manager.entity._ +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{QueryJobListVo, TaskCoreNumVo, VersionDetailVo} import com.webank.wedatasphere.streamis.jobmanager.manager.exception.{JobCreateErrorException, JobFetchErrorException} import com.webank.wedatasphere.streamis.jobmanager.manager.transform.JobContentParser import com.webank.wedatasphere.streamis.jobmanager.manager.transform.entity.StreamisTransformJobContent @@ -128,6 +128,9 @@ class DefaultStreamJobService extends StreamJobService with Logging { override def createStreamJob(metaJsonInfo: MetaJsonInfo, userName: String): StreamJobVersion = { if(StringUtils.isBlank(metaJsonInfo.getJobType)) throw new JobCreateErrorException(30030, s"jobType is needed.") + else if(!JobConf.SUPPORTED_JOB_TYPES.getValue.contains(metaJsonInfo.getJobType)) { + throw new JobCreateErrorException(30030, s"jobType ${metaJsonInfo.getJobType} is not supported.") + } if(metaJsonInfo.getJobContent == null || metaJsonInfo.getJobContent.isEmpty) throw new JobCreateErrorException(30030, s"jobContent is needed.") val job = streamJobMapper.getCurrentJob(metaJsonInfo.getProjectName, metaJsonInfo.getJobName) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkJarJobContentParser.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkJarJobContentParser.scala new file mode 100644 index 000000000..d730753c2 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkJarJobContentParser.scala @@ -0,0 +1,78 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.transform.parser + +import java.util + +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.{StreamJob, StreamJobVersion, StreamJobVersionFiles, StreamisFile} +import com.webank.wedatasphere.streamis.jobmanager.manager.exception.JobExecuteErrorException +import com.webank.wedatasphere.streamis.jobmanager.manager.transform.entity.{StreamisJarTransformJobContent, StreamisTransformJobContent} +import org.apache.commons.lang.StringUtils +import org.apache.linkis.common.utils.JsonUtils +import org.apache.linkis.manager.label.entity.engine.RunType +import org.apache.linkis.manager.label.entity.engine.RunType.RunType +import org.springframework.stereotype.Component + +import scala.collection.JavaConverters._ + +/** + * + * @date 2022-10-19 + * @author enjoyyin + * @since 0.5.0 + */ +@Component +class SparkJarJobContentParser extends AbstractJobContentParser { + + override val jobType: String = "spark.jar" + override val runType: RunType = RunType.JAR + + override def parseTo(job: StreamJob, jobVersion: StreamJobVersion): StreamisTransformJobContent = { + val createFile: String => StreamisFile = fileName => { + val file = new StreamJobVersionFiles() + file.setFileName(fileName) + file.setCreateBy(job.getCreateBy) + file.setCreateTime(job.getCreateTime) + file.setJobId(job.getId) + file.setJobVersionId(jobVersion.getId) + file.setVersion(jobVersion.getVersion) + file.setStorePath("") + file.setStoreType("") + file + } + val transformJobContent = new StreamisJarTransformJobContent + val jobContent = JsonUtils.jackson.readValue(jobVersion.getJobContent, classOf[util.Map[String, Object]]) + jobContent.get("main.class.jar") match { + case mainClassJar: String => + transformJobContent.setMainClassJar(createFile(mainClassJar)) + case _ => throw new JobExecuteErrorException(30500, "main.class.jar is needed.") + } + jobContent.get("main.class") match { + case mainClass: String => + transformJobContent.setMainClass(mainClass) + case _ => throw new JobExecuteErrorException(30500, "main.class is needed.") + } + jobContent.get("args") match { + case args: util.List[String] => + transformJobContent.setArgs(args) + case _ => + } + jobContent.get("hdfs.jars") match { + case hdfsJars: util.List[String] => + transformJobContent.setHdfsJars(hdfsJars) + case _ => + } + jobContent.get("dependency.jars") match { + case dependencyJars: util.List[String] => + val parsedDependencyJars = dependencyJars.asScala.filter(StringUtils.isNotBlank).map(createFile).asJava + transformJobContent.setDependencyJars(parsedDependencyJars) + case _ => + } + jobContent.get("resources") match { + case resources: util.List[String] => + val parsedResources = resources.asScala.filter(StringUtils.isNotBlank).map(createFile).asJava + transformJobContent.setResources(parsedResources) + case _ => + } + transformJobContent + } + +} diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index c6c9b5a97..bba1dc579 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -51,6 +51,7 @@ import javax.servlet.http.HttpServletRequest; import java.io.IOException; import java.util.*; +import java.util.stream.Collectors; @RequestMapping(path = "/streamis/streamJobManager/job") @RestController @@ -77,7 +78,7 @@ public Message getJobList(HttpServletRequest req, @RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "jobName", required = false) String jobName, @RequestParam(value = "jobStatus", required = false) Integer jobStatus, - @RequestParam(value = "jobCreator", required = false) String jobCreator) throws JobException { + @RequestParam(value = "jobCreator", required = false) String jobCreator) { String username = SecurityFilter.getLoginUsername(req); if(StringUtils.isBlank(projectName)){ return Message.error("Project name cannot be empty(项目名不能为空，请指定)"); @@ -100,7 +101,7 @@ public Message getJobList(HttpServletRequest req, } @RequestMapping(path = "/createOrUpdate", method = RequestMethod.POST) - public Message createOrUpdate(HttpServletRequest req, @Validated @RequestBody MetaJsonInfo metaJsonInfo) throws Exception { + public Message createOrUpdate(HttpServletRequest req, @Validated @RequestBody MetaJsonInfo metaJsonInfo) { String username = SecurityFilter.getLoginUsername(req); String projectName = metaJsonInfo.getProjectName(); if (StringUtils.isBlank(projectName)){ @@ -142,6 +143,11 @@ public Message executeJob(HttpServletRequest req, @RequestBody Map jobList = streamJobService.getByProList(streamJobs.get(0).getProjectName(), username, null, 0, null); - Optional copyJob = jobList.getList().stream().min((job1, job2) -> { - if (job1.getStatus() > 0) { - return 0; + List copyJobs = jobList.getList().stream().filter(job -> !job.getJobType().startsWith("spark") && job.getStatus() > 0) + .collect(Collectors.toList()); + if(copyJobs.isEmpty()) { + return Message.error("no Flink Job has submitted, the register to Streamis cannot be succeeded."); + } + int index = 0; + while(streamTask == null && index < copyJobs.size()) { + StreamTask copyTask = streamTaskService.getLatestTaskByJobId(copyJobs.get(index).getId()); + if(copyTask == null) { + index ++; } else { - return 1; + LOG.warn("Streamis Job {} will bind the linkisJobInfo from history Flink Job {} with linkisJobId: {}, linkisJobInfo: {}.", + jobName, copyJobs.get(index).getName(), copyTask.getLinkisJobId(), copyTask.getLinkisJobInfo()); + streamTask = streamTaskService.createTask(streamJobs.get(0).getId(), (Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue(), username); + streamTask.setLinkisJobId(copyTask.getLinkisJobId()); + streamTask.setLinkisJobInfo(copyTask.getLinkisJobInfo()); } - }); - if(!copyJob.isPresent()) { - return Message.error("If no Flink Job has submitted, the register to Streamis cannot be succeeded."); } - StreamTask copyTask = streamTaskService.getLatestTaskByJobId(copyJob.get().getId()); - LOG.warn("Streamis Job {} will bind the linkisJobInfo from history Flink Job {} with linkisJobId: {}, linkisJobInfo: {}.", - jobName, copyJob.get().getName(), copyTask.getLinkisJobId(), copyTask.getLinkisJobInfo()); - streamTask = streamTaskService.createTask(streamJobs.get(0).getId(), (Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue(), username); - streamTask.setLinkisJobId(copyTask.getLinkisJobId()); - streamTask.setLinkisJobInfo(copyTask.getLinkisJobInfo()); + if(streamTask == null) { + return Message.error("no Flink Job has submitted, the register to Streamis cannot be succeeded."); + } } streamTask.setStartTime(new Date()); streamTask.setLastUpdateTime(new Date()); @@ -320,11 +336,11 @@ public Message stopTask(HttpServletRequest req, } else if(streamJobs.size() > 1) { return Message.error("Too many Streamis Job named " + jobName + ", we cannot distinguish between them."); } else if(!"spark.jar".equals(streamJobs.get(0).getJobType())) { - return Message.error("Only spark.jar Job support to add new tasks."); + return Message.error("Only spark.jar Job support to stop task."); } if (!streamJobService.hasPermission(streamJobs.get(0), username) && !this.privilegeService.hasEditPrivilege(req, streamJobs.get(0).getProjectName())) { - return Message.error("Have no permission to add new task for StreamJob [" + jobName + "]."); + return Message.error("Have no permission to stop task for StreamJob [" + jobName + "]."); } // 如果存在正在运行的，将其停止掉 StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJobs.get(0).getId()); @@ -340,12 +356,17 @@ public Message stopTask(HttpServletRequest req, @RequestMapping(path = "/progress", method = RequestMethod.GET) public Message progressJob(HttpServletRequest req, @RequestParam(value = "jobId", required = false) Long jobId, - @RequestParam(value = "version", required = false) String version) throws IOException, JobException { + @RequestParam(value = "version", required = false) String version) throws JobException { String username = SecurityFilter.getLoginUsername(req); if (jobId == null) { throw JobExceptionManager.createException(30301, "jobId"); } StreamJob streamJob = this.streamJobService.getJobById(jobId); + if(streamJob == null) { + return Message.error("not exists job " + jobId); + } else if(!JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES().getValue().contains(streamJob.getJobType())) { + return Message.error("Job " + streamJob.getName() + " is not supported to get progress."); + } if (!streamJobService.hasPermission(streamJob, username) && !this.privilegeService.hasAccessPrivilege(req, streamJob.getProjectName())) { return Message.error("Have no permission to view the progress of StreamJob [" + jobId + "]"); @@ -371,7 +392,6 @@ public Message uploadDetailsJob(HttpServletRequest req, @RequestParam(value = "j public Message getAlert(HttpServletRequest req, @RequestParam(value = "jobId", required = false) Long jobId, @RequestParam(value = "version", required = false) String version) { String username = SecurityFilter.getLoginUsername(req); - return Message.ok().data("list", streamJobService.getAlert(username, jobId, version)); } @@ -436,6 +456,11 @@ public Message snapshot(@PathVariable("jobId")Long jobId, HttpServletRequest req try{ String username = SecurityFilter.getLoginUsername(request); StreamJob streamJob = this.streamJobService.getJobById(jobId); + if(streamJob == null) { + return Message.error("not exists job " + jobId); + } else if(!JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES().getValue().contains(streamJob.getJobType())) { + return Message.error("Job " + streamJob.getName() + " is not supported to do snapshot."); + } if (!streamJobService.hasPermission(streamJob, username) && !this.privilegeService.hasEditPrivilege(request, streamJob.getProjectName())){ return Message.error("Have no permission to do snapshot for StreamJob [" + jobId + "]"); From 798c20fff2e497f17771b2fd56e9bcd90737b638 Mon Sep 17 00:00:00 2001 From: davidhua Date: Thu, 20 Oct 2022 03:11:32 +0800 Subject: [PATCH 007/240] Complete the log collector (80%). --- .../streamis-job-log-collector/pom.xml | 7 + .../collector/StreamisLogAppenderConfig.java | 4 + .../log/collector/StreamisRpcLogAppender.java | 5 + .../log/collector/cache/LogCache.java | 43 ++ .../sender/AbstractRpcLogSender.java | 428 ++++++++++++++++++ .../log/collector/sender/RpcLogSender.java | 33 ++ .../log/collector/sender/RpcSenderConfig.java | 34 ++ .../log/collector/sender/SendLogCache.java | 20 + .../sender/SendLogCacheConsumer.java | 99 ++++ .../sender/SendLogExceptionStrategy.java | 62 +++ .../sender/StreamisRpcLogSender.java | 6 + .../sender/{ => buf}/AbstractSendBuffer.java | 18 +- .../sender/{ => buf}/ImmutableSendBuffer.java | 2 +- .../sender/{ => buf}/SendBuffer.java | 19 +- .../sender/http/AbstractHttpLogSender.java | 24 + 15 files changed, 800 insertions(+), 4 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/cache/LogCache.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java rename streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/{ => buf}/AbstractSendBuffer.java (89%) rename streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/{ => buf}/ImmutableSendBuffer.java (99%) rename streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/{ => buf}/SendBuffer.java (83%) create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml index aa20062f4..9549d3439 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml @@ -16,4 +16,11 @@ 8 + + + com.webank.wedatasphere.streamis + streamis-job-log-common + 0.2.0 + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderConfig.java new file mode 100644 index 000000000..5c0ff730d --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderConfig.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector; + +public class StreamisLogAppenderConfig { +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java new file mode 100644 index 000000000..d326e6e04 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java @@ -0,0 +1,5 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector; + +public class StreamisRpcLogAppender { + +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/cache/LogCache.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/cache/LogCache.java new file mode 100644 index 000000000..f11556cc8 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/cache/LogCache.java @@ -0,0 +1,43 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.cache; + +import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Log cache + * @param element + */ +public interface LogCache { + + /** + * Cache log + * @param logElement log element + */ + void cacheLog(E logElement) throws InterruptedException; + + /** + * Drain log elements into collection + * @param elements elements + * @param maxElements max elements size + * @return count + */ + int drainLogsTo(List elements, int maxElements); + + /** + * Take log element + * @return log element + */ + E takeLog(long timeout, TimeUnit unit) throws InterruptedException; + + /** + * If the cache is full + * @return + */ + boolean isCacheable(); + /** + * Release the resource + */ + void destroy(); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java new file mode 100644 index 000000000..3c010878b --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java @@ -0,0 +1,428 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.cache.LogCache; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.ImmutableSendBuffer; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Abstract rpc log sender + * @param + * @param + */ +public abstract class AbstractRpcLogSender implements RpcLogSender{ + + /** + * Size of log cache + */ + int cacheSize; + + /** + * The buffer size of sender + */ + int sendBufSize; + + /** + * Max thread num of send + */ + int maxCacheConsume; + /** + * Connect config + */ + protected RpcSenderConfig rpcSenderConfig; + + /** + * Rpc log context + */ + private volatile RpcLogContext rpcLogContext; + + + public AbstractRpcLogSender(RpcSenderConfig rpcSenderConfig, int cacheSize, int sendBufSize){ + this(rpcSenderConfig, cacheSize, sendBufSize, Integer.MAX_VALUE); + } + + public AbstractRpcLogSender(RpcSenderConfig rpcSenderConfig, int cacheSize, int sendBufSize, int maxCacheConsume){ + this.rpcSenderConfig = rpcSenderConfig; + if (sendBufSize > cacheSize) { + throw new IllegalArgumentException("Size of send buffer is larger than cache size"); + } + this.cacheSize = cacheSize; + this.sendBufSize = sendBufSize; + this.maxCacheConsume = maxCacheConsume; + } + + @Override + public LogCache getOrCreateLogCache() { + getOrCreateRpcLogContext(); + return null; + } + + @Override + public void sendLog(T log) { + // Just send it into log cache + try { + getOrCreateLogCache().cacheLog(log); + } catch (InterruptedException e) { + // Exception handler + } + } + + @Override + public void syncSendLog(T log) { + + } + + @Override + public void close() { + + } + + /** + * Aggregate send buffer for sending + * @param sendBuffer send buffer + * @return E aggregated entity + */ + protected abstract E aggregateBuffer(SendBuffer sendBuffer); + + /** + * Sending operation + * @param aggregatedEntity agg entity + * @param rpcSenderConfig rpc sender config + */ + protected abstract void doSend(E aggregatedEntity, RpcSenderConfig rpcSenderConfig); + + /** + * Send log exception strategy + * @return exception strategy + */ + protected abstract SendLogExceptionStrategy getSendLogExceptionStrategy(); + + protected RpcLogContext getOrCreateRpcLogContext(){ + if (null == this.rpcLogContext){ + synchronized (this){ + if (null == this.rpcLogContext){ + SendLogCache logCache = new QueuedSendLogCache(this.cacheSize, false); + this.rpcLogContext = new RpcLogContext(logCache); + // Start cache consumer + this.rpcLogContext.startCacheConsumer(); + } + } + + } + return this.rpcLogContext; + } + + private class RpcLogContext{ + + private static final String RPC_LOG_CACHE_CONSUMER = "RpcLog-Cache-Consumer-Thread-"; + /** + * Send log cache + */ + private final SendLogCache logCache; + + /** + * Consume pool + */ + private final ThreadPoolExecutor consumePool; + + /** + * Count of the consumers + */ + private int consumers = 0; + + public RpcLogContext(SendLogCache logCache){ + this.logCache = logCache; + this.consumePool = new ThreadPoolExecutor(0, maxCacheConsume, + 60L, TimeUnit.SECONDS, + new SynchronousQueue<>(), new ThreadFactory() { + private final ThreadGroup group = Thread.currentThread().getThreadGroup(); + private final AtomicInteger threadNum = new AtomicInteger(1); + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(group, r, RPC_LOG_CACHE_CONSUMER + + threadNum.getAndIncrement(), 0); + if (t.isDaemon()) { + t.setDaemon(false); + } + if (t.getPriority() != Thread.NORM_PRIORITY) { + t.setPriority(Thread.NORM_PRIORITY); + } + return t; + } + }); + } + + public synchronized void startCacheConsumer(){ + if (consumers >= maxCacheConsume){ + throw new IllegalStateException("Over the limit number of cache consumers: [" + maxCacheConsume + "]"); + } + String id = UUID.randomUUID().toString(); + SendBuffer sendBuffer = new ImmutableSendBuffer<>(sendBufSize); + this.consumePool.submit(new SendLogCacheConsumer(id, logCache, sendBuffer, rpcSenderConfig) { + @Override + protected void onFlushAndSend(SendBuffer sendBuffer) { + // First to aggregate the buffer + E aggEntity = aggregateBuffer(sendBuffer); + Optional.ofNullable(getSendLogExceptionStrategy()).ifPresent( + strategy -> strategy.doSend(() -> { + doSend(aggEntity, rpcSenderConfig); + return null; + }, sendBuffer)); + } + }); + this.consumers ++; + } + + public SendLogCache getLogCache(){ + return this.logCache; + } + + public void destroyCacheConsumer(){ + + } + } + /** + * Act as ArrayBlockingQueue (jdk 1.8) + */ + private class QueuedSendLogCache implements SendLogCache{ + + // Queued items + final Object[] items; + + // Take index + int takeIndex; + + // Put index + int putIndex; + + // Count + int count; + + // Reentrant lock + final ReentrantLock lock; + + // Condition for waiting takes + private final Condition notEmpty; + + // Condition for waiting puts(cacheLog) + private final Condition notFull; + + public QueuedSendLogCache(int capacity, boolean fair) { + this.items = new Object[capacity]; + lock = new ReentrantLock(fair); + this.notEmpty = lock.newCondition(); + this.notFull = lock.newCondition(); + } + + @Override + public void cacheLog(T logElement) throws InterruptedException { + // Skip the null element + if (Objects.nonNull(logElement)){ + final ReentrantLock lock = this.lock; + lock.lockInterruptibly(); + try{ + while (count == items.length){ + notFull.await(); + } + enqueue(logElement); + }finally{ + lock.unlock(); + } + } + } + + @Override + public int drainLogsTo(List elements, int maxElements) { + if (Objects.nonNull(elements) && maxElements > 0){ + final Object[] items = this.items; + final ReentrantLock lock = this.lock; + lock.lock(); + try{ + int n = Math.min(maxElements, count); + int take = takeIndex; + int i = 0; + try { + while (i < n){ + @SuppressWarnings("unchecked") + T x = (T) items[take]; + elements.add(x); + items[take] = null; + if (++ take == items.length) + take = 0; + i++; + } + return n; + }finally { + restoreInvariants(i, take, false); + } + } finally { + lock.unlock(); + } + } + return 0; + } + + // Equal to the poll method in ArrayBlockingQuue + @Override + public T takeLog(long timeout, TimeUnit unit) throws InterruptedException { + long nanos = unit.toNanos(timeout); + final ReentrantLock lock = this.lock; + T element; + lock.lockInterruptibly(); + try{ + while (count == 0){ + if (nanos < 0){ + return null; + } + nanos = notEmpty.awaitNanos(nanos); + } + element = dequeue(); + } finally { + lock.unlock(); + } + return element; + } + + @Override + public boolean isCacheable() { + final ReentrantLock lock = this.lock; + lock.lock(); + try { + return count < items.length; + }finally { + lock.unlock(); + } + } + + // The same as the clear() method, + @Override + public void destroy() { + final Object[] items = this.items; + final ReentrantLock lock = this.lock; + lock.lock(); + try { + int k = count; + if (k > 0) { + final int putIndex = this.putIndex; + int i = takeIndex; + do { + items[i] = null; + if (++i == items.length) + i = 0; + } while (i != putIndex); + takeIndex = putIndex; + count = 0; + for (; k > 0 && lock.hasWaiters(notFull); k--) + notFull.signal(); + } + } finally { + lock.unlock(); + } + } + + /** + * Drain the elements into send buffer + * @param sendBuffer send buffer + * @param maxElements max element size + * @return int + */ + @Override + public int drainLogsTo(SendBuffer sendBuffer, int maxElements) { + if (Objects.nonNull(sendBuffer) && maxElements > 0){ + final Object[] items = this.items; + final ReentrantLock lock = this.lock; + lock.lock(); + try{ + int n = Math.min(maxElements, count); + int take = takeIndex; + int i = 0; + int send; + try { + while (n > 0) { + int len = items.length - take; + int send0 = Math.min(n, len); + // Copy the array element to buffer directly + send = sendBuf(sendBuffer, this.items, take, send0); + n -= send; + if ((take = take + send) >= items.length) { + take = 0; + } + i += send; + if (send < send0 || send <= 0) { + break; + } + } + return i; + } finally { + if (i > 0){ + restoreInvariants(i, take, true); + } + } + }finally { + lock.unlock(); + } + } + return 0; + } + + @SuppressWarnings("unchecked") + private int sendBuf(SendBuffer sendBuffer, Object[] items, int takeIndex, int len){ + int send = sendBuffer.writeBuf((T[]) items, takeIndex, len); + if (send < len){ + // Buffer full exception + } + // Allow data loss + return send; + } + + private void restoreInvariants(int i, int take, boolean clearItems){ + this.count -= i; + if (clearItems){ + int index = this.takeIndex; + for (; i > 0; i --){ + this.items[index] = null; + if (++index == items.length){ + index = 0; + } + } + //At last index equals take + } + this.takeIndex = take; + for (; i > 0 && lock.hasWaiters(notFull); i--){ + notFull.signal(); + } + } + // Inserts element at current put position, advances, and signals. Call only when holding lock. + private void enqueue(T element){ + this.items[putIndex] = element; + if (++putIndex >= items.length){ + putIndex = 0; + } + count ++; + notEmpty.signal(); + } + + // Extracts element at current take position, advances, and signals. Call only when holding lock. + private T dequeue(){ + @SuppressWarnings("unchecked") + T element = (T)this.items[takeIndex]; + this.items[takeIndex] = null; + if ( ++ takeIndex == items.length){ + this.takeIndex = 0; + } + count --; + // Not need to support iterator + notFull.signal(); + return element; + } + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java new file mode 100644 index 000000000..25f512643 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.cache.LogCache; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; + +/** + * Rpc Log sender + */ +public interface RpcLogSender { + + /** + * Produce log cache + * @return log cache + */ + LogCache getOrCreateLogCache(); + + /** + * Send log (async) + * @param log log element + */ + void sendLog(T log); + + /** + * Send log (sync) + * @param log log element + */ + void syncSendLog(T log); + + /** + * Close sender + */ + void close(); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java new file mode 100644 index 000000000..a0d3a77e5 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; + +/** + * Rpc sender configuration + */ +public class RpcSenderConfig { + + /** + * Send address + */ + private String sendAddress; + /** + * Size of send cache + */ + private int sendCacheSize = 150; + + /** + * Size of send buffer + */ + private int sendBufferSize = 50; + + /** + * Expire time of send buffer + */ + private long sendBufferExpireTimeInSec = -1; + + public long getSendBufferExpireTimeInSec() { + return sendBufferExpireTimeInSec; + } + + public void setSendBufferExpireTimeInSec(long sendBufferExpireTimeInSec) { + this.sendBufferExpireTimeInSec = sendBufferExpireTimeInSec; + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java new file mode 100644 index 000000000..200c573d9 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.cache.LogCache; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; + +/** + * Send log cache + * @param + */ +public interface SendLogCache extends LogCache { + + /** + * Drain the logs into send buffer + * @param sendBuffer send buffer + * @param maxElements max element size + * @return count + */ + int drainLogsTo(SendBuffer sendBuffer, int maxElements);; +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java new file mode 100644 index 000000000..f0d8f5866 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java @@ -0,0 +1,99 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; + +import java.util.concurrent.TimeUnit; + +/** + * Send log consumer + * Consume the log elements from cache and put into send buffer + * @param + */ +public abstract class SendLogCacheConsumer implements Runnable{ + + private boolean isTerminated = false; + + /** + * Buffer expire time in milliseconds + */ + private final long bufferExpireTimeInMills; + /** + * Send log cache + */ + private final SendLogCache cache; + + /** + * Send buffer + */ + private final SendBuffer sendBuffer; + + private final String id; + public SendLogCacheConsumer(String id, SendLogCache cache, + SendBuffer sendBuffer, + RpcSenderConfig rpcSenderConfig){ + this.id = id; + this.cache = cache; + this.sendBuffer = sendBuffer; + this.bufferExpireTimeInMills = rpcSenderConfig.getSendBufferExpireTimeInSec() > 0 ? TimeUnit.SECONDS + .toMillis(rpcSenderConfig.getSendBufferExpireTimeInSec()) : -1; + + } + + @Override + public void run() { + int remain; + long expireTimeInMills = requireNewFlushTime(); + while (!this.isTerminated) { + try { + if ((expireTimeInMills > 0 && expireTimeInMills >= System.currentTimeMillis()) + || (remain = this.sendBuffer.remaining()) <= 0) { + // Transient to the read mode + sendBuffer.flip(); + onFlushAndSend(sendBuffer); + expireTimeInMills = requireNewFlushTime(); + if (sendBuffer.isReadMode()) { + // Clear the buffer and transient to the write mode, otherwise continue writing + sendBuffer.clear(); + } + remain = this.sendBuffer.remaining(); + } + if (remain > 0) { + int inBuf = this.cache.drainLogsTo(sendBuffer, remain); + if (inBuf < remain) { + // Means that the cache is empty, take and wait the log element + long waitTime = expireTimeInMills - System.currentTimeMillis(); + if (waitTime > 0) { + T logElement = this.cache.takeLog(waitTime, TimeUnit.MILLISECONDS); + if (null != logElement) { + sendBuffer.writeBuf(logElement); + } + } + } + } + } catch (Throwable e){ + if (this.isTerminated && e instanceof InterruptedException){ + return; + } else { + System.err.println("SendLogCacheConsumer[" + Thread.currentThread().getName() + "] occurred exception [" + e.getLocalizedMessage() + "]"); + // For the unknown exception clear the cache + sendBuffer.clear(); + expireTimeInMills = requireNewFlushTime(); + } + } + } + } + + public void shutdown(){ + this.isTerminated = true; + } + + private long requireNewFlushTime(){ + return bufferExpireTimeInMills > 0 ? System.currentTimeMillis() + bufferExpireTimeInMills : -1; + } + /** + * When the buffer is full or reach the idle time, invoke the method + * @param sendBuffer send buffer + */ + protected abstract void onFlushAndSend(SendBuffer sendBuffer); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java new file mode 100644 index 000000000..d3c2fafd9 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java @@ -0,0 +1,62 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; + +import java.util.Objects; +import java.util.concurrent.Callable; + +/** + * Strategy control the action on exception + */ +public abstract class SendLogExceptionStrategy { + + protected final RpcLogSender sender; + + public SendLogExceptionStrategy(RpcLogSender sender){ + this.sender = sender; + } + /** + * Retry count + * @return retry + */ + abstract int retryCount(); + + /** + * + * @param e exception + * @return boolean + */ + abstract RetryDescription onException(Exception e, SendBuffer sendBuffer); + + V doSend(Callable sendOperation, SendBuffer sendBuffer){ + int retryCount = retryCount(); + int count = 0; + RetryDescription retryDescription; + while (++count <= retryCount) { + try { + return sendOperation.call(); + } catch (Exception e) { + retryDescription = onException(e, sendBuffer); + if (Objects.isNull(retryDescription) || !retryDescription.canRetry) { + break; + } + + } + } + return null; + } + + private static class RetryDescription{ + + private final boolean canRetry; + + public RetryDescription(boolean canRetry){ + this.canRetry = canRetry; + } + + public boolean isCanRetry() { + return canRetry; + } + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java new file mode 100644 index 000000000..06d70407a --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java @@ -0,0 +1,6 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; + + + +public class StreamisRpcLogSender{ +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractSendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java similarity index 89% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractSendBuffer.java rename to streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java index 2f778b231..83e04ec7d 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractSendBuffer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf; /** * Abstract sender buffer; @@ -31,6 +31,17 @@ public AbstractSendBuffer(int capacity){ public AbstractSendBuffer(){ this(Integer.MAX_VALUE); } + + @Override + public boolean isReadMode() { + return accessFlag == Flag.READ_MODE; + } + + @Override + public boolean isWriteMode() { + return accessFlag == Flag.WRITE_MODE; + } + @Override public int capacity() { return this.capacity; @@ -49,6 +60,11 @@ public void flip() { this.accessFlag = Flag.READ_MODE; } + @Override + public void rewind() { + position = 0; + } + @Override public void clear() { limit(this.capacity); diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/ImmutableSendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java similarity index 99% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/ImmutableSendBuffer.java rename to streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java index 3587079fa..eefe50ae7 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/ImmutableSendBuffer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf; import java.util.Arrays; import java.util.function.Function; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java similarity index 83% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendBuffer.java rename to streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java index e8bee928b..28e3a6356 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendBuffer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf; import java.util.function.Function; @@ -14,6 +14,17 @@ public interface SendBuffer { */ int capacity(); + /** + * Is read mode + * @return boolean + */ + boolean isReadMode(); + + /** + * Is write mode + * @return boolean + */ + boolean isWriteMode(); /** * Scale-up or scale-in * @param newCapacity new capacity @@ -27,10 +38,14 @@ public interface SendBuffer { int remaining(); /** - * transient between write-mode and read-mode + * Transient between write-mode and read-mode */ void flip(); + /** + * Restart from the beginning of window + */ + void rewind(); /** * Clear to reuse the buffer */ diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java new file mode 100644 index 000000000..b8bba66b8 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.AbstractRpcLogSender; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.RpcSenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.SendLogExceptionStrategy; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; + +public abstract class AbstractHttpLogSender extends AbstractRpcLogSender { + + /** + * Retry strategy + */ + private SendLogExceptionStrategy sendRetryStrategy; + + public AbstractHttpLogSender(RpcSenderConfig rpcSenderConfig, int cacheSize, int sendBufSize) { + super(rpcSenderConfig, cacheSize, sendBufSize); + + } + + @Override + protected void doSend(E aggregatedEntity, RpcSenderConfig rpcSenderConfig) { + + } +} From f95e06ae156aa1c906c425db2684fc8d58296cde Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Thu, 20 Oct 2022 11:00:13 +0800 Subject: [PATCH 008/240] fix the bug for adding task for existed jobs. --- .../streamis/jobmanager/restful/api/JobRestfulApi.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index bba1dc579..e174947a8 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -281,11 +281,11 @@ public Message addTask(HttpServletRequest req, } else if(streamTask == null) { // 这里取个巧，从该工程该用户有权限的Job中找到一个Flink的历史作业，作为这个Spark Streaming作业的jobId和jobInfo // 替换掉JobInfo中的 yarn 信息，这样我们前端就可以在不修改任何逻辑的情况下正常展示Spark Streaming作业了 - PageInfo jobList = streamJobService.getByProList(streamJobs.get(0).getProjectName(), username, null, 0, null); - List copyJobs = jobList.getList().stream().filter(job -> !job.getJobType().startsWith("spark") && job.getStatus() > 0) + PageInfo jobList = streamJobService.getByProList(streamJobs.get(0).getProjectName(), username, null, null, null); + List copyJobs = jobList.getList().stream().filter(job -> !job.getJobType().startsWith("spark.")) .collect(Collectors.toList()); if(copyJobs.isEmpty()) { - return Message.error("no Flink Job has submitted, the register to Streamis cannot be succeeded."); + return Message.error("no Flink Job has been submitted, the register to Streamis cannot be succeeded."); } int index = 0; while(streamTask == null && index < copyJobs.size()) { @@ -301,7 +301,7 @@ public Message addTask(HttpServletRequest req, } } if(streamTask == null) { - return Message.error("no Flink Job has submitted, the register to Streamis cannot be succeeded."); + return Message.error("no Flink task has been executed, the register to Streamis cannot be succeeded."); } } streamTask.setStartTime(new Date()); From 04d5ae8dc975077c5f2ea3d1295eba5d0dead904 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Thu, 20 Oct 2022 17:31:30 +0800 Subject: [PATCH 009/240] 1. Add the ability for updating task for existed jobs. 2. Spark Streaming Job will not ask Linkis for status. --- .../manager/service/TaskMonitorService.scala | 104 +++++++++--------- .../jobmanager/restful/api/JobRestfulApi.java | 95 +++++++++++++--- 2 files changed, 134 insertions(+), 65 deletions(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskMonitorService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskMonitorService.scala index edf36bb94..f6f250e6a 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskMonitorService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskMonitorService.scala @@ -18,6 +18,7 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.service import java.util import java.util.Date import java.util.concurrent.{Future, TimeUnit} + import com.google.common.collect.Sets import com.webank.wedatasphere.streamis.jobmanager.launcher.JobLauncherAutoConfiguration import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants @@ -30,7 +31,6 @@ import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamJobMapper, StreamTaskMapper} import com.webank.wedatasphere.streamis.jobmanager.manager.entity.{StreamJob, StreamTask} import com.webank.wedatasphere.streamis.jobmanager.manager.utils.StreamTaskUtils - import javax.annotation.{PostConstruct, PreDestroy, Resource} import org.apache.commons.lang.exception.ExceptionUtils import org.apache.linkis.common.exception.ErrorException @@ -84,58 +84,64 @@ class TaskMonitorService extends Logging { return } streamTasks.filter(shouldMonitor).foreach { streamTask => - streamTask.setLastUpdateTime(new Date) - streamTaskMapper.updateTask(streamTask) val job = streamJobMapper.getJobById(streamTask.getJobId) - info(s"Try to update status of StreamJob-${job.getName}.") - val retryHandler = new RetryHandler {} - retryHandler.setRetryNum(3) - retryHandler.setRetryMaxPeriod(2000) - retryHandler.addRetryException(classOf[ErrorException]) - var jobInfo:JobInfo = null - Utils.tryCatch { - jobInfo = retryHandler.retry(refresh(streamTask, jobLaunchManager), s"Task-Monitor-${job.getName}") - } { ex => { - error(s"Fetch StreamJob-${job.getName} failed, maybe the Linkis cluster is wrong, please be noticed!", ex) - val errorMsg = ExceptionUtils.getRootCauseMessage(ex) - if (errorMsg != null && errorMsg.contains("Not exists EngineConn")) { - streamTask.setStatus(JobConf.FLINK_JOB_STATUS_FAILED.getValue) - streamTask.setErrDesc("Not exists EngineConn.") - } else { - // 连续三次还是出现异常，说明Linkis的Manager已经不能正常提供服务，告警并不再尝试获取状态，等待下次尝试 - val users = getAlertUsers(job) - users.add(job.getCreateBy) - alert(jobService.getAlertLevel(job), s"请求LinkisManager失败，Linkis集群出现异常，请关注！影响任务[${job.getName}]", users, streamTask) - } - } - } - streamTaskMapper.updateTask(streamTask) - if(streamTask.getStatus == JobConf.FLINK_JOB_STATUS_FAILED.getValue) { - warn(s"StreamJob-${job.getName} is failed, please be noticed.") - var extraMessage = "" - Option(jobInfo) match { - case Some(flinkJobInfo: FlinkJobInfo) => - extraMessage = s",${flinkJobInfo.getApplicationId}" - case _ => - } - // Need to add restart feature if user sets the restart parameters. - var alertMsg = s"Streamis 流式应用[${job.getName}${extraMessage}]已经失败, 请登陆Streamis查看应用日志." - this.streamJobConfMapper.getRawConfValue(job.getId, JobConfKeyConstants.FAIL_RESTART_SWITCH.getValue) match { - case "ON" => - alertMsg = s"${alertMsg} 现将自动拉起该应用" - Utils.tryCatch{ - info(s"Start to reLaunch the StreamisJob [${job.getName}], now to submit and schedule it...") - // Use submit user to start job - val future: Future[String] = streamTaskService.asyncExecute(job.getId, 0L, job.getSubmitUser, true) - }{ - case e:Exception => - warn(s"Fail to reLaunch the StreamisJob [${job.getName}]", e) - } - case _ => - } + if(!JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES.getValue.contains(job.getJobType)) { val userList = Sets.newHashSet(job.getSubmitUser, job.getCreateBy) userList.addAll(getAlertUsers(job)) + val alertMsg = s"Spark Streaming应用[${job.getName}]已经超过 ${Utils.msDurationToString(System.currentTimeMillis - streamTask.getLastUpdateTime.getTime)} 没有更新状态, 请及时确认应用是否正常！" alert(jobService.getAlertLevel(job), alertMsg, new util.ArrayList[String](userList), streamTask) + } else { + streamTask.setLastUpdateTime(new Date) + streamTaskMapper.updateTask(streamTask) + info(s"Try to update status of StreamJob-${job.getName}.") + val retryHandler = new RetryHandler {} + retryHandler.setRetryNum(3) + retryHandler.setRetryMaxPeriod(2000) + retryHandler.addRetryException(classOf[ErrorException]) + var jobInfo:JobInfo = null + Utils.tryCatch { + jobInfo = retryHandler.retry(refresh(streamTask, jobLaunchManager), s"Task-Monitor-${job.getName}") + } { ex => + error(s"Fetch StreamJob-${job.getName} failed, maybe the Linkis cluster is wrong, please be noticed!", ex) + val errorMsg = ExceptionUtils.getRootCauseMessage(ex) + if (errorMsg != null && errorMsg.contains("Not exists EngineConn")) { + streamTask.setStatus(JobConf.FLINK_JOB_STATUS_FAILED.getValue) + streamTask.setErrDesc("Not exists EngineConn.") + } else { + // 连续三次还是出现异常，说明Linkis的Manager已经不能正常提供服务，告警并不再尝试获取状态，等待下次尝试 + val users = getAlertUsers(job) + users.add(job.getCreateBy) + alert(jobService.getAlertLevel(job), s"请求LinkisManager失败，Linkis集群出现异常，请关注！影响任务[${job.getName}]", users, streamTask) + } + } + streamTaskMapper.updateTask(streamTask) + if(streamTask.getStatus == JobConf.FLINK_JOB_STATUS_FAILED.getValue) { + warn(s"StreamJob-${job.getName} is failed, please be noticed.") + var extraMessage = "" + Option(jobInfo) match { + case Some(flinkJobInfo: FlinkJobInfo) => + extraMessage = s",${flinkJobInfo.getApplicationId}" + case _ => + } + // Need to add restart feature if user sets the restart parameters. + var alertMsg = s"Streamis 流式应用[${job.getName}${extraMessage}]已经失败, 请登陆Streamis查看应用日志." + this.streamJobConfMapper.getRawConfValue(job.getId, JobConfKeyConstants.FAIL_RESTART_SWITCH.getValue) match { + case "ON" => + alertMsg = s"${alertMsg} 现将自动拉起该应用" + Utils.tryCatch{ + info(s"Start to reLaunch the StreamisJob [${job.getName}], now to submit and schedule it...") + // Use submit user to start job + val future: Future[String] = streamTaskService.asyncExecute(job.getId, 0L, job.getSubmitUser, true) + }{ + case e:Exception => + warn(s"Fail to reLaunch the StreamisJob [${job.getName}]", e) + } + case _ => + } + val userList = Sets.newHashSet(job.getSubmitUser, job.getCreateBy) + userList.addAll(getAlertUsers(job)) + alert(jobService.getAlertLevel(job), alertMsg, new util.ArrayList[String](userList), streamTask) + } } } info("All StreamTasks status have updated.") diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index e174947a8..37ed9bf1a 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -22,6 +22,7 @@ import com.webank.wedatasphere.streamis.jobmanager.exception.JobExceptionManager; import com.webank.wedatasphere.streamis.jobmanager.launcher.job.JobInfo; import com.webank.wedatasphere.streamis.jobmanager.launcher.job.manager.JobLaunchManager; +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.state.JobStateInfo; import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.entity.LogRequestPayload; import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.FlinkJobInfo; import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf; @@ -51,6 +52,7 @@ import javax.servlet.http.HttpServletRequest; import java.io.IOException; import java.util.*; +import java.util.function.Function; import java.util.stream.Collectors; @RequestMapping(path = "/streamis/streamJobManager/job") @@ -277,6 +279,7 @@ public Message addTask(HttpServletRequest req, if(streamTask != null && JobConf.isRunning(streamTask.getStatus())) { LOG.warn("Streamis Job {} exists running task, update its status from Running to stopped at first.", jobName); streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); + streamTask.setErrDesc("stopped by App's new task."); streamTaskService.updateTask(streamTask); } else if(streamTask == null) { // 这里取个巧，从该工程该用户有权限的Job中找到一个Flink的历史作业，作为这个Spark Streaming作业的jobId和jobInfo @@ -306,21 +309,65 @@ public Message addTask(HttpServletRequest req, } streamTask.setStartTime(new Date()); streamTask.setLastUpdateTime(new Date()); + StreamTask finalStreamTask = streamTask; + return withFlinkJobInfo(jobName, streamTask.getLinkisJobInfo(), flinkJobInfo -> { + flinkJobInfo.setApplicationId(appId); + flinkJobInfo.setApplicationUrl(appUrl); + flinkJobInfo.setName(jobName); + flinkJobInfo.setStatus(JobConf.getStatusString((Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue())); + StreamTaskUtils.refreshInfo(finalStreamTask, flinkJobInfo); + streamTaskService.updateTask(finalStreamTask); + LOG.info("Streamis Job {} has added a new task successfully.", jobName); + return Message.ok(); + }); + } + + private Message withFlinkJobInfo(String jobName, String flinkJobInfoStr, Function flinkJobInfoFunction) { FlinkJobInfo flinkJobInfo; try { - flinkJobInfo = DWSHttpClient.jacksonJson().readValue(streamTask.getLinkisJobInfo(), FlinkJobInfo.class); + flinkJobInfo = DWSHttpClient.jacksonJson().readValue(flinkJobInfoStr, FlinkJobInfo.class); } catch (JsonProcessingException e) { - LOG.error("Job {} deserialize the jobInfo from history Job failed!", jobName, e); - return Message.error("Deserialize the jobInfo from history Job failed!"); - } - flinkJobInfo.setApplicationId(appId); - flinkJobInfo.setApplicationUrl(appUrl); - flinkJobInfo.setName(jobName); - flinkJobInfo.setStatus(JobConf.getStatusString((Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue())); - StreamTaskUtils.refreshInfo(streamTask, flinkJobInfo); - streamTaskService.updateTask(streamTask); - LOG.info("Streamis Job {} has added a new task successfully.", jobName); - return Message.ok(); + LOG.error("Job {} deserialize the flinkJobInfo string to object failed!", jobName, e); + return Message.error("Deserialize the flinkJobInfo string to object failed!"); + } + return flinkJobInfoFunction.apply(flinkJobInfo); + } + + @RequestMapping(path = "/updateTask", method = RequestMethod.GET) + public Message updateTask(HttpServletRequest req, + @RequestParam(value = "jobName") String jobName, + @RequestParam(value = "appId") String appId, + @RequestParam(value = "metrics") String metrics) { + String username = SecurityFilter.getLoginUsername(req); + LOG.info("User {} try to update task for Streamis job {} with appId: {}.", username, jobName, appId); + List streamJobs = streamJobService.getJobByName(jobName); + if(CollectionUtils.isEmpty(streamJobs)) { + return Message.error("Not exits Streamis job " + jobName); + } else if(streamJobs.size() > 1) { + return Message.error("Too many Streamis Job named " + jobName + ", we cannot distinguish between them."); + } else if(!"spark.jar".equals(streamJobs.get(0).getJobType())) { + return Message.error("Only spark.jar Job support to update task."); + } + StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJobs.get(0).getId()); + if(streamTask == null) { + LOG.warn("Job {} is not exists running task, ignore to update its metrics.", jobName); + return Message.ok("not exists running task, ignore it."); + } + return withFlinkJobInfo(jobName, streamTask.getLinkisJobInfo(), flinkJobInfo -> { + if(!flinkJobInfo.getApplicationId().equals(appId)) { + LOG.warn("Job {} with running task is not equals to the request appId: {}, ignore to update its metrics.", + jobName, flinkJobInfo.getApplicationId(), appId); + return Message.ok("the request appId is not equals to the running task appId " + flinkJobInfo.getApplicationId()); + } + JobStateInfo jobStateInfo = new JobStateInfo(); + jobStateInfo.setTimestamp(System.currentTimeMillis()); + jobStateInfo.setLocation(metrics); + flinkJobInfo.setJobStates(new JobStateInfo[]{jobStateInfo}); + StreamTaskUtils.refreshInfo(streamTask, flinkJobInfo); + streamTaskService.updateTask(streamTask); + LOG.info("Streamis Job {} has updated the task metrics successfully.", jobName); + return Message.ok(); + }); } @RequestMapping(path = "/stopTask", method = RequestMethod.GET) @@ -345,13 +392,23 @@ public Message stopTask(HttpServletRequest req, // 如果存在正在运行的，将其停止掉 StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJobs.get(0).getId()); if(streamTask != null && JobConf.isRunning(streamTask.getStatus())) { - LOG.warn("Streamis Job {} is exists running task, update its status to stopped.", jobName); - streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); - streamTaskService.updateTask(streamTask); + return withFlinkJobInfo(jobName, streamTask.getLinkisJobInfo(), flinkJobInfo -> { + if(flinkJobInfo.getApplicationId().equals(appId)) { + LOG.warn("Streamis Job {} is exists running task, update its status to stopped.", jobName); + streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); + streamTask.setErrDesc("stopped by App itself."); + streamTaskService.updateTask(streamTask); + return Message.ok(); + } else { + LOG.warn("Job {} with running task is not equals to the request appId: {}, ignore to stop it.", + jobName, flinkJobInfo.getApplicationId(), appId); + return Message.ok("the request appId is not equals to the running task appId " + flinkJobInfo.getApplicationId()); + } + }); } else { LOG.warn("Streamis Job {} is not exists running task, ignore to stop it.", jobName); + return Message.ok(); } - return Message.ok(); } @RequestMapping(path = "/progress", method = RequestMethod.GET) @@ -411,6 +468,12 @@ public Message getLog(HttpServletRequest req, logType = StringUtils.isBlank(logType) ? "client" : logType; String username = SecurityFilter.getLoginUsername(req); StreamJob streamJob = this.streamJobService.getJobById(jobId); + if(streamJob == null) { + return Message.error("not exists job " + jobId); + } else if(!JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES().getValue().contains(streamJob.getJobType()) && + "client".equals(logType)) { + return Message.error("Job " + streamJob.getName() + " is not supported to get client logs."); + } if (!streamJobService.hasPermission(streamJob, username) && !this.privilegeService.hasAccessPrivilege(req, streamJob.getProjectName())) { return Message.error("Have no permission to fetch logs from StreamJob [" + jobId + "]"); From 90d6937a0af2e57c206e117f60f00e1d2a8b0c56 Mon Sep 17 00:00:00 2001 From: davidhua Date: Thu, 20 Oct 2022 19:12:02 +0800 Subject: [PATCH 010/240] RPC(Http) module classes in collector --- .../streamis-job-log-collector/pom.xml | 20 +++ .../sender/AbstractRpcLogSender.java | 77 +++++--- .../log/collector/sender/RpcSenderConfig.java | 170 +++++++++++++++++- .../sender/SendLogCacheConsumer.java | 19 +- .../sender/SendLogExceptionStrategy.java | 6 +- .../sender/http/AbstractHttpLogSender.java | 34 +++- .../collector/sender/http/HttpClientTool.java | 71 ++++++++ .../log/collector/sender/http/Json.java | 130 ++++++++++++++ .../sender/http/entities/Resource.java | 17 ++ .../http/request/AbstractHttpAction.java | 62 +++++++ .../sender/http/request/EntityPostAction.java | 80 +++++++++ .../sender/http/request/GetAction.java | 4 + .../sender/http/request/HttpAction.java | 38 ++++ .../sender/http/request/PostAction.java | 41 +++++ .../jobmanager/log/entities/LogElement.java | 3 +- .../log/entities/StreamisLogEvents.java | 19 ++ 16 files changed, 752 insertions(+), 39 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/entities/Resource.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/HttpAction.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/PostAction.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml index 9549d3439..bbc6a69c0 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml @@ -14,6 +14,9 @@ 8 8 + 4.5.13 + 4.5.4 + 2.13.2.2 @@ -22,5 +25,22 @@ streamis-job-log-common 0.2.0 + + + org.apache.httpcomponents + httpclient + ${httpclient.version} + + + org.apache.httpcomponents + httpmime + ${httpmine.version} + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson-databind.version} + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java index 3c010878b..6b4d36580 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java @@ -5,10 +5,7 @@ import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.UUID; +import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Condition; @@ -139,8 +136,17 @@ private class RpcLogContext{ */ private int consumers = 0; + /** + * Futures of consumers + */ + private final Map> sendLogCacheConsumers = new ConcurrentHashMap<>(); + /** + * Context lock + */ + private final ReentrantLock ctxLock; public RpcLogContext(SendLogCache logCache){ this.logCache = logCache; + this.ctxLock = new ReentrantLock(); this.consumePool = new ThreadPoolExecutor(0, maxCacheConsume, 60L, TimeUnit.SECONDS, new SynchronousQueue<>(), new ThreadFactory() { @@ -161,33 +167,58 @@ public Thread newThread(Runnable r) { }); } - public synchronized void startCacheConsumer(){ - if (consumers >= maxCacheConsume){ - throw new IllegalStateException("Over the limit number of cache consumers: [" + maxCacheConsume + "]"); - } - String id = UUID.randomUUID().toString(); - SendBuffer sendBuffer = new ImmutableSendBuffer<>(sendBufSize); - this.consumePool.submit(new SendLogCacheConsumer(id, logCache, sendBuffer, rpcSenderConfig) { - @Override - protected void onFlushAndSend(SendBuffer sendBuffer) { - // First to aggregate the buffer - E aggEntity = aggregateBuffer(sendBuffer); - Optional.ofNullable(getSendLogExceptionStrategy()).ifPresent( - strategy -> strategy.doSend(() -> { - doSend(aggEntity, rpcSenderConfig); - return null; - }, sendBuffer)); + public void startCacheConsumer(){ + this.ctxLock.lock(); + try { + if (consumers >= maxCacheConsume) { + throw new IllegalStateException("Over the limit number of cache consumers: [" + maxCacheConsume + "]"); } - }); - this.consumers ++; + String id = UUID.randomUUID().toString(); + SendBuffer sendBuffer = new ImmutableSendBuffer<>(sendBufSize); + SendLogCacheConsumer consumer = new SendLogCacheConsumer(id, logCache, sendBuffer, rpcSenderConfig) { + @Override + protected void onFlushAndSend(SendBuffer sendBuffer) { + // First to aggregate the buffer + E aggEntity = aggregateBuffer(sendBuffer); + Optional.ofNullable(getSendLogExceptionStrategy()).ifPresent( + strategy -> strategy.doSend(() -> { + doSend(aggEntity, rpcSenderConfig); + return null; + }, sendBuffer)); + } + }; + Future future = this.consumePool.submit(consumer); + consumer.setFuture(future); + sendLogCacheConsumers.put(id, consumer); + this.consumers++; + } finally { + this.ctxLock.unlock(); + } } public SendLogCache getLogCache(){ return this.logCache; } - public void destroyCacheConsumer(){ + /** + * Destroy cache consumer + * @param id id + */ + public void destroyCacheConsumer(String id){ + SendLogCacheConsumer consumer = sendLogCacheConsumers.remove(id); + consumer.shutdown(); + } + /** + * Destroy all the consumers + */ + public void destroyCacheConsumers(){ + this.ctxLock.lock(); + try { + sendLogCacheConsumers.forEach( (key, consumer)-> consumer.shutdown()); + } finally { + this.ctxLock.unlock(); + } } } /** diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java index a0d3a77e5..d7a307f2a 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java @@ -8,27 +8,181 @@ public class RpcSenderConfig { /** * Send address */ - private String sendAddress; + private String address; + + /** + * Key of token-code + */ + private String tokenCodeKey = "Token-Code"; + + /** + * Key of token-user + */ + private String tokenUserKey = "Token-User"; + + /** + * Token user + */ + private String tokenUser = System.getProperty("user.name"); + + /** + * Token code + */ + private String tokenCode = "STREAM_LOG"; + + /** + * Timeout of connecting + */ + private int connectionTimeout = 3000; + + /** + * Timeout of reading from socket + */ + private int socketTimeout = 15000; + /** * Size of send cache */ - private int sendCacheSize = 150; + private int cacheSize = 150; /** * Size of send buffer */ - private int sendBufferSize = 50; + private int bufferSize = 50; /** * Expire time of send buffer */ - private long sendBufferExpireTimeInSec = -1; + private long bufferExpireTimeInSec = -1; + + /** + * Retry count of sending + */ + private int sendRetryCnt = 3; + + /** + * The time for server recovery + */ + private int serverRecoveryTimeInSec = 5; + /** + * Retry max delay time of sender + */ + private int maxDelayTimeInSec; + + /** + * Max number of consuming thread + */ + private int maxConsumeThread = 10; + + public long getBufferExpireTimeInSec() { + return bufferExpireTimeInSec; + } + + public void setBufferExpireTimeInSec(long bufferExpireTimeInSec) { + this.bufferExpireTimeInSec = bufferExpireTimeInSec; + } + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public int getCacheSize() { + return cacheSize; + } + + public void setCacheSize(int cacheSize) { + this.cacheSize = cacheSize; + } + + public int getBufferSize() { + return bufferSize; + } + + public void setBufferSize(int bufferSize) { + this.bufferSize = bufferSize; + } + + public int getSendRetryCnt() { + return sendRetryCnt; + } + + public void setSendRetryCnt(int sendRetryCnt) { + this.sendRetryCnt = sendRetryCnt; + } + + public int getConnectionTimeout() { + return connectionTimeout; + } + + public void setConnectionTimeout(int connectionTimeout) { + this.connectionTimeout = connectionTimeout; + } + + public int getSocketTimeout() { + return socketTimeout; + } + + public void setSocketTimeout(int socketTimeout) { + this.socketTimeout = socketTimeout; + } + + public int getMaxConsumeThread() { + return maxConsumeThread; + } + + public void setMaxConsumeThread(int maxConsumeThread) { + this.maxConsumeThread = maxConsumeThread; + } + + public String getTokenCodeKey() { + return tokenCodeKey; + } + + public void setTokenCodeKey(String tokenCodeKey) { + this.tokenCodeKey = tokenCodeKey; + } + + public String getTokenCode() { + return tokenCode; + } + + public void setTokenCode(String tokenCode) { + this.tokenCode = tokenCode; + } + + public int getMaxDelayTimeInSec() { + return maxDelayTimeInSec; + } + + public void setMaxDelayTimeInSec(int maxDelayTimeInSec) { + this.maxDelayTimeInSec = maxDelayTimeInSec; + } + + public String getTokenUserKey() { + return tokenUserKey; + } + + public void setTokenUserKey(String tokenUserKey) { + this.tokenUserKey = tokenUserKey; + } + + public String getTokenUser() { + return tokenUser; + } + + public void setTokenUser(String tokenUser) { + this.tokenUser = tokenUser; + } - public long getSendBufferExpireTimeInSec() { - return sendBufferExpireTimeInSec; + public int getServerRecoveryTimeInSec() { + return serverRecoveryTimeInSec; } - public void setSendBufferExpireTimeInSec(long sendBufferExpireTimeInSec) { - this.sendBufferExpireTimeInSec = sendBufferExpireTimeInSec; + public void setServerRecoveryTimeInSec(int serverRecoveryTimeInSec) { + this.serverRecoveryTimeInSec = serverRecoveryTimeInSec; } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java index f0d8f5866..e851943a7 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java @@ -3,6 +3,7 @@ import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; /** @@ -29,14 +30,20 @@ public abstract class SendLogCacheConsumer implements Runn private final SendBuffer sendBuffer; private final String id; + + /** + * Future for execution + */ + private Future future; + public SendLogCacheConsumer(String id, SendLogCache cache, SendBuffer sendBuffer, RpcSenderConfig rpcSenderConfig){ this.id = id; this.cache = cache; this.sendBuffer = sendBuffer; - this.bufferExpireTimeInMills = rpcSenderConfig.getSendBufferExpireTimeInSec() > 0 ? TimeUnit.SECONDS - .toMillis(rpcSenderConfig.getSendBufferExpireTimeInSec()) : -1; + this.bufferExpireTimeInMills = rpcSenderConfig.getBufferExpireTimeInSec() > 0 ? TimeUnit.SECONDS + .toMillis(rpcSenderConfig.getBufferExpireTimeInSec()) : -1; } @@ -88,6 +95,14 @@ public void shutdown(){ this.isTerminated = true; } + public Future getFuture() { + return future; + } + + public void setFuture(Future future) { + this.future = future; + } + private long requireNewFlushTime(){ return bufferExpireTimeInMills > 0 ? System.currentTimeMillis() + bufferExpireTimeInMills : -1; } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java index d3c2fafd9..c8952ff1d 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java @@ -20,14 +20,14 @@ public SendLogExceptionStrategy(RpcLogSender sender){ * Retry count * @return retry */ - abstract int retryCount(); + public abstract int retryCount(); /** * * @param e exception * @return boolean */ - abstract RetryDescription onException(Exception e, SendBuffer sendBuffer); + public abstract RetryDescription onException(Exception e, SendBuffer sendBuffer); V doSend(Callable sendOperation, SendBuffer sendBuffer){ int retryCount = retryCount(); @@ -47,7 +47,7 @@ V doSend(Callable sendOperation, SendBuffer sendBuffer){ return null; } - private static class RetryDescription{ + protected static class RetryDescription{ private final boolean canRetry; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java index b8bba66b8..7f2c68d04 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java @@ -3,22 +3,54 @@ import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.AbstractRpcLogSender; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.RpcSenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.SendLogExceptionStrategy; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; +import org.apache.http.client.HttpClient; + +import java.util.concurrent.atomic.AtomicInteger; public abstract class AbstractHttpLogSender extends AbstractRpcLogSender { /** * Retry strategy */ - private SendLogExceptionStrategy sendRetryStrategy; + private SendLogExceptionStrategy sendRetryStrategy; + + /** + * Exception counter + */ + private AtomicInteger exceptionCounter = new AtomicInteger(); + /** + * Hold the global http client + */ + private HttpClient globalHttpClient; public AbstractHttpLogSender(RpcSenderConfig rpcSenderConfig, int cacheSize, int sendBufSize) { super(rpcSenderConfig, cacheSize, sendBufSize); + this.globalHttpClient = HttpClientTool.createHttpClient(rpcSenderConfig); + this.sendRetryStrategy = new SendLogExceptionStrategy(this) { + @Override + public int retryCount() { + return rpcSenderConfig.getSendRetryCnt(); + } + + @Override + public SendLogExceptionStrategy.RetryDescription onException(Exception e, SendBuffer sendBuffer) { + return null; + } + }; } @Override protected void doSend(E aggregatedEntity, RpcSenderConfig rpcSenderConfig) { + if (aggregatedEntity instanceof LogElement){ + long timestamp = ((LogElement) aggregatedEntity).getLogTimeStamp(); + if (System.currentTimeMillis() - timestamp > rpcSenderConfig.getMaxDelayTimeInSec() * 1000L){ + // Abort the entity + return; + } + } } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java new file mode 100644 index 000000000..df4eb31ea --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java @@ -0,0 +1,71 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.RpcSenderConfig; +import org.apache.http.Header; +import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicHeader; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Http clients + */ +public class HttpClientTool { + + /** + * Connect timeout + */ + public static final int DEFAULT_CONNECT_TIMEOUT = 3000; + + /** + * Socket timeout + */ + public static final int DEFAULT_SOCKET_TIMEOUT = 15000; + + /** + * Max connections + */ + public static final int DEFAULT_MAX_CONN = 10; + + /** + * Create http client + * @param rpcSenderConfig rpc sender config + * @return http client + */ + public static HttpClient createHttpClient(RpcSenderConfig rpcSenderConfig){ + int connectTimeout = rpcSenderConfig.getConnectionTimeout() > 0? rpcSenderConfig.getConnectionTimeout() : DEFAULT_CONNECT_TIMEOUT; + int socketTimeout = rpcSenderConfig.getSocketTimeout() > 0? rpcSenderConfig.getSocketTimeout() : DEFAULT_SOCKET_TIMEOUT; + RequestConfig requestConfig = RequestConfig.custom() + .setConnectTimeout(connectTimeout) + .setConnectionRequestTimeout(socketTimeout) + .setSocketTimeout(socketTimeout) + .build(); + int maxConn = rpcSenderConfig.getMaxConsumeThread() > 0? rpcSenderConfig.getMaxConsumeThread() : DEFAULT_MAX_CONN; + HttpClientBuilder clientBuilder = HttpClients.custom(); + String tokenValue = rpcSenderConfig.getTokenCode(); + List
defaultHeaders = new ArrayList<>(); + if (null != tokenValue && !tokenValue.trim().equals("")){ + defaultHeaders.add(new BasicHeader(rpcSenderConfig.getTokenCodeKey(), tokenValue)); + } + clientBuilder.setDefaultRequestConfig(requestConfig).setDefaultHeaders(defaultHeaders) + .useSystemProperties().setMaxConnTotal(maxConn); + CloseableHttpClient httpClient = clientBuilder.build(); + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + @Override + public void run() { + try { + httpClient.close(); + } catch (IOException e) { + // Ignore + } + } + })); + return httpClient; + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java new file mode 100644 index 000000000..8c493d042 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java @@ -0,0 +1,130 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.*; +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; + +/** + * Json utils + */ +public class Json { + private static final String PREFIX = "["; + private static final String SUFFIX = "]"; + + private static ObjectMapper mapper; + + static{ + mapper = new ObjectMapper(); + mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true); + mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); + mapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true); + mapper.configure(DeserializationFeature.READ_ENUMS_USING_TO_STRING, true); + mapper.configure(SerializationFeature.WRITE_ENUMS_USING_TO_STRING, true); + mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true); + //empty beans allowed + mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + //cancel to scape non ascii + mapper.configure(JsonGenerator.Feature.ESCAPE_NON_ASCII, false); + } + private Json(){} + + @SuppressWarnings("unchecked") + public static T fromJson(String json, Class clazz, Class... parameters) throws Exception{ + if(null != json && !json.trim().equals("")){ + try{ + if(parameters.length > 0){ + return (T)mapper.readValue(json, mapper.getTypeFactory().constructParametricType(clazz, parameters)); + } + if(json.startsWith(PREFIX) + && json.endsWith(SUFFIX)){ + JavaType javaType = mapper.getTypeFactory() + .constructParametricType(ArrayList.class, clazz); + return mapper.readValue(json, javaType); + } + return (T)mapper.readValue(json, clazz); + } catch (Exception e) { + String message = "Unable to deserialize to object from string(json) in type: [" + + (null != clazz ? clazz.getSimpleName() : "UNKNOWN") + "], parameters size: " + parameters.length; + throw new RuntimeException(message, e); + } + } + return null; + } + + public static T fromJson(InputStream stream, Class clazz, Class... parameters) throws Exception{ + StringBuilder builder = new StringBuilder(); + String jsonStr = null; + try{ + BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)); + while((jsonStr = reader.readLine()) != null){ + builder.append(jsonStr); + } + reader.close(); + }catch(Exception e){ + String message = "Unable to deserialize to object from stream(json) in type: [" + + (null != clazz ? clazz.getSimpleName() : "UNKNOWN") + "], parameters size: " + parameters.length; + throw new RuntimeException(message, e); + } + return fromJson(builder.toString(), clazz, parameters); + } + + public static String toJson(Object obj, Class model) { + ObjectWriter writer = mapper.writer(); + if(null != obj){ + try{ + if(null != model){ + writer = writer.withView(model); + } + return writer.writeValueAsString(obj); + } catch (JsonProcessingException e) { + String message = "Unable to serialize the object in type: ["+ (null != model ? model.getSimpleName() : "UNKNOWN") + "]"; + throw new RuntimeException(message, e); + } + } + return null; + } + + /** + * Convert object using serialization and deserialization + * + * @param simpleObj simpleObj + * @param tClass type class + * @param T + * @return result + */ + @SuppressWarnings("unchecked") + public static T convert(Object simpleObj, Class tClass, Class... parameters) { + try { + if (parameters.length > 0) { + return mapper.convertValue(simpleObj, mapper.getTypeFactory().constructParametricType(tClass, parameters)); + } + return (T) mapper.convertValue(simpleObj, tClass); + } catch (Exception e) { + String message = "Fail to process method 'convert(" + simpleObj + ": " + simpleObj.getClass().getSimpleName() + + ", " + tClass.getSimpleName() + ": "+ Class.class + ", ...: " + Class.class + ")"; + throw new RuntimeException(message, e); + } + } + + public static T convert(Object simpleObj, JavaType javaType) { + try { + return mapper.convertValue(simpleObj, javaType); + } catch (Exception e) { + String message = "Fail to process method 'convert(" + simpleObj + ": " + simpleObj.getClass().getSimpleName() + + ", " + javaType.getTypeName() + ": "+ JavaType.class + ")"; + throw new RuntimeException(message, e); + } + } + + public static ObjectMapper getMapper(){ + return mapper; + } +} + diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/entities/Resource.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/entities/Resource.java new file mode 100644 index 000000000..eaa355e92 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/entities/Resource.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.entities; + +import java.io.File; +import java.util.List; + +/** + * Entity with resources + */ +public interface Resource { + + /** + * Resources related + * @return file list + */ + List getResources(); + +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java new file mode 100644 index 000000000..8b0472e08 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java @@ -0,0 +1,62 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request; + +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpRequestBase; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.Map; + +/** + * Abstract implement + * @param + */ +public abstract class AbstractHttpAction implements HttpAction { + + protected String uri; + + protected String user; + + public AbstractHttpAction(String uri){ + this.uri = uri; + } + + @Override + public String uri() { + return uri; + } + + /** + * Request method + * @return method + */ + protected abstract T getRequestMethod(); + + private Map requestHeaders = new HashMap<>(); + + private Map requestPayload = new HashMap<>(); + + @Override + public Map getRequestHeaders() { + return this.requestHeaders; + } + + @Override + public Map getRequestPayload() { + return this.requestPayload; + } + + @Override + public HttpResponse execute(HttpClient httpClient) throws IOException { + HttpRequestBase requestBase = getRequestMethod(); + try{ + requestBase.setURI(new URI(uri)); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("URI maybe has wrong format", e); + } + return httpClient.execute(requestBase); + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java new file mode 100644 index 000000000..f8bfc5e4c --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java @@ -0,0 +1,80 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.ValueNode; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.Json; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.entities.Resource; +import org.apache.http.HttpEntity; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.entity.mime.MultipartEntityBuilder; +import java.nio.charset.StandardCharsets; +import java.util.*; + +/** + * Post action for entity + */ +public class EntityPostAction extends PostAction{ + + private static final String RESOURCE_FIELD_NAME = "resources"; + + private T entity; + + public EntityPostAction(String uri, T entity) { + super(uri); + if (Objects.isNull(entity)){ + throw new IllegalArgumentException("Entity cannot be null in action"); + } + this.entity = entity; + } + + @Override + HttpEntity getHttpEntity() { + try { + if (entity instanceof Resource) { + MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create(); + String entityStr = Json.toJson(entity, null); + JsonNode rootNode = Json.getMapper().readTree(entityStr); + addEntityBody(entityBuilder, "", rootNode); + Optional.ofNullable(((Resource) entity).getResources()).ifPresent(resources -> { + for(int i = 0; i < resources.size(); i ++){ + entityBuilder.addBinaryBody(RESOURCE_FIELD_NAME + "[" + i +"]", resources.get(i)); + } + }); + return entityBuilder.build(); + } else { + StringEntity stringEntity; + stringEntity = new StringEntity(Objects.requireNonNull(Json.toJson(entity, null)), "UTF-8"); + stringEntity.setContentType(ContentType.APPLICATION_JSON.toString()); + return stringEntity; + } + } catch (Exception e) { + throw new IllegalArgumentException("Cannot deserialize the entity: " + entity.getClass().getName(), e); + } + } + + private void addEntityBody(MultipartEntityBuilder builder, String prefix, JsonNode node){ + if (node instanceof ObjectNode){ + if (null != prefix && !prefix.trim().equals("")){ + prefix += "."; + } + Iterator> fields = node.fields(); + while (fields.hasNext()){ + Map.Entry entry = fields.next(); + addEntityBody(builder, prefix + entry.getKey(), entry.getValue()); + } + } else if (node instanceof ArrayNode){ + ArrayNode arrayNode = ((ArrayNode)node); + for (int i = 0 ; i < arrayNode.size(); i ++){ + addEntityBody(builder, prefix + "[" + i + "]", arrayNode.get(i)); + } + } else if (node instanceof ValueNode){ + ContentType strContent = ContentType.create("text/plain", StandardCharsets.UTF_8); + System.out.println("p: " + prefix + ", data: " + node.asText()); + builder.addTextBody(prefix, node.asText(), strContent); + } + } + +} \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java new file mode 100644 index 000000000..f5a8a5fef --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request; + +public class GetAction { +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/HttpAction.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/HttpAction.java new file mode 100644 index 000000000..87435f8a3 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/HttpAction.java @@ -0,0 +1,38 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request; + +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; + +import java.io.IOException; +import java.util.Map; + +/** + * Http action + */ +public interface HttpAction { + + /** + * URI path + * @return path + */ + String uri(); + + /** + * Request headers + * @return map + */ + Map getRequestHeaders(); + + /** + * Request pay load(body) + * @return map + */ + Map getRequestPayload(); + + /** + * Execute http action + * @return http response + */ + HttpResponse execute(HttpClient httpClient) throws IOException; + +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/PostAction.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/PostAction.java new file mode 100644 index 000000000..9b3ea003f --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/PostAction.java @@ -0,0 +1,41 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.Json; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; + +import java.util.Objects; + +/** + * Post action + */ +public class PostAction extends AbstractHttpAction { + + public PostAction(String uri) { + super(uri); + } + + /** + * Http entity (use stringEntity and "application/json" default) + * @return http entity + */ + HttpEntity getHttpEntity(){ + StringEntity stringEntity; + try{ + stringEntity = new StringEntity(Objects.requireNonNull(Json.toJson(getRequestPayload(), null)), "UTF-8"); + } catch (Exception e){ + throw new IllegalArgumentException("Cannot deserialize the requestPayLoad: " + getRequestPayload().toString(), e); + } + stringEntity.setContentType(ContentType.APPLICATION_JSON.toString()); + return stringEntity; + } + + @Override + protected HttpPost getRequestMethod() { + HttpPost httpPost = new HttpPost(); + httpPost.setEntity(getHttpEntity()); + return httpPost; + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java index a2b0a7807..8a497b61e 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java @@ -15,10 +15,9 @@ public interface LogElement { /** * Log time - * @param unit unit * @return log time */ - long getLogTime(TimeUnit unit); + long getLogTimeStamp(); /** * Get content diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java new file mode 100644 index 000000000..f194fb289 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.entities; + +public class StreamisLogEvents implements LogElement{ + @Override + public int getSequenceId() { + return 0; + } + + @Override + public long getLogTimeStamp() { + return 0; + } + + + @Override + public String[] getContents() { + return new String[0]; + } +} From 4e0a7a543d9ece38331e0badf93b23480db573f6 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Thu, 20 Oct 2022 21:15:11 +0800 Subject: [PATCH 011/240] 1. Add the ability for updating task for existed jobs. 2. Spark Streaming Job will not ask Linkis for status. --- .../jobmanager/restful/api/JobRestfulApi.java | 139 +++++++++++++----- 1 file changed, 104 insertions(+), 35 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 37ed9bf1a..7b9f5d9c2 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -39,6 +39,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.linkis.common.utils.Utils; import org.apache.linkis.httpclient.dws.DWSHttpClient; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; @@ -50,7 +51,6 @@ import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; -import java.io.IOException; import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; @@ -194,44 +194,107 @@ public Message killJob(HttpServletRequest req, @RequestMapping(path = "/details", method = RequestMethod.GET) public Message detailsJob(HttpServletRequest req, @RequestParam(value = "jobId", required = false) Long jobId, - @RequestParam(value = "version", required = false) String version) throws JobException { + @RequestParam(value = "version", required = false) String version) throws JobException, JsonProcessingException { if (jobId == null) { JobExceptionManager.createException(30301, "jobId"); } - // TODO This is just sample datas, waiting for it completed. We have planned it to a later release, welcome all partners to join us to realize this powerful feature. + + StreamJob streamJob = streamJobService.getJobById(jobId); + if(streamJob == null) { + return Message.error("not exists job " + jobId); + } + FlinkJobInfo flinkJobInfo = streamTaskService.getTaskJobInfo(jobId,version); JobDetailsVo jobDetailsVO = new JobDetailsVo(); List dataNumberDTOS = new ArrayList<>(); - JobDetailsVo.DataNumberDTO dataNumberDTO = new JobDetailsVo.DataNumberDTO(); - dataNumberDTO.setDataName("kafka topic"); - dataNumberDTO.setDataNumber(109345); - dataNumberDTOS.add(dataNumberDTO); - List loadConditionDTOs = new ArrayList<>(); - JobDetailsVo.LoadConditionDTO loadConditionDTO = new JobDetailsVo.LoadConditionDTO(); - loadConditionDTO.setType("jobManager"); - loadConditionDTO.setHost("localhost"); - loadConditionDTO.setMemory("1.5"); - loadConditionDTO.setTotalMemory("2.0"); - loadConditionDTO.setGcLastTime("2020-08-01"); - loadConditionDTO.setGcLastConsume("1"); - loadConditionDTO.setGcTotalTime("2min"); - loadConditionDTOs.add(loadConditionDTO); - List realTimeTrafficDTOS = new ArrayList<>(); - JobDetailsVo.RealTimeTrafficDTO realTimeTrafficDTO = new JobDetailsVo.RealTimeTrafficDTO(); - realTimeTrafficDTO.setSourceKey("kafka topic"); - realTimeTrafficDTO.setSourceSpeed("100 Records/S"); - realTimeTrafficDTO.setTransformKey("transform"); - realTimeTrafficDTO.setSinkKey("hbase key"); - realTimeTrafficDTO.setSinkSpeed("10 Records/S"); - realTimeTrafficDTOS.add(realTimeTrafficDTO); - + JobStateInfo[] jobStateInfos = flinkJobInfo.getJobStates(); + if(JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES().getValue().contains(streamJob.getJobType()) || + (jobStateInfos == null || jobStateInfos.length == 0)) { + // TODO This is just sample datas, waiting for it completed. We have planned it to a later release, welcome all partners to join us to realize this powerful feature. + JobDetailsVo.DataNumberDTO dataNumberDTO = new JobDetailsVo.DataNumberDTO(); + dataNumberDTO.setDataName("kafka topic"); + dataNumberDTO.setDataNumber(109345); + dataNumberDTOS.add(dataNumberDTO); + + JobDetailsVo.LoadConditionDTO loadConditionDTO = new JobDetailsVo.LoadConditionDTO(); + loadConditionDTO.setType("jobManager"); + loadConditionDTO.setHost("localhost"); + loadConditionDTO.setMemory("1.5"); + loadConditionDTO.setTotalMemory("2.0"); + loadConditionDTO.setGcLastTime("2020-08-01"); + loadConditionDTO.setGcLastConsume("1"); + loadConditionDTO.setGcTotalTime("2min"); + loadConditionDTOs.add(loadConditionDTO); + + JobDetailsVo.RealTimeTrafficDTO realTimeTrafficDTO = new JobDetailsVo.RealTimeTrafficDTO(); + realTimeTrafficDTO.setSourceKey("kafka topic"); + realTimeTrafficDTO.setSourceSpeed("100 Records/S"); + realTimeTrafficDTO.setTransformKey("transform"); + realTimeTrafficDTO.setSinkKey("hbase key"); + realTimeTrafficDTO.setSinkSpeed("10 Records/S"); + realTimeTrafficDTOS.add(realTimeTrafficDTO); - jobDetailsVO.setLinkisJobInfo(streamTaskService.getTaskJobInfo(jobId,version)); + } else { + String metricsStr = jobStateInfos[0].getLocation(); + Map metricsMap = DWSHttpClient.jacksonJson().readValue(metricsStr, Map.class); + JobDetailsVo.DataNumberDTO dataNumberDTO = new JobDetailsVo.DataNumberDTO(); + dataNumberDTO.setDataName("waitingBatchs"); + dataNumberDTO.setDataNumber(Math.toIntExact((Long) metricsMap.get("waitingBatchs"))); + dataNumberDTOS.add(dataNumberDTO); + List> executors = (List>) metricsMap.get("executors"); + if(executors != null && !executors.isEmpty()) { + executors.forEach(map -> { + JobDetailsVo.LoadConditionDTO loadConditionDTO = new JobDetailsVo.LoadConditionDTO(); + loadConditionDTO.setType((String) map.get("type")); + loadConditionDTO.setHost((String) map.get("host")); + loadConditionDTO.setMemory((String) map.get("memory")); + loadConditionDTO.setTotalMemory((String) map.get("totalMemory")); + loadConditionDTO.setGcLastTime((String) map.get("gcLastTime")); + loadConditionDTO.setGcLastConsume((String) map.get("gcLastConsume")); + loadConditionDTO.setGcTotalTime((String) map.get("gcTotalTime")); + loadConditionDTOs.add(loadConditionDTO); + }); + } else { + JobDetailsVo.LoadConditionDTO loadConditionDTO = new JobDetailsVo.LoadConditionDTO(); + loadConditionDTO.setType("sparkAppMaster"); + loadConditionDTO.setHost(""); + loadConditionDTO.setMemory(""); + loadConditionDTO.setTotalMemory(""); + loadConditionDTO.setGcLastTime(""); + loadConditionDTO.setGcLastConsume(""); + loadConditionDTO.setGcTotalTime(""); + loadConditionDTOs.add(loadConditionDTO); + } + JobDetailsVo.RealTimeTrafficDTO realTimeTrafficDTO = new JobDetailsVo.RealTimeTrafficDTO(); + List> batchMetrics = (List>) metricsMap.get("batchMetrics"); + if(batchMetrics != null && !batchMetrics.isEmpty()) { + batchMetrics.stream().max(Comparator.comparing(map -> String.valueOf(map.get("batchTime")))).ifPresent(batchMetric -> { + realTimeTrafficDTO.setSourceKey((String) metricsMap.getOrDefault("source", "")); + realTimeTrafficDTO.setSourceSpeed(batchMetric.get("inputRecords") + " Records"); + realTimeTrafficDTO.setTransformKey("processing"); + realTimeTrafficDTO.setSinkKey((String) metricsMap.getOrDefault("sink", "")); + String totalDelay = ""; + if(batchMetric.containsKey("totalDelay") && batchMetric.get("totalDelay") != null) { + totalDelay = Utils.msDurationToString((long) batchMetric.get("totalDelay")) + " totalDelay"; + } else if(batchMetric.containsKey("taskExecuteTime") && batchMetric.get("taskExecuteTime") != null) { + totalDelay = Utils.msDurationToString((long) batchMetric.get("taskExecuteTime")) + " executeTime(Last Batch)"; + } + realTimeTrafficDTO.setSinkSpeed(totalDelay); + }); + } else { + realTimeTrafficDTO.setSourceKey(""); + realTimeTrafficDTO.setSourceSpeed(" Records/S"); + realTimeTrafficDTO.setTransformKey(""); + realTimeTrafficDTO.setSinkKey(""); + realTimeTrafficDTO.setSinkSpeed(" Records/S"); + } + realTimeTrafficDTOS.add(realTimeTrafficDTO); + } jobDetailsVO.setDataNumber(dataNumberDTOS); jobDetailsVO.setLoadCondition(loadConditionDTOs); jobDetailsVO.setRealTimeTraffic(realTimeTrafficDTOS); - + jobDetailsVO.setLinkisJobInfo(flinkJobInfo); return Message.ok().data("details", jobDetailsVO); } @@ -276,12 +339,7 @@ public Message addTask(HttpServletRequest req, } // 如果存在正在运行的，先将其停止掉 StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJobs.get(0).getId()); - if(streamTask != null && JobConf.isRunning(streamTask.getStatus())) { - LOG.warn("Streamis Job {} exists running task, update its status from Running to stopped at first.", jobName); - streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); - streamTask.setErrDesc("stopped by App's new task."); - streamTaskService.updateTask(streamTask); - } else if(streamTask == null) { + if(streamTask == null) { // 这里取个巧，从该工程该用户有权限的Job中找到一个Flink的历史作业，作为这个Spark Streaming作业的jobId和jobInfo // 替换掉JobInfo中的 yarn 信息，这样我们前端就可以在不修改任何逻辑的情况下正常展示Spark Streaming作业了 PageInfo jobList = streamJobService.getByProList(streamJobs.get(0).getProjectName(), username, null, null, null); @@ -306,6 +364,17 @@ public Message addTask(HttpServletRequest req, if(streamTask == null) { return Message.error("no Flink task has been executed, the register to Streamis cannot be succeeded."); } + } else { + if(JobConf.isRunning(streamTask.getStatus())) { + LOG.warn("Streamis Job {} exists running task, update its status from Running to stopped at first.", jobName); + streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); + streamTask.setErrDesc("stopped by App's new task."); + streamTaskService.updateTask(streamTask); + } + StreamTask newStreamTask = streamTaskService.createTask(streamJobs.get(0).getId(), (Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue(), username); + streamTask.setId(newStreamTask.getId()); + streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue()); + streamTask.setSubmitUser(username); } streamTask.setStartTime(new Date()); streamTask.setLastUpdateTime(new Date()); @@ -359,7 +428,7 @@ public Message updateTask(HttpServletRequest req, jobName, flinkJobInfo.getApplicationId(), appId); return Message.ok("the request appId is not equals to the running task appId " + flinkJobInfo.getApplicationId()); } - JobStateInfo jobStateInfo = new JobStateInfo(); + JobStateInfo jobStateInfo = new JobStateInfo(); jobStateInfo.setTimestamp(System.currentTimeMillis()); jobStateInfo.setLocation(metrics); flinkJobInfo.setJobStates(new JobStateInfo[]{jobStateInfo}); From ecbabfd24cece052490e158227f9ac7f8117c3bc Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Thu, 20 Oct 2022 21:24:47 +0800 Subject: [PATCH 012/240] 1. Add the ability for updating task for existed jobs. 2. Spark Streaming Job will not ask Linkis for status. --- .../streamis/jobmanager/restful/api/JobRestfulApi.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 7b9f5d9c2..57d5473b5 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -373,7 +373,9 @@ public Message addTask(HttpServletRequest req, } StreamTask newStreamTask = streamTaskService.createTask(streamJobs.get(0).getId(), (Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue(), username); streamTask.setId(newStreamTask.getId()); - streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue()); + streamTask.setVersion(newStreamTask.getVersion()); + streamTask.setErrDesc(""); + streamTask.setStatus(newStreamTask.getStatus()); streamTask.setSubmitUser(username); } streamTask.setStartTime(new Date()); @@ -383,7 +385,7 @@ public Message addTask(HttpServletRequest req, flinkJobInfo.setApplicationId(appId); flinkJobInfo.setApplicationUrl(appUrl); flinkJobInfo.setName(jobName); - flinkJobInfo.setStatus(JobConf.getStatusString((Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue())); + flinkJobInfo.setStatus(JobConf.getStatusString(finalStreamTask.getStatus())); StreamTaskUtils.refreshInfo(finalStreamTask, flinkJobInfo); streamTaskService.updateTask(finalStreamTask); LOG.info("Streamis Job {} has added a new task successfully.", jobName); From fad94ff916a4f4cf6c252b929a303a6560f18a2b Mon Sep 17 00:00:00 2001 From: davidhua Date: Thu, 20 Oct 2022 22:28:24 +0800 Subject: [PATCH 013/240] The strategy to retry the request and compact the buffer in log collector. --- .../log/collector/ExceptionListener.java | 15 ++++ .../sender/AbstractRpcLogSender.java | 25 ++++-- .../log/collector/sender/RpcLogSender.java | 6 ++ .../log/collector/sender/RpcSenderConfig.java | 1 + .../sender/buf/ImmutableSendBuffer.java | 8 +- .../sender/http/AbstractHttpLogSender.java | 80 ++++++++++++++++--- .../jobmanager/log/entities/LogElement.java | 6 ++ .../log/entities/StreamisLogEvents.java | 5 ++ 8 files changed, 125 insertions(+), 21 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/ExceptionListener.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/ExceptionListener.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/ExceptionListener.java new file mode 100644 index 000000000..4c9ac6ea8 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/ExceptionListener.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector; + +/** + * Exception listener + */ +public interface ExceptionListener { + + /** + * Listen the exception + * @param subject the subject that throws the exception + * @param t Throwable + * @param message message + */ + void onException(Object subject, Throwable t, String message); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java index 6b4d36580..a30ff98c6 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.ExceptionListener; import com.webank.wedatasphere.streamis.jobmanager.log.collector.cache.LogCache; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.ImmutableSendBuffer; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; @@ -42,7 +43,11 @@ public abstract class AbstractRpcLogSender implements R */ private volatile RpcLogContext rpcLogContext; - + protected boolean isTerminated = false; + /** + * Use the listener instead of log4j structure + */ + protected ExceptionListener exceptionListener; public AbstractRpcLogSender(RpcSenderConfig rpcSenderConfig, int cacheSize, int sendBufSize){ this(rpcSenderConfig, cacheSize, sendBufSize, Integer.MAX_VALUE); } @@ -59,8 +64,7 @@ public AbstractRpcLogSender(RpcSenderConfig rpcSenderConfig, int cacheSize, int @Override public LogCache getOrCreateLogCache() { - getOrCreateRpcLogContext(); - return null; + return getOrCreateRpcLogContext().getLogCache(); } @Override @@ -69,7 +73,9 @@ public void sendLog(T log) { try { getOrCreateLogCache().cacheLog(log); } catch (InterruptedException e) { - // Exception handler + // Invoke exception listener + Optional.ofNullable(exceptionListener).ifPresent(listener -> + listener.onException(this, e, null)); } } @@ -78,6 +84,11 @@ public void syncSendLog(T log) { } + @Override + public void setExceptionListener(ExceptionListener listener) { + this.exceptionListener = listener; + } + @Override public void close() { @@ -95,7 +106,7 @@ public void close() { * @param aggregatedEntity agg entity * @param rpcSenderConfig rpc sender config */ - protected abstract void doSend(E aggregatedEntity, RpcSenderConfig rpcSenderConfig); + protected abstract void doSend(E aggregatedEntity, RpcSenderConfig rpcSenderConfig) throws Exception; /** * Send log exception strategy @@ -302,7 +313,7 @@ public int drainLogsTo(List elements, int maxElements) { return 0; } - // Equal to the poll method in ArrayBlockingQuue + // Equal to the poll method in ArrayBlockingQueue @Override public T takeLog(long timeout, TimeUnit unit) throws InterruptedException { long nanos = unit.toNanos(timeout); @@ -410,6 +421,8 @@ private int sendBuf(SendBuffer sendBuffer, Object[] items, int takeIndex, int int send = sendBuffer.writeBuf((T[]) items, takeIndex, len); if (send < len){ // Buffer full exception + exceptionListener.onException(this, null, "The sender buffer is full," + + " expected: [" + len + "], actual: [" + send + "]"); } // Allow data loss return send; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java index 25f512643..8254f0a34 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.ExceptionListener; import com.webank.wedatasphere.streamis.jobmanager.log.collector.cache.LogCache; import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; @@ -26,6 +27,11 @@ public interface RpcLogSender { */ void syncSendLog(T log); + /** + * Exception listener + * @param listener listener + */ + void setExceptionListener(ExceptionListener listener); /** * Close sender */ diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java index d7a307f2a..aac1a9578 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java @@ -64,6 +64,7 @@ public class RpcSenderConfig { * The time for server recovery */ private int serverRecoveryTimeInSec = 5; + /** * Retry max delay time of sender */ diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java index eefe50ae7..3c61985f6 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java @@ -1,5 +1,7 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvent; + import java.util.Arrays; import java.util.function.Function; @@ -30,14 +32,13 @@ public void capacity(String newCapacity) { } @Override + @SuppressWarnings("all") public int writeBuf(E[] elements, int srcIndex, int length) { if (srcIndex < elements.length){ int startPos = nextPosition(Math.min(elements.length - srcIndex, length), Flag.WRITE_MODE); if (startPos >= 0){ int writes = position() - startPos; - for (int i = srcIndex; i < writes; i ++){ - buf[startPos++] = elements[i]; - } + System.arraycopy(this.buf, startPos, elements, srcIndex, writes); return writes; } } @@ -68,4 +69,5 @@ public E readBuf() { public SendBuffer compact(Function dropAble) { return null; } + } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java index 7f2c68d04..bebfd64b7 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java @@ -1,34 +1,57 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.AbstractRpcLogSender; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.RpcLogSender; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.RpcSenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.SendLogExceptionStrategy; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request.EntityPostAction; import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; +import org.apache.http.HttpException; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; import org.apache.http.client.HttpClient; +import org.apache.http.conn.ConnectTimeoutException; +import javax.net.ssl.SSLException; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.net.UnknownHostException; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; public abstract class AbstractHttpLogSender extends AbstractRpcLogSender { /** * Retry strategy */ - private SendLogExceptionStrategy sendRetryStrategy; + private final SendLogExceptionStrategy sendRetryStrategy; /** * Exception counter */ - private AtomicInteger exceptionCounter = new AtomicInteger(); + private final AtomicInteger exceptionCounter = new AtomicInteger(); /** * Hold the global http client */ - private HttpClient globalHttpClient; + private final HttpClient globalHttpClient; + + /** + * Recover time point + */ + private final AtomicLong serverRecoveryTimePoint = new AtomicLong(-1L); public AbstractHttpLogSender(RpcSenderConfig rpcSenderConfig, int cacheSize, int sendBufSize) { super(rpcSenderConfig, cacheSize, sendBufSize); this.globalHttpClient = HttpClientTool.createHttpClient(rpcSenderConfig); + final RpcLogSender sender = this; this.sendRetryStrategy = new SendLogExceptionStrategy(this) { + + private final Class[] retryOnExceptions = new Class[]{ + InterruptedIOException.class, UnknownHostException.class, + ConnectTimeoutException.class, SSLException.class}; @Override public int retryCount() { return rpcSenderConfig.getSendRetryCnt(); @@ -36,21 +59,54 @@ public int retryCount() { @Override public SendLogExceptionStrategy.RetryDescription onException(Exception e, SendBuffer sendBuffer) { - - return null; + boolean shouldRetry = false; + // Limit of exception number is the same as the retry times + if (exceptionCounter.incrementAndGet() > retryCount()){ + serverRecoveryTimePoint.set(System.currentTimeMillis() + + TimeUnit.SECONDS.toMillis(rpcSenderConfig.getServerRecoveryTimeInSec())); + } else { + for (Class retryOnException : retryOnExceptions) { + if (retryOnException.isAssignableFrom(e.getClass())) { + shouldRetry = true; + break; + } + } + } + if (shouldRetry && !sender.getOrCreateLogCache().isCacheable()){ + // Means that the cache is full + // Set the position of buffer to 0 + sendBuffer.rewind(); + // Compact the buffer and transient to write mode; + sendBuffer.compact( element -> element.mark() > 1); + shouldRetry = false; + } + exceptionListener.onException(sender, e, null); + return new RetryDescription(shouldRetry); } }; } @Override - protected void doSend(E aggregatedEntity, RpcSenderConfig rpcSenderConfig) { - if (aggregatedEntity instanceof LogElement){ - long timestamp = ((LogElement) aggregatedEntity).getLogTimeStamp(); - if (System.currentTimeMillis() - timestamp > rpcSenderConfig.getMaxDelayTimeInSec() * 1000L){ - // Abort the entity - return; + protected SendLogExceptionStrategy getSendLogExceptionStrategy() { + return this.sendRetryStrategy; + } + + @Override + protected void doSend(E aggregatedEntity, RpcSenderConfig rpcSenderConfig) throws IOException { + if (System.currentTimeMillis() >= serverRecoveryTimePoint.get()) { + if (aggregatedEntity instanceof LogElement) { + long timestamp = ((LogElement) aggregatedEntity).getLogTimeStamp(); + if (System.currentTimeMillis() - timestamp > rpcSenderConfig.getMaxDelayTimeInSec() * 1000L) { + // Abort the entity + return; + } } + EntityPostAction postAction = new EntityPostAction<>(rpcSenderConfig.getAddress(), aggregatedEntity); + postAction.getRequestHeaders().put(rpcSenderConfig.getTokenUserKey(), rpcSenderConfig.getTokenUser()); + // Ignore the response + postAction.execute(this.globalHttpClient); + // Init the counter + this.exceptionCounter.set(0); } - } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java index 8a497b61e..b0694580c 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java @@ -25,4 +25,10 @@ public interface LogElement { */ String[] getContents(); + /** + * The importance of log + * 0: useless, 1: normal, 2:important + * @return + */ + int mark(); } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java index f194fb289..3a1766ee6 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java @@ -16,4 +16,9 @@ public long getLogTimeStamp() { public String[] getContents() { return new String[0]; } + + @Override + public int mark() { + return 0; + } } From cd9fcc7624c4f21c441deea83f787dfd5679e292 Mon Sep 17 00:00:00 2001 From: davidhua Date: Fri, 21 Oct 2022 03:26:35 +0800 Subject: [PATCH 014/240] Complete the log collector (95%) --- .../streamis-job-log-collector/pom.xml | 33 ++- .../collector/StreamisLogAppenderConfig.java | 4 - .../log/collector/StreamisRpcLogAppender.java | 97 ++++++++- .../log/collector/config/RpcAuthConfig.java | 92 +++++++++ .../collector/config/RpcLogSenderConfig.java | 172 ++++++++++++++++ .../collector/config/SendBufferConfig.java | 53 +++++ .../collector/config/SendLogCacheConfig.java | 55 +++++ .../config/StreamisLogAppenderConfig.java | 49 +++++ .../StreamisLogAppenderConfigBuilder.java | 24 +++ .../sender/AbstractRpcLogSender.java | 23 ++- .../log/collector/sender/RpcSenderConfig.java | 189 ------------------ .../sender/SendLogCacheConsumer.java | 11 +- .../sender/StreamisRpcLogSender.java | 33 ++- .../sender/buf/AbstractSendBuffer.java | 16 +- .../sender/buf/ImmutableSendBuffer.java | 39 +++- .../sender/http/AbstractHttpLogSender.java | 17 +- .../collector/sender/http/HttpClientTool.java | 11 +- .../log/collector/sender/http/Json.java | 1 + .../streamis-job-log-common/pom.xml | 9 + .../jobmanager/log/entities/LogElement.java | 1 - .../log/entities/StreamisLogEvent.java | 57 +++++- .../log/entities/StreamisLogEvents.java | 50 ++++- 22 files changed, 794 insertions(+), 242 deletions(-) delete mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java delete mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml index bbc6a69c0..5fd394795 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml @@ -16,7 +16,8 @@ 8 4.5.13 4.5.4 - 2.13.2.2 + 2.17.1 + 1.7.15 @@ -36,11 +37,33 @@ httpmime ${httpmine.version} - + - com.fasterxml.jackson.core - jackson-databind - ${jackson-databind.version} + org.slf4j + slf4j-api + ${slf4j.version} + provided + + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + provided + + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + provided + + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + provided \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderConfig.java deleted file mode 100644 index 5c0ff730d..000000000 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderConfig.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.collector; - -public class StreamisLogAppenderConfig { -} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java index d326e6e04..a13de8d83 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java @@ -1,5 +1,100 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector; -public class StreamisRpcLogAppender { +import com.webank.wedatasphere.streamis.jobmanager.log.collector.cache.LogCache; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfigBuilder; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.StreamisRpcLogSender; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvent; +import org.apache.logging.log4j.core.Filter; +import org.apache.logging.log4j.core.Layout; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.config.Property; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginElement; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.layout.PatternLayout; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.ServiceLoader; +import java.util.concurrent.TimeUnit; + +/** + * Streamis rpc log appender + */ +@Plugin(name = "StreamRpcLog", category = "Core", elementType = "appender", printObject = true) +public class StreamisRpcLogAppender extends AbstractAppender { + + private static final String DEFAULT_APPENDER_NAME = "StreamRpcLog"; + + /** + * Appender config + */ + private StreamisLogAppenderConfig appenderConfig; + + /** + * Rpc log sender + */ + private StreamisRpcLogSender rpcLogSender; + + /** + * Cache + */ + private LogCache logCache; + protected StreamisRpcLogAppender(String name, Filter filter, + Layout layout, + boolean ignoreExceptions, Property[] properties, + StreamisLogAppenderConfig appenderConfig) { + super(name, filter, layout, ignoreExceptions, properties); + this.appenderConfig = appenderConfig; + this.rpcLogSender = new StreamisRpcLogSender(this.appenderConfig.getApplicationName(), + this.appenderConfig.getSenderConfig()); + this.logCache = this.rpcLogSender.getOrCreateLogCache(); + Runtime.getRuntime().addShutdownHook(new Thread(() -> this.rpcLogSender.close())); + } + + @Override + public void append(LogEvent event) { + String content = new String(getLayout().toByteArray(event)); + // Transform to stream log event; + StreamisLogEvent logEvent = new StreamisLogEvent(content, System.currentTimeMillis()); + try { + this.logCache.cacheLog(logEvent); + } catch (InterruptedException e) { + LOGGER.error("StreamisRpcLogAppender: {} interrupted when cache the log into the RPC sender, message: {}", this.getName(), e.getMessage()); + } + } + + @PluginFactory + public static StreamisRpcLogAppender createAppender(@PluginAttribute("name") String name, + @PluginAttribute("appName") String applicationName, + @PluginAttribute("ignoreExceptions") boolean ignoreExceptions, + @PluginElement("Filter") final Filter filter, + @PluginElement("Layout") Layout layout, + @PluginElement("RpcLogSender")RpcLogSenderConfig rpcLogSenderConfig){ + if (null == name || name.trim().equals("")){ + name = DEFAULT_APPENDER_NAME; + } + if (Objects.isNull(layout)){ + layout = PatternLayout.createDefaultLayout(); + } + // Search the config builder + List configBuilders = new ArrayList<>(); + StreamisLogAppenderConfig logAppenderConfig = null; + ServiceLoader.load(StreamisLogAppenderConfigBuilder.class, + StreamisRpcLogAppender.class.getClassLoader()).iterator().forEachRemaining(configBuilders::add); + if (!configBuilders.isEmpty()){ + logAppenderConfig = configBuilders.get(0).build(applicationName, filter, rpcLogSenderConfig); + } + if (Objects.isNull(logAppenderConfig)){ + logAppenderConfig = new StreamisLogAppenderConfig(applicationName, filter, rpcLogSenderConfig); + } + return new StreamisRpcLogAppender(name, filter, layout, ignoreExceptions, Property.EMPTY_ARRAY, logAppenderConfig); + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java new file mode 100644 index 000000000..e88eb91e9 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java @@ -0,0 +1,92 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; + +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; + +/** + * Authentication config + */ +@Plugin( + name = "AuthConfig", + category = "Core", + printObject = true +) +public class RpcAuthConfig { + /** + * Key of token-code + */ + private String tokenCodeKey = "Token-Code"; + + /** + * Key of token-user + */ + private String tokenUserKey = "Token-User"; + + /** + * Token user + */ + private String tokenUser = System.getProperty("user.name"); + + /** + * Token code + */ + private String tokenCode = "STREAM-LOG"; + + public RpcAuthConfig(){ + + } + + public RpcAuthConfig(String tokenCodeKey, String tokenCode, String tokenUserKey, String tokenUser){ + if (null != tokenCodeKey) { + this.tokenCodeKey = tokenCodeKey; + } + if (null != tokenCode){ + this.tokenCode = tokenCode; + } + if (null != tokenUserKey){ + this.tokenUserKey = tokenUserKey; + } + if (null != tokenUser){ + this.tokenUser = tokenUser; + } + } + + @PluginFactory + public static RpcAuthConfig createRpcAuthConfig(@PluginAttribute("tokenCodeKey") String tokenCodeKey, + @PluginAttribute("tokenCode") String tokenCode, + @PluginAttribute("tokenUserKey") String tokenUserKey, @PluginAttribute("tokenUser") String tokenUser){ + return new RpcAuthConfig(tokenCodeKey, tokenCode, tokenUserKey, tokenUser); + } + public String getTokenCodeKey() { + return tokenCodeKey; + } + + public void setTokenCodeKey(String tokenCodeKey) { + this.tokenCodeKey = tokenCodeKey; + } + + public String getTokenUserKey() { + return tokenUserKey; + } + + public void setTokenUserKey(String tokenUserKey) { + this.tokenUserKey = tokenUserKey; + } + + public String getTokenUser() { + return tokenUser; + } + + public void setTokenUser(String tokenUser) { + this.tokenUser = tokenUser; + } + + public String getTokenCode() { + return tokenCode; + } + + public void setTokenCode(String tokenCode) { + this.tokenCode = tokenCode; + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java new file mode 100644 index 000000000..10a063946 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java @@ -0,0 +1,172 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; + +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginElement; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.util.Integers; + +import java.util.Objects; + +/** + * Rpc sender configuration + */ +@Plugin( + name = "RpcLogSender", + category = "Core", + printObject = true +) +public class RpcLogSenderConfig { + + /** + * Send address + */ + private String address; + + /** + * Timeout of connecting + */ + private int connectionTimeout = 3000; + + /** + * Timeout of reading from socket + */ + private int socketTimeout = 15000; + + /** + * Retry count of sending + */ + private int sendRetryCnt = 3; + + /** + * The time for server recovery + */ + private int serverRecoveryTimeInSec = 5; + + /** + * Retry max delay time of sender + */ + private int maxDelayTimeInSec = 60; + + /** + * Auth config + */ + private RpcAuthConfig authConfig = new RpcAuthConfig(); + + /** + * Cache config + */ + private SendLogCacheConfig cacheConfig = new SendLogCacheConfig(); + + /** + * Buffer config + */ + private SendBufferConfig bufferConfig = new SendBufferConfig(); + + public RpcLogSenderConfig(){ + + } + + public RpcLogSenderConfig(String address, int sendRetryCnt, int connectionTimeout, int socketTimeout, + int serverRecoveryTimeInSec, int maxDelayTimeInSec, + RpcAuthConfig authConfig, SendLogCacheConfig cacheConfig, SendBufferConfig bufferConfig){ + this.address = address; + this.sendRetryCnt = sendRetryCnt; + this.connectionTimeout = connectionTimeout; + this.socketTimeout = socketTimeout; + this.serverRecoveryTimeInSec = serverRecoveryTimeInSec; + this.maxDelayTimeInSec = maxDelayTimeInSec; + if (Objects.nonNull(authConfig)){ + this.authConfig = authConfig; + } + if (Objects.nonNull(cacheConfig)){ + this.cacheConfig = cacheConfig; + } + if (Objects.nonNull(bufferConfig)){ + this.bufferConfig = bufferConfig; + } + } + @PluginFactory + public static RpcLogSenderConfig createConfig( + @PluginAttribute("address") String address, @PluginAttribute("sendRetryCnt") String sendRetryCnt, + @PluginAttribute("connectionTimeout") String connectionTimeout, @PluginAttribute("socketTimeout") String socketTimeout, + @PluginAttribute("serverRecoveryTimeInSec") String serverRecoveryTimeInSec, @PluginAttribute("maxDelayTimeInSec") String maxDelayTimeInSec, + @PluginElement("AuthConfig")RpcAuthConfig authConfig, @PluginElement("SendLogCache") SendLogCacheConfig cacheConfig, + @PluginElement("SendBuffer")SendBufferConfig bufferConfig){ + return new RpcLogSenderConfig(address, Integers.parseInt(sendRetryCnt, 3), + Integers.parseInt(connectionTimeout, 3000), Integers.parseInt(socketTimeout, 15000), + Integers.parseInt(serverRecoveryTimeInSec, 5), Integers.parseInt(maxDelayTimeInSec, 60), + authConfig, cacheConfig, bufferConfig); + } + public RpcAuthConfig getAuthConfig() { + return authConfig; + } + + public void setAuthConfig(RpcAuthConfig authConfig) { + this.authConfig = authConfig; + } + + public SendLogCacheConfig getCacheConfig() { + return cacheConfig; + } + + public void setCacheConfig(SendLogCacheConfig cacheConfig) { + this.cacheConfig = cacheConfig; + } + + public SendBufferConfig getBufferConfig() { + return bufferConfig; + } + + public void setBufferConfig(SendBufferConfig bufferConfig) { + this.bufferConfig = bufferConfig; + } + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public int getSendRetryCnt() { + return sendRetryCnt; + } + + public void setSendRetryCnt(int sendRetryCnt) { + this.sendRetryCnt = sendRetryCnt; + } + + public int getConnectionTimeout() { + return connectionTimeout; + } + + public void setConnectionTimeout(int connectionTimeout) { + this.connectionTimeout = connectionTimeout; + } + + public int getSocketTimeout() { + return socketTimeout; + } + + public void setSocketTimeout(int socketTimeout) { + this.socketTimeout = socketTimeout; + } + + public int getMaxDelayTimeInSec() { + return maxDelayTimeInSec; + } + + public void setMaxDelayTimeInSec(int maxDelayTimeInSec) { + this.maxDelayTimeInSec = maxDelayTimeInSec; + } + + public int getServerRecoveryTimeInSec() { + return serverRecoveryTimeInSec; + } + + public void setServerRecoveryTimeInSec(int serverRecoveryTimeInSec) { + this.serverRecoveryTimeInSec = serverRecoveryTimeInSec; + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java new file mode 100644 index 000000000..651caa616 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java @@ -0,0 +1,53 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; + +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.util.Integers; + +@Plugin( + name = "SendBuffer", + category = "Core", + printObject = true +) +public class SendBufferConfig { + /** + * Size of send buffer + */ + private int size = 50; + + /** + * Expire time of send buffer + */ + private long expireTimeInSec = 2; + + public SendBufferConfig(){ + + } + + public SendBufferConfig(int size, long expireTimeInSec){ + this.size = size; + this.expireTimeInSec = expireTimeInSec; + } + + @PluginFactory + public static SendBufferConfig createBufferConfig( + @PluginAttribute("size") String size, @PluginAttribute("expireTimeInSec") String expireTimeInSec){ + return new SendBufferConfig(Integers.parseInt(size, 50), Integers.parseInt(expireTimeInSec, 2)); + } + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + public long getExpireTimeInSec() { + return expireTimeInSec; + } + + public void setExpireTimeInSec(long expireTimeInSec) { + this.expireTimeInSec = expireTimeInSec; + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java new file mode 100644 index 000000000..6921584a5 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java @@ -0,0 +1,55 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; + +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.util.Integers; + +/** + * Cache config + */ +@Plugin( + name = "SendLogCache", + category = "Core", + printObject = true +) +public class SendLogCacheConfig { + /** + * Size of send cache + */ + private int size = 150; + + /** + * Max number of consuming thread + */ + private int maxConsumeThread = 10; + + public SendLogCacheConfig(){ + + } + + public SendLogCacheConfig(int size, int maxConsumeThread){ + this.size = size; + this.maxConsumeThread = maxConsumeThread; + } + @PluginFactory + public static SendLogCacheConfig createCacheConfig( + @PluginAttribute("size") String size, @PluginAttribute("maxConsumeThread") String maxConsumeThread){ + return new SendLogCacheConfig(Integers.parseInt(size, 150), Integers.parseInt(maxConsumeThread, 10)); + } + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + public int getMaxConsumeThread() { + return maxConsumeThread; + } + + public void setMaxConsumeThread(int maxConsumeThread) { + this.maxConsumeThread = maxConsumeThread; + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java new file mode 100644 index 000000000..40a6a7d0d --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java @@ -0,0 +1,49 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; + +import org.apache.logging.log4j.core.Filter; + +/** + * Appender config + */ +public class StreamisLogAppenderConfig { + + private String applicationName; + + /** + * Filter in log4j + */ + private Filter filter; + + private RpcLogSenderConfig senderConfig; + + public StreamisLogAppenderConfig(String applicationName, Filter filter, + RpcLogSenderConfig rpcLogSenderConfig){ + this.applicationName = applicationName; + this.filter = filter; + this.senderConfig = null != rpcLogSenderConfig? rpcLogSenderConfig : new RpcLogSenderConfig(); + } + + public String getApplicationName() { + return applicationName; + } + + public void setApplicationName(String applicationName) { + this.applicationName = applicationName; + } + + public Filter getFilter() { + return filter; + } + + public void setFilter(Filter filter) { + this.filter = filter; + } + + public RpcLogSenderConfig getSenderConfig() { + return senderConfig; + } + + public void setSenderConfig(RpcLogSenderConfig senderConfig) { + this.senderConfig = senderConfig; + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java new file mode 100644 index 000000000..cecc589a1 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; + +import org.apache.logging.log4j.core.Filter; + +import java.util.Map; + +/** + * Config builder for appender + */ +public abstract class StreamisLogAppenderConfigBuilder { + + + /** + * Build method + * @param applicationName application name + * @param filter filter + * @param senderConfig sender config + * @return + */ + public abstract StreamisLogAppenderConfig build(String applicationName, + Filter filter, RpcLogSenderConfig senderConfig); + + abstract Map loadConfigProps(); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java index a30ff98c6..5aa1bdcc4 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java @@ -2,6 +2,8 @@ import com.webank.wedatasphere.streamis.jobmanager.log.collector.ExceptionListener; import com.webank.wedatasphere.streamis.jobmanager.log.collector.cache.LogCache; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.SendLogCacheConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.ImmutableSendBuffer; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; @@ -36,7 +38,7 @@ public abstract class AbstractRpcLogSender implements R /** * Connect config */ - protected RpcSenderConfig rpcSenderConfig; + protected RpcLogSenderConfig rpcSenderConfig; /** * Rpc log context @@ -48,18 +50,18 @@ public abstract class AbstractRpcLogSender implements R * Use the listener instead of log4j structure */ protected ExceptionListener exceptionListener; - public AbstractRpcLogSender(RpcSenderConfig rpcSenderConfig, int cacheSize, int sendBufSize){ - this(rpcSenderConfig, cacheSize, sendBufSize, Integer.MAX_VALUE); - } - public AbstractRpcLogSender(RpcSenderConfig rpcSenderConfig, int cacheSize, int sendBufSize, int maxCacheConsume){ + public AbstractRpcLogSender(RpcLogSenderConfig rpcSenderConfig){ this.rpcSenderConfig = rpcSenderConfig; + SendLogCacheConfig cacheConfig = rpcSenderConfig.getCacheConfig(); + this.cacheSize = cacheConfig.getSize(); + this.maxCacheConsume = cacheConfig.getMaxConsumeThread(); + this.sendBufSize = rpcSenderConfig.getBufferConfig().getSize(); + if (sendBufSize > cacheSize) { throw new IllegalArgumentException("Size of send buffer is larger than cache size"); } - this.cacheSize = cacheSize; - this.sendBufSize = sendBufSize; - this.maxCacheConsume = maxCacheConsume; + } @Override @@ -91,7 +93,8 @@ public void setExceptionListener(ExceptionListener listener) { @Override public void close() { - + getOrCreateRpcLogContext().destroyCacheConsumers(); + this.isTerminated = true; } /** @@ -106,7 +109,7 @@ public void close() { * @param aggregatedEntity agg entity * @param rpcSenderConfig rpc sender config */ - protected abstract void doSend(E aggregatedEntity, RpcSenderConfig rpcSenderConfig) throws Exception; + protected abstract void doSend(E aggregatedEntity, RpcLogSenderConfig rpcSenderConfig) throws Exception; /** * Send log exception strategy diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java deleted file mode 100644 index aac1a9578..000000000 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcSenderConfig.java +++ /dev/null @@ -1,189 +0,0 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; - -/** - * Rpc sender configuration - */ -public class RpcSenderConfig { - - /** - * Send address - */ - private String address; - - /** - * Key of token-code - */ - private String tokenCodeKey = "Token-Code"; - - /** - * Key of token-user - */ - private String tokenUserKey = "Token-User"; - - /** - * Token user - */ - private String tokenUser = System.getProperty("user.name"); - - /** - * Token code - */ - private String tokenCode = "STREAM_LOG"; - - /** - * Timeout of connecting - */ - private int connectionTimeout = 3000; - - /** - * Timeout of reading from socket - */ - private int socketTimeout = 15000; - - /** - * Size of send cache - */ - private int cacheSize = 150; - - /** - * Size of send buffer - */ - private int bufferSize = 50; - - /** - * Expire time of send buffer - */ - private long bufferExpireTimeInSec = -1; - - /** - * Retry count of sending - */ - private int sendRetryCnt = 3; - - /** - * The time for server recovery - */ - private int serverRecoveryTimeInSec = 5; - - /** - * Retry max delay time of sender - */ - private int maxDelayTimeInSec; - - /** - * Max number of consuming thread - */ - private int maxConsumeThread = 10; - - public long getBufferExpireTimeInSec() { - return bufferExpireTimeInSec; - } - - public void setBufferExpireTimeInSec(long bufferExpireTimeInSec) { - this.bufferExpireTimeInSec = bufferExpireTimeInSec; - } - - public String getAddress() { - return address; - } - - public void setAddress(String address) { - this.address = address; - } - - public int getCacheSize() { - return cacheSize; - } - - public void setCacheSize(int cacheSize) { - this.cacheSize = cacheSize; - } - - public int getBufferSize() { - return bufferSize; - } - - public void setBufferSize(int bufferSize) { - this.bufferSize = bufferSize; - } - - public int getSendRetryCnt() { - return sendRetryCnt; - } - - public void setSendRetryCnt(int sendRetryCnt) { - this.sendRetryCnt = sendRetryCnt; - } - - public int getConnectionTimeout() { - return connectionTimeout; - } - - public void setConnectionTimeout(int connectionTimeout) { - this.connectionTimeout = connectionTimeout; - } - - public int getSocketTimeout() { - return socketTimeout; - } - - public void setSocketTimeout(int socketTimeout) { - this.socketTimeout = socketTimeout; - } - - public int getMaxConsumeThread() { - return maxConsumeThread; - } - - public void setMaxConsumeThread(int maxConsumeThread) { - this.maxConsumeThread = maxConsumeThread; - } - - public String getTokenCodeKey() { - return tokenCodeKey; - } - - public void setTokenCodeKey(String tokenCodeKey) { - this.tokenCodeKey = tokenCodeKey; - } - - public String getTokenCode() { - return tokenCode; - } - - public void setTokenCode(String tokenCode) { - this.tokenCode = tokenCode; - } - - public int getMaxDelayTimeInSec() { - return maxDelayTimeInSec; - } - - public void setMaxDelayTimeInSec(int maxDelayTimeInSec) { - this.maxDelayTimeInSec = maxDelayTimeInSec; - } - - public String getTokenUserKey() { - return tokenUserKey; - } - - public void setTokenUserKey(String tokenUserKey) { - this.tokenUserKey = tokenUserKey; - } - - public String getTokenUser() { - return tokenUser; - } - - public void setTokenUser(String tokenUser) { - this.tokenUser = tokenUser; - } - - public int getServerRecoveryTimeInSec() { - return serverRecoveryTimeInSec; - } - - public void setServerRecoveryTimeInSec(int serverRecoveryTimeInSec) { - this.serverRecoveryTimeInSec = serverRecoveryTimeInSec; - } -} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java index e851943a7..8a3a05a61 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; @@ -38,12 +39,13 @@ public abstract class SendLogCacheConsumer implements Runn public SendLogCacheConsumer(String id, SendLogCache cache, SendBuffer sendBuffer, - RpcSenderConfig rpcSenderConfig){ + RpcLogSenderConfig rpcSenderConfig){ this.id = id; this.cache = cache; this.sendBuffer = sendBuffer; - this.bufferExpireTimeInMills = rpcSenderConfig.getBufferExpireTimeInSec() > 0 ? TimeUnit.SECONDS - .toMillis(rpcSenderConfig.getBufferExpireTimeInSec()) : -1; + long expireTimeInSec = rpcSenderConfig.getBufferConfig().getExpireTimeInSec(); + this.bufferExpireTimeInMills = expireTimeInSec > 0 ? TimeUnit.SECONDS + .toMillis(expireTimeInSec) : -1; } @@ -93,6 +95,9 @@ public void run() { public void shutdown(){ this.isTerminated = true; + if (null != this.future){ + this.future.cancel(true); + } } public Future getFuture() { diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java index 06d70407a..02348c3af 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java @@ -1,6 +1,37 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.AbstractHttpLogSender; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvent; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvents; -public class StreamisRpcLogSender{ +/** + * Log sender for streamis + */ +public class StreamisRpcLogSender extends AbstractHttpLogSender { + + /** + * Each sender register an application + */ + private final String applicationName; + + public StreamisRpcLogSender(String applicationName, RpcLogSenderConfig rpcSenderConfig) { + super(rpcSenderConfig); + this.applicationName = applicationName; + } + + /** + * Aggregate to streamis log events + * @param sendBuffer send buffer + * @return + */ + @Override + protected StreamisLogEvents aggregateBuffer(SendBuffer sendBuffer) { + int remain = sendBuffer.remaining(); + StreamisLogEvent[] logEvents = new StreamisLogEvent[remain]; + sendBuffer.readBuf(logEvents, 0, logEvents.length); + return new StreamisLogEvents(applicationName, logEvents); + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java index 83e04ec7d..138817053 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java @@ -55,6 +55,7 @@ public int remaining() { @Override public void flip() { + checkFlag(Flag.WRITE_MODE); this.limit = this.position; this.position = 0; this.accessFlag = Flag.READ_MODE; @@ -94,9 +95,7 @@ final void limit(int newLimit){ * @return the current position value */ final int nextPosition(int offset, Flag accessFlag){ - if (this.accessFlag != accessFlag){ - throw new IllegalStateException("Illegal access flag [" + accessFlag + "] for send buffer"); - } + checkFlag(accessFlag); int p = position; // Reach the limit, return -1 value if (p >= limit){ @@ -108,6 +107,14 @@ final int nextPosition(int offset, Flag accessFlag){ return p; } + final void checkFlag(Flag accessFlag){ + if (this.accessFlag != accessFlag){ + throw new IllegalStateException("Illegal access flag [" + accessFlag + "] for send buffer"); + } + } + final void setFlag(Flag accessFlag){ + this.accessFlag = accessFlag; + } /** * * @return the current position @@ -116,6 +123,9 @@ final int position(){ return this.position; } + final void position(int position){ + this.position = position; + } /** * Do the actual clear */ diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java index 3c61985f6..4de1e6753 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java @@ -1,7 +1,5 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf; -import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvent; - import java.util.Arrays; import java.util.function.Function; @@ -38,7 +36,7 @@ public int writeBuf(E[] elements, int srcIndex, int length) { int startPos = nextPosition(Math.min(elements.length - srcIndex, length), Flag.WRITE_MODE); if (startPos >= 0){ int writes = position() - startPos; - System.arraycopy(this.buf, startPos, elements, srcIndex, writes); + System.arraycopy(elements, srcIndex, this.buf, startPos, writes); return writes; } } @@ -46,8 +44,17 @@ public int writeBuf(E[] elements, int srcIndex, int length) { } @Override + @SuppressWarnings("all") public int readBuf(E[] elements, int srcIndex, int length) { - return 0; + if (srcIndex < elements.length){ + int startPos = nextPosition(Math.min(elements.length - srcIndex, length), Flag.READ_MODE); + if (startPos >= 0){ + int reads = position() - startPos; + System.arraycopy(this.buf, startPos, elements, srcIndex, reads); + return reads; + } + } + return -1; } @Override @@ -61,13 +68,35 @@ public int writeBuf(E element) { } @Override + @SuppressWarnings("unchecked") public E readBuf() { + int startPos = nextPosition(1, Flag.READ_MODE); + if (startPos >= 0){ + return (E)buf[startPos]; + } return null; } @Override + @SuppressWarnings("unchecked") public SendBuffer compact(Function dropAble) { - return null; + checkFlag(Flag.READ_MODE); + int offset = 0; + int compact = position() - 1; + for(int i = position(); i < capacity; i ++){ + Object element = buf[i]; + if (dropAble.apply((E)element)){ + buf[i] = null; + offset ++; + } else { + compact = i - offset; + buf[compact] = element; + } + } + position(compact + 1); + limit(this.capacity); + setFlag(Flag.WRITE_MODE); + return this; } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java index bebfd64b7..187152728 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java @@ -1,16 +1,13 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcAuthConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.AbstractRpcLogSender; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.RpcLogSender; -import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.RpcSenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.SendLogExceptionStrategy; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request.EntityPostAction; import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; -import org.apache.http.HttpException; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.StatusLine; import org.apache.http.client.HttpClient; import org.apache.http.conn.ConnectTimeoutException; @@ -43,10 +40,9 @@ public abstract class AbstractHttpLogSender extends Abs */ private final AtomicLong serverRecoveryTimePoint = new AtomicLong(-1L); - public AbstractHttpLogSender(RpcSenderConfig rpcSenderConfig, int cacheSize, int sendBufSize) { - super(rpcSenderConfig, cacheSize, sendBufSize); + public AbstractHttpLogSender(RpcLogSenderConfig rpcSenderConfig) { + super(rpcSenderConfig); this.globalHttpClient = HttpClientTool.createHttpClient(rpcSenderConfig); - final RpcLogSender sender = this; this.sendRetryStrategy = new SendLogExceptionStrategy(this) { private final Class[] retryOnExceptions = new Class[]{ @@ -92,7 +88,7 @@ protected SendLogExceptionStrategy getSendLogExceptionStrategy() { } @Override - protected void doSend(E aggregatedEntity, RpcSenderConfig rpcSenderConfig) throws IOException { + protected void doSend(E aggregatedEntity, RpcLogSenderConfig rpcSenderConfig) throws IOException { if (System.currentTimeMillis() >= serverRecoveryTimePoint.get()) { if (aggregatedEntity instanceof LogElement) { long timestamp = ((LogElement) aggregatedEntity).getLogTimeStamp(); @@ -102,7 +98,8 @@ protected void doSend(E aggregatedEntity, RpcSenderConfig rpcSenderConfig) throw } } EntityPostAction postAction = new EntityPostAction<>(rpcSenderConfig.getAddress(), aggregatedEntity); - postAction.getRequestHeaders().put(rpcSenderConfig.getTokenUserKey(), rpcSenderConfig.getTokenUser()); + RpcAuthConfig authConfig = rpcSenderConfig.getAuthConfig(); + postAction.getRequestHeaders().put(authConfig.getTokenUserKey(), authConfig.getTokenUser()); // Ignore the response postAction.execute(this.globalHttpClient); // Init the counter diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java index df4eb31ea..e67bec6da 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java @@ -1,6 +1,6 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http; -import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.RpcSenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; import org.apache.http.Header; import org.apache.http.client.HttpClient; import org.apache.http.client.config.RequestConfig; @@ -38,7 +38,7 @@ public class HttpClientTool { * @param rpcSenderConfig rpc sender config * @return http client */ - public static HttpClient createHttpClient(RpcSenderConfig rpcSenderConfig){ + public static HttpClient createHttpClient(RpcLogSenderConfig rpcSenderConfig){ int connectTimeout = rpcSenderConfig.getConnectionTimeout() > 0? rpcSenderConfig.getConnectionTimeout() : DEFAULT_CONNECT_TIMEOUT; int socketTimeout = rpcSenderConfig.getSocketTimeout() > 0? rpcSenderConfig.getSocketTimeout() : DEFAULT_SOCKET_TIMEOUT; RequestConfig requestConfig = RequestConfig.custom() @@ -46,12 +46,13 @@ public static HttpClient createHttpClient(RpcSenderConfig rpcSenderConfig){ .setConnectionRequestTimeout(socketTimeout) .setSocketTimeout(socketTimeout) .build(); - int maxConn = rpcSenderConfig.getMaxConsumeThread() > 0? rpcSenderConfig.getMaxConsumeThread() : DEFAULT_MAX_CONN; + int maxConsumeThread = rpcSenderConfig.getCacheConfig().getMaxConsumeThread(); + int maxConn = maxConsumeThread > 0? maxConsumeThread : DEFAULT_MAX_CONN; HttpClientBuilder clientBuilder = HttpClients.custom(); - String tokenValue = rpcSenderConfig.getTokenCode(); + String tokenValue = rpcSenderConfig.getAuthConfig().getTokenCode(); List
defaultHeaders = new ArrayList<>(); if (null != tokenValue && !tokenValue.trim().equals("")){ - defaultHeaders.add(new BasicHeader(rpcSenderConfig.getTokenCodeKey(), tokenValue)); + defaultHeaders.add(new BasicHeader(rpcSenderConfig.getAuthConfig().getTokenCodeKey(), tokenValue)); } clientBuilder.setDefaultRequestConfig(requestConfig).setDefaultHeaders(defaultHeaders) .useSystemProperties().setMaxConnTotal(maxConn); diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java index 8c493d042..40b20b50e 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java @@ -126,5 +126,6 @@ public static T convert(Object simpleObj, JavaType javaType) { public static ObjectMapper getMapper(){ return mapper; } + } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml index 51d283f53..a675aa404 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml @@ -14,6 +14,15 @@ 8 8 + 2.13.2.2 + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson-databind.version} + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java index b0694580c..e253d2a32 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java @@ -1,6 +1,5 @@ package com.webank.wedatasphere.streamis.jobmanager.log.entities; -import java.util.concurrent.TimeUnit; /** * Element defined of log diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java index 0c73d83d2..5c82e22cb 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java @@ -1,4 +1,59 @@ package com.webank.wedatasphere.streamis.jobmanager.log.entities; -public class StreamisLogEvent { +import com.fasterxml.jackson.annotation.JsonIgnore; + +import java.io.Serializable; + +/** + * Log event for streamis + */ +public class StreamisLogEvent implements LogElement, Serializable { + + /** + * Log time + */ + private final long logTimeInMills; + + /** + * Log content + */ + private final String content; + + /** + * Mark + */ + private int mark; + + public StreamisLogEvent(String content, long logTimeInMills){ + this.content = content; + this.logTimeInMills = logTimeInMills; + } + @Override + public int getSequenceId() { + return 0; + } + + @Override + public long getLogTimeStamp() { + return this.logTimeInMills; + } + + @Override + @JsonIgnore + public String[] getContents() { + return new String[]{content}; + } + + public String getContent() { + return content; + } + + @Override + public int mark() { + return this.mark; + } + + public void mark(int mark){ + this.mark = mark; + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java index 3a1766ee6..972d1a492 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java @@ -1,6 +1,35 @@ package com.webank.wedatasphere.streamis.jobmanager.log.entities; -public class StreamisLogEvents implements LogElement{ +import com.fasterxml.jackson.annotation.JsonIgnore; + +import java.io.Serializable; + +public class StreamisLogEvents implements LogElement, Serializable { + + /** + * Application name + */ + private String appName; + /** + * Log time + */ + private final long logTimeInMills; + + private final StreamisLogEvent[] events; + + public StreamisLogEvents(String applicationName, StreamisLogEvent[] events){ + this.appName = applicationName; + this.events = events; + long maxTime = -1; + for(StreamisLogEvent event : events){ + long time = event.getLogTimeStamp(); + if (time > maxTime){ + maxTime = time; + } + } + this.logTimeInMills = maxTime; + } + @Override public int getSequenceId() { return 0; @@ -8,17 +37,30 @@ public int getSequenceId() { @Override public long getLogTimeStamp() { - return 0; + return this.logTimeInMills; } @Override + @JsonIgnore public String[] getContents() { - return new String[0]; + String[] contents = new String[events.length]; + for(int i = 0 ; i < contents.length; i++){ + contents[i] = events[i].getContent(); + } + return contents; } @Override public int mark() { - return 0; + return 1; + } + + public String getAppName() { + return appName; + } + + public StreamisLogEvent[] getEvents() { + return events; } } From a607bf11c92689aa6a86301aa6eb6e75a645db4d Mon Sep 17 00:00:00 2001 From: davidhua Date: Fri, 21 Oct 2022 10:49:32 +0800 Subject: [PATCH 015/240] Fix the problem in send buffer and consumer. --- .../log/collector/StreamisRpcLogAppender.java | 1 + .../StreamisLogAppenderConfigBuilder.java | 9 +++-- .../sender/AbstractRpcLogSender.java | 2 +- .../sender/SendLogCacheConsumer.java | 18 ++++++--- .../sender/StreamisRpcLogSender.java | 9 +++-- .../sender/buf/AbstractSendBuffer.java | 2 + .../sender/buf/ImmutableSendBuffer.java | 4 +- .../log/collector/sender/buf/SendBuffer.java | 4 +- .../sender/http/AbstractHttpLogSender.java | 7 +++- .../collector/StreamisLogAppenderTest.java | 16 ++++++++ .../src/test/resources/log4j2.xml | 37 +++++++++++++++++++ 11 files changed, 91 insertions(+), 18 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java index a13de8d83..737885e1a 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java @@ -18,6 +18,7 @@ import org.apache.logging.log4j.core.layout.PatternLayout; import java.io.Serializable; +import java.nio.Buffer; import java.util.ArrayList; import java.util.List; import java.util.Objects; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java index cecc589a1..f3374ac48 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java @@ -17,8 +17,11 @@ public abstract class StreamisLogAppenderConfigBuilder { * @param senderConfig sender config * @return */ - public abstract StreamisLogAppenderConfig build(String applicationName, - Filter filter, RpcLogSenderConfig senderConfig); + public StreamisLogAppenderConfig build(String applicationName, + Filter filter, RpcLogSenderConfig senderConfig){ + return null; + } - abstract Map loadConfigProps(); + + public abstract Map loadConfigProps(); } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java index 5aa1bdcc4..d90eb1a7f 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java @@ -421,7 +421,7 @@ public int drainLogsTo(SendBuffer sendBuffer, int maxElements) { @SuppressWarnings("unchecked") private int sendBuf(SendBuffer sendBuffer, Object[] items, int takeIndex, int len){ - int send = sendBuffer.writeBuf((T[]) items, takeIndex, len); + int send = sendBuffer.writeBuf(items, takeIndex, len); if (send < len){ // Buffer full exception exceptionListener.onException(this, null, "The sender buffer is full," + diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java index 8a3a05a61..52a7b72cb 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java @@ -53,13 +53,16 @@ public SendLogCacheConsumer(String id, SendLogCache cache, public void run() { int remain; long expireTimeInMills = requireNewFlushTime(); + int capacity = sendBuffer.capacity(); while (!this.isTerminated) { try { - if ((expireTimeInMills > 0 && expireTimeInMills >= System.currentTimeMillis()) - || (remain = this.sendBuffer.remaining()) <= 0) { + remain = this.sendBuffer.remaining(); + if ((expireTimeInMills > 0 && expireTimeInMills <= System.currentTimeMillis()) || remain <= 0) { // Transient to the read mode - sendBuffer.flip(); - onFlushAndSend(sendBuffer); + if (remain < capacity) { + sendBuffer.flip(); + onFlushAndSend(sendBuffer); + } expireTimeInMills = requireNewFlushTime(); if (sendBuffer.isReadMode()) { // Clear the buffer and transient to the write mode, otherwise continue writing @@ -84,11 +87,16 @@ public void run() { if (this.isTerminated && e instanceof InterruptedException){ return; } else { - System.err.println("SendLogCacheConsumer[" + Thread.currentThread().getName() + "] occurred exception [" + e.getLocalizedMessage() + "]"); + System.err.println("SendLogCacheConsumer[" + Thread.currentThread().getName() + "] occurred exception [" + e.getLocalizedMessage() + "]"); // For the unknown exception clear the cache sendBuffer.clear(); expireTimeInMills = requireNewFlushTime(); } + try { + Thread.sleep(500); + } catch (InterruptedException ex) { + // Ignore + } } } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java index 02348c3af..b9ec15537 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java @@ -30,8 +30,11 @@ public StreamisRpcLogSender(String applicationName, RpcLogSenderConfig rpcSender @Override protected StreamisLogEvents aggregateBuffer(SendBuffer sendBuffer) { int remain = sendBuffer.remaining(); - StreamisLogEvent[] logEvents = new StreamisLogEvent[remain]; - sendBuffer.readBuf(logEvents, 0, logEvents.length); - return new StreamisLogEvents(applicationName, logEvents); + if (remain > 0) { + StreamisLogEvent[] logEvents = new StreamisLogEvent[remain]; + sendBuffer.readBuf(logEvents, 0, logEvents.length); + return new StreamisLogEvents(applicationName, logEvents); + } + return null; } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java index 138817053..1b42ad957 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java @@ -103,6 +103,8 @@ final int nextPosition(int offset, Flag accessFlag){ } if (p + offset > limit){ this.position = limit; + } else { + this.position = p + offset; } return p; } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java index 4de1e6753..0e64c4ffa 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java @@ -31,7 +31,7 @@ public void capacity(String newCapacity) { @Override @SuppressWarnings("all") - public int writeBuf(E[] elements, int srcIndex, int length) { + public int writeBuf(Object[] elements, int srcIndex, int length) { if (srcIndex < elements.length){ int startPos = nextPosition(Math.min(elements.length - srcIndex, length), Flag.WRITE_MODE); if (startPos >= 0){ @@ -45,7 +45,7 @@ public int writeBuf(E[] elements, int srcIndex, int length) { @Override @SuppressWarnings("all") - public int readBuf(E[] elements, int srcIndex, int length) { + public int readBuf(Object[] elements, int srcIndex, int length) { if (srcIndex < elements.length){ int startPos = nextPosition(Math.min(elements.length - srcIndex, length), Flag.READ_MODE); if (startPos >= 0){ diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java index 28e3a6356..0a98580fb 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java @@ -64,7 +64,7 @@ public interface SendBuffer { * @param length the length to read * @return write num */ - int writeBuf(E[] elements, int srcIndex, int length); + int writeBuf(Object[] elements, int srcIndex, int length); /** * Read buffer element @@ -79,7 +79,7 @@ public interface SendBuffer { * @param length the length to write * @return read num */ - int readBuf(E[] elements, int srcIndex, int length); + int readBuf(Object[] elements, int srcIndex, int length); /** * Compact the buffer, avoid the useless elements diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java index 187152728..d247ba4ad 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java @@ -15,6 +15,7 @@ import java.io.IOException; import java.io.InterruptedIOException; import java.net.UnknownHostException; +import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -55,6 +56,7 @@ public int retryCount() { @Override public SendLogExceptionStrategy.RetryDescription onException(Exception e, SendBuffer sendBuffer) { + e.printStackTrace(); boolean shouldRetry = false; // Limit of exception number is the same as the retry times if (exceptionCounter.incrementAndGet() > retryCount()){ @@ -62,7 +64,7 @@ public SendLogExceptionStrategy.RetryDescription onException(Exception e, SendBu TimeUnit.SECONDS.toMillis(rpcSenderConfig.getServerRecoveryTimeInSec())); } else { for (Class retryOnException : retryOnExceptions) { - if (retryOnException.isAssignableFrom(e.getClass())) { + if (retryOnException.equals(e.getClass())) { shouldRetry = true; break; } @@ -76,7 +78,7 @@ public SendLogExceptionStrategy.RetryDescription onException(Exception e, SendBu sendBuffer.compact( element -> element.mark() > 1); shouldRetry = false; } - exceptionListener.onException(sender, e, null); + Optional.ofNullable(exceptionListener).ifPresent(listener -> listener.onException(sender, e, null)); return new RetryDescription(shouldRetry); } }; @@ -97,6 +99,7 @@ protected void doSend(E aggregatedEntity, RpcLogSenderConfig rpcSenderConfig) th return; } } + EntityPostAction postAction = new EntityPostAction<>(rpcSenderConfig.getAddress(), aggregatedEntity); RpcAuthConfig authConfig = rpcSenderConfig.getAuthConfig(); postAction.getRequestHeaders().put(authConfig.getTokenUserKey(), authConfig.getTokenUser()); diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java new file mode 100644 index 000000000..189e573ae --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StreamisLogAppenderTest { + private static final Logger LOG = LoggerFactory.getLogger(StreamisLogAppenderTest.class); + public static void main(String[] args) throws InterruptedException { + while(true){ + for(int i = 0; i < 100; i ++){ + LOG.info("Stream Log appender test"); + } + Thread.sleep(1000); + } + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml new file mode 100644 index 000000000..d806c5198 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml @@ -0,0 +1,37 @@ + + + + + + + + + + + + + ` + + + + + + + + \ No newline at end of file From 563c819d3e534346f29a989b06ca439bed51aea7 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Fri, 21 Oct 2022 12:38:08 +0800 Subject: [PATCH 016/240] Optimize the architecture of fetch Job details. --- .../service/DefaultStreamTaskService.scala | 31 ++++-- .../manager/service/StreamTaskService.scala | 6 +- .../manager/transform/TaskMetricsParser.scala | 18 ++++ .../parser/AbstractTaskMetricsParser.scala | 37 +++++++ .../parser/FlinkTaskMetricsParser.scala | 48 +++++++++ .../parser/SparkTaskMetricsParser.scala | 83 ++++++++++++++ .../jobmanager/restful/api/JobRestfulApi.java | 101 ++---------------- 7 files changed, 218 insertions(+), 106 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/TaskMetricsParser.scala create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/AbstractTaskMetricsParser.scala create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/FlinkTaskMetricsParser.scala create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala index 3bc49354f..d03b0b9e0 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala @@ -15,6 +15,10 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.service +import java.util +import java.util.concurrent.Future +import java.util.{Calendar, function} + import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants import com.webank.wedatasphere.streamis.jobmanager.launcher.dao.StreamJobConfMapper import com.webank.wedatasphere.streamis.jobmanager.launcher.job.manager.JobLaunchManager @@ -27,17 +31,18 @@ import com.webank.wedatasphere.streamis.jobmanager.manager.SpringContextHolder import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf.FLINK_JOB_STATUS_FAILED import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamJobMapper, StreamTaskMapper} -import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamTask -import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{ExecResultVo, JobProgressVo, JobStatusVo, PauseResultVo, ScheduleResultVo, StreamTaskListVo} -import com.webank.wedatasphere.streamis.jobmanager.manager.exception.{JobErrorException, JobExecuteErrorException, JobFetchErrorException, JobPauseErrorException, JobTaskErrorException} +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo._ +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.{StreamJob, StreamTask} +import com.webank.wedatasphere.streamis.jobmanager.manager.exception.{JobExecuteErrorException, JobFetchErrorException, JobPauseErrorException, JobTaskErrorException} import com.webank.wedatasphere.streamis.jobmanager.manager.scheduler.FutureScheduler import com.webank.wedatasphere.streamis.jobmanager.manager.scheduler.events.AbstractStreamisSchedulerEvent.StreamisEventInfo -import com.webank.wedatasphere.streamis.jobmanager.manager.scheduler.events.{AbstractStreamisSchedulerEvent, StreamisPhaseInSchedulerEvent} +import com.webank.wedatasphere.streamis.jobmanager.manager.scheduler.events.StreamisPhaseInSchedulerEvent import com.webank.wedatasphere.streamis.jobmanager.manager.scheduler.events.StreamisPhaseInSchedulerEvent.ScheduleCommand import com.webank.wedatasphere.streamis.jobmanager.manager.transform.exception.TransformFailedErrorException -import com.webank.wedatasphere.streamis.jobmanager.manager.transform.{StreamisTransformJobBuilder, Transform} +import com.webank.wedatasphere.streamis.jobmanager.manager.transform.{StreamisTransformJobBuilder, TaskMetricsParser, Transform} import com.webank.wedatasphere.streamis.jobmanager.manager.util.DateUtils import com.webank.wedatasphere.streamis.jobmanager.manager.utils.StreamTaskUtils +import javax.annotation.Resource import org.apache.commons.lang.StringUtils import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.httpclient.dws.DWSHttpClient @@ -47,10 +52,6 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional -import java.util -import java.util.{Calendar, Date, function} -import java.util.concurrent.Future -import javax.annotation.Resource import scala.collection.JavaConverters._ @@ -60,6 +61,7 @@ class DefaultStreamTaskService extends StreamTaskService with Logging{ @Autowired private var streamTaskMapper:StreamTaskMapper=_ @Autowired private var streamJobMapper:StreamJobMapper=_ @Autowired private var streamisTransformJobBuilders: Array[StreamisTransformJobBuilder] = _ + @Autowired private var taskMetricsParser: Array[TaskMetricsParser] = _ @Resource private var jobLaunchManager: JobLaunchManager[_ <: JobInfo] = _ @@ -721,4 +723,15 @@ class DefaultStreamTaskService extends StreamTaskService with Logging{ } } + override def getJobDetailsVO(streamJob: StreamJob, version: String): JobDetailsVo = { + val flinkJobInfo = getTaskJobInfo(streamJob.getId, version) + val jobStateInfos = flinkJobInfo.getJobStates + val metricsStr = if (JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES.getValue.contains(streamJob.getJobType)) null + else if(jobStateInfos == null || jobStateInfos.length == 0) null + else jobStateInfos(0).getLocation + taskMetricsParser.find(_.canParse(streamJob)).map(_.parse(metricsStr)).filter { jobDetailsVO => + jobDetailsVO.setLinkisJobInfo(flinkJobInfo) + true + }.getOrElse(throw new JobFetchErrorException(30030, s"Cannot find a TaskMetricsParser to parse job details.")) + } } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamTaskService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamTaskService.scala index 65c584804..26591d23f 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamTaskService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamTaskService.scala @@ -18,8 +18,8 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.service import com.webank.wedatasphere.streamis.jobmanager.launcher.job.state.JobState import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.entity.LogRequestPayload import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.FlinkJobInfo -import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamTask -import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{ExecResultVo, JobProgressVo, JobStatusVo, PauseResultVo, StreamTaskListVo} +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.{StreamJob, StreamTask} +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{ExecResultVo, JobDetailsVo, JobProgressVo, JobStatusVo, PauseResultVo, StreamTaskListVo} import java.util import java.util.concurrent.Future /** @@ -154,4 +154,6 @@ trait StreamTaskService { def getStateInfo(taskId: Long): JobState + def getJobDetailsVO(streamJob: StreamJob, version: String): JobDetailsVo + } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/TaskMetricsParser.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/TaskMetricsParser.scala new file mode 100644 index 000000000..1e7ad7b68 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/TaskMetricsParser.scala @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.transform + +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJob +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.JobDetailsVo + +/** + * + * @date 2022-10-21 + * @author enjoyyin + * @since 0.5.0 + */ +trait TaskMetricsParser { + + def canParse(streamJob: StreamJob): Boolean + + def parse(metrics: String): JobDetailsVo + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/AbstractTaskMetricsParser.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/AbstractTaskMetricsParser.scala new file mode 100644 index 000000000..eb0de6d6e --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/AbstractTaskMetricsParser.scala @@ -0,0 +1,37 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.transform.parser + +import java.util + +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.JobDetailsVo +import com.webank.wedatasphere.streamis.jobmanager.manager.transform.TaskMetricsParser +import org.apache.commons.lang3.StringUtils +import org.apache.linkis.httpclient.dws.DWSHttpClient + +/** + * + * @date 2022-10-21 + * @author enjoyyin + * @since 0.5.0 + */ +trait AbstractTaskMetricsParser extends TaskMetricsParser { + + override def parse(metrics: String): JobDetailsVo = { + val jobDetailsVO = new JobDetailsVo + val dataNumberDTOS = new util.ArrayList[JobDetailsVo.DataNumberDTO] + val loadConditionDTOs = new util.ArrayList[JobDetailsVo.LoadConditionDTO] + val realTimeTrafficDTOS = new util.ArrayList[JobDetailsVo.RealTimeTrafficDTO] + jobDetailsVO.setDataNumber(dataNumberDTOS) + jobDetailsVO.setLoadCondition(loadConditionDTOs) + jobDetailsVO.setRealTimeTraffic(realTimeTrafficDTOS) + val metricsMap = if(StringUtils.isNotBlank(metrics)) DWSHttpClient.jacksonJson.readValue(metrics, classOf[util.Map[String, Object]]) + else new util.HashMap[String, Object](0) + parse(metricsMap, dataNumberDTOS, loadConditionDTOs, realTimeTrafficDTOS) + jobDetailsVO + } + + protected def parse(metricsMap: util.Map[String, Object], + dataNumberDTOS: util.List[JobDetailsVo.DataNumberDTO], + loadConditionDTOs: util.List[JobDetailsVo.LoadConditionDTO], + realTimeTrafficDTOS: util.List[JobDetailsVo.RealTimeTrafficDTO]): Unit + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/FlinkTaskMetricsParser.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/FlinkTaskMetricsParser.scala new file mode 100644 index 000000000..48eb8ad06 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/FlinkTaskMetricsParser.scala @@ -0,0 +1,48 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.transform.parser + +import java.util + +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJob +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.JobDetailsVo +import org.springframework.stereotype.Component + +/** + * + * @date 2022-10-21 + * @author enjoyyin + * @since 0.5.0 + */ +@Component +class FlinkTaskMetricsParser extends AbstractTaskMetricsParser { + + override def canParse(streamJob: StreamJob): Boolean = streamJob.getJobType.startsWith("flink.") + + override def parse(metricsMap: util.Map[String, Object], + dataNumberDTOS: util.List[JobDetailsVo.DataNumberDTO], + loadConditionDTOs: util.List[JobDetailsVo.LoadConditionDTO], + realTimeTrafficDTOS: util.List[JobDetailsVo.RealTimeTrafficDTO]): Unit = { + // TODO This is just sample datas, waiting for it completed. We have planned it to a later release, welcome all partners to join us to realize this powerful feature. + val dataNumberDTO = new JobDetailsVo.DataNumberDTO + dataNumberDTO.setDataName("kafka topic") + dataNumberDTO.setDataNumber(109345) + dataNumberDTOS.add(dataNumberDTO) + + val loadConditionDTO = new JobDetailsVo.LoadConditionDTO + loadConditionDTO.setType("jobManager") + loadConditionDTO.setHost("localhost") + loadConditionDTO.setMemory("1.5") + loadConditionDTO.setTotalMemory("2.0") + loadConditionDTO.setGcLastTime("2020-08-01") + loadConditionDTO.setGcLastConsume("1") + loadConditionDTO.setGcTotalTime("2min") + loadConditionDTOs.add(loadConditionDTO) + + val realTimeTrafficDTO = new JobDetailsVo.RealTimeTrafficDTO + realTimeTrafficDTO.setSourceKey("kafka topic") + realTimeTrafficDTO.setSourceSpeed("100 Records/S") + realTimeTrafficDTO.setTransformKey("transform") + realTimeTrafficDTO.setSinkKey("hbase key") + realTimeTrafficDTO.setSinkSpeed("10 Records/S") + realTimeTrafficDTOS.add(realTimeTrafficDTO) + } +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala new file mode 100644 index 000000000..392986b53 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala @@ -0,0 +1,83 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.transform.parser + +import java.util + +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJob +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.JobDetailsVo +import org.apache.linkis.common.utils.Utils +import org.springframework.stereotype.Component + +import scala.collection.JavaConverters._ + +/** + * + * @date 2022-10-21 + * @author enjoyyin + * @since 0.5.0 + */ +@Component +class SparkTaskMetricsParser extends AbstractTaskMetricsParser { + + override protected def parse(metricsMap: util.Map[String, Object], + dataNumberDTOS: util.List[JobDetailsVo.DataNumberDTO], + loadConditionDTOs: util.List[JobDetailsVo.LoadConditionDTO], + realTimeTrafficDTOS: util.List[JobDetailsVo.RealTimeTrafficDTO]): Unit = { + val addDataNumberDTO: String => () = key => { + val batch = new JobDetailsVo.DataNumberDTO + batch.setDataName(key) + batch.setDataNumber(metricsMap.get(key).toString.toInt) + dataNumberDTOS.add(batch) + } + addDataNumberDTO("waitingBatchs") + addDataNumberDTO("runningBatchs") + addDataNumberDTO("completedBatchs") + metricsMap.get("executors") match { + case executors: util.List[util.Map[String, AnyRef]] if !executors.isEmpty => + executors.asScala.foreach { executor => + val loadConditionDTO = new JobDetailsVo.LoadConditionDTO + loadConditionDTO.setType(executor.get("type").asInstanceOf[String]) + loadConditionDTO.setHost(executor.get("host").asInstanceOf[String]) + loadConditionDTO.setMemory(executor.get("memory").asInstanceOf[String]) + loadConditionDTO.setTotalMemory(executor.get("totalMemory").asInstanceOf[String]) + loadConditionDTO.setGcLastTime(executor.get("gcLastTime").asInstanceOf[String]) + loadConditionDTO.setGcLastConsume(executor.get("gcLastConsume").asInstanceOf[String]) + loadConditionDTO.setGcTotalTime(executor.get("gcTotalTime").asInstanceOf[String]) + loadConditionDTOs.add(loadConditionDTO) + } + case _ => + val loadConditionDTO = new JobDetailsVo.LoadConditionDTO + loadConditionDTO.setType("Driver") + loadConditionDTO.setHost("") + loadConditionDTO.setMemory("") + loadConditionDTO.setTotalMemory("") + loadConditionDTO.setGcLastTime("") + loadConditionDTO.setGcLastConsume("") + loadConditionDTO.setGcTotalTime("") + loadConditionDTOs.add(loadConditionDTO) + } + val realTimeTrafficDTO = new JobDetailsVo.RealTimeTrafficDTO + metricsMap.get("batchMetrics") match { + case batchMetrics: util.List[util.Map[String, Object]] if !batchMetrics.isEmpty => + val batchMetric = batchMetrics.asScala.maxBy(_.get("batchTime").asInstanceOf[String]) + realTimeTrafficDTO.setSourceKey(metricsMap.getOrDefault("source", "").asInstanceOf[String]) + realTimeTrafficDTO.setSourceSpeed(batchMetric.get("inputRecords") + " Records") + realTimeTrafficDTO.setTransformKey("processing") + realTimeTrafficDTO.setSinkKey(metricsMap.getOrDefault("sink", "").asInstanceOf[String]) + val sinkSpeed = if (batchMetric.containsKey("totalDelay") && batchMetric.get("totalDelay") != null) + Utils.msDurationToString(batchMetric.get("totalDelay").asInstanceOf[Long]) + " totalDelay" + else if (batchMetric.containsKey("taskExecuteTime") && batchMetric.get("taskExecuteTime") != null) + Utils.msDurationToString(batchMetric.get("taskExecuteTime").asInstanceOf[Long]) + " executeTime(Last Batch)" + else "" + realTimeTrafficDTO.setSinkSpeed(sinkSpeed) + case _ => + realTimeTrafficDTO.setSourceKey("") + realTimeTrafficDTO.setSourceSpeed(" Records/S") + realTimeTrafficDTO.setTransformKey("") + realTimeTrafficDTO.setSinkKey("") + realTimeTrafficDTO.setSinkSpeed(" Records/S") + } + realTimeTrafficDTOS.add(realTimeTrafficDTO) + } + + override def canParse(streamJob: StreamJob): Boolean = streamJob.getJobType.startsWith("spark.") +} diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 57d5473b5..acb1ac529 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -39,7 +39,6 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.linkis.common.utils.Utils; import org.apache.linkis.httpclient.dws.DWSHttpClient; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; @@ -198,104 +197,16 @@ public Message detailsJob(HttpServletRequest req, @RequestParam(value = "jobId", if (jobId == null) { JobExceptionManager.createException(30301, "jobId"); } - + String username = SecurityFilter.getLoginUsername(req); StreamJob streamJob = streamJobService.getJobById(jobId); + if (!streamJobService.hasPermission(streamJob, username) && + !this.privilegeService.hasAccessPrivilege(req, streamJob.getProjectName())) { + return Message.error("Have no permission to get Job details of StreamJob [" + jobId + "]"); + } if(streamJob == null) { return Message.error("not exists job " + jobId); } - FlinkJobInfo flinkJobInfo = streamTaskService.getTaskJobInfo(jobId,version); - JobDetailsVo jobDetailsVO = new JobDetailsVo(); - List dataNumberDTOS = new ArrayList<>(); - List loadConditionDTOs = new ArrayList<>(); - List realTimeTrafficDTOS = new ArrayList<>(); - JobStateInfo[] jobStateInfos = flinkJobInfo.getJobStates(); - if(JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES().getValue().contains(streamJob.getJobType()) || - (jobStateInfos == null || jobStateInfos.length == 0)) { - // TODO This is just sample datas, waiting for it completed. We have planned it to a later release, welcome all partners to join us to realize this powerful feature. - JobDetailsVo.DataNumberDTO dataNumberDTO = new JobDetailsVo.DataNumberDTO(); - dataNumberDTO.setDataName("kafka topic"); - dataNumberDTO.setDataNumber(109345); - dataNumberDTOS.add(dataNumberDTO); - - JobDetailsVo.LoadConditionDTO loadConditionDTO = new JobDetailsVo.LoadConditionDTO(); - loadConditionDTO.setType("jobManager"); - loadConditionDTO.setHost("localhost"); - loadConditionDTO.setMemory("1.5"); - loadConditionDTO.setTotalMemory("2.0"); - loadConditionDTO.setGcLastTime("2020-08-01"); - loadConditionDTO.setGcLastConsume("1"); - loadConditionDTO.setGcTotalTime("2min"); - loadConditionDTOs.add(loadConditionDTO); - - JobDetailsVo.RealTimeTrafficDTO realTimeTrafficDTO = new JobDetailsVo.RealTimeTrafficDTO(); - realTimeTrafficDTO.setSourceKey("kafka topic"); - realTimeTrafficDTO.setSourceSpeed("100 Records/S"); - realTimeTrafficDTO.setTransformKey("transform"); - realTimeTrafficDTO.setSinkKey("hbase key"); - realTimeTrafficDTO.setSinkSpeed("10 Records/S"); - realTimeTrafficDTOS.add(realTimeTrafficDTO); - - } else { - String metricsStr = jobStateInfos[0].getLocation(); - Map metricsMap = DWSHttpClient.jacksonJson().readValue(metricsStr, Map.class); - JobDetailsVo.DataNumberDTO dataNumberDTO = new JobDetailsVo.DataNumberDTO(); - dataNumberDTO.setDataName("waitingBatchs"); - dataNumberDTO.setDataNumber(Math.toIntExact((Long) metricsMap.get("waitingBatchs"))); - dataNumberDTOS.add(dataNumberDTO); - List> executors = (List>) metricsMap.get("executors"); - if(executors != null && !executors.isEmpty()) { - executors.forEach(map -> { - JobDetailsVo.LoadConditionDTO loadConditionDTO = new JobDetailsVo.LoadConditionDTO(); - loadConditionDTO.setType((String) map.get("type")); - loadConditionDTO.setHost((String) map.get("host")); - loadConditionDTO.setMemory((String) map.get("memory")); - loadConditionDTO.setTotalMemory((String) map.get("totalMemory")); - loadConditionDTO.setGcLastTime((String) map.get("gcLastTime")); - loadConditionDTO.setGcLastConsume((String) map.get("gcLastConsume")); - loadConditionDTO.setGcTotalTime((String) map.get("gcTotalTime")); - loadConditionDTOs.add(loadConditionDTO); - }); - } else { - JobDetailsVo.LoadConditionDTO loadConditionDTO = new JobDetailsVo.LoadConditionDTO(); - loadConditionDTO.setType("sparkAppMaster"); - loadConditionDTO.setHost(""); - loadConditionDTO.setMemory(""); - loadConditionDTO.setTotalMemory(""); - loadConditionDTO.setGcLastTime(""); - loadConditionDTO.setGcLastConsume(""); - loadConditionDTO.setGcTotalTime(""); - loadConditionDTOs.add(loadConditionDTO); - } - JobDetailsVo.RealTimeTrafficDTO realTimeTrafficDTO = new JobDetailsVo.RealTimeTrafficDTO(); - List> batchMetrics = (List>) metricsMap.get("batchMetrics"); - if(batchMetrics != null && !batchMetrics.isEmpty()) { - batchMetrics.stream().max(Comparator.comparing(map -> String.valueOf(map.get("batchTime")))).ifPresent(batchMetric -> { - realTimeTrafficDTO.setSourceKey((String) metricsMap.getOrDefault("source", "")); - realTimeTrafficDTO.setSourceSpeed(batchMetric.get("inputRecords") + " Records"); - realTimeTrafficDTO.setTransformKey("processing"); - realTimeTrafficDTO.setSinkKey((String) metricsMap.getOrDefault("sink", "")); - String totalDelay = ""; - if(batchMetric.containsKey("totalDelay") && batchMetric.get("totalDelay") != null) { - totalDelay = Utils.msDurationToString((long) batchMetric.get("totalDelay")) + " totalDelay"; - } else if(batchMetric.containsKey("taskExecuteTime") && batchMetric.get("taskExecuteTime") != null) { - totalDelay = Utils.msDurationToString((long) batchMetric.get("taskExecuteTime")) + " executeTime(Last Batch)"; - } - realTimeTrafficDTO.setSinkSpeed(totalDelay); - }); - } else { - realTimeTrafficDTO.setSourceKey(""); - realTimeTrafficDTO.setSourceSpeed(" Records/S"); - realTimeTrafficDTO.setTransformKey(""); - realTimeTrafficDTO.setSinkKey(""); - realTimeTrafficDTO.setSinkSpeed(" Records/S"); - } - realTimeTrafficDTOS.add(realTimeTrafficDTO); - } - jobDetailsVO.setDataNumber(dataNumberDTOS); - jobDetailsVO.setLoadCondition(loadConditionDTOs); - jobDetailsVO.setRealTimeTraffic(realTimeTrafficDTOS); - jobDetailsVO.setLinkisJobInfo(flinkJobInfo); - return Message.ok().data("details", jobDetailsVO); + return Message.ok().data("details", streamTaskService.getJobDetailsVO(streamJob, version)); } @RequestMapping(path = "/execute/history", method = RequestMethod.GET) From fbb875442a6a2ebe56188080a8067668e27fc54e Mon Sep 17 00:00:00 2001 From: davidhua Date: Fri, 21 Oct 2022 13:41:36 +0800 Subject: [PATCH 017/240] Add config autowired interface to set the log appender params. --- .../collector/StreamisConfigAutowired.java | 15 +++++++++ .../log/collector/StreamisRpcLogAppender.java | 22 +++++++------ .../config/StreamisLogAppenderConfig.java | 33 ++++++++++++------- .../StreamisLogAppenderConfigBuilder.java | 27 --------------- .../src/test/resources/log4j2.xml | 4 +-- .../log/server/storage/JobLogStorage.java | 4 +-- 6 files changed, 53 insertions(+), 52 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisConfigAutowired.java delete mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisConfigAutowired.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisConfigAutowired.java new file mode 100644 index 000000000..25ccfa7d0 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisConfigAutowired.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; + +/** + * Streamis config autowired + */ +public interface StreamisConfigAutowired { + + /** + * Log appender config + * @param builder builder + */ + StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Builder builder); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java index 737885e1a..1b66ef6ea 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java @@ -3,7 +3,6 @@ import com.webank.wedatasphere.streamis.jobmanager.log.collector.cache.LogCache; import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; -import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfigBuilder; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.StreamisRpcLogSender; import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvent; import org.apache.logging.log4j.core.Filter; @@ -18,12 +17,10 @@ import org.apache.logging.log4j.core.layout.PatternLayout; import java.io.Serializable; -import java.nio.Buffer; import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.ServiceLoader; -import java.util.concurrent.TimeUnit; /** * Streamis rpc log appender @@ -84,16 +81,21 @@ public static StreamisRpcLogAppender createAppender(@PluginAttribute("name") Str if (Objects.isNull(layout)){ layout = PatternLayout.createDefaultLayout(); } - // Search the config builder - List configBuilders = new ArrayList<>(); + // Search the config autowired class + List configAutowiredEntities = new ArrayList<>(); StreamisLogAppenderConfig logAppenderConfig = null; - ServiceLoader.load(StreamisLogAppenderConfigBuilder.class, - StreamisRpcLogAppender.class.getClassLoader()).iterator().forEachRemaining(configBuilders::add); - if (!configBuilders.isEmpty()){ - logAppenderConfig = configBuilders.get(0).build(applicationName, filter, rpcLogSenderConfig); + ServiceLoader.load(StreamisConfigAutowired.class, + StreamisRpcLogAppender.class.getClassLoader()).iterator().forEachRemaining(configAutowiredEntities::add); + StreamisLogAppenderConfig.Builder builder = new StreamisLogAppenderConfig.Builder(applicationName, filter, rpcLogSenderConfig); + for (StreamisConfigAutowired autowired : configAutowiredEntities){ + logAppenderConfig = autowired.logAppenderConfig(builder); } if (Objects.isNull(logAppenderConfig)){ - logAppenderConfig = new StreamisLogAppenderConfig(applicationName, filter, rpcLogSenderConfig); + logAppenderConfig = builder.build(); + } + applicationName = logAppenderConfig.getApplicationName(); + if (null == applicationName || applicationName.trim().equals("")){ + throw new IllegalArgumentException("Application name cannot be empty"); } return new StreamisRpcLogAppender(name, filter, layout, ignoreExceptions, Property.EMPTY_ARRAY, logAppenderConfig); } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java index 40a6a7d0d..14792664e 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java @@ -23,27 +23,38 @@ public StreamisLogAppenderConfig(String applicationName, Filter filter, this.senderConfig = null != rpcLogSenderConfig? rpcLogSenderConfig : new RpcLogSenderConfig(); } + public static class Builder{ + private String applicationName; + + private Filter filter; + + private RpcLogSenderConfig rpcLogSenderConfig; + + public Builder(String applicationName, Filter filter, + RpcLogSenderConfig rpcLogSenderConfig){ + this.applicationName = applicationName; + this.filter = filter; + this.rpcLogSenderConfig = rpcLogSenderConfig; + } + + StreamisLogAppenderConfig.Builder setAppName(){ + return null; + } + + public StreamisLogAppenderConfig build(){ + return null; + } + } public String getApplicationName() { return applicationName; } - public void setApplicationName(String applicationName) { - this.applicationName = applicationName; - } - public Filter getFilter() { return filter; } - public void setFilter(Filter filter) { - this.filter = filter; - } - public RpcLogSenderConfig getSenderConfig() { return senderConfig; } - public void setSenderConfig(RpcLogSenderConfig senderConfig) { - this.senderConfig = senderConfig; - } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java deleted file mode 100644 index f3374ac48..000000000 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfigBuilder.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; - -import org.apache.logging.log4j.core.Filter; - -import java.util.Map; - -/** - * Config builder for appender - */ -public abstract class StreamisLogAppenderConfigBuilder { - - - /** - * Build method - * @param applicationName application name - * @param filter filter - * @param senderConfig sender config - * @return - */ - public StreamisLogAppenderConfig build(String applicationName, - Filter filter, RpcLogSenderConfig senderConfig){ - return null; - } - - - public abstract Map loadConfigProps(); -} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml index d806c5198..56a3bba82 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml @@ -20,8 +20,8 @@ - + diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java index e951a1e7a..66afa1a2b 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java @@ -10,9 +10,9 @@ public interface JobLogStorage { /** * Create buckets - * @param jobName job name + * @param appName application name * @param bucketConfig bucket config * @return config */ - JobLogBucket getOrCreateBucket(String jobName, JobLogBucketConfig bucketConfig); + JobLogBucket getOrCreateBucket(String appName, JobLogBucketConfig bucketConfig); } From 90f0748ae3eb9fe54d44425be994021772e161db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?v=5Fboweicai=28=E8=94=A1=E5=8D=9A=E4=BC=9F=29?= Date: Fri, 21 Oct 2022 14:30:15 +0800 Subject: [PATCH 018/240] =?UTF-8?q?fix:=20=E5=8E=86=E5=8F=B2=E6=97=A5?= =?UTF-8?q?=E5=BF=97=EF=BC=8C=E5=85=B3=E9=97=AD=E5=BC=B9=E7=AA=97=E6=97=B6?= =?UTF-8?q?=E9=87=8D=E7=BD=AEendline=E6=95=B0=E6=8D=AE=EF=BC=8C=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E6=97=A5=E5=BF=97=E5=88=86=E9=A1=B5bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/src/apps/streamis/module/logDetail/index.vue | 1 + 1 file changed, 1 insertion(+) diff --git a/web/src/apps/streamis/module/logDetail/index.vue b/web/src/apps/streamis/module/logDetail/index.vue index 17f3a64da..24cce2725 100644 --- a/web/src/apps/streamis/module/logDetail/index.vue +++ b/web/src/apps/streamis/module/logDetail/index.vue @@ -151,6 +151,7 @@ export default { }, cancel() { this.fromLine = 1 + this.endLine = 0 this.spinShow = false this.query = { ignoreKeywords: '', From 5e74ed31a97810460181bc007a0efc5512cb7f81 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Fri, 21 Oct 2022 14:59:12 +0800 Subject: [PATCH 019/240] fix the compilation bug. --- .../manager/transform/parser/SparkTaskMetricsParser.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala index 392986b53..ebe8fa9c5 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala @@ -22,7 +22,7 @@ class SparkTaskMetricsParser extends AbstractTaskMetricsParser { dataNumberDTOS: util.List[JobDetailsVo.DataNumberDTO], loadConditionDTOs: util.List[JobDetailsVo.LoadConditionDTO], realTimeTrafficDTOS: util.List[JobDetailsVo.RealTimeTrafficDTO]): Unit = { - val addDataNumberDTO: String => () = key => { + val addDataNumberDTO: String => Unit = key => { val batch = new JobDetailsVo.DataNumberDTO batch.setDataName(key) batch.setDataNumber(metricsMap.get(key).toString.toInt) From 07c7f8bc127a0e0e784de9c7d480ee72c562a42f Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Fri, 21 Oct 2022 20:54:02 +0800 Subject: [PATCH 020/240] fix the long to int parse bug. --- .../manager/transform/parser/SparkTaskMetricsParser.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala index ebe8fa9c5..a8e6bb7c3 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala @@ -64,9 +64,9 @@ class SparkTaskMetricsParser extends AbstractTaskMetricsParser { realTimeTrafficDTO.setTransformKey("processing") realTimeTrafficDTO.setSinkKey(metricsMap.getOrDefault("sink", "").asInstanceOf[String]) val sinkSpeed = if (batchMetric.containsKey("totalDelay") && batchMetric.get("totalDelay") != null) - Utils.msDurationToString(batchMetric.get("totalDelay").asInstanceOf[Long]) + " totalDelay" + Utils.msDurationToString(batchMetric.get("totalDelay").toString.toInt) + " totalDelay" else if (batchMetric.containsKey("taskExecuteTime") && batchMetric.get("taskExecuteTime") != null) - Utils.msDurationToString(batchMetric.get("taskExecuteTime").asInstanceOf[Long]) + " executeTime(Last Batch)" + Utils.msDurationToString(batchMetric.get("taskExecuteTime").toString.toInt) + " executeTime(Last Batch)" else "" realTimeTrafficDTO.setSinkSpeed(sinkSpeed) case _ => From 00f0056c0486369f6ad0f735ee7d0ee2ee52727e Mon Sep 17 00:00:00 2001 From: davidhua Date: Sat, 22 Oct 2022 11:24:26 +0800 Subject: [PATCH 021/240] Complete the log collector (100%) --- .../log/collector/StreamisRpcLogAppender.java | 8 +- .../config/StreamisLogAppenderConfig.java | 203 +++++++++++++++++- .../sender/http/AbstractHttpLogSender.java | 19 +- 3 files changed, 206 insertions(+), 24 deletions(-) diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java index 1b66ef6ea..e086ad17f 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java @@ -33,17 +33,17 @@ public class StreamisRpcLogAppender extends AbstractAppender { /** * Appender config */ - private StreamisLogAppenderConfig appenderConfig; + private final StreamisLogAppenderConfig appenderConfig; /** * Rpc log sender */ - private StreamisRpcLogSender rpcLogSender; + private final StreamisRpcLogSender rpcLogSender; /** * Cache */ - private LogCache logCache; + private final LogCache logCache; protected StreamisRpcLogAppender(String name, Filter filter, Layout layout, boolean ignoreExceptions, Property[] properties, @@ -53,7 +53,7 @@ protected StreamisRpcLogAppender(String name, Filter filter, this.rpcLogSender = new StreamisRpcLogSender(this.appenderConfig.getApplicationName(), this.appenderConfig.getSenderConfig()); this.logCache = this.rpcLogSender.getOrCreateLogCache(); - Runtime.getRuntime().addShutdownHook(new Thread(() -> this.rpcLogSender.close())); + Runtime.getRuntime().addShutdownHook(new Thread(this.rpcLogSender::close)); } @Override diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java index 14792664e..4f60e0a61 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java @@ -1,22 +1,27 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; import org.apache.logging.log4j.core.Filter; +import org.apache.logging.log4j.core.filter.CompositeFilter; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; /** * Appender config */ public class StreamisLogAppenderConfig { - private String applicationName; + private final String applicationName; /** * Filter in log4j */ - private Filter filter; + private final Filter filter; - private RpcLogSenderConfig senderConfig; + private final RpcLogSenderConfig senderConfig; - public StreamisLogAppenderConfig(String applicationName, Filter filter, + StreamisLogAppenderConfig(String applicationName, Filter filter, RpcLogSenderConfig rpcLogSenderConfig){ this.applicationName = applicationName; this.filter = filter; @@ -24,26 +29,202 @@ public StreamisLogAppenderConfig(String applicationName, Filter filter, } public static class Builder{ + /** + * Application name + */ private String applicationName; - private Filter filter; + /** + * Filter rules + */ + private final List filters = new ArrayList<>(); - private RpcLogSenderConfig rpcLogSenderConfig; + /** + * Sender config + */ + private final RpcLogSenderConfig rpcLogSenderConfig; public Builder(String applicationName, Filter filter, RpcLogSenderConfig rpcLogSenderConfig){ this.applicationName = applicationName; - this.filter = filter; - this.rpcLogSenderConfig = rpcLogSenderConfig; + this.filters.add(filter); + this.rpcLogSenderConfig = Optional.ofNullable(rpcLogSenderConfig).orElse(new RpcLogSenderConfig()); + } + + /** + * Set application name + * @param applicationName application name + * @return builder + */ + StreamisLogAppenderConfig.Builder setAppName(String applicationName){ + this.applicationName = applicationName; + return this; + } + + /** + * Set filter + * @param filter filter + * @return builder + */ + StreamisLogAppenderConfig.Builder setFilter(Filter filter){ + this.filters.clear(); + this.filters.add(filter); + return this; + } + + /** + * Append filter + * @param filter filter + * @return builder + */ + StreamisLogAppenderConfig.Builder withFilter(Filter filter){ + filters.add(filter); + return this; + } + + /** + * Rpc connect timeout + * @param connectionTimeout connection timeout + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcConnTimeout(int connectionTimeout){ + this.rpcLogSenderConfig.setConnectionTimeout(connectionTimeout); + return this; + } + + /** + * Rpc socket timeout + * @param socketTimeout socket timeout + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcSocketTimeout(int socketTimeout){ + this.rpcLogSenderConfig.setSocketTimeout(socketTimeout); + return this; + } + + /** + * Rpc send retry count + * @param sendRetryCnt send retry count + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcSendRetryCnt(int sendRetryCnt){ + this.rpcLogSenderConfig.setSendRetryCnt(sendRetryCnt); + return this; + } + + /** + * Rpc server recovery time in seconds + * @param serverRecoveryTimeInSec server recovery time + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcServerRecoveryTimeInSec(int serverRecoveryTimeInSec){ + this.rpcLogSenderConfig.setServerRecoveryTimeInSec(serverRecoveryTimeInSec); + return this; } - StreamisLogAppenderConfig.Builder setAppName(){ - return null; + /** + * Rpc max delay time in seconds + * @param maxDelayTimeInSec max delay time in seconds + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcMaxDelayTimeInSec(int maxDelayTimeInSec){ + this.rpcLogSenderConfig.setMaxDelayTimeInSec(maxDelayTimeInSec); + return this; } + /** + * Rpc auth token code key + * @param tokenCodeKey key of token code + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcAuthTokenCodeKey(String tokenCodeKey){ + this.rpcLogSenderConfig.getAuthConfig().setTokenCodeKey(tokenCodeKey); + return this; + } + + /** + * Rpc auth token user key + * @param tokenUserKey key of token user + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcAuthTokenUserKey(String tokenUserKey){ + this.rpcLogSenderConfig.getAuthConfig().setTokenUserKey(tokenUserKey); + return this; + } + + /** + * Rpc auth token user + * @param tokenUser token user + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcAuthTokenUser(String tokenUser){ + this.rpcLogSenderConfig.getAuthConfig().setTokenUser(tokenUser); + return this; + } + + /** + * Rpc auth token code + * @param tokenCode token code + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcAuthTokenCode(String tokenCode){ + this.rpcLogSenderConfig.getAuthConfig().setTokenCode(tokenCode); + return this; + } + + /** + * Rpc cache size + * @param cacheSize cache size + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcCacheSize(int cacheSize){ + this.rpcLogSenderConfig.getCacheConfig().setSize(cacheSize); + return this; + } + + /** + * Rpc cache max consume thread + * @param maxConsumeThread max consume thread + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcCacheMaxConsumeThread(int maxConsumeThread){ + this.rpcLogSenderConfig.getCacheConfig().setMaxConsumeThread(maxConsumeThread); + return this; + } + + /** + * Rpc buffer size + * @param bufferSize buffer size + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcBufferSize(int bufferSize){ + this.rpcLogSenderConfig.getBufferConfig().setSize(bufferSize); + return this; + } + + /** + * Rpc buffer expire time in seconds + * @param expireTimeInSec expire time + * @return builder + */ + StreamisLogAppenderConfig.Builder setRpcBufferExpireTimeInSec(int expireTimeInSec){ + this.rpcLogSenderConfig.getBufferConfig().setExpireTimeInSec(expireTimeInSec); + return this; + } + /** + * Build method + * @return config + */ public StreamisLogAppenderConfig build(){ - return null; + Filter logFilter = null; + if (filters.size() > 1){ + logFilter = CompositeFilter.createFilters(filters.toArray(new Filter[0])); + } else if (!filters.isEmpty()){ + logFilter = filters.get(0); + } + return new StreamisLogAppenderConfig(applicationName, logFilter, rpcLogSenderConfig); } + } public String getApplicationName() { return applicationName; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java index d247ba4ad..a0385ecb4 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java @@ -2,7 +2,6 @@ import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcAuthConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.AbstractRpcLogSender; -import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.RpcLogSender; import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.SendLogExceptionStrategy; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; @@ -99,14 +98,16 @@ protected void doSend(E aggregatedEntity, RpcLogSenderConfig rpcSenderConfig) th return; } } - - EntityPostAction postAction = new EntityPostAction<>(rpcSenderConfig.getAddress(), aggregatedEntity); - RpcAuthConfig authConfig = rpcSenderConfig.getAuthConfig(); - postAction.getRequestHeaders().put(authConfig.getTokenUserKey(), authConfig.getTokenUser()); - // Ignore the response - postAction.execute(this.globalHttpClient); - // Init the counter - this.exceptionCounter.set(0); + String address = rpcSenderConfig.getAddress(); + if (null != address && !address.trim().equals("")) { + EntityPostAction postAction = new EntityPostAction<>(rpcSenderConfig.getAddress(), aggregatedEntity); + RpcAuthConfig authConfig = rpcSenderConfig.getAuthConfig(); + postAction.getRequestHeaders().put(authConfig.getTokenUserKey(), authConfig.getTokenUser()); + // Ignore the response + postAction.execute(this.globalHttpClient); + // Init the counter + this.exceptionCounter.set(0); + } } } } From 26d24d2698accc6d426bf7347c29c351688d13ec Mon Sep 17 00:00:00 2001 From: davidhua Date: Sat, 22 Oct 2022 21:12:59 +0800 Subject: [PATCH 022/240] Add Flink log collector and the transform in streamis which resolves the internal configuration --- pom.xml | 1 + .../flink-streamis-log-collector/pom.xml | 67 ++++++++++ .../src/main/assembly/package.xml | 19 +++ .../flink/FlinkStreamisConfigAutowired.java | 104 +++++++++++++++ .../flink/FlinkStreamisConfigDefine.java | 120 ++++++++++++++++++ ....jobmanager.plugin.StreamisConfigAutowired | 1 + .../flink/FlinkConfigurationLoadTest.java | 25 ++++ .../src/test/resources/flink-conf.yaml | 0 .../streamis-job-log-collector/pom.xml | 9 +- .../log/collector/StreamisRpcLogAppender.java | 9 +- .../config/StreamisLogAppenderConfig.java | 42 +++--- .../sender/SendLogCacheConsumer.java | 1 + .../log/collector/sender/http/Json.java | 18 +++ .../StreamisConfigAutowired.java | 4 +- .../collector/StreamisLogAppenderTest.java | 22 +++- .../log/entities/StreamisLogEvents.java | 16 ++- .../jobmanager/manager/conf/JobConf.scala | 17 ++- .../AbstractStreamisTransformJobBuilder.scala | 18 ++- .../impl/FlinkInternalConfigTransform.scala | 42 ++++++ 19 files changed, 501 insertions(+), 34 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/assembly/package.xml create mode 100644 streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java create mode 100644 streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java create mode 100644 streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired create mode 100644 streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java create mode 100644 streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/resources/flink-conf.yaml rename streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/{log/collector => plugin}/StreamisConfigAutowired.java (75%) create mode 100644 streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala diff --git a/pom.xml b/pom.xml index bd535bf6f..580cce56f 100644 --- a/pom.xml +++ b/pom.xml @@ -46,6 +46,7 @@ 1.1.3 + 4.12 1.1.0 0.2.0 2.11.12 diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/pom.xml index 5207e035a..6bbee8ba8 100644 --- a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/pom.xml @@ -16,13 +16,80 @@ 8 1.12.2 + 2.17.1 + 1.7.15 + + com.webank.wedatasphere.streamis + streamis-job-log-collector + ${streamis.version} + org.apache.flink flink-java ${flink.version} + provided + + + org.apache.flink + flink-yarn_2.11 + ${flink.version} + provided + + + + junit + junit + ${junit.version} + test + + + + org.slf4j + slf4j-api + ${slf4j.version} + provided + + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + provided + + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + provided + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + + + assemble + + single + + + install + + + + + src/main/assembly/package.xml + + false + + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/assembly/package.xml b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/assembly/package.xml new file mode 100644 index 000000000..8da27bf2c --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/assembly/package.xml @@ -0,0 +1,19 @@ + + + package + + + jar + + false + + + / + true + runtime + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java new file mode 100644 index 000000000..a862d080f --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java @@ -0,0 +1,104 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.flink; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.GlobalConfiguration; +import org.apache.flink.yarn.configuration.YarnConfigOptions; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.Filter; +import org.apache.logging.log4j.core.filter.LevelMatchFilter; +import org.apache.logging.log4j.core.filter.RegexFilter; + +import java.util.Enumeration; +import java.util.List; +import java.util.Properties; + +import static com.webank.wedatasphere.streamis.jobmanager.log.collector.flink.FlinkStreamisConfigDefine.*; + +/** + * Autoconfigure the streamis config inf Flink environment + */ +public class FlinkStreamisConfigAutowired implements StreamisConfigAutowired { + + /** + * Flink configuration + */ + private Configuration configuration; + + public FlinkStreamisConfigAutowired(){ + // First to load configuration + this.configuration = loadConfiguration(); + } + @Override + public StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Builder builder) throws Exception{ + String applicationName = this.configuration.getString(YarnConfigOptions.APPLICATION_NAME); + if (StringUtils.isNotBlank(applicationName)){ + builder.setAppName(applicationName); + } + String gateway = this.configuration.getString(LOG_GATEWAY_ADDRESS); + if (StringUtils.isNotBlank(gateway)){ + if (gateway.endsWith("/")){ + gateway = gateway.substring(0, gateway.length() - 1); + } + gateway += this.configuration.getString(LOG_COLLECT_PATH, "/"); + builder.setRpcAddress(gateway); + } + List filterStrategies = this.configuration.get(LOG_FILTER_STRATEGIES); + for(String filterStrategy : filterStrategies){ + if ("LevelMatch".equals(filterStrategy)){ + builder.withFilter(LevelMatchFilter.newBuilder() + .setLevel(Level.getLevel(this.configuration.getString(LOG_FILTER_LEVEL_MATCH))).build()); + } else if ("RegexMatch".equals(filterStrategy)){ + builder.withFilter(RegexFilter.createFilter( this.configuration.getString(LOG_FILTER_REGEX), + null, true, Filter.Result.ACCEPT, Filter.Result.DENY)); + } + } + return builder.setRpcConnTimeout(this.configuration.getInteger(LOG_RPC_CONN_TIMEOUT)) + .setRpcSocketTimeout(this.configuration.getInteger(LOG_RPC_SOCKET_TIMEOUT)) + .setRpcSendRetryCnt(this.configuration.getInteger(LOG_RPC_SEND_RETRY_COUNT)) + .setRpcServerRecoveryTimeInSec(this.configuration.getInteger(LOG_RPC_SERVER_RECOVERY_TIME)) + .setRpcMaxDelayTimeInSec(this.configuration.getInteger(LOG_RPC_MAX_DELAY_TIME)) + .setRpcAuthTokenCodeKey(this.configuration.getString(LOG_RPC_AUTH_TOKEN_CODE_KEY)) + .setRpcAuthTokenUserKey(this.configuration.getString(LOG_RPC_AUTH_TOKEN_USER_KEY)) + .setRpcAuthTokenCode(this.configuration.getString(LOG_RPC_AUTH_TOKEN_CODE)) + .setRpcAuthTokenUser(this.configuration.getString(LOG_RPC_AUTH_TOKEN_USER)) + .setRpcCacheSize(this.configuration.getInteger(LOG_RPC_CACHE_SIZE)) + .setRpcCacheMaxConsumeThread(this.configuration.getInteger(LOG_PRC_CACHE_MAX_CONSUME_THREAD)) + .setRpcBufferSize(this.configuration.getInteger(LOG_RPC_BUFFER_SIZE)) + .setRpcBufferExpireTimeInSec(this.configuration.getInteger(LOG_RPC_BUFFER_EXPIRE_TIME)).build(); + } + + /** + * According to : + * String launchCommand = + * BootstrapTools.getTaskManagerShellCommand( + * flinkConfig, + * tmParams, + * ".", + * ApplicationConstants.LOG_DIR_EXPANSION_VAR, + * hasLogback, + * hasLog4j, + * hasKrb5, + * taskManagerMainClass, + * taskManagerDynamicProperties); + * the configuration directory of Flink yarn container is always ".", + * @return configuration + */ + private Configuration loadConfiguration(){ + String configDir = System.getenv("FLINK_CONF_DIR"); + if (null == configDir){ + configDir = "."; + } + Properties properties = System.getProperties(); + Enumeration enumeration = properties.propertyNames(); + Configuration dynamicConfiguration = new Configuration(); + while(enumeration.hasMoreElements()){ + String prop = String.valueOf(enumeration.nextElement()); + dynamicConfiguration.setString(prop, properties.getProperty(prop)); + } + return GlobalConfiguration.loadConfiguration(configDir, dynamicConfiguration); + } + +} diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java new file mode 100644 index 000000000..d40834fa5 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java @@ -0,0 +1,120 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.flink; + +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; + +import java.util.List; + +/** + * Config definition + */ +public class FlinkStreamisConfigDefine { + + /** + * Gateway address of log module for streamis + */ + public static final ConfigOption LOG_GATEWAY_ADDRESS = ConfigOptions.key("stream.log.gateway.address") + .stringType().noDefaultValue().withDescription("The gateway address ex: http://127.0.0.1:8080"); + + /** + * Entrypoint path of collecting log + */ + public static final ConfigOption LOG_COLLECT_PATH = ConfigOptions.key("stream.log.collect.path") + .stringType().defaultValue("/api/rest_j/v1/streamis/streamJobManager/log/collect/events").withDescription("The entrypoint path of collecting log"); + + /** + * Connection timeout(in milliseconds) in log RPC module + */ + public static final ConfigOption LOG_RPC_CONN_TIMEOUT = ConfigOptions.key("stream.log.rpc.connect-timeout") + .intType().defaultValue(3000).withDescription("Connection timeout(ms) in log RPC module"); + + /** + * Socket timeout(in milliseconds) in log RPC module + */ + public static final ConfigOption LOG_RPC_SOCKET_TIMEOUT = ConfigOptions.key("stream.log.rpc.socket-timeout") + .intType().defaultValue(15000).withDescription("Socket timeout(ms) in log RPC module"); + + /** + * Max retry count of sending message in log RPC module + */ + public static final ConfigOption LOG_RPC_SEND_RETRY_COUNT = ConfigOptions.key("stream.log.rpc.send-retry-count") + .intType().defaultValue(3).withDescription("Max retry count of sending message in log RPC module"); + + /** + * Server recovery time(in seconds) in log RPC module + */ + public static final ConfigOption LOG_RPC_SERVER_RECOVERY_TIME = ConfigOptions.key("stream.log.rpc.server-recovery-time-in-sec") + .intType().defaultValue(5).withDescription("Server recovery time(sec) in log RPC module"); + + /** + * Max delay time(in seconds) in log RPC module. if reach the limit, the message will be dropped + */ + public static final ConfigOption LOG_RPC_MAX_DELAY_TIME = ConfigOptions.key("stream.log.rpc.max-delay-time") + .intType().defaultValue(60).withDescription("Max delay time(sec) in log RPC module"); + + /** + * Token code key in log RPC auth module + */ + public static final ConfigOption LOG_RPC_AUTH_TOKEN_CODE_KEY = ConfigOptions.key("stream.log.rpc.auth.token-code-key") + .stringType().defaultValue("Token-Code").withDescription("Token code key in log RPC auth module"); + + /** + * Token user key in log RPC auth module + */ + public static final ConfigOption LOG_RPC_AUTH_TOKEN_USER_KEY = ConfigOptions.key("stream.log.rpc.auth.token-user-key") + .stringType().defaultValue("Token-User").withDescription("Token user key in log RPC auth module"); + + /** + * Token code in log RPC auth module + */ + public static final ConfigOption LOG_RPC_AUTH_TOKEN_CODE = ConfigOptions.key("stream.log.rpc.auth.token-code") + .stringType().defaultValue("STREAM-LOG").withDescription("Token code in log RPC auth module"); + + /** + * Token user in log RPC auth module + */ + public static final ConfigOption LOG_RPC_AUTH_TOKEN_USER = ConfigOptions.key("stream.log.rpc.auth.token-user") + .stringType().defaultValue(System.getProperty("user.name")).withDescription("Token user in log RPC auth module"); + + /** + * Cache size in log RPC module + */ + public static final ConfigOption LOG_RPC_CACHE_SIZE = ConfigOptions.key("stream.log.rpc.cache.size") + .intType().defaultValue(150).withDescription("Cache size in log RPC module"); + + /** + * Max cache consume threads in log RPC module + */ + public static final ConfigOption LOG_PRC_CACHE_MAX_CONSUME_THREAD = ConfigOptions.key("stream.log.rpc.cache.max-consume-thread") + .intType().defaultValue(10).withDescription("Max cache consume threads in log RPC module"); + + /** + * Buffer size in log RPC module + */ + public static final ConfigOption LOG_RPC_BUFFER_SIZE = ConfigOptions.key("stream.log.rpc.buffer.size") + .intType().defaultValue(50).withDescription("Buffer size in log RPC module"); + + /** + * Buffer expire time(sec) in log RPC module + */ + public static final ConfigOption LOG_RPC_BUFFER_EXPIRE_TIME = ConfigOptions.key("stream.log.rpc.buffer.expire-time-in-sec") + .intType().defaultValue(2).withDescription("Buffer expire time (sec) in log RPC module"); + + /** + * Log filter strategy list + */ + public static final ConfigOption> LOG_FILTER_STRATEGIES = ConfigOptions.key("stream.log.filter.strategies") + .stringType().asList().defaultValues("LevelMatch").withDescription("Log filter strategy list"); + + /** + * Level value of LevelMatch filter strategy + */ + public static final ConfigOption LOG_FILTER_LEVEL_MATCH = ConfigOptions.key("stream.log.filter.level-match.level") + .stringType().defaultValue("ERROR").withDescription("Level value of LevelMatch filter strategy"); + + /** + * Regex value of RegexMatch filter strategy + */ + public static final ConfigOption LOG_FILTER_REGEX = ConfigOptions.key("stream.log.filter.regex.value") + .stringType().defaultValue(".*").withDescription("Regex value of RegexMatch filter strategy"); +} diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired new file mode 100644 index 000000000..dc13253b7 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired @@ -0,0 +1 @@ +com.webank.wedatasphere.streamis.jobmanager.log.collector.flink.FlinkStreamisConfigAutowired \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java new file mode 100644 index 000000000..d04988099 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java @@ -0,0 +1,25 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.flink; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.GlobalConfiguration; +import org.junit.Test; + +import java.io.File; +import java.util.Enumeration; +import java.util.Objects; +import java.util.Properties; + +public class FlinkConfigurationLoadTest { + @Test + public void loadConfiguration() { + String configDir = Objects.requireNonNull(FlinkConfigurationLoadTest.class.getResource("/")).getFile(); + Properties properties = System.getProperties(); + Enumeration enumeration = properties.propertyNames(); + Configuration dynamicConfiguration = new Configuration(); + while(enumeration.hasMoreElements()){ + String prop = String.valueOf(enumeration.nextElement()); + dynamicConfiguration.setString(prop, properties.getProperty(prop)); + } + GlobalConfiguration.loadConfiguration(configDir, dynamicConfiguration); + } +} diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/resources/flink-conf.yaml b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/resources/flink-conf.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml index 5fd394795..5c4822539 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml @@ -24,7 +24,7 @@ com.webank.wedatasphere.streamis streamis-job-log-common - 0.2.0 + ${streamis.version} @@ -65,5 +65,12 @@ ${log4j.version} provided + + + junit + junit + ${junit.version} + test + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java index e086ad17f..5b30c36ce 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java @@ -4,7 +4,9 @@ import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.StreamisRpcLogSender; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.Json; import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvent; +import com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired; import org.apache.logging.log4j.core.Filter; import org.apache.logging.log4j.core.Layout; import org.apache.logging.log4j.core.LogEvent; @@ -27,9 +29,7 @@ */ @Plugin(name = "StreamRpcLog", category = "Core", elementType = "appender", printObject = true) public class StreamisRpcLogAppender extends AbstractAppender { - private static final String DEFAULT_APPENDER_NAME = "StreamRpcLog"; - /** * Appender config */ @@ -52,6 +52,8 @@ protected StreamisRpcLogAppender(String name, Filter filter, this.appenderConfig = appenderConfig; this.rpcLogSender = new StreamisRpcLogSender(this.appenderConfig.getApplicationName(), this.appenderConfig.getSenderConfig()); + this.rpcLogSender.setExceptionListener((subject, t, message) -> + LOGGER.error((null != subject? subject.getClass().getSimpleName() : "") + ": " + message, t)); this.logCache = this.rpcLogSender.getOrCreateLogCache(); Runtime.getRuntime().addShutdownHook(new Thread(this.rpcLogSender::close)); } @@ -74,7 +76,7 @@ public static StreamisRpcLogAppender createAppender(@PluginAttribute("name") Str @PluginAttribute("ignoreExceptions") boolean ignoreExceptions, @PluginElement("Filter") final Filter filter, @PluginElement("Layout") Layout layout, - @PluginElement("RpcLogSender")RpcLogSenderConfig rpcLogSenderConfig){ + @PluginElement("RpcLogSender")RpcLogSenderConfig rpcLogSenderConfig) throws Exception{ if (null == name || name.trim().equals("")){ name = DEFAULT_APPENDER_NAME; } @@ -97,6 +99,7 @@ public static StreamisRpcLogAppender createAppender(@PluginAttribute("name") Str if (null == applicationName || applicationName.trim().equals("")){ throw new IllegalArgumentException("Application name cannot be empty"); } + LOGGER.info("StreamisRpcLogAppender: init with config {}", Json.toJson(logAppenderConfig, null)); return new StreamisRpcLogAppender(name, filter, layout, ignoreExceptions, Property.EMPTY_ARRAY, logAppenderConfig); } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java index 4f60e0a61..0253838ed 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java @@ -56,7 +56,7 @@ public Builder(String applicationName, Filter filter, * @param applicationName application name * @return builder */ - StreamisLogAppenderConfig.Builder setAppName(String applicationName){ + public StreamisLogAppenderConfig.Builder setAppName(String applicationName){ this.applicationName = applicationName; return this; } @@ -66,7 +66,7 @@ StreamisLogAppenderConfig.Builder setAppName(String applicationName){ * @param filter filter * @return builder */ - StreamisLogAppenderConfig.Builder setFilter(Filter filter){ + public StreamisLogAppenderConfig.Builder setFilter(Filter filter){ this.filters.clear(); this.filters.add(filter); return this; @@ -77,17 +77,27 @@ StreamisLogAppenderConfig.Builder setFilter(Filter filter){ * @param filter filter * @return builder */ - StreamisLogAppenderConfig.Builder withFilter(Filter filter){ + public StreamisLogAppenderConfig.Builder withFilter(Filter filter){ filters.add(filter); return this; } + /** + * Rpc address + * @param address address + * @return builder + */ + public StreamisLogAppenderConfig.Builder setRpcAddress(String address){ + this.rpcLogSenderConfig.setAddress(address); + return this; + } + /** * Rpc connect timeout * @param connectionTimeout connection timeout * @return builder */ - StreamisLogAppenderConfig.Builder setRpcConnTimeout(int connectionTimeout){ + public StreamisLogAppenderConfig.Builder setRpcConnTimeout(int connectionTimeout){ this.rpcLogSenderConfig.setConnectionTimeout(connectionTimeout); return this; } @@ -97,7 +107,7 @@ StreamisLogAppenderConfig.Builder setRpcConnTimeout(int connectionTimeout){ * @param socketTimeout socket timeout * @return builder */ - StreamisLogAppenderConfig.Builder setRpcSocketTimeout(int socketTimeout){ + public StreamisLogAppenderConfig.Builder setRpcSocketTimeout(int socketTimeout){ this.rpcLogSenderConfig.setSocketTimeout(socketTimeout); return this; } @@ -107,7 +117,7 @@ StreamisLogAppenderConfig.Builder setRpcSocketTimeout(int socketTimeout){ * @param sendRetryCnt send retry count * @return builder */ - StreamisLogAppenderConfig.Builder setRpcSendRetryCnt(int sendRetryCnt){ + public StreamisLogAppenderConfig.Builder setRpcSendRetryCnt(int sendRetryCnt){ this.rpcLogSenderConfig.setSendRetryCnt(sendRetryCnt); return this; } @@ -117,7 +127,7 @@ StreamisLogAppenderConfig.Builder setRpcSendRetryCnt(int sendRetryCnt){ * @param serverRecoveryTimeInSec server recovery time * @return builder */ - StreamisLogAppenderConfig.Builder setRpcServerRecoveryTimeInSec(int serverRecoveryTimeInSec){ + public StreamisLogAppenderConfig.Builder setRpcServerRecoveryTimeInSec(int serverRecoveryTimeInSec){ this.rpcLogSenderConfig.setServerRecoveryTimeInSec(serverRecoveryTimeInSec); return this; } @@ -127,7 +137,7 @@ StreamisLogAppenderConfig.Builder setRpcServerRecoveryTimeInSec(int serverRecove * @param maxDelayTimeInSec max delay time in seconds * @return builder */ - StreamisLogAppenderConfig.Builder setRpcMaxDelayTimeInSec(int maxDelayTimeInSec){ + public StreamisLogAppenderConfig.Builder setRpcMaxDelayTimeInSec(int maxDelayTimeInSec){ this.rpcLogSenderConfig.setMaxDelayTimeInSec(maxDelayTimeInSec); return this; } @@ -137,7 +147,7 @@ StreamisLogAppenderConfig.Builder setRpcMaxDelayTimeInSec(int maxDelayTimeInSec) * @param tokenCodeKey key of token code * @return builder */ - StreamisLogAppenderConfig.Builder setRpcAuthTokenCodeKey(String tokenCodeKey){ + public StreamisLogAppenderConfig.Builder setRpcAuthTokenCodeKey(String tokenCodeKey){ this.rpcLogSenderConfig.getAuthConfig().setTokenCodeKey(tokenCodeKey); return this; } @@ -147,7 +157,7 @@ StreamisLogAppenderConfig.Builder setRpcAuthTokenCodeKey(String tokenCodeKey){ * @param tokenUserKey key of token user * @return builder */ - StreamisLogAppenderConfig.Builder setRpcAuthTokenUserKey(String tokenUserKey){ + public StreamisLogAppenderConfig.Builder setRpcAuthTokenUserKey(String tokenUserKey){ this.rpcLogSenderConfig.getAuthConfig().setTokenUserKey(tokenUserKey); return this; } @@ -157,7 +167,7 @@ StreamisLogAppenderConfig.Builder setRpcAuthTokenUserKey(String tokenUserKey){ * @param tokenUser token user * @return builder */ - StreamisLogAppenderConfig.Builder setRpcAuthTokenUser(String tokenUser){ + public StreamisLogAppenderConfig.Builder setRpcAuthTokenUser(String tokenUser){ this.rpcLogSenderConfig.getAuthConfig().setTokenUser(tokenUser); return this; } @@ -167,7 +177,7 @@ StreamisLogAppenderConfig.Builder setRpcAuthTokenUser(String tokenUser){ * @param tokenCode token code * @return builder */ - StreamisLogAppenderConfig.Builder setRpcAuthTokenCode(String tokenCode){ + public StreamisLogAppenderConfig.Builder setRpcAuthTokenCode(String tokenCode){ this.rpcLogSenderConfig.getAuthConfig().setTokenCode(tokenCode); return this; } @@ -177,7 +187,7 @@ StreamisLogAppenderConfig.Builder setRpcAuthTokenCode(String tokenCode){ * @param cacheSize cache size * @return builder */ - StreamisLogAppenderConfig.Builder setRpcCacheSize(int cacheSize){ + public StreamisLogAppenderConfig.Builder setRpcCacheSize(int cacheSize){ this.rpcLogSenderConfig.getCacheConfig().setSize(cacheSize); return this; } @@ -187,7 +197,7 @@ StreamisLogAppenderConfig.Builder setRpcCacheSize(int cacheSize){ * @param maxConsumeThread max consume thread * @return builder */ - StreamisLogAppenderConfig.Builder setRpcCacheMaxConsumeThread(int maxConsumeThread){ + public StreamisLogAppenderConfig.Builder setRpcCacheMaxConsumeThread(int maxConsumeThread){ this.rpcLogSenderConfig.getCacheConfig().setMaxConsumeThread(maxConsumeThread); return this; } @@ -197,7 +207,7 @@ StreamisLogAppenderConfig.Builder setRpcCacheMaxConsumeThread(int maxConsumeThre * @param bufferSize buffer size * @return builder */ - StreamisLogAppenderConfig.Builder setRpcBufferSize(int bufferSize){ + public StreamisLogAppenderConfig.Builder setRpcBufferSize(int bufferSize){ this.rpcLogSenderConfig.getBufferConfig().setSize(bufferSize); return this; } @@ -207,7 +217,7 @@ StreamisLogAppenderConfig.Builder setRpcBufferSize(int bufferSize){ * @param expireTimeInSec expire time * @return builder */ - StreamisLogAppenderConfig.Builder setRpcBufferExpireTimeInSec(int expireTimeInSec){ + public StreamisLogAppenderConfig.Builder setRpcBufferExpireTimeInSec(int expireTimeInSec){ this.rpcLogSenderConfig.getBufferConfig().setExpireTimeInSec(expireTimeInSec); return this; } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java index 52a7b72cb..fac98b90a 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java @@ -87,6 +87,7 @@ public void run() { if (this.isTerminated && e instanceof InterruptedException){ return; } else { + e.printStackTrace(); System.err.println("SendLogCacheConsumer[" + Thread.currentThread().getName() + "] occurred exception [" + e.getLocalizedMessage() + "]"); // For the unknown exception clear the cache sendBuffer.clear(); diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java index 40b20b50e..911ccb381 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java @@ -91,6 +91,24 @@ public static String toJson(Object obj, Class model) { return null; } + public static String toJson(Object obj, Class model, boolean beautify){ + ObjectWriter writer = mapper.writer(); + if(null != obj){ + try{ + if(null != model){ + writer = writer.withView(model); + } + if(beautify){ + return writer.withDefaultPrettyPrinter().writeValueAsString(obj); + } + return writer.writeValueAsString(obj); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + return null; + } + /** * Convert object using serialization and deserialization * diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisConfigAutowired.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/plugin/StreamisConfigAutowired.java similarity index 75% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisConfigAutowired.java rename to streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/plugin/StreamisConfigAutowired.java index 25ccfa7d0..980a6aec5 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisConfigAutowired.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/plugin/StreamisConfigAutowired.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.collector; +package com.webank.wedatasphere.streamis.jobmanager.plugin; import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; @@ -11,5 +11,5 @@ public interface StreamisConfigAutowired { * Log appender config * @param builder builder */ - StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Builder builder); + StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Builder builder) throws Exception; } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java index 189e573ae..d4a7e61ce 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java @@ -1,16 +1,28 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector; +import org.apache.logging.log4j.LogManager; +import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class StreamisLogAppenderTest { private static final Logger LOG = LoggerFactory.getLogger(StreamisLogAppenderTest.class); - public static void main(String[] args) throws InterruptedException { - while(true){ - for(int i = 0; i < 100; i ++){ - LOG.info("Stream Log appender test"); + @Test + public void appenderLog() throws InterruptedException { + org.apache.logging.log4j.core.Logger logger = (org.apache.logging.log4j.core.Logger) LogManager.getLogger("Test: Hello-world"); + logger.info(""); + int total = 1000; + int tps = 100; + long timer = System.currentTimeMillis() + 1000; + for(int i = 0; i < total; i ++){ + if (i > 0 && i % tps == 0){ + long sleep = timer - System.currentTimeMillis(); + if (sleep > 0){ + Thread.sleep(sleep); + } + timer = System.currentTimeMillis() + 1000; } - Thread.sleep(1000); + LOG.info("Stream Log appender test, sequence id: " + i); } } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java index 972d1a492..fa0b1c929 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java @@ -21,13 +21,19 @@ public StreamisLogEvents(String applicationName, StreamisLogEvent[] events){ this.appName = applicationName; this.events = events; long maxTime = -1; - for(StreamisLogEvent event : events){ - long time = event.getLogTimeStamp(); - if (time > maxTime){ - maxTime = time; + StreamisLogEvent lastEvent = events[events.length - 1]; + if (null == lastEvent) { + for (StreamisLogEvent event : events) { + long time = event.getLogTimeStamp(); + if (time > maxTime) { + maxTime = time; + } } + this.logTimeInMills = maxTime; + }else { + this.logTimeInMills = lastEvent.getLogTimeStamp(); } - this.logTimeInMills = maxTime; + } @Override diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala index 73697bdbb..33c4827b8 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala @@ -15,7 +15,7 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.conf -import org.apache.linkis.common.conf.{CommonVars, TimeType} +import org.apache.linkis.common.conf.{CommonVars, Configuration, TimeType} import com.webank.wedatasphere.streamis.jobmanager.manager.exception.JobExecuteErrorException @@ -29,6 +29,21 @@ object JobConf { val STREAMIS_JOB_PARAM_BLANK_PLACEHOLDER: CommonVars[String] = CommonVars("wds.streamis.job.param.blank.placeholder", "\u0001") + /** + * Config group for streamis internal configuration + */ + val STREAMIS_INTERNAL_CONFIG_GROUP: CommonVars[String] = CommonVars("wds.streamis.job.internal.config.group", "wds.streamis.internal.params") + + /** + * Gateway for stream job log module + */ + val STREAMIS_JOB_LOG_GATEWAY: CommonVars[String] = CommonVars("wds.streamis.job.log.gateway", Configuration.getGateWayURL()) + + /** + * Path for collecting stream job log + */ + val STREAMIS_JOB_LOG_COLLECT_PATH: CommonVars[String] = CommonVars("wds.streamis.job.log.collect.path", "/api/rest_j/v1/streamis/streamJobManager/log/collect/events") + val FLINK_JOB_STATUS_NOT_STARTED: CommonVars[Int] = CommonVars("wds.streamis.job.status.not-started", 0,"Not Started") val FLINK_JOB_STATUS_COMPLETED: CommonVars[Int] = CommonVars("wds.streamis.job.status.completed", 1,"Completed") diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala index 039fbd3e2..6d44dced6 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala @@ -18,12 +18,15 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.transform.builder import org.apache.linkis.common.conf.CommonVars import org.apache.linkis.manager.label.entity.engine.RunType.RunType import com.webank.wedatasphere.streamis.jobmanager.launcher.service.StreamJobConfService +import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf import com.webank.wedatasphere.streamis.jobmanager.manager.dao.StreamJobMapper import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJob import com.webank.wedatasphere.streamis.jobmanager.manager.transform.StreamisTransformJobBuilder import com.webank.wedatasphere.streamis.jobmanager.manager.transform.entity.{StreamisJobEngineConnImpl, StreamisTransformJob, StreamisTransformJobContent, StreamisTransformJobImpl} import org.springframework.beans.factory.annotation.Autowired +import java.util +import scala.collection.JavaConverters.mapAsJavaMapConverter /** * Created by enjoyyin on 2021/9/22. */ @@ -39,7 +42,12 @@ abstract class AbstractStreamisTransformJobBuilder extends StreamisTransformJobB override def build(streamJob: StreamJob): StreamisTransformJob = { val transformJob = createStreamisTransformJob() transformJob.setStreamJob(streamJob) - transformJob.setConfigMap(streamJobConfService.getJobConfig(streamJob.getId)) + val jobConfig: util.Map[String, Any] = Option(streamJobConfService.getJobConfig(streamJob.getId)) + .getOrElse(new util.HashMap[String, Any]()) + // Put and overwrite internal group, users cannot customize the internal configuration + jobConfig.put(JobConf.STREAMIS_INTERNAL_CONFIG_GROUP.getValue, new util.HashMap[String, Any]()) + internalLogConfig(jobConfig) + transformJob.setConfigMap(jobConfig) // transformJob.setConfig(configurationService.getFullTree(streamJob.getId)) val streamJobVersions = streamJobMapper.getJobVersions(streamJob.getId) // 无需判断streamJobVersions是否非空，因为TaskService已经判断了 @@ -48,6 +56,14 @@ abstract class AbstractStreamisTransformJobBuilder extends StreamisTransformJobB transformJob } + /** + * Log internal configuration + * @param internal internal config group + */ + private def internalLogConfig(internal: util.Map[String, Any]): Unit = { + internal.put(JobConf.STREAMIS_JOB_LOG_GATEWAY.key, JobConf.STREAMIS_JOB_LOG_GATEWAY.getValue) + internal.put(JobConf.STREAMIS_JOB_LOG_COLLECT_PATH.key, JobConf.STREAMIS_JOB_LOG_COLLECT_PATH.getValue) + } } abstract class AbstractFlinkStreamisTransformJobBuilder extends AbstractStreamisTransformJobBuilder{ diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala new file mode 100644 index 000000000..70e9631af --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala @@ -0,0 +1,42 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.transform.impl +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.LaunchJob +import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf +import com.webank.wedatasphere.streamis.jobmanager.manager.transform.impl.FlinkInternalConfigTransform.INTERNAL_CONFIG_MAP +import org.apache.linkis.common.conf.CommonVars + +import java.util +import scala.collection.JavaConverters.{mapAsJavaMapConverter, mapAsScalaMapConverter} + +/** + * Flink internal config transform + */ +class FlinkInternalConfigTransform extends FlinkConfigTransform { + + /** + * Config group name + * + * @return + */ + override protected def configGroup(): String = JobConf.STREAMIS_INTERNAL_CONFIG_GROUP.getValue + + override protected def transform(internalConfig: util.Map[String, Any], job: LaunchJob): LaunchJob = { + transformConfig(internalConfig.asScala.map{ + case (key, value) => + (FlinkConfigTransform.FLINK_CONFIG_PREFIX + (INTERNAL_CONFIG_MAP.get(key) match { + case Some(mappingKey) => mappingKey + case _ => value + }), value) + }.asJava, job) + } +} + +object FlinkInternalConfigTransform { + + private val FLINK_LOG_GATEWAY_CONFIG_NAME = CommonVars("wds.streamis.flink.config.name.log-gateway", "stream.log.gateway.address").getValue + + private val FLINK_LOG_COLLECT_PATH_CONFIG_NAME = CommonVars("wds.streamis.flink.config.name.log-collect-path", "stream.log.collect.path").getValue + + val INTERNAL_CONFIG_MAP = Map(JobConf.STREAMIS_JOB_LOG_GATEWAY.key -> FLINK_LOG_GATEWAY_CONFIG_NAME, + JobConf.STREAMIS_JOB_LOG_COLLECT_PATH.key -> FLINK_LOG_COLLECT_PATH_CONFIG_NAME + ) +} From 6c680e37945709d6e491ec22791fe2c666d94a62 Mon Sep 17 00:00:00 2001 From: davidhua Date: Sun, 23 Oct 2022 02:30:01 +0800 Subject: [PATCH 023/240] Complete stream log server to store the log events. --- .../streamis-job-log-server/pom.xml | 2 +- .../log/server/config/StreamJobLogConfig.java | 46 +++ .../log/server/restful/JobLogRestfulApi.java | 48 +++ .../service/DefaultStreamisJobLogService.java | 35 +++ .../server/service/StreamisJobLogService.java | 16 + .../log/server/storage/JobLogStorage.java | 13 +- .../server/storage/StreamisJobLogStorage.java | 127 +++++++- .../server/storage/bucket/JobLogBucket.java | 5 + .../storage/bucket/JobLogBucketConfig.java | 42 +-- .../storage/bucket/JobLogBucketState.java | 14 +- .../storage/bucket/Log4j2JobLogBucket.java | 274 +++++++++++++++++- streamis-server/pom.xml | 6 + .../src/main/resources/linkis.properties | 3 +- 13 files changed, 598 insertions(+), 33 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/config/StreamJobLogConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/service/DefaultStreamisJobLogService.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/service/StreamisJobLogService.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml index 7f2a5bc10..ec75cea68 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml @@ -9,7 +9,7 @@ 4.0.0 - streamis-job-log-service + streamis-job-log-server 8 diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/config/StreamJobLogConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/config/StreamJobLogConfig.java new file mode 100644 index 000000000..c857d5249 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/config/StreamJobLogConfig.java @@ -0,0 +1,46 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.config; + +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.conf.TimeType; + +/** + * Store the configuration defined for job log + */ +public class StreamJobLogConfig { + + /** + * Bucket monitor name + */ + public static final CommonVars BUCKET_MONITOR_NAME = CommonVars.apply("wds.stream.job.log.storage.bucket.monitor.name", "Log-Storage-Bucket-Monitor"); + + /** + * Bucket monitor interval + */ + public static final CommonVars BUCKET_MONITOR_INTERVAL = CommonVars.apply("wds.stream.job.log.storage.bucket.monitor.interval", new TimeType("2m")); + + /** + * Bucket max idle time + */ + public static final CommonVars BUCKET_MAX_IDLE_TIME = CommonVars.apply("wds.stream.job.log.storage.bucket.max-idle-time", new TimeType("12h")); + + /** + * Bucket root path + */ + public static final CommonVars BUCKET_ROOT_PATH = CommonVars.apply("wds.stream.job.log.storage.bucket.root-path", "/data/stream/log"); + /** + * Max active part size in bucket + */ + public static final CommonVars BUCKET_MAX_ACTIVE_PART_SIZE = CommonVars.apply("wds.stream.job.log.storage.bucket.max-active-part-size", 100L); + + /** + * Compression of part in bucket + */ + public static final CommonVars BUCKET_PART_COMPRESS = CommonVars.apply("wds.stream.job.log.storage.bucket.part-compress", "gz"); + + /** + * Bucket layout + */ + public static final CommonVars BUCKET_LAYOUT = CommonVars.apply("wds.stream.job.log.storage.bucket.layout", "%msg"); + + public static final CommonVars BUCKET_PART_HOLD_DAY = CommonVars.apply("wds.stream.job.log.storage.bucket.part-hold-day", 30); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java new file mode 100644 index 000000000..7b67c4b2c --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java @@ -0,0 +1,48 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.restful; + +import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvents; +import com.webank.wedatasphere.streamis.jobmanager.log.server.exception.StreamJobLogException; +import com.webank.wedatasphere.streamis.jobmanager.log.server.service.StreamisJobLogService; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import javax.annotation.Resource; +import javax.servlet.http.HttpServletRequest; + +@RestController +@RequestMapping(path = "/streamis/streamJobManager/log") +public class JobLogRestfulApi { + + private static final Logger LOG = LoggerFactory.getLogger(JobLogRestfulApi.class); + + @Resource + private StreamisJobLogService streamisJobLogService; + + @RequestMapping(value = "/collect/events", method = RequestMethod.POST) + public Message collectEvents(@RequestBody StreamisLogEvents events, HttpServletRequest request){ + Message result; + try{ + if (StringUtils.isBlank(events.getAppName())){ + return Message.ok("Ignore the stream log events without application name"); + } + String userName = SecurityFilter.getLoginUsername(request); + if (StringUtils.isBlank(userName)){ + throw new StreamJobLogException(-1, "The request should has token user"); + } + this.streamisJobLogService.store(userName, events); + result = Message.ok(); + }catch (Exception e){ + String message = "Fail to collect stream log events, message: " + e.getMessage(); + result = Message.error(message); + } + return result; + } + +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/service/DefaultStreamisJobLogService.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/service/DefaultStreamisJobLogService.java new file mode 100644 index 000000000..8fea4dab6 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/service/DefaultStreamisJobLogService.java @@ -0,0 +1,35 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.service; + +import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvents; +import com.webank.wedatasphere.streamis.jobmanager.log.server.storage.JobLogStorage; +import com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket.JobLogBucket; +import com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket.JobLogBucketConfig; +import org.springframework.stereotype.Service; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; + +/** + * Default implement + */ +@Service +public class DefaultStreamisJobLogService implements StreamisJobLogService{ + + @Resource + private JobLogStorage jobLogStorage; + + private JobLogBucketConfig jobLogBucketConfig; + + @PostConstruct + public void init(){ + jobLogBucketConfig = new JobLogBucketConfig(); + } + @Override + public void store(String user, StreamisLogEvents events) { + JobLogBucket jobLogBucket = jobLogStorage.getOrCreateBucket(user, events.getAppName(), jobLogBucketConfig); + // If cannot get log bucket, drop the events + if (null != jobLogBucket){ + jobLogBucket.getBucketStorageWriter().write(events); + } + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/service/StreamisJobLogService.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/service/StreamisJobLogService.java new file mode 100644 index 000000000..e8f8bfe4e --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/service/StreamisJobLogService.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.service; + +import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvents; + +/** + * Job log service + */ +public interface StreamisJobLogService { + + /** + * Store log events + * @param user user own + * @param events events + */ + void store(String user, StreamisLogEvents events); +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java index 66afa1a2b..e70590338 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java @@ -10,9 +10,20 @@ public interface JobLogStorage { /** * Create buckets + * @param userName user own * @param appName application name * @param bucketConfig bucket config * @return config */ - JobLogBucket getOrCreateBucket(String appName, JobLogBucketConfig bucketConfig); + JobLogBucket getOrCreateBucket(String userName, String appName, JobLogBucketConfig bucketConfig); + + /** + * Init method + */ + void init(); + + /** + * Destroy method + */ + void destroy(); } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java index b357c30c6..86851e5b7 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java @@ -1,4 +1,129 @@ package com.webank.wedatasphere.streamis.jobmanager.log.server.storage; -public class StreamisJobLogStorage { +import com.webank.wedatasphere.streamis.jobmanager.log.server.config.StreamJobLogConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket.JobLogBucket; +import com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket.JobLogBucketConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket.JobLogBucketState; +import org.apache.linkis.common.utils.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.text.SimpleDateFormat; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static com.webank.wedatasphere.streamis.jobmanager.log.server.config.StreamJobLogConfig.BUCKET_MONITOR_INTERVAL; + +/** + * Job log storage + */ +@Component +public class StreamisJobLogStorage implements JobLogStorage{ + + private static final Logger LOG = LoggerFactory.getLogger(StreamisJobLogStorage.class); + /** + * Buckets + */ + private final Map buckets = new ConcurrentHashMap<>(); + + /** + * Constructor cache + */ + private final Map> bucketConstructors = new ConcurrentHashMap<>(); + /** + * To monitor the status of buckets + */ + private Future monitorThread; + + @Override + public JobLogBucket getOrCreateBucket(String userName, String appName, JobLogBucketConfig bucketConfig) { + String bucketName = toBucketName(userName, appName); + return buckets.computeIfAbsent(bucketName, name -> { + Class bucketClass = bucketConfig.getBucketClass(); + if (Objects.nonNull(bucketClass)) { + Constructor constructor = bucketConstructors.computeIfAbsent(bucketClass.getName(), className -> { + Constructor[] constructors = bucketClass.getConstructors(); + Constructor matchConstructor = null; + for (Constructor constructor1 : constructors) { + Class[] inputParams = constructor1.getParameterTypes(); + if (inputParams.length >= 2 && inputParams[0].equals(String.class) + && inputParams[1].equals(JobLogBucketConfig.class)) { + matchConstructor = constructor1; + break; + } + } + return matchConstructor; + }); + if (Objects.nonNull(constructor)) { + try { + return (JobLogBucket) constructor.newInstance(bucketName, bucketConfig); + } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { + LOG.warn("Cannot create storage log bucket from [{}]", bucketClass.getName(), e); + } + } + } + return null; + }); + } + + @Override + @PostConstruct + public synchronized void init() { + if (Objects.isNull(monitorThread)){ + monitorThread = Utils.defaultScheduler().scheduleAtFixedRate(() -> { + Thread.currentThread().setName(StreamJobLogConfig.BUCKET_MONITOR_NAME.getValue()); + long maxIdleTime = StreamJobLogConfig.BUCKET_MAX_IDLE_TIME.getValue().toLong(); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + if (buckets.size() > 0) { + StringBuilder builder = new StringBuilder("Buckets in LogStorage: [\n"); + buckets.forEach((bucketName, bucket) -> { + JobLogBucketState bucketState = bucket.getBucketState(); + builder.append("bucket: [ name: ") + .append(bucketName) + .append(", path: ").append(bucketState.getBucketPath()) + .append(", parts: ").append(bucketState.getBucketParts()) + .append(", write-rate: ").append(bucketState.getBucketWriteRate()).append("/s") + .append(", last-write-time: ").append(dateFormat.format(bucketState.getBucketWriteTime())) + .append(" ]\n"); + if (bucketState.getBucketWriteTime() + maxIdleTime <= System.currentTimeMillis()){ + LOG.info("Close the idle bucket: [ name: {}, last-write-time: {} ]", + bucketName, dateFormat.format(bucketState.getBucketWriteTime())); + bucket.close(); + } + + }); + LOG.info(builder.toString()); + } + + },BUCKET_MONITOR_INTERVAL.getValue().toLong(), BUCKET_MONITOR_INTERVAL.getValue().toLong(), TimeUnit.MILLISECONDS); + } + } + + /** + * Bucket name + * @param userName username + * @param appName app name + * @return bucket name + */ + private String toBucketName(String userName, String appName){ + return userName + "." + appName; + } + + @Override + @PreDestroy + public void destroy() { + // Fist to close all the bucket + buckets.forEach((bucketName, bucket) -> bucket.close()); + if (null != monitorThread){ + monitorThread.cancel(true); + } + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucket.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucket.java index c20c08944..b55ce818b 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucket.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucket.java @@ -17,6 +17,11 @@ public interface JobLogBucket { */ JobLogStorageWriter getBucketStorageWriter(); + /** + * Bucket name + * @return bucket name + */ + String getBucketName(); /** * Close the bucket */ diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java index 2dc82e40f..831cccc65 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketConfig.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket; +import com.webank.wedatasphere.streamis.jobmanager.log.server.config.StreamJobLogConfig; import com.webank.wedatasphere.streamis.jobmanager.log.server.exception.StreamJobLogException; import org.apache.linkis.common.conf.CommonVars; @@ -19,7 +20,7 @@ public JobLogBucketConfig(){ this.bucketClass = (Class) defaultBucketClass; } } catch (ClassNotFoundException e) { -// throw new StreamJobLogException.Runtime(-1, "", e); + throw new StreamJobLogException.Runtime(-1, "Cannot find the bucket class, message: " + e.getMessage()); } } @@ -31,7 +32,7 @@ public JobLogBucketConfig(){ /** * Root path for bucket */ - private String bucketRootPath; + private String bucketRootPath = StreamJobLogConfig.BUCKET_ROOT_PATH.getValue(); /** * Attribute @@ -39,29 +40,24 @@ public JobLogBucketConfig(){ protected Map attributes = new HashMap<>(); /** - * Max size of bucket active part + * Max size of bucket active part (MB) */ - private int maxBucketActivePartSize; - - /** - * Max number of bucket part - */ - private int maxBucketPartNum; + private long maxBucketActivePartSize = StreamJobLogConfig.BUCKET_MAX_ACTIVE_PART_SIZE.getValue(); /** * The compress format used for bucket parts */ - private String bucketPartCompress; + private String bucketPartCompress = StreamJobLogConfig.BUCKET_PART_COMPRESS.getValue(); /** - * Max hold time in minutes for bucket part + * Max hold time in days for bucket part */ - private long bucketPartHoldTimeInMin; + private int bucketPartHoldTimeInDay = StreamJobLogConfig.BUCKET_PART_HOLD_DAY.getValue(); /** * Layout pattern */ - private String LogLayOutPattern = "%msg%n"; + private String LogLayOutPattern = StreamJobLogConfig.BUCKET_LAYOUT.getValue(); public Class getBucketClass() { return bucketClass; @@ -87,22 +83,14 @@ public void setAttributes(Map attributes) { this.attributes = attributes; } - public int getMaxBucketActivePartSize() { + public long getMaxBucketActivePartSize() { return maxBucketActivePartSize; } - public void setMaxBucketActivePartSize(int maxBucketActivePartSize) { + public void setMaxBucketActivePartSize(long maxBucketActivePartSize) { this.maxBucketActivePartSize = maxBucketActivePartSize; } - public int getMaxBucketPartNum() { - return maxBucketPartNum; - } - - public void setMaxBucketPartNum(int maxBucketPartNum) { - this.maxBucketPartNum = maxBucketPartNum; - } - public String getBucketPartCompress() { return bucketPartCompress; } @@ -111,12 +99,12 @@ public void setBucketPartCompress(String bucketPartCompress) { this.bucketPartCompress = bucketPartCompress; } - public long getBucketPartHoldTimeInMin() { - return bucketPartHoldTimeInMin; + public int getBucketPartHoldTimeInDay() { + return bucketPartHoldTimeInDay; } - public void setBucketPartHoldTimeInMin(long bucketPartHoldTimeInMin) { - this.bucketPartHoldTimeInMin = bucketPartHoldTimeInMin; + public void setBucketPartHoldTimeInDay(int bucketPartHoldTimeInDay) { + this.bucketPartHoldTimeInDay = bucketPartHoldTimeInDay; } public String getLogLayOutPattern() { diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketState.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketState.java index bdd64f265..8051e6d13 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketState.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/JobLogBucketState.java @@ -11,9 +11,21 @@ public interface JobLogBucketState { */ String getBucketPath(); + /** + * Write rate + * @return rate + */ double getBucketWriteRate(); + /** + * Bucket parts + * @return number + */ int getBucketParts(); - int getBucketWriteTime(); + /** + * Last rite time + * @return time + */ + long getBucketWriteTime(); } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java index 21c3303f5..4f298ecf4 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java @@ -1,4 +1,276 @@ package com.webank.wedatasphere.streamis.jobmanager.log.server.storage.bucket; -public class Log4j2JobLogBucket { +import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.linkis.common.conf.CommonVars; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.Appender; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.RollingFileAppender; +import org.apache.logging.log4j.core.appender.rolling.*; +import org.apache.logging.log4j.core.appender.rolling.action.*; +import org.apache.logging.log4j.core.config.AppenderRef; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; +import org.apache.logging.log4j.core.layout.PatternLayout; +import org.checkerframework.checker.units.qual.A; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Job log bucket for log4j + */ +public class Log4j2JobLogBucket implements JobLogBucket{ + + private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(Log4j2JobLogBucket.class); + + private static final String DEFAULT_FILE_PATTERN_SUFFIX = ".%d{yyyy-MM-dd}-%i"; + + private static final CommonVars ROLLOVER_MAX = CommonVars.apply("wds.stream.job.log.storage.bucket.log4j.rollover-max", 20); + /** + * Bucket name + */ + private final String bucketName; + + /** + * Logger context + */ + private final LoggerContext loggerContext; + + /** + * Logger entity + */ + private final Logger logger; + + /** + * Storage writer + */ + private final JobLogStorageWriter jobLogStorageWriter; + + /** + * Bucket state + */ + private final JobLogBucketState jobLogBucketState; + + /** + * Last write time; + */ + private long lastWriteTime; + + /** + * Prev Interval time + */ + private long preIntervalTime; + + /** + * Interval counter + */ + private final AtomicLong intervalCounter = new AtomicLong(0); + + /** + * Store the write rate + */ + private double writeRate; + public Log4j2JobLogBucket(String bucketName, JobLogBucketConfig config){ + this.bucketName = bucketName; + // Create logger context + this.loggerContext = (LoggerContext) LogManager.getContext(false); + this.logger = initLogger(this.bucketName, config, this.loggerContext); + this.jobLogStorageWriter = createStorageWriter(); + this.jobLogBucketState = createBucketState(); + } + @Override + public JobLogBucketState getBucketState() { + return this.jobLogBucketState; + } + + @Override + public JobLogStorageWriter getBucketStorageWriter() { + return this.jobLogStorageWriter; + } + + @Override + public String getBucketName() { + return this.bucketName; + } + + @Override + public void close() { + Configuration log4jConfig = this.loggerContext.getConfiguration(); + // First to stop appender + log4jConfig.getAppender(this.bucketName).stop(); + log4jConfig.getLoggerConfig(this.bucketName).removeAppender(this.bucketName); + log4jConfig.removeLogger(this.bucketName); + loggerContext.updateLoggers(); + } + + private synchronized Logger initLogger(String bucketName, JobLogBucketConfig config, LoggerContext loggerContext){ + Configuration log4jConfig = loggerContext.getConfiguration(); + String fileName = resolveFileName(config.getBucketRootPath(), bucketName); + RollingFileAppender appender = RollingFileAppender.newBuilder() + .setLayout(PatternLayout.newBuilder().withPattern(config.getLogLayOutPattern()).build()) + .setName(bucketName) +// .withFileOwner() + .withFileName(fileName) + .withFilePattern(resolveFilePattern(fileName, config.getBucketPartCompress())) + .withStrategy(createRolloverStrategy(log4jConfig, fileName, ROLLOVER_MAX.getValue(), config.getBucketPartHoldTimeInDay())) + .setConfiguration(log4jConfig) + .build(); + appender.start(); + log4jConfig.addAppender(appender); + LoggerConfig loggerConfig = LoggerConfig.newBuilder().withLevel(Level.ALL) + .withRefs(new AppenderRef[]{ + AppenderRef.createAppenderRef(bucketName, null, null) + }) + .withLoggerName(bucketName).withConfig(log4jConfig).build(); + loggerConfig.addAppender(appender, null, null); + log4jConfig.addLogger(bucketName, loggerConfig); + // Should we update the logger context ? + loggerContext.updateLoggers(); + return loggerContext.getLogger(bucketName); + } + + /** + * Create storage writer + * @return storage writer + */ + private JobLogStorageWriter createStorageWriter(){ + return new JobLogStorageWriter() { + @Override + public void write(LogElement logEl) { + String[] contents = logEl.getContents(); + if (null != contents){ + for(String content : contents){ + write(content); + } + } + } + + @Override + public void write(String logLine) { + logger.info(logLine); + long currentTime = System.currentTimeMillis(); + long intervalCnt = intervalCounter.getAndIncrement(); + long intervalTime = (currentTime - preIntervalTime)/1000; + // Per minute accumulate the rate + if ( intervalTime >= 60){ + writeRate = (double)intervalCnt / (double)intervalTime; + preIntervalTime = currentTime; + intervalCounter.set(0); + } + lastWriteTime = currentTime; + + } + + @Override + public void close() { + // Ignore + } + }; + } + + /** + * Create bucket state + * @return bucket state + */ + private JobLogBucketState createBucketState(){ + return new JobLogBucketState() { + private String bucketPath; + @Override + public String getBucketPath() { + if (StringUtils.isBlank(bucketPath)) { + Appender appender = loggerContext.getConfiguration().getAppender(bucketName); + if (appender instanceof RollingFileAppender) { + bucketPath = new File(((RollingFileAppender) appender).getFileName()).getParent(); + } + } + return this.bucketPath; + } + + @Override + public double getBucketWriteRate() { + return writeRate; + } + + @Override + public int getBucketParts() { + AtomicInteger parts = new AtomicInteger(-1); + String bucketPath = getBucketPath(); + if (StringUtils.isNotBlank(bucketPath)){ + Optional.ofNullable(new File(bucketPath).list()).ifPresent(list -> parts.set(list.length)); + } + return parts.get(); + } + + @Override + public long getBucketWriteTime() { + return lastWriteTime; + } + }; + } + /** + * Create rollover strategy + * @param configuration configuration + * @param fileName file name + * @param rolloverMax rollover max inf file pattern + * @param fileHoldDay file hold day time + * @return strategy + */ + private RolloverStrategy createRolloverStrategy(Configuration configuration, + String fileName, int rolloverMax, int fileHoldDay){ + DefaultRolloverStrategy.Builder builder = DefaultRolloverStrategy.newBuilder(); + if (rolloverMax > 0){ + builder.withMax(rolloverMax + ""); + } + if (fileHoldDay > 0){ + // Create the actions to delete old file + builder.withCustomActions(new Action[]{ + DeleteAction.createDeleteAction(new File(fileName).getParent(), false, 2, false, null, + new PathCondition[]{ + IfFileName.createNameCondition(null, ".*"), + IfLastModified.createAgeCondition(Duration.parse(fileHoldDay + "d")) + }, + null, configuration) + } + ); + } + return builder.build(); + } + /** + * Ex: /data/stream/log/hadoop/{projectName}/{jobName}/{projectName}.{jobName}.log + * @param bucketRootPath bucket root path + * @param bucketName bucket name + * @return file name with absolute path + */ + private String resolveFileName(String bucketRootPath, String bucketName){ + // {projectName}.{jobName} + String fileName = FilenameUtils.normalize(bucketName.substring(bucketName.indexOf("."))); + String basePath = bucketRootPath; + if (!basePath.endsWith("/")){ + basePath += "/"; + } + basePath += fileName.replace(".", "/"); + return basePath + "/" + fileName; + } + + /** + * Resolve file pattern + * @param fileName file name + * @param format format + * @return file pattern + */ + private String resolveFilePattern(String fileName, String format){ + String filePattern = fileName + Log4j2JobLogBucket.DEFAULT_FILE_PATTERN_SUFFIX; + if (StringUtils.isNotBlank(format)){ + filePattern = filePattern + (format.startsWith(".") ? format : "." +format); + } + return filePattern; + } } diff --git a/streamis-server/pom.xml b/streamis-server/pom.xml index 701457496..1bf8ecfba 100644 --- a/streamis-server/pom.xml +++ b/streamis-server/pom.xml @@ -69,6 +69,12 @@ ${streamis.version} + + + com.webank.wedatasphere.streamis + streamis-job-log-server + ${streamis.version} + diff --git a/streamis-server/src/main/resources/linkis.properties b/streamis-server/src/main/resources/linkis.properties index b80410c10..b09dbbc4a 100644 --- a/streamis-server/src/main/resources/linkis.properties +++ b/streamis-server/src/main/resources/linkis.properties @@ -34,7 +34,8 @@ wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.streamis.datasou com.webank.wedatasphere.streamis.project.server.restful,\ com.webank.wedatasphere.streamis.jobmanager.restful.api,\ com.webank.wedatasphere.streamis.datasource.execute.rest,\ - com.webank.wedatasphere.streamis.projectmanager.restful.api + com.webank.wedatasphere.streamis.projectmanager.restful.api,\ + com.webank.wedatasphere.streamis.jobmanager.log.server.restful ##mybatis wds.linkis.server.mybatis.mapperLocations=\ classpath*:com/webank/wedatasphere/streamis/datasource/manager/dao/impl/*.xml,\ From 0a08d0f0faf8ab9269233275b702b4927943cfba Mon Sep 17 00:00:00 2001 From: davidhua Date: Sun, 23 Oct 2022 09:29:49 +0800 Subject: [PATCH 024/240] Fix the problem in job builder. --- .../builder/AbstractStreamisTransformJobBuilder.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala index 6d44dced6..fedc86591 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala @@ -45,8 +45,9 @@ abstract class AbstractStreamisTransformJobBuilder extends StreamisTransformJobB val jobConfig: util.Map[String, Any] = Option(streamJobConfService.getJobConfig(streamJob.getId)) .getOrElse(new util.HashMap[String, Any]()) // Put and overwrite internal group, users cannot customize the internal configuration - jobConfig.put(JobConf.STREAMIS_INTERNAL_CONFIG_GROUP.getValue, new util.HashMap[String, Any]()) - internalLogConfig(jobConfig) + val internalGroup = new util.HashMap[String, Any]() + jobConfig.put(JobConf.STREAMIS_INTERNAL_CONFIG_GROUP.getValue, internalGroup) + internalLogConfig(internalGroup) transformJob.setConfigMap(jobConfig) // transformJob.setConfig(configurationService.getFullTree(streamJob.getId)) val streamJobVersions = streamJobMapper.getJobVersions(streamJob.getId) From 86590fa23b01a58e9b0ea09f65aa4fbc3062cd53 Mon Sep 17 00:00:00 2001 From: davidhua Date: Sun, 23 Oct 2022 16:31:26 +0800 Subject: [PATCH 025/240] Move the internal group config into JobConfKeyConstants. --- .../jobmanager/launcher/conf/JobConfKeyConstants.scala | 4 ++++ .../streamis/jobmanager/manager/conf/JobConf.scala | 5 ----- .../builder/AbstractStreamisTransformJobBuilder.scala | 3 ++- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/conf/JobConfKeyConstants.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/conf/JobConfKeyConstants.scala index f756daecc..187b02288 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/conf/JobConfKeyConstants.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/conf/JobConfKeyConstants.scala @@ -23,6 +23,10 @@ import org.apache.linkis.common.conf.CommonVars */ object JobConfKeyConstants { + /** + * Config group for streamis internal configuration + */ + val GROUP_INTERNAL: CommonVars[String] = CommonVars("wds.streamis.job.internal.config.group", "wds.streamis.internal.params") /** * Group: Flink extra */ diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala index 33c4827b8..41b91e348 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobConf.scala @@ -29,11 +29,6 @@ object JobConf { val STREAMIS_JOB_PARAM_BLANK_PLACEHOLDER: CommonVars[String] = CommonVars("wds.streamis.job.param.blank.placeholder", "\u0001") - /** - * Config group for streamis internal configuration - */ - val STREAMIS_INTERNAL_CONFIG_GROUP: CommonVars[String] = CommonVars("wds.streamis.job.internal.config.group", "wds.streamis.internal.params") - /** * Gateway for stream job log module */ diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala index fedc86591..622961329 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala @@ -15,6 +15,7 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.transform.builder +import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants import org.apache.linkis.common.conf.CommonVars import org.apache.linkis.manager.label.entity.engine.RunType.RunType import com.webank.wedatasphere.streamis.jobmanager.launcher.service.StreamJobConfService @@ -46,7 +47,7 @@ abstract class AbstractStreamisTransformJobBuilder extends StreamisTransformJobB .getOrElse(new util.HashMap[String, Any]()) // Put and overwrite internal group, users cannot customize the internal configuration val internalGroup = new util.HashMap[String, Any]() - jobConfig.put(JobConf.STREAMIS_INTERNAL_CONFIG_GROUP.getValue, internalGroup) + jobConfig.put(JobConfKeyConstants.GROUP_INTERNAL.getValue, internalGroup) internalLogConfig(internalGroup) transformJob.setConfigMap(jobConfig) // transformJob.setConfig(configurationService.getFullTree(streamJob.getId)) From 717ef2452bc07881fec53351463f22089786aea2 Mon Sep 17 00:00:00 2001 From: davidhua Date: Sun, 23 Oct 2022 16:35:52 +0800 Subject: [PATCH 026/240] Move the internal group config into JobConfKeyConstants. --- .../manager/transform/impl/FlinkInternalConfigTransform.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala index 70e9631af..e2d0a7848 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala @@ -1,4 +1,5 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.transform.impl +import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants import com.webank.wedatasphere.streamis.jobmanager.launcher.job.LaunchJob import com.webank.wedatasphere.streamis.jobmanager.manager.conf.JobConf import com.webank.wedatasphere.streamis.jobmanager.manager.transform.impl.FlinkInternalConfigTransform.INTERNAL_CONFIG_MAP @@ -17,7 +18,7 @@ class FlinkInternalConfigTransform extends FlinkConfigTransform { * * @return */ - override protected def configGroup(): String = JobConf.STREAMIS_INTERNAL_CONFIG_GROUP.getValue + override protected def configGroup(): String = JobConfKeyConstants.GROUP_INTERNAL.getValue override protected def transform(internalConfig: util.Map[String, Any], job: LaunchJob): LaunchJob = { transformConfig(internalConfig.asScala.map{ From 23b6924670c4aa683c2a65e01332492d85933fb7 Mon Sep 17 00:00:00 2001 From: davidhua Date: Sun, 23 Oct 2022 18:08:51 +0800 Subject: [PATCH 027/240] Make the constraints in transforms be configurable. --- .../impl/FlinkCheckpointConfigTransform.scala | 3 ++- .../impl/FlinkInternalConfigTransform.scala | 14 ++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkCheckpointConfigTransform.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkCheckpointConfigTransform.scala index 01426f656..7d797aaf5 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkCheckpointConfigTransform.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkCheckpointConfigTransform.scala @@ -23,6 +23,7 @@ import com.webank.wedatasphere.streamis.jobmanager.launcher.job.LaunchJob import com.webank.wedatasphere.streamis.jobmanager.launcher.job.manager.JobLaunchManager import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.state.Checkpoint import com.webank.wedatasphere.streamis.jobmanager.manager.transform.impl.FlinkCheckpointConfigTransform.CHECKPOINT_PATH_CONFIG_NAME +import org.apache.linkis.common.conf.CommonVars import org.apache.linkis.common.utils.Logging import scala.collection.JavaConverters._ @@ -62,5 +63,5 @@ class FlinkCheckpointConfigTransform extends FlinkConfigTransform with Logging{ } object FlinkCheckpointConfigTransform{ - val CHECKPOINT_PATH_CONFIG_NAME = "state.checkpoints.dir" + private val CHECKPOINT_PATH_CONFIG_NAME = CommonVars("wds.streamis.flink.config.name.checkpoint-path", "state.checkpoints.dir").getValue } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala index e2d0a7848..be9782734 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala @@ -32,12 +32,18 @@ class FlinkInternalConfigTransform extends FlinkConfigTransform { } object FlinkInternalConfigTransform { + /** + * Defined in FlinkStreamisConfigDefine.LOG_GATEWAY_ADDRESS of 'flink-streamis-log-collector' + */ + private val LOG_GATEWAY_CONFIG_NAME = CommonVars("wds.streamis.flink.config.name.log-gateway", "stream.log.gateway.address").getValue - private val FLINK_LOG_GATEWAY_CONFIG_NAME = CommonVars("wds.streamis.flink.config.name.log-gateway", "stream.log.gateway.address").getValue + /** + * Defined in FlinkStreamisConfigDefine.LOG_GATEWAY_ADDRESS of 'flink-streamis-log-collector' + */ + private val LOG_COLLECT_PATH_CONFIG_NAME = CommonVars("wds.streamis.flink.config.name.log-collect-path", "stream.log.collect.path").getValue - private val FLINK_LOG_COLLECT_PATH_CONFIG_NAME = CommonVars("wds.streamis.flink.config.name.log-collect-path", "stream.log.collect.path").getValue - val INTERNAL_CONFIG_MAP = Map(JobConf.STREAMIS_JOB_LOG_GATEWAY.key -> FLINK_LOG_GATEWAY_CONFIG_NAME, - JobConf.STREAMIS_JOB_LOG_COLLECT_PATH.key -> FLINK_LOG_COLLECT_PATH_CONFIG_NAME + val INTERNAL_CONFIG_MAP = Map(JobConf.STREAMIS_JOB_LOG_GATEWAY.key -> LOG_GATEWAY_CONFIG_NAME, + JobConf.STREAMIS_JOB_LOG_COLLECT_PATH.key -> LOG_COLLECT_PATH_CONFIG_NAME ) } From c2f7834f7713f1bc92425da92d146ff08ea46175 Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 24 Oct 2022 01:26:02 +0800 Subject: [PATCH 028/240] Fix serialization and configuration problem. --- .../flink/FlinkStreamisConfigAutowired.java | 27 +++++++++----- .../log/collector/StreamisRpcLogAppender.java | 4 +- .../config/StreamisLogAppenderConfig.java | 5 ++- .../sender/SendLogExceptionStrategy.java | 1 - .../sender/http/AbstractHttpLogSender.java | 37 +++++++++++++++++-- .../http/request/AbstractHttpAction.java | 1 + .../sender/http/request/EntityPostAction.java | 1 - .../log/entities/StreamisLogEvent.java | 21 +++++++++-- .../log/entities/StreamisLogEvents.java | 22 ++++++++++- .../storage/bucket/Log4j2JobLogBucket.java | 7 ++-- 10 files changed, 101 insertions(+), 25 deletions(-) diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java index a862d080f..88c38a4b3 100644 --- a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java @@ -5,6 +5,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; +import org.apache.flink.runtime.util.EnvironmentInformation; import org.apache.flink.yarn.configuration.YarnConfigOptions; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.Filter; @@ -29,11 +30,13 @@ public class FlinkStreamisConfigAutowired implements StreamisConfigAutowired { public FlinkStreamisConfigAutowired(){ // First to load configuration - this.configuration = loadConfiguration(); + // We should sleep and wait for append of the flink-yaml.conf } @Override public StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Builder builder) throws Exception{ - String applicationName = this.configuration.getString(YarnConfigOptions.APPLICATION_NAME); + this.configuration = loadConfiguration(); + String applicationName = + this.configuration.getString(YarnConfigOptions.APPLICATION_NAME); if (StringUtils.isNotBlank(applicationName)){ builder.setAppName(applicationName); } @@ -48,13 +51,17 @@ public StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Bui List filterStrategies = this.configuration.get(LOG_FILTER_STRATEGIES); for(String filterStrategy : filterStrategies){ if ("LevelMatch".equals(filterStrategy)){ - builder.withFilter(LevelMatchFilter.newBuilder() + builder.withFilter(LevelMatchFilter.newBuilder().setOnMatch(Filter.Result.ACCEPT).setOnMismatch(Filter.Result.DENY) .setLevel(Level.getLevel(this.configuration.getString(LOG_FILTER_LEVEL_MATCH))).build()); } else if ("RegexMatch".equals(filterStrategy)){ builder.withFilter(RegexFilter.createFilter( this.configuration.getString(LOG_FILTER_REGEX), null, true, Filter.Result.ACCEPT, Filter.Result.DENY)); } } + String hadoopUser = EnvironmentInformation.getHadoopUser(); + if (hadoopUser.equals("") || hadoopUser.equals("")){ + hadoopUser = ""; + } return builder.setRpcConnTimeout(this.configuration.getInteger(LOG_RPC_CONN_TIMEOUT)) .setRpcSocketTimeout(this.configuration.getInteger(LOG_RPC_SOCKET_TIMEOUT)) .setRpcSendRetryCnt(this.configuration.getInteger(LOG_RPC_SEND_RETRY_COUNT)) @@ -63,7 +70,8 @@ public StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Bui .setRpcAuthTokenCodeKey(this.configuration.getString(LOG_RPC_AUTH_TOKEN_CODE_KEY)) .setRpcAuthTokenUserKey(this.configuration.getString(LOG_RPC_AUTH_TOKEN_USER_KEY)) .setRpcAuthTokenCode(this.configuration.getString(LOG_RPC_AUTH_TOKEN_CODE)) - .setRpcAuthTokenUser(this.configuration.getString(LOG_RPC_AUTH_TOKEN_USER)) + .setRpcAuthTokenUser(this.configuration.getString(LOG_RPC_AUTH_TOKEN_USER, + hadoopUser)) .setRpcCacheSize(this.configuration.getInteger(LOG_RPC_CACHE_SIZE)) .setRpcCacheMaxConsumeThread(this.configuration.getInteger(LOG_PRC_CACHE_MAX_CONSUME_THREAD)) .setRpcBufferSize(this.configuration.getInteger(LOG_RPC_BUFFER_SIZE)) @@ -86,11 +94,12 @@ public StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Bui * the configuration directory of Flink yarn container is always ".", * @return configuration */ - private Configuration loadConfiguration(){ - String configDir = System.getenv("FLINK_CONF_DIR"); - if (null == configDir){ - configDir = "."; - } + private synchronized Configuration loadConfiguration(){ +// String configDir = System.getenv("FLINK_CONF_DIR"); +// if (null == configDir){ +// configDir = "."; +// } + String configDir = "."; Properties properties = System.getProperties(); Enumeration enumeration = properties.propertyNames(); Configuration dynamicConfiguration = new Configuration(); diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java index 5b30c36ce..26c304d9b 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java @@ -99,8 +99,8 @@ public static StreamisRpcLogAppender createAppender(@PluginAttribute("name") Str if (null == applicationName || applicationName.trim().equals("")){ throw new IllegalArgumentException("Application name cannot be empty"); } - LOGGER.info("StreamisRpcLogAppender: init with config {}", Json.toJson(logAppenderConfig, null)); - return new StreamisRpcLogAppender(name, filter, layout, ignoreExceptions, Property.EMPTY_ARRAY, logAppenderConfig); + System.out.println("StreamisRpcLogAppender: init with config" + Json.toJson(logAppenderConfig, null)); + return new StreamisRpcLogAppender(name, logAppenderConfig.getFilter(), layout, ignoreExceptions, Property.EMPTY_ARRAY, logAppenderConfig); } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java index 0253838ed..e334c6be2 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java @@ -5,6 +5,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Optional; /** @@ -47,7 +48,9 @@ public static class Builder{ public Builder(String applicationName, Filter filter, RpcLogSenderConfig rpcLogSenderConfig){ this.applicationName = applicationName; - this.filters.add(filter); + if (Objects.nonNull(filter)) { + this.filters.add(filter); + } this.rpcLogSenderConfig = Optional.ofNullable(rpcLogSenderConfig).orElse(new RpcLogSenderConfig()); } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java index c8952ff1d..d33b7d2e4 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java @@ -41,7 +41,6 @@ V doSend(Callable sendOperation, SendBuffer sendBuffer){ if (Objects.isNull(retryDescription) || !retryDescription.canRetry) { break; } - } } return null; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java index a0385ecb4..827e6666e 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java @@ -7,13 +7,22 @@ import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request.EntityPostAction; import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; +import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; +import org.apache.http.client.HttpResponseException; +import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.conn.ConnectTimeoutException; +import org.apache.logging.log4j.core.util.IOUtils; import javax.net.ssl.SSLException; +import javax.sound.midi.SysexMessage; +import javax.xml.ws.Response; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStreamReader; import java.io.InterruptedIOException; import java.net.UnknownHostException; +import java.nio.charset.StandardCharsets; import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -55,7 +64,6 @@ public int retryCount() { @Override public SendLogExceptionStrategy.RetryDescription onException(Exception e, SendBuffer sendBuffer) { - e.printStackTrace(); boolean shouldRetry = false; // Limit of exception number is the same as the retry times if (exceptionCounter.incrementAndGet() > retryCount()){ @@ -68,6 +76,11 @@ public SendLogExceptionStrategy.RetryDescription onException(Exception e, SendBu break; } } + if (!shouldRetry && e instanceof HttpResponseException){ + if (((HttpResponseException) e).getStatusCode() < 500){ + shouldRetry = true; + } + } } if (shouldRetry && !sender.getOrCreateLogCache().isCacheable()){ // Means that the cache is full @@ -103,8 +116,26 @@ protected void doSend(E aggregatedEntity, RpcLogSenderConfig rpcSenderConfig) th EntityPostAction postAction = new EntityPostAction<>(rpcSenderConfig.getAddress(), aggregatedEntity); RpcAuthConfig authConfig = rpcSenderConfig.getAuthConfig(); postAction.getRequestHeaders().put(authConfig.getTokenUserKey(), authConfig.getTokenUser()); - // Ignore the response - postAction.execute(this.globalHttpClient); + HttpResponse response = null; + try { + response = postAction.execute(this.globalHttpClient); + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode > 200){ + throw new HttpResponseException(statusCode, + IOUtils.toString(new InputStreamReader( + response.getEntity().getContent(), StandardCharsets.UTF_8))); + } + }finally { + // Close the response and release the conn + if (null != response){ + if (response instanceof CloseableHttpResponse){ + ((CloseableHttpResponse)response).close(); + } else { + // Destroy the stream + response.getEntity().getContent().close(); + } + } + } // Init the counter this.exceptionCounter.set(0); } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java index 8b0472e08..143f72b8e 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java @@ -57,6 +57,7 @@ public HttpResponse execute(HttpClient httpClient) throws IOException { } catch (URISyntaxException e) { throw new IllegalArgumentException("URI maybe has wrong format", e); } + requestHeaders.forEach(requestBase::setHeader); return httpClient.execute(requestBase); } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java index f8bfc5e4c..583483a6f 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java @@ -72,7 +72,6 @@ private void addEntityBody(MultipartEntityBuilder builder, String prefix, JsonNo } } else if (node instanceof ValueNode){ ContentType strContent = ContentType.create("text/plain", StandardCharsets.UTF_8); - System.out.println("p: " + prefix + ", data: " + node.asText()); builder.addTextBody(prefix, node.asText(), strContent); } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java index 5c82e22cb..3aea21d6f 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java @@ -12,18 +12,21 @@ public class StreamisLogEvent implements LogElement, Serializable { /** * Log time */ - private final long logTimeInMills; + private long logTimeInMills; /** * Log content */ - private final String content; + private String content; /** * Mark */ private int mark; + public StreamisLogEvent(){ + + } public StreamisLogEvent(String content, long logTimeInMills){ this.content = content; this.logTimeInMills = logTimeInMills; @@ -53,7 +56,19 @@ public int mark() { return this.mark; } - public void mark(int mark){ + public void setLogTimeStamp(long logTimeInMills) { + this.logTimeInMills = logTimeInMills; + } + + public void setContent(String content) { + this.content = content; + } + + public void setMark(int mark) { this.mark = mark; } + + public void setSequenceId(int sequenceId){ + // Ignore + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java index fa0b1c929..8e58aa55d 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java @@ -13,10 +13,12 @@ public class StreamisLogEvents implements LogElement, Serializable { /** * Log time */ - private final long logTimeInMills; + private long logTimeInMills; - private final StreamisLogEvent[] events; + private StreamisLogEvent[] events; + public StreamisLogEvents(){ + } public StreamisLogEvents(String applicationName, StreamisLogEvent[] events){ this.appName = applicationName; this.events = events; @@ -69,4 +71,20 @@ public String getAppName() { public StreamisLogEvent[] getEvents() { return events; } + + public void setAppName(String appName) { + this.appName = appName; + } + + public void setLogTimeStamp(long logTimeInMills) { + this.logTimeInMills = logTimeInMills; + } + + public void setEvents(StreamisLogEvent[] events) { + this.events = events; + } + + public void setSequenceId(int sequenceId){ + // Ignore + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java index 4f298ecf4..04cdfd430 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java @@ -120,12 +120,13 @@ private synchronized Logger initLogger(String bucketName, JobLogBucketConfig con // .withFileOwner() .withFileName(fileName) .withFilePattern(resolveFilePattern(fileName, config.getBucketPartCompress())) + .withPolicy(SizeBasedTriggeringPolicy.createPolicy(config.getMaxBucketActivePartSize() + "MB")) .withStrategy(createRolloverStrategy(log4jConfig, fileName, ROLLOVER_MAX.getValue(), config.getBucketPartHoldTimeInDay())) .setConfiguration(log4jConfig) .build(); appender.start(); log4jConfig.addAppender(appender); - LoggerConfig loggerConfig = LoggerConfig.newBuilder().withLevel(Level.ALL) + LoggerConfig loggerConfig = LoggerConfig.newBuilder().withAdditivity(false).withLevel(Level.ALL) .withRefs(new AppenderRef[]{ AppenderRef.createAppenderRef(bucketName, null, null) }) @@ -251,13 +252,13 @@ private RolloverStrategy createRolloverStrategy(Configuration configuration, */ private String resolveFileName(String bucketRootPath, String bucketName){ // {projectName}.{jobName} - String fileName = FilenameUtils.normalize(bucketName.substring(bucketName.indexOf("."))); + String fileName = FilenameUtils.normalize(bucketName); String basePath = bucketRootPath; if (!basePath.endsWith("/")){ basePath += "/"; } basePath += fileName.replace(".", "/"); - return basePath + "/" + fileName; + return basePath + "/" + fileName.substring(bucketName.indexOf(".") + 1) + ".log"; } /** From 3ede78a5a357290d3cbf1326448f86555b1b5a8f Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Mon, 24 Oct 2022 16:42:44 +0800 Subject: [PATCH 029/240] Optimize the update task and stop task logic. --- .../jobmanager/restful/api/JobRestfulApi.java | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index acb1ac529..7f2e396e3 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -175,19 +175,22 @@ public Message killJob(HttpServletRequest req, StreamJob streamJob = this.streamJobService.getJobById(jobId); if(streamJob == null) { return Message.error("not exists job " + jobId); - } else if(!JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES().getValue().contains(streamJob.getJobType())) { - return Message.error("Job " + streamJob.getName() + " is not supported to stop."); } if (!streamJobService.hasPermission(streamJob, userName) && !this.privilegeService.hasEditPrivilege(req, streamJob.getProjectName())) { return Message.error("Have no permission to kill/stop StreamJob [" + jobId + "]"); } - try { - PauseResultVo resultVo = streamTaskService.pause(jobId, 0L, userName, Objects.nonNull(snapshot)? snapshot : false); - return snapshot? Message.ok().data("path", resultVo.getSnapshotPath()) : Message.ok(); - } catch (Exception e) { - LOG.error("{} kill job {} failed!", userName, jobId, e); - return Message.error(ExceptionUtils.getRootCauseMessage(e)); + if(JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES().getValue().contains(streamJob.getJobType())) { + try { + PauseResultVo resultVo = streamTaskService.pause(jobId, 0L, userName, Objects.nonNull(snapshot)? snapshot : false); + return snapshot? Message.ok().data("path", resultVo.getSnapshotPath()) : Message.ok(); + } catch (Exception e) { + LOG.error("{} kill job {} failed!", userName, jobId, e); + return Message.error(ExceptionUtils.getRootCauseMessage(e)); + } + } else { + LOG.error("{} try to kill not-supported-management job {} with name {}.", userName, jobId, streamJob.getName()); + return tryStopTask(req, userName, streamJob, null); } } @@ -334,6 +337,9 @@ public Message updateTask(HttpServletRequest req, if(streamTask == null) { LOG.warn("Job {} is not exists running task, ignore to update its metrics.", jobName); return Message.ok("not exists running task, ignore it."); + } else if(JobConf.isCompleted(streamTask.getStatus())) { + LOG.warn("The task of job {} is completed, ignore to update its metrics.", jobName); + return Message.ok("Task is completed, ignore to update its metrics."); } return withFlinkJobInfo(jobName, streamTask.getLinkisJobInfo(), flinkJobInfo -> { if(!flinkJobInfo.getApplicationId().equals(appId)) { @@ -367,28 +373,32 @@ public Message stopTask(HttpServletRequest req, } else if(!"spark.jar".equals(streamJobs.get(0).getJobType())) { return Message.error("Only spark.jar Job support to stop task."); } - if (!streamJobService.hasPermission(streamJobs.get(0), username) && - !this.privilegeService.hasEditPrivilege(req, streamJobs.get(0).getProjectName())) { - return Message.error("Have no permission to stop task for StreamJob [" + jobName + "]."); + return tryStopTask(req, username, streamJobs.get(0), appId); + } + + private Message tryStopTask(HttpServletRequest req, String username, StreamJob streamJob, String appId) { + if (!streamJobService.hasPermission(streamJob, username) && + !this.privilegeService.hasEditPrivilege(req, streamJob.getProjectName())) { + return Message.error("Have no permission to stop task for StreamJob [" + streamJob.getName() + "]."); } // 如果存在正在运行的，将其停止掉 - StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJobs.get(0).getId()); + StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJob.getId()); if(streamTask != null && JobConf.isRunning(streamTask.getStatus())) { - return withFlinkJobInfo(jobName, streamTask.getLinkisJobInfo(), flinkJobInfo -> { - if(flinkJobInfo.getApplicationId().equals(appId)) { - LOG.warn("Streamis Job {} is exists running task, update its status to stopped.", jobName); + return withFlinkJobInfo(streamJob.getName(), streamTask.getLinkisJobInfo(), flinkJobInfo -> { + if(appId == null || flinkJobInfo.getApplicationId().equals(appId)) { + LOG.warn("Streamis Job {} is exists running task, update its status to stopped.", streamJob.getName()); streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); streamTask.setErrDesc("stopped by App itself."); streamTaskService.updateTask(streamTask); return Message.ok(); } else { LOG.warn("Job {} with running task is not equals to the request appId: {}, ignore to stop it.", - jobName, flinkJobInfo.getApplicationId(), appId); + streamJob.getName(), flinkJobInfo.getApplicationId(), appId); return Message.ok("the request appId is not equals to the running task appId " + flinkJobInfo.getApplicationId()); } }); } else { - LOG.warn("Streamis Job {} is not exists running task, ignore to stop it.", jobName); + LOG.warn("Streamis Job {} is not exists running task, ignore to stop it.", streamJob.getName()); return Message.ok(); } } From 2429e6fad1442bfcaea316679f56e291c6cf1504 Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 24 Oct 2022 17:30:50 +0800 Subject: [PATCH 030/240] Fix the problem in fetching user info when the flink application doesn't have hadoop dependencies. --- .../log/collector/flink/FlinkStreamisConfigAutowired.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java index 88c38a4b3..a42be88c1 100644 --- a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java @@ -60,7 +60,7 @@ public StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Bui } String hadoopUser = EnvironmentInformation.getHadoopUser(); if (hadoopUser.equals("") || hadoopUser.equals("")){ - hadoopUser = ""; + hadoopUser = System.getProperty("user.name"); } return builder.setRpcConnTimeout(this.configuration.getInteger(LOG_RPC_CONN_TIMEOUT)) .setRpcSocketTimeout(this.configuration.getInteger(LOG_RPC_SOCKET_TIMEOUT)) From 935100dda2d5496977d0c65bd25e3aa1e4625fb9 Mon Sep 17 00:00:00 2001 From: davidhua Date: Tue, 25 Oct 2022 11:40:06 +0800 Subject: [PATCH 031/240] Add new strategy named ThresholdFilter --- .../log/collector/flink/FlinkStreamisConfigAutowired.java | 6 +++++- .../log/collector/flink/FlinkStreamisConfigDefine.java | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java index a42be88c1..c146d23b6 100644 --- a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.core.Filter; import org.apache.logging.log4j.core.filter.LevelMatchFilter; import org.apache.logging.log4j.core.filter.RegexFilter; +import org.apache.logging.log4j.core.filter.ThresholdFilter; import java.util.Enumeration; import java.util.List; @@ -53,7 +54,10 @@ public StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Bui if ("LevelMatch".equals(filterStrategy)){ builder.withFilter(LevelMatchFilter.newBuilder().setOnMatch(Filter.Result.ACCEPT).setOnMismatch(Filter.Result.DENY) .setLevel(Level.getLevel(this.configuration.getString(LOG_FILTER_LEVEL_MATCH))).build()); - } else if ("RegexMatch".equals(filterStrategy)){ + }else if ("ThresholdFilter".equals(filterStrategy)){ + builder.withFilter(ThresholdFilter.createFilter(Level + .getLevel(this.configuration.getString(LOG_FILTER_THRESHOLD_MATCH)), Filter.Result.ACCEPT, Filter.Result.DENY)); + }else if ("RegexMatch".equals(filterStrategy)){ builder.withFilter(RegexFilter.createFilter( this.configuration.getString(LOG_FILTER_REGEX), null, true, Filter.Result.ACCEPT, Filter.Result.DENY)); } diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java index d40834fa5..379f15e0e 100644 --- a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java +++ b/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java @@ -112,6 +112,11 @@ public class FlinkStreamisConfigDefine { public static final ConfigOption LOG_FILTER_LEVEL_MATCH = ConfigOptions.key("stream.log.filter.level-match.level") .stringType().defaultValue("ERROR").withDescription("Level value of LevelMatch filter strategy"); + /** + * Level value of ThresholdMatch filter strategy + */ + public static final ConfigOption LOG_FILTER_THRESHOLD_MATCH = ConfigOptions.key("stream.log.filter.threshold.level") + .stringType().defaultValue("ERROR").withDescription("Level value of ThresholdMatch filter strategy"); /** * Regex value of RegexMatch filter strategy */ From c28e0edd1b05725bb4701ac7bce665a889b35795 Mon Sep 17 00:00:00 2001 From: davidhua Date: Tue, 25 Oct 2022 13:46:01 +0800 Subject: [PATCH 032/240] Fix the problem when closing bucket. --- .../jobmanager/log/server/storage/StreamisJobLogStorage.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java index 86851e5b7..abe312b23 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java @@ -97,6 +97,8 @@ public synchronized void init() { LOG.info("Close the idle bucket: [ name: {}, last-write-time: {} ]", bucketName, dateFormat.format(bucketState.getBucketWriteTime())); bucket.close(); + // Delete the bucket + buckets.remove(bucketName); } }); From e7f55eb2ef8f66010edd57697fbe64754e3f0ca4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?v=5Fboweicai=28=E8=94=A1=E5=8D=9A=E4=BC=9F=29?= Date: Tue, 25 Oct 2022 15:49:27 +0800 Subject: [PATCH 033/240] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E5=9C=A8?= =?UTF-8?q?=E5=88=87=E6=8D=A2=E6=97=A5=E5=BF=97=E7=B1=BB=E5=9E=8B=E6=97=B6?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E6=97=A5=E5=BF=97=E7=BF=BB=E9=A1=B5=E6=8C=89?= =?UTF-8?q?=E9=92=AE=E5=A4=B1=E6=95=88=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/src/apps/streamis/module/logDetail/index.vue | 1 + 1 file changed, 1 insertion(+) diff --git a/web/src/apps/streamis/module/logDetail/index.vue b/web/src/apps/streamis/module/logDetail/index.vue index 24cce2725..ee297746a 100644 --- a/web/src/apps/streamis/module/logDetail/index.vue +++ b/web/src/apps/streamis/module/logDetail/index.vue @@ -177,6 +177,7 @@ export default { }, handleQuery() { this.fromLine = 1 + this.endLine = 0 this.getDatas() } } From f3d38072f88a479a70ff7cf1f93ce5fcc549c424 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?v=5Fboweicai=28=E8=94=A1=E5=8D=9A=E4=BC=9F=29?= Date: Tue, 25 Oct 2022 17:04:17 +0800 Subject: [PATCH 034/240] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8Dstreamis?= =?UTF-8?q?=E4=BD=9C=E4=B8=9A=E7=9A=84Flink=E5=8F=82=E6=95=B0=E4=BB=85?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E4=B8=80=E4=B8=AA=E6=97=B6=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E6=97=A0=E6=B3=95=E4=BF=9D=E5=AD=98=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/src/apps/streamis/module/jobConfig/index.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/apps/streamis/module/jobConfig/index.vue b/web/src/apps/streamis/module/jobConfig/index.vue index b6bd86f46..860d5af89 100644 --- a/web/src/apps/streamis/module/jobConfig/index.vue +++ b/web/src/apps/streamis/module/jobConfig/index.vue @@ -227,7 +227,7 @@ export default { }); if ((this.diyMap[key] || []).length <= 1) { const only = (this.diyMap[key] || [])[0] || {}; - emptyWarning = !((!only.key || !only.key.trim()) && (!only.value || !only.value.trim())) + emptyWarning = (!only.key || !only.key.trim()) && (!only.value || !only.value.trim()) } }); console.log('configuration', configuration, this.valueMap) From 7d1fb7354cd29827e8c102c9d16e6b0391a49b35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?v=5Fboweicai=28=E8=94=A1=E5=8D=9A=E4=BC=9F=29?= Date: Tue, 25 Oct 2022 18:44:51 +0800 Subject: [PATCH 035/240] =?UTF-8?q?fix:=20=E6=97=A0=E6=9D=83=E9=99=90?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E5=81=9C=E6=AD=A2=E5=B7=A5=E4=BD=9C=E6=B5=81?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E6=97=B6=E5=9C=A8=E5=BC=B9=E7=AA=97=E4=B8=AD?= =?UTF-8?q?=E4=BD=93=E7=8E=B0=E5=A4=B1=E8=B4=A5=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/src/apps/streamis/module/jobList/index.vue | 1 + 1 file changed, 1 insertion(+) diff --git a/web/src/apps/streamis/module/jobList/index.vue b/web/src/apps/streamis/module/jobList/index.vue index c02f0ea0a..64551087a 100644 --- a/web/src/apps/streamis/module/jobList/index.vue +++ b/web/src/apps/streamis/module/jobList/index.vue @@ -742,6 +742,7 @@ export default { this.queryProcess(bulk_sbj); } catch (error) { console.warn(error); + this.modalContent = '停止任务失败，失败信息：' + error // this.modalLoading = false } }, From 3945d5e5ef75b38929094d9e3a5bb20528a4d278 Mon Sep 17 00:00:00 2001 From: davidhua Date: Wed, 26 Oct 2022 05:10:46 +0800 Subject: [PATCH 036/240] Split the log collector into log4j1 and log4j2; Avoid the jackson module in log collector. --- pom.xml | 2 +- .../flink-streamis-log-collector/pom.xml | 0 .../src/main/assembly/package.xml | 0 .../flink/FlinkStreamisConfigAutowired.java | 25 ++- .../flink/FlinkStreamisConfigDefine.java | 0 ....jobmanager.plugin.StreamisConfigAutowired | 0 .../flink/FlinkConfigurationLoadTest.java | 1 - .../src/test/resources/flink-conf.yaml | 0 .../streamis-job-log-collector-core/pom.xml | 39 ++++ .../log/collector/ExceptionListener.java | 0 .../log/collector/cache/LogCache.java | 0 .../log/collector/config/RpcAuthConfig.java | 22 +- .../collector/config/RpcLogSenderConfig.java | 38 ++-- .../collector/config/SendBufferConfig.java | 24 +-- .../collector/config/SendLogCacheConfig.java | 25 +-- .../config/StreamisLogAppenderConfig.java | 67 +------ .../sender/AbstractRpcLogSender.java | 0 .../log/collector/sender/RpcLogSender.java | 0 .../log/collector/sender/SendLogCache.java | 0 .../sender/SendLogCacheConsumer.java | 0 .../sender/SendLogExceptionStrategy.java | 0 .../sender/StreamisRpcLogSender.java | 5 + .../sender/buf/AbstractSendBuffer.java | 0 .../sender/buf/ImmutableSendBuffer.java | 0 .../log/collector/sender/buf/SendBuffer.java | 0 .../sender/http/AbstractHttpLogSender.java | 41 +++- .../collector/sender/http/HttpClientTool.java | 0 .../sender/http/entities/Resource.java | 0 .../http/request/AbstractHttpAction.java | 0 .../sender/http/request/GetAction.java | 0 .../sender/http/request/HttpAction.java | 0 .../sender/http/request/StringPostAction.java | 29 +++ .../plugin/StreamisConfigAutowired.java | 0 .../streamis-job-log-collector/pom.xml | 15 +- .../log/collector/StreamisRpcLogAppender.java | 10 +- .../log4j2/StreamisLog4j2AppenderConfig.java | 88 ++++++++ .../log4j2/config/RpcAuthConfig.java | 31 +++ .../log4j2/config/RpcLogSenderConfig.java | 37 ++++ .../log4j2/config/SendBufferConfig.java | 28 +++ .../log4j2/config/SendLogCacheConfig.java | 27 +++ .../collector/StreamisLogAppenderTest.java | 3 - .../src/test/resources/log4j2.xml | 4 +- .../streamis-job-log-collector1x/pom.xml | 55 +++++ .../log/collector/StreamisRpcLogAppender.java | 189 ++++++++++++++++++ .../log4j1/StreamisLog4jAppenderConfig.java | 95 +++++++++ .../collector/StreamisLogAppenderTest.java | 27 +++ .../src/test/resources/log4j.properties | 43 ++++ streamis-jobmanager/streamis-job-log/pom.xml | 6 +- .../log/collector/sender/http/Json.java | 149 -------------- .../sender/http/request/EntityPostAction.java | 79 -------- .../sender/http/request/PostAction.java | 41 ---- .../streamis-job-log-common/pom.xml | 12 +- .../jobmanager/log/entities/LogElement.java | 1 + .../log/entities/StreamisLogEvent.java | 14 +- .../log/entities/StreamisLogEvents.java | 25 ++- .../jobmanager/log/json/JsonTool.java | 45 +++++ .../server/entities/StreamisLogEvents.java | 12 ++ .../log/server/restful/JobLogRestfulApi.java | 2 +- .../storage/bucket/Log4j2JobLogBucket.java | 1 + .../jobmanager/log/LogStorageTest.java | 4 + 60 files changed, 907 insertions(+), 454 deletions(-) rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/flink-streamis-log-collector/pom.xml (100%) rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/flink-streamis-log-collector/src/main/assembly/package.xml (100%) rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java (79%) rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java (100%) rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/flink-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired (100%) rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java (97%) rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/flink-streamis-log-collector/src/test/resources/flink-conf.yaml (100%) create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/ExceptionListener.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/cache/LogCache.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java (68%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java (72%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java (54%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java (55%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java (77%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java (91%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java (84%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/entities/Resource.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java (100%) rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/HttpAction.java (100%) create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/StringPostAction.java rename streamis-jobmanager/streamis-job-log/{streamis-job-log-collector => job-log-collector/streamis-job-log-collector-core}/src/main/java/com/webank/wedatasphere/streamis/jobmanager/plugin/StreamisConfigAutowired.java (100%) rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/streamis-job-log-collector/pom.xml (77%) rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java (93%) create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/StreamisLog4j2AppenderConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/RpcAuthConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/RpcLogSenderConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/SendBufferConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/SendLogCacheConfig.java rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java (80%) rename streamis-jobmanager/streamis-job-log/{ => job-log-collector}/streamis-job-log-collector/src/test/resources/log4j2.xml (91%) create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j1/StreamisLog4jAppenderConfig.java create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/test/resources/log4j.properties delete mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java delete mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java delete mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/PostAction.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/json/JsonTool.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/entities/StreamisLogEvents.java create mode 100644 streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/test/com/webank/wedatasphere/streamis/jobmanager/log/LogStorageTest.java diff --git a/pom.xml b/pom.xml index 580cce56f..0fc4d2fba 100644 --- a/pom.xml +++ b/pom.xml @@ -53,7 +53,7 @@ 1.8 3.3.3 2.8.5 - 2.11.3 + 2.13.2 3.1.1 4.5.4 4.5.4 diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml similarity index 100% rename from streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/pom.xml rename to streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/assembly/package.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/assembly/package.xml similarity index 100% rename from streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/assembly/package.xml rename to streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/assembly/package.xml diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java similarity index 79% rename from streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java index c146d23b6..ba1756006 100644 --- a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigAutowired.java @@ -1,6 +1,7 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.flink; import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.log4j2.StreamisLog4j2AppenderConfig; import com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired; import org.apache.commons.lang3.StringUtils; import org.apache.flink.configuration.Configuration; @@ -49,17 +50,19 @@ public StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Bui gateway += this.configuration.getString(LOG_COLLECT_PATH, "/"); builder.setRpcAddress(gateway); } - List filterStrategies = this.configuration.get(LOG_FILTER_STRATEGIES); - for(String filterStrategy : filterStrategies){ - if ("LevelMatch".equals(filterStrategy)){ - builder.withFilter(LevelMatchFilter.newBuilder().setOnMatch(Filter.Result.ACCEPT).setOnMismatch(Filter.Result.DENY) - .setLevel(Level.getLevel(this.configuration.getString(LOG_FILTER_LEVEL_MATCH))).build()); - }else if ("ThresholdFilter".equals(filterStrategy)){ - builder.withFilter(ThresholdFilter.createFilter(Level - .getLevel(this.configuration.getString(LOG_FILTER_THRESHOLD_MATCH)), Filter.Result.ACCEPT, Filter.Result.DENY)); - }else if ("RegexMatch".equals(filterStrategy)){ - builder.withFilter(RegexFilter.createFilter( this.configuration.getString(LOG_FILTER_REGEX), - null, true, Filter.Result.ACCEPT, Filter.Result.DENY)); + if (builder instanceof StreamisLog4j2AppenderConfig.Builder) { + List filterStrategies = this.configuration.get(LOG_FILTER_STRATEGIES); + for (String filterStrategy : filterStrategies) { + if ("LevelMatch".equals(filterStrategy)) { + ((StreamisLog4j2AppenderConfig.Builder)builder).withFilter(LevelMatchFilter.newBuilder().setOnMatch(Filter.Result.ACCEPT).setOnMismatch(Filter.Result.DENY) + .setLevel(Level.getLevel(this.configuration.getString(LOG_FILTER_LEVEL_MATCH))).build()); + } else if ("ThresholdFilter".equals(filterStrategy)) { + ((StreamisLog4j2AppenderConfig.Builder)builder).withFilter(ThresholdFilter.createFilter(Level + .getLevel(this.configuration.getString(LOG_FILTER_THRESHOLD_MATCH)), Filter.Result.ACCEPT, Filter.Result.DENY)); + } else if ("RegexMatch".equals(filterStrategy)) { + ((StreamisLog4j2AppenderConfig.Builder)builder).withFilter(RegexFilter.createFilter(this.configuration.getString(LOG_FILTER_REGEX), + null, true, Filter.Result.ACCEPT, Filter.Result.DENY)); + } } } String hadoopUser = EnvironmentInformation.getHadoopUser(); diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkStreamisConfigDefine.java diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired similarity index 100% rename from streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired rename to streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java similarity index 97% rename from streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java index d04988099..6958957a7 100644 --- a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/flink/FlinkConfigurationLoadTest.java @@ -4,7 +4,6 @@ import org.apache.flink.configuration.GlobalConfiguration; import org.junit.Test; -import java.io.File; import java.util.Enumeration; import java.util.Objects; import java.util.Properties; diff --git a/streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/resources/flink-conf.yaml b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/test/resources/flink-conf.yaml similarity index 100% rename from streamis-jobmanager/streamis-job-log/flink-streamis-log-collector/src/test/resources/flink-conf.yaml rename to streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/src/test/resources/flink-conf.yaml diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml new file mode 100644 index 000000000..bc68d0796 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml @@ -0,0 +1,39 @@ + + + + streamis-job-log + com.webank.wedatasphere.streamis + 0.2.0 + + 4.0.0 + + streamis-job-log-collector-core + + + 8 + 8 + 4.5.13 + 4.5.4 + + + + + com.webank.wedatasphere.streamis + streamis-job-log-common + ${streamis.version} + + + + org.apache.httpcomponents + httpclient + ${httpclient.version} + + + org.apache.httpcomponents + httpmime + ${httpmine.version} + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/ExceptionListener.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/ExceptionListener.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/ExceptionListener.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/ExceptionListener.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/cache/LogCache.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/cache/LogCache.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/cache/LogCache.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/cache/LogCache.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java similarity index 68% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java index e88eb91e9..ebf9b7f68 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcAuthConfig.java @@ -1,17 +1,9 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; -import org.apache.logging.log4j.core.config.plugins.Plugin; -import org.apache.logging.log4j.core.config.plugins.PluginAttribute; -import org.apache.logging.log4j.core.config.plugins.PluginFactory; /** * Authentication config */ -@Plugin( - name = "AuthConfig", - category = "Core", - printObject = true -) public class RpcAuthConfig { /** * Key of token-code @@ -52,12 +44,6 @@ public RpcAuthConfig(String tokenCodeKey, String tokenCode, String tokenUserKey, } } - @PluginFactory - public static RpcAuthConfig createRpcAuthConfig(@PluginAttribute("tokenCodeKey") String tokenCodeKey, - @PluginAttribute("tokenCode") String tokenCode, - @PluginAttribute("tokenUserKey") String tokenUserKey, @PluginAttribute("tokenUser") String tokenUser){ - return new RpcAuthConfig(tokenCodeKey, tokenCode, tokenUserKey, tokenUser); - } public String getTokenCodeKey() { return tokenCodeKey; } @@ -89,4 +75,12 @@ public String getTokenCode() { public void setTokenCode(String tokenCode) { this.tokenCode = tokenCode; } + + @Override + public String toString() { + return "RpcAuthConfig{" + + ", tokenUserKey='" + tokenUserKey + '\'' + + ", tokenUser='" + tokenUser + '\'' + + '}'; + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java similarity index 72% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java index 10a063946..0fbc563d7 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/RpcLogSenderConfig.java @@ -1,21 +1,11 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; -import org.apache.logging.log4j.core.config.plugins.Plugin; -import org.apache.logging.log4j.core.config.plugins.PluginAttribute; -import org.apache.logging.log4j.core.config.plugins.PluginElement; -import org.apache.logging.log4j.core.config.plugins.PluginFactory; -import org.apache.logging.log4j.core.util.Integers; import java.util.Objects; /** * Rpc sender configuration */ -@Plugin( - name = "RpcLogSender", - category = "Core", - printObject = true -) public class RpcLogSenderConfig { /** @@ -86,18 +76,7 @@ public RpcLogSenderConfig(String address, int sendRetryCnt, int connectionTimeou this.bufferConfig = bufferConfig; } } - @PluginFactory - public static RpcLogSenderConfig createConfig( - @PluginAttribute("address") String address, @PluginAttribute("sendRetryCnt") String sendRetryCnt, - @PluginAttribute("connectionTimeout") String connectionTimeout, @PluginAttribute("socketTimeout") String socketTimeout, - @PluginAttribute("serverRecoveryTimeInSec") String serverRecoveryTimeInSec, @PluginAttribute("maxDelayTimeInSec") String maxDelayTimeInSec, - @PluginElement("AuthConfig")RpcAuthConfig authConfig, @PluginElement("SendLogCache") SendLogCacheConfig cacheConfig, - @PluginElement("SendBuffer")SendBufferConfig bufferConfig){ - return new RpcLogSenderConfig(address, Integers.parseInt(sendRetryCnt, 3), - Integers.parseInt(connectionTimeout, 3000), Integers.parseInt(socketTimeout, 15000), - Integers.parseInt(serverRecoveryTimeInSec, 5), Integers.parseInt(maxDelayTimeInSec, 60), - authConfig, cacheConfig, bufferConfig); - } + public RpcAuthConfig getAuthConfig() { return authConfig; } @@ -169,4 +148,19 @@ public int getServerRecoveryTimeInSec() { public void setServerRecoveryTimeInSec(int serverRecoveryTimeInSec) { this.serverRecoveryTimeInSec = serverRecoveryTimeInSec; } + + @Override + public String toString() { + return "RpcLogSenderConfig{" + + "address='" + address + '\'' + + ", connectionTimeout=" + connectionTimeout + + ", socketTimeout=" + socketTimeout + + ", sendRetryCnt=" + sendRetryCnt + + ", serverRecoveryTimeInSec=" + serverRecoveryTimeInSec + + ", maxDelayTimeInSec=" + maxDelayTimeInSec + + ", authConfig=" + authConfig + + ", cacheConfig=" + cacheConfig + + ", bufferConfig=" + bufferConfig + + '}'; + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java similarity index 54% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java index 651caa616..6be0ae826 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendBufferConfig.java @@ -1,15 +1,6 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; -import org.apache.logging.log4j.core.config.plugins.Plugin; -import org.apache.logging.log4j.core.config.plugins.PluginAttribute; -import org.apache.logging.log4j.core.config.plugins.PluginFactory; -import org.apache.logging.log4j.core.util.Integers; - -@Plugin( - name = "SendBuffer", - category = "Core", - printObject = true -) + public class SendBufferConfig { /** * Size of send buffer @@ -30,11 +21,6 @@ public SendBufferConfig(int size, long expireTimeInSec){ this.expireTimeInSec = expireTimeInSec; } - @PluginFactory - public static SendBufferConfig createBufferConfig( - @PluginAttribute("size") String size, @PluginAttribute("expireTimeInSec") String expireTimeInSec){ - return new SendBufferConfig(Integers.parseInt(size, 50), Integers.parseInt(expireTimeInSec, 2)); - } public int getSize() { return size; } @@ -50,4 +36,12 @@ public long getExpireTimeInSec() { public void setExpireTimeInSec(long expireTimeInSec) { this.expireTimeInSec = expireTimeInSec; } + + @Override + public String toString() { + return "SendBufferConfig{" + + "size=" + size + + ", expireTimeInSec=" + expireTimeInSec + + '}'; + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java similarity index 55% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java index 6921584a5..1caaedb21 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/SendLogCacheConfig.java @@ -1,18 +1,9 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; -import org.apache.logging.log4j.core.config.plugins.Plugin; -import org.apache.logging.log4j.core.config.plugins.PluginAttribute; -import org.apache.logging.log4j.core.config.plugins.PluginFactory; -import org.apache.logging.log4j.core.util.Integers; - /** * Cache config */ -@Plugin( - name = "SendLogCache", - category = "Core", - printObject = true -) + public class SendLogCacheConfig { /** * Size of send cache @@ -32,11 +23,7 @@ public SendLogCacheConfig(int size, int maxConsumeThread){ this.size = size; this.maxConsumeThread = maxConsumeThread; } - @PluginFactory - public static SendLogCacheConfig createCacheConfig( - @PluginAttribute("size") String size, @PluginAttribute("maxConsumeThread") String maxConsumeThread){ - return new SendLogCacheConfig(Integers.parseInt(size, 150), Integers.parseInt(maxConsumeThread, 10)); - } + public int getSize() { return size; } @@ -52,4 +39,12 @@ public int getMaxConsumeThread() { public void setMaxConsumeThread(int maxConsumeThread) { this.maxConsumeThread = maxConsumeThread; } + + @Override + public String toString() { + return "SendLogCacheConfig{" + + "size=" + size + + ", maxConsumeThread=" + maxConsumeThread + + '}'; + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java similarity index 77% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java index e334c6be2..35680913f 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/config/StreamisLogAppenderConfig.java @@ -1,10 +1,6 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector.config; -import org.apache.logging.log4j.core.Filter; -import org.apache.logging.log4j.core.filter.CompositeFilter; - import java.util.ArrayList; -import java.util.List; import java.util.Objects; import java.util.Optional; @@ -13,19 +9,13 @@ */ public class StreamisLogAppenderConfig { - private final String applicationName; + protected final String applicationName; - /** - * Filter in log4j - */ - private final Filter filter; - private final RpcLogSenderConfig senderConfig; + protected final RpcLogSenderConfig senderConfig; - StreamisLogAppenderConfig(String applicationName, Filter filter, - RpcLogSenderConfig rpcLogSenderConfig){ + protected StreamisLogAppenderConfig(String applicationName, RpcLogSenderConfig rpcLogSenderConfig){ this.applicationName = applicationName; - this.filter = filter; this.senderConfig = null != rpcLogSenderConfig? rpcLogSenderConfig : new RpcLogSenderConfig(); } @@ -33,24 +23,17 @@ public static class Builder{ /** * Application name */ - private String applicationName; - - /** - * Filter rules - */ - private final List filters = new ArrayList<>(); + protected String applicationName; /** * Sender config */ - private final RpcLogSenderConfig rpcLogSenderConfig; + protected final RpcLogSenderConfig rpcLogSenderConfig; - public Builder(String applicationName, Filter filter, + public Builder(String applicationName, RpcLogSenderConfig rpcLogSenderConfig){ this.applicationName = applicationName; - if (Objects.nonNull(filter)) { - this.filters.add(filter); - } + this.rpcLogSenderConfig = Optional.ofNullable(rpcLogSenderConfig).orElse(new RpcLogSenderConfig()); } @@ -64,26 +47,7 @@ public StreamisLogAppenderConfig.Builder setAppName(String applicationName){ return this; } - /** - * Set filter - * @param filter filter - * @return builder - */ - public StreamisLogAppenderConfig.Builder setFilter(Filter filter){ - this.filters.clear(); - this.filters.add(filter); - return this; - } - /** - * Append filter - * @param filter filter - * @return builder - */ - public StreamisLogAppenderConfig.Builder withFilter(Filter filter){ - filters.add(filter); - return this; - } /** * Rpc address @@ -224,28 +188,15 @@ public StreamisLogAppenderConfig.Builder setRpcBufferExpireTimeInSec(int expireT this.rpcLogSenderConfig.getBufferConfig().setExpireTimeInSec(expireTimeInSec); return this; } - /** - * Build method - * @return config - */ + public StreamisLogAppenderConfig build(){ - Filter logFilter = null; - if (filters.size() > 1){ - logFilter = CompositeFilter.createFilters(filters.toArray(new Filter[0])); - } else if (!filters.isEmpty()){ - logFilter = filters.get(0); - } - return new StreamisLogAppenderConfig(applicationName, logFilter, rpcLogSenderConfig); + return new StreamisLogAppenderConfig(applicationName, rpcLogSenderConfig); } - } public String getApplicationName() { return applicationName; } - public Filter getFilter() { - return filter; - } public RpcLogSenderConfig getSenderConfig() { return senderConfig; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcLogSender.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCache.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogCacheConsumer.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/SendLogExceptionStrategy.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java similarity index 91% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java index b9ec15537..d019c29f5 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/StreamisRpcLogSender.java @@ -37,4 +37,9 @@ protected StreamisLogEvents aggregateBuffer(SendBuffer sendBuf } return null; } + + @Override + protected String convertToJsonString(StreamisLogEvents aggregatedEntity) { + return aggregatedEntity.toJson(); + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/AbstractSendBuffer.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/ImmutableSendBuffer.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/buf/SendBuffer.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java similarity index 84% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java index 827e6666e..c693d0152 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/AbstractHttpLogSender.java @@ -5,23 +5,18 @@ import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.SendLogExceptionStrategy; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.buf.SendBuffer; -import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request.EntityPostAction; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request.StringPostAction; import com.webank.wedatasphere.streamis.jobmanager.log.entities.LogElement; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.HttpResponseException; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.conn.ConnectTimeoutException; -import org.apache.logging.log4j.core.util.IOUtils; import javax.net.ssl.SSLException; -import javax.sound.midi.SysexMessage; -import javax.xml.ws.Response; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.InterruptedIOException; +import java.io.*; import java.net.UnknownHostException; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.Optional; import java.util.concurrent.TimeUnit; @@ -113,7 +108,7 @@ protected void doSend(E aggregatedEntity, RpcLogSenderConfig rpcSenderConfig) th } String address = rpcSenderConfig.getAddress(); if (null != address && !address.trim().equals("")) { - EntityPostAction postAction = new EntityPostAction<>(rpcSenderConfig.getAddress(), aggregatedEntity); + StringPostAction postAction = new StringPostAction(rpcSenderConfig.getAddress(), convertToJsonString(aggregatedEntity)); RpcAuthConfig authConfig = rpcSenderConfig.getAuthConfig(); postAction.getRequestHeaders().put(authConfig.getTokenUserKey(), authConfig.getTokenUser()); HttpResponse response = null; @@ -122,8 +117,7 @@ protected void doSend(E aggregatedEntity, RpcLogSenderConfig rpcSenderConfig) th int statusCode = response.getStatusLine().getStatusCode(); if (statusCode > 200){ throw new HttpResponseException(statusCode, - IOUtils.toString(new InputStreamReader( - response.getEntity().getContent(), StandardCharsets.UTF_8))); + convertToString(response.getEntity().getContent(), StandardCharsets.UTF_8)); } }finally { // Close the response and release the conn @@ -141,4 +135,29 @@ protected void doSend(E aggregatedEntity, RpcLogSenderConfig rpcSenderConfig) th } } } + + /** + * Convert input to string + * @param inputStream input stream + * @param charset charset + * @return string value + * @throws IOException + */ + private String convertToString(InputStream inputStream, Charset charset) throws IOException { + StringBuilder builder = new StringBuilder(); + try(BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, charset))){ + String line; + while((line = reader.readLine()) != null){ + builder.append(line); + } + } + return builder.toString(); + } + + /** + * Convert the entity to json + * @param aggregatedEntity aggregated entity + * @return json string + */ + protected abstract String convertToJsonString(E aggregatedEntity); } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/HttpClientTool.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/entities/Resource.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/entities/Resource.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/entities/Resource.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/entities/Resource.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/AbstractHttpAction.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/GetAction.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/HttpAction.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/HttpAction.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/HttpAction.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/HttpAction.java diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/StringPostAction.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/StringPostAction.java new file mode 100644 index 000000000..6ce0d8cdf --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/StringPostAction.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request; + +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; + +/** + * Use string to request + */ +public class StringPostAction extends AbstractHttpAction { + + /** + * Raw string value + */ + private final String rawString; + public StringPostAction(String uri, String rawString) { + super(uri); + this.rawString = rawString; + } + + @Override + protected HttpPost getRequestMethod() { + HttpPost httpPost = new HttpPost(); + StringEntity stringEntity = new StringEntity(rawString, "UTF-8"); + stringEntity.setContentType(ContentType.APPLICATION_JSON.toString()); + httpPost.setEntity(stringEntity); + return httpPost; + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/plugin/StreamisConfigAutowired.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/plugin/StreamisConfigAutowired.java similarity index 100% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/plugin/StreamisConfigAutowired.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/plugin/StreamisConfigAutowired.java diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml similarity index 77% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml index 5c4822539..e2e556fc8 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml @@ -14,8 +14,6 @@ 8 8 - 4.5.13 - 4.5.4 2.17.1 1.7.15 @@ -23,20 +21,9 @@ com.webank.wedatasphere.streamis - streamis-job-log-common + streamis-job-log-collector-core ${streamis.version} - - - org.apache.httpcomponents - httpclient - ${httpclient.version} - - - org.apache.httpcomponents - httpmime - ${httpmine.version} - org.slf4j diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java similarity index 93% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java index 26c304d9b..c4674106f 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java @@ -3,8 +3,8 @@ import com.webank.wedatasphere.streamis.jobmanager.log.collector.cache.LogCache; import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.log4j2.StreamisLog4j2AppenderConfig; import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.StreamisRpcLogSender; -import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.Json; import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvent; import com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired; import org.apache.logging.log4j.core.Filter; @@ -85,12 +85,12 @@ public static StreamisRpcLogAppender createAppender(@PluginAttribute("name") Str } // Search the config autowired class List configAutowiredEntities = new ArrayList<>(); - StreamisLogAppenderConfig logAppenderConfig = null; + StreamisLog4j2AppenderConfig logAppenderConfig = null; ServiceLoader.load(StreamisConfigAutowired.class, StreamisRpcLogAppender.class.getClassLoader()).iterator().forEachRemaining(configAutowiredEntities::add); - StreamisLogAppenderConfig.Builder builder = new StreamisLogAppenderConfig.Builder(applicationName, filter, rpcLogSenderConfig); + StreamisLog4j2AppenderConfig.Builder builder = new StreamisLog4j2AppenderConfig.Builder(applicationName, filter, rpcLogSenderConfig); for (StreamisConfigAutowired autowired : configAutowiredEntities){ - logAppenderConfig = autowired.logAppenderConfig(builder); + logAppenderConfig = (StreamisLog4j2AppenderConfig) autowired.logAppenderConfig(builder); } if (Objects.isNull(logAppenderConfig)){ logAppenderConfig = builder.build(); @@ -99,7 +99,7 @@ public static StreamisRpcLogAppender createAppender(@PluginAttribute("name") Str if (null == applicationName || applicationName.trim().equals("")){ throw new IllegalArgumentException("Application name cannot be empty"); } - System.out.println("StreamisRpcLogAppender: init with config" + Json.toJson(logAppenderConfig, null)); + System.out.println("StreamisRpcLogAppender: init with config => " + logAppenderConfig); return new StreamisRpcLogAppender(name, logAppenderConfig.getFilter(), layout, ignoreExceptions, Property.EMPTY_ARRAY, logAppenderConfig); } diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/StreamisLog4j2AppenderConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/StreamisLog4j2AppenderConfig.java new file mode 100644 index 000000000..f68d77210 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/StreamisLog4j2AppenderConfig.java @@ -0,0 +1,88 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.log4j2; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; +import org.apache.logging.log4j.core.Filter; +import org.apache.logging.log4j.core.filter.CompositeFilter; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * Appender config for log4j2 + */ +public class StreamisLog4j2AppenderConfig extends StreamisLogAppenderConfig { + /** + * Filter in log4j2 + */ + private final Filter filter; + + public StreamisLog4j2AppenderConfig(String applicationName, Filter filter, + RpcLogSenderConfig rpcLogSenderConfig){ + super(applicationName, rpcLogSenderConfig); + this.filter = filter; + } + + public static class Builder extends StreamisLogAppenderConfig.Builder { + + /** + * Filter rules + */ + private final List filters = new ArrayList<>(); + + public Builder(String applicationName, Filter filter, RpcLogSenderConfig rpcLogSenderConfig) { + super(applicationName, rpcLogSenderConfig); + if (Objects.nonNull(filter)) { + this.filters.add(filter); + } + } + + /** + * Set filter + * @param filter filter + * @return builder + */ + public StreamisLog4j2AppenderConfig.Builder setFilter(Filter filter){ + this.filters.clear(); + this.filters.add(filter); + return this; + } + + /** + * Append filter + * @param filter filter + * @return builder + */ + public StreamisLog4j2AppenderConfig.Builder withFilter(Filter filter){ + filters.add(filter); + return this; + } + + /** + * Build method + * @return config + */ + public StreamisLog4j2AppenderConfig build(){ + Filter logFilter = null; + if (filters.size() > 1){ + logFilter = CompositeFilter.createFilters(filters.toArray(new Filter[0])); + } else if (!filters.isEmpty()){ + logFilter = filters.get(0); + } + return new StreamisLog4j2AppenderConfig(applicationName, logFilter, rpcLogSenderConfig); + } + } + public Filter getFilter() { + return filter; + } + + @Override + public String toString() { + return "StreamisLog4j2AppenderConfig{" + + "applicationName='" + applicationName + '\'' + + ", senderConfig=" + senderConfig + + ", filter=" + filter + + '}'; + } +} diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/RpcAuthConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/RpcAuthConfig.java new file mode 100644 index 000000000..87a10ba85 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/RpcAuthConfig.java @@ -0,0 +1,31 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.log4j2.config; + +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; + +/** + * AuthConfig Element in log4j2 + */ +@Plugin( + name = "AuthConfig", + category = "Core", + printObject = true +) +public class RpcAuthConfig extends com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcAuthConfig { + + public RpcAuthConfig(){ + super(); + } + public RpcAuthConfig(String tokenCodeKey, String tokenCode, String tokenUserKey, String tokenUser) { + super(tokenCodeKey, tokenCode, tokenUserKey, tokenUser); + } + + @PluginFactory + public static RpcAuthConfig createRpcAuthConfig(@PluginAttribute("tokenCodeKey") String tokenCodeKey, + @PluginAttribute("tokenCode") String tokenCode, + @PluginAttribute("tokenUserKey") String tokenUserKey, @PluginAttribute("tokenUser") String tokenUser){ + return new RpcAuthConfig(tokenCodeKey, tokenCode, tokenUserKey, tokenUser); + } + +} diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/RpcLogSenderConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/RpcLogSenderConfig.java new file mode 100644 index 000000000..7be526465 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/RpcLogSenderConfig.java @@ -0,0 +1,37 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.log4j2.config; + +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginElement; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.util.Integers; + +/** + * Rpc sender configuration + */ +@Plugin( + name = "RpcLogSender", + category = "Core", + printObject = true +) +public class RpcLogSenderConfig extends com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig { + + public RpcLogSenderConfig(String address, int sendRetryCnt, int connectionTimeout, int socketTimeout, int serverRecoveryTimeInSec, int maxDelayTimeInSec, + RpcAuthConfig authConfig, SendLogCacheConfig cacheConfig, SendBufferConfig bufferConfig) { + super(address, sendRetryCnt, connectionTimeout, socketTimeout, serverRecoveryTimeInSec, maxDelayTimeInSec, authConfig, cacheConfig, bufferConfig); + } + + @PluginFactory + public static RpcLogSenderConfig createConfig( + @PluginAttribute("address") String address, @PluginAttribute("sendRetryCnt") String sendRetryCnt, + @PluginAttribute("connectionTimeout") String connectionTimeout, @PluginAttribute("socketTimeout") String socketTimeout, + @PluginAttribute("serverRecoveryTimeInSec") String serverRecoveryTimeInSec, @PluginAttribute("maxDelayTimeInSec") String maxDelayTimeInSec, + @PluginElement("AuthConfig")RpcAuthConfig authConfig, @PluginElement("SendLogCache") SendLogCacheConfig cacheConfig, + @PluginElement("SendBuffer")SendBufferConfig bufferConfig){ + return new RpcLogSenderConfig(address, Integers.parseInt(sendRetryCnt, 3), + Integers.parseInt(connectionTimeout, 3000), Integers.parseInt(socketTimeout, 15000), + Integers.parseInt(serverRecoveryTimeInSec, 5), Integers.parseInt(maxDelayTimeInSec, 60), + authConfig, cacheConfig, bufferConfig); + } + +} diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/SendBufferConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/SendBufferConfig.java new file mode 100644 index 000000000..936accd72 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/SendBufferConfig.java @@ -0,0 +1,28 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.log4j2.config; + +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.util.Integers; + +@Plugin( + name = "SendBuffer", + category = "Core", + printObject = true +) +public class SendBufferConfig extends com.webank.wedatasphere.streamis.jobmanager.log.collector.config.SendBufferConfig { + + public SendBufferConfig() { + } + + public SendBufferConfig(int size, long expireTimeInSec) { + super(size, expireTimeInSec); + } + + @PluginFactory + public static SendBufferConfig createBufferConfig( + @PluginAttribute("size") String size, @PluginAttribute("expireTimeInSec") String expireTimeInSec){ + return new SendBufferConfig(Integers.parseInt(size, 50), + Integers.parseInt(expireTimeInSec, 2)); + } +} diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/SendLogCacheConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/SendLogCacheConfig.java new file mode 100644 index 000000000..f4a63c49c --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j2/config/SendLogCacheConfig.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.log4j2.config; + +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.util.Integers; + +/** + * Cache config + */ +@Plugin( + name = "SendLogCache", + category = "Core", + printObject = true +) +public class SendLogCacheConfig extends com.webank.wedatasphere.streamis.jobmanager.log.collector.config.SendLogCacheConfig { + + public SendLogCacheConfig(int size, int maxConsumeThread) { + super(size, maxConsumeThread); + } + + @PluginFactory + public static SendLogCacheConfig createCacheConfig( + @PluginAttribute("size") String size, @PluginAttribute("maxConsumeThread") String maxConsumeThread){ + return new SendLogCacheConfig(Integers.parseInt(size, 150), Integers.parseInt(maxConsumeThread, 10)); + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java similarity index 80% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java index d4a7e61ce..2cec0ec46 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java @@ -1,6 +1,5 @@ package com.webank.wedatasphere.streamis.jobmanager.log.collector; -import org.apache.logging.log4j.LogManager; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -9,8 +8,6 @@ public class StreamisLogAppenderTest { private static final Logger LOG = LoggerFactory.getLogger(StreamisLogAppenderTest.class); @Test public void appenderLog() throws InterruptedException { - org.apache.logging.log4j.core.Logger logger = (org.apache.logging.log4j.core.Logger) LogManager.getLogger("Test: Hello-world"); - logger.info(""); int total = 1000; int tps = 100; long timer = System.currentTimeMillis() + 1000; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/test/resources/log4j2.xml similarity index 91% rename from streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml rename to streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/test/resources/log4j2.xml index 56a3bba82..ee3f4125a 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/test/resources/log4j2.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/src/test/resources/log4j2.xml @@ -21,9 +21,9 @@ + socketTimeout="20000" serverRecoveryTimeInSec="5" maxDelayTimeInSec="60"> - + ` diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml new file mode 100644 index 000000000..43792639c --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml @@ -0,0 +1,55 @@ + + + + streamis-job-log + com.webank.wedatasphere.streamis + 0.2.0 + + 4.0.0 + + streamis-job-log-collector1x + + + 8 + 8 + 1.2.17 + 1.7.12 + + + + com.webank.wedatasphere.streamis + streamis-job-log-collector-core + ${streamis.version} + + + + org.slf4j + slf4j-api + ${slf4j.version} + provided + + + + org.slf4j + slf4j-log4j12 + ${slf4j.version} + provided + + + + log4j + log4j + ${log4j.version} + provided + + + + junit + junit + ${junit.version} + test + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java new file mode 100644 index 000000000..ccbaf02fc --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisRpcLogAppender.java @@ -0,0 +1,189 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.cache.LogCache; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.log4j1.StreamisLog4jAppenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.StreamisRpcLogSender; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvent; +import com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired; +import org.apache.log4j.AppenderSkeleton; +import org.apache.log4j.Level; +import org.apache.log4j.SimpleLayout; +import org.apache.log4j.helpers.LogLog; +import org.apache.log4j.spi.LoggingEvent; + +import java.util.*; + +/** + * Rpc appender for log4j1 + */ +public class StreamisRpcLogAppender extends AppenderSkeleton { + + /** + * Application name + */ + private String applicationName; + + /** + * Appender config + */ + private StreamisLog4jAppenderConfig appenderConfig; + + /** + * Rpc log sender + */ + private StreamisRpcLogSender rpcLogSender; + + /** + * Rpc log sender config + */ + private RpcLogSenderConfig rpcLogSenderConfig = new RpcLogSenderConfig(); + + + /** + * Cache + */ + private LogCache logCache; + + + @Override + protected void append(LoggingEvent loggingEvent) { + String content = super.getLayout().format(loggingEvent); + // Transform to stream log event; + StreamisLogEvent logEvent = new StreamisLogEvent(content, System.currentTimeMillis()); + if (Objects.nonNull(logCache)){ + try { + this.logCache.cacheLog(logEvent); + } catch (InterruptedException e) { + LogLog.error("StreamisRpcLogAppender: " + this.getName() + + " interrupted when cache the log into the RPC sender, message: " + e.getMessage()); + } + } + } + + @Override + public void close() { + if (Objects.nonNull(this.rpcLogSender)){ + this.rpcLogSender.close(); + } + } + + @Override + public boolean requiresLayout() { + return true; + } + + @Override + public void activateOptions() { + if (Objects.nonNull(this.logCache)){ + return; + } + if (Objects.isNull(getLayout())){ + setLayout(new SimpleLayout()); + } + // Search the config autowired class + List configAutowiredEntities = new ArrayList<>(); + StreamisLog4jAppenderConfig logAppenderConfig = null; + ServiceLoader.load(StreamisConfigAutowired.class, + StreamisRpcLogAppender.class.getClassLoader()).iterator().forEachRemaining(configAutowiredEntities::add); + StreamisLog4jAppenderConfig.Builder builder = new StreamisLog4jAppenderConfig.Builder(this.applicationName, + getThreshold(), getFilter(), rpcLogSenderConfig); + for (StreamisConfigAutowired autowired : configAutowiredEntities){ + try { + logAppenderConfig = (StreamisLog4jAppenderConfig) autowired.logAppenderConfig(builder); + } catch (Exception e) { + LogLog.warn("Unable to autowired the config from: " +autowired.getClass().getName(), e); + } + } + if (Objects.isNull(logAppenderConfig)){ + logAppenderConfig = builder.build(); + } + this.applicationName = logAppenderConfig.getApplicationName(); + if (null == applicationName || applicationName.trim().equals("")){ + throw new IllegalArgumentException("Application name cannot be empty"); + } + this.appenderConfig = logAppenderConfig; + // Set the threshold to error default + setThreshold(Optional.ofNullable(logAppenderConfig.getThreshold()).orElse(Level.ERROR)); + // First to clear the filters + clearFilters(); + // Then to add filter + logAppenderConfig.getFilters().forEach(this :: addFilter); + System.out.println("StreamisRpcLogAppender: init with config => " + logAppenderConfig); + this.rpcLogSender = new StreamisRpcLogSender(this.appenderConfig.getApplicationName(), + this.appenderConfig.getSenderConfig()); + this.rpcLogSender.setExceptionListener((subject, t, message) -> + LogLog.error((null != subject? subject.getClass().getSimpleName() : "") + ": " + message, t)); + this.logCache = this.rpcLogSender.getOrCreateLogCache(); + } + + + public String getAppName() { + return applicationName; + } + + /** + * Application name + * @param applicationName name + */ + public void setAppName(String applicationName) { + this.applicationName = applicationName; + } + + public void setRpcAddress(String address){ + this.rpcLogSenderConfig.setAddress(address); + } + + public void setRpcConnTimeout(int connectionTimeout){ + this.rpcLogSenderConfig.setConnectionTimeout(connectionTimeout); + } + + public void setRpcSocketTimeout(int socketTimeout){ + this.rpcLogSenderConfig.setSocketTimeout(socketTimeout); + } + public void setRpcSendRetryCnt(int sendRetryCnt){ + this.rpcLogSenderConfig.setSendRetryCnt(sendRetryCnt); + } + + public void setRpcServerRecoveryTimeInSec(int serverRecoveryTimeInSec){ + this.rpcLogSenderConfig.setServerRecoveryTimeInSec(serverRecoveryTimeInSec); + } + + public void setRpcMaxDelayTimeInSec(int maxDelayTimeInSec){ + this.rpcLogSenderConfig.setMaxDelayTimeInSec(maxDelayTimeInSec); + } + // Authentication + public void setRpcAuthTokenCodeKey(String tokenCodeKey){ + this.rpcLogSenderConfig.getAuthConfig().setTokenCodeKey(tokenCodeKey); + } + + public void setRpcAuthTokenUserKey(String tokenUserKey){ + this.rpcLogSenderConfig.getAuthConfig().setTokenUserKey(tokenUserKey); + } + + public void setRpcAuthTokenUser(String tokenUser){ + this.rpcLogSenderConfig.getAuthConfig().setTokenUser(tokenUser); + } + + public void setRpcAuthTokenCode(String tokenCode){ + this.rpcLogSenderConfig.getAuthConfig().setTokenCode(tokenCode); + } + + // Cache configuration + public void setRpcCacheSize(int cacheSize){ + this.rpcLogSenderConfig.getCacheConfig().setSize(cacheSize); + } + + public void setRpcCacheMaxConsumeThread(int maxConsumeThread){ + this.rpcLogSenderConfig.getCacheConfig().setMaxConsumeThread(maxConsumeThread); + } + + // Buffer configuration + public void setRpcBufferSize(int bufferSize){ + this.rpcLogSenderConfig.getBufferConfig().setSize(bufferSize); + } + + public void setRpcBufferExpireTimeInSec(int expireTimeInSec){ + this.rpcLogSenderConfig.getBufferConfig().setExpireTimeInSec(expireTimeInSec); + } +} diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j1/StreamisLog4jAppenderConfig.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j1/StreamisLog4jAppenderConfig.java new file mode 100644 index 000000000..84489c155 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/log4j1/StreamisLog4jAppenderConfig.java @@ -0,0 +1,95 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.log4j1; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.RpcLogSenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; +import org.apache.log4j.Priority; +import org.apache.log4j.spi.Filter; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * Appender config for log4j1 + */ +public class StreamisLog4jAppenderConfig extends StreamisLogAppenderConfig { + + /** + * Filter in log4j1 + */ + private final List filters = new ArrayList<>(); + /** + * + */ + private final Priority threshold; + + protected StreamisLog4jAppenderConfig(String applicationName, Priority threshold, List filters, + RpcLogSenderConfig rpcLogSenderConfig) { + super(applicationName, rpcLogSenderConfig); + this.threshold = threshold; + this.filters.addAll(filters); + } + + public static class Builder extends StreamisLogAppenderConfig.Builder{ + + /** + * Filter rules + */ + private final List filters = new ArrayList<>(); + + /** + * Threshold + */ + private Priority threshold; + + public Builder(String applicationName, Priority threshold, Filter filter,RpcLogSenderConfig rpcLogSenderConfig) { + super(applicationName, rpcLogSenderConfig); + this.threshold = threshold; + if (Objects.nonNull(filter)) { + this.filters.add(filter); + } + } + + public StreamisLog4jAppenderConfig.Builder setFilter(Filter filter){ + this.filters.clear(); + this.filters.add(filter); + return this; + } + + public StreamisLog4jAppenderConfig.Builder withFilter(Filter filter){ + filters.add(filter); + return this; + } + + /** + * Set threshold + * @param threshold threshold + * @return builder + */ + public StreamisLog4jAppenderConfig.Builder threshold(Priority threshold){ + this.threshold = threshold; + return this; + } + public StreamisLog4jAppenderConfig build(){ + return new StreamisLog4jAppenderConfig(applicationName, threshold, filters, rpcLogSenderConfig); + } + } + + public List getFilters() { + return filters; + } + + public Priority getThreshold() { + return threshold; + } + + @Override + public String toString() { + return "StreamisLog4jAppenderConfig{" + + "applicationName='" + applicationName + '\'' + + ", senderConfig=" + senderConfig + + ", filters=" + filters + + ", threshold=" + threshold + + '}'; + } +} diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java new file mode 100644 index 000000000..0dcca02c9 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/test/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/StreamisLogAppenderTest.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector; + +import org.apache.log4j.PropertyConfigurator; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StreamisLogAppenderTest { + private static final Logger LOG = LoggerFactory.getLogger(StreamisLogAppenderTest.class); + @Test + public void appenderLog() throws InterruptedException { + PropertyConfigurator.configure(StreamisLogAppenderTest.class.getResource("/log4j.properties").getPath()); + int total = 1000; + int tps = 100; + long timer = System.currentTimeMillis() + 1000; + for(int i = 0; i < total; i ++){ + if (i > 0 && i % tps == 0){ + long sleep = timer - System.currentTimeMillis(); + if (sleep > 0){ + Thread.sleep(sleep); + } + timer = System.currentTimeMillis() + 1000; + } + LOG.info("Stream Log appender test, sequence id: " + i); + } + } +} diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/test/resources/log4j.properties b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/test/resources/log4j.properties new file mode 100644 index 000000000..85e2a9167 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/src/test/resources/log4j.properties @@ -0,0 +1,43 @@ +# +# Copyright 2021 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### set log levels ### + +log4j.rootCategory=INFO,stream + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + +log4j.appender.stream=com.webank.wedatasphere.streamis.jobmanager.log.collector.StreamisRpcLogAppender +log4j.appender.stream.appName=stream_applicatioin +log4j.appender.stream.Threshold=INFO +log4j.appender.stream.layout=org.apache.log4j.PatternLayout +log4j.appender.stream.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.stream.rpcConnTimeout=3000 +log4j.appender.stream.rpcSocketTimeout=15000 +log4j.appender.stream.rpcSendRetryCnt=3 +log4j.appender.stream.rpcServerRecoveryTimeInSec=5 +log4j.appender.stream.rpcMaxDelayTimeInSec=60 +log4j.appender.stream.rpcAuthTokenCodeKey= +log4j.appender.stream.rpcAuthTokenUserKey= +log4j.appender.stream.rpcAuthTokenUser= +log4j.appender.stream.rpcAuthTokenCode= +log4j.appender.stream.rpcCacheSize=200 +log4j.appender.stream.rpcCacheMaxConsumeThread=1 +log4j.appender.stream.rpcBufferSize=50 +log4j.appender.stream.rpcBufferExpireTimeInSec=2 \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/pom.xml b/streamis-jobmanager/streamis-job-log/pom.xml index 3446a972b..0c8b087f1 100644 --- a/streamis-jobmanager/streamis-job-log/pom.xml +++ b/streamis-jobmanager/streamis-job-log/pom.xml @@ -12,8 +12,10 @@ streamis-job-log pom - streamis-job-log-collector - flink-streamis-log-collector + job-log-collector/streamis-job-log-collector-core + job-log-collector/streamis-job-log-collector + job-log-collector/streamis-job-log-collector1x + job-log-collector/flink-streamis-log-collector streamis-job-log-server streamis-job-log-common diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java deleted file mode 100644 index 911ccb381..000000000 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/Json.java +++ /dev/null @@ -1,149 +0,0 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.*; -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; - -/** - * Json utils - */ -public class Json { - private static final String PREFIX = "["; - private static final String SUFFIX = "]"; - - private static ObjectMapper mapper; - - static{ - mapper = new ObjectMapper(); - mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true); - mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); - mapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true); - mapper.configure(DeserializationFeature.READ_ENUMS_USING_TO_STRING, true); - mapper.configure(SerializationFeature.WRITE_ENUMS_USING_TO_STRING, true); - mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true); - //empty beans allowed - mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - //cancel to scape non ascii - mapper.configure(JsonGenerator.Feature.ESCAPE_NON_ASCII, false); - } - private Json(){} - - @SuppressWarnings("unchecked") - public static T fromJson(String json, Class clazz, Class... parameters) throws Exception{ - if(null != json && !json.trim().equals("")){ - try{ - if(parameters.length > 0){ - return (T)mapper.readValue(json, mapper.getTypeFactory().constructParametricType(clazz, parameters)); - } - if(json.startsWith(PREFIX) - && json.endsWith(SUFFIX)){ - JavaType javaType = mapper.getTypeFactory() - .constructParametricType(ArrayList.class, clazz); - return mapper.readValue(json, javaType); - } - return (T)mapper.readValue(json, clazz); - } catch (Exception e) { - String message = "Unable to deserialize to object from string(json) in type: [" + - (null != clazz ? clazz.getSimpleName() : "UNKNOWN") + "], parameters size: " + parameters.length; - throw new RuntimeException(message, e); - } - } - return null; - } - - public static T fromJson(InputStream stream, Class clazz, Class... parameters) throws Exception{ - StringBuilder builder = new StringBuilder(); - String jsonStr = null; - try{ - BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)); - while((jsonStr = reader.readLine()) != null){ - builder.append(jsonStr); - } - reader.close(); - }catch(Exception e){ - String message = "Unable to deserialize to object from stream(json) in type: [" + - (null != clazz ? clazz.getSimpleName() : "UNKNOWN") + "], parameters size: " + parameters.length; - throw new RuntimeException(message, e); - } - return fromJson(builder.toString(), clazz, parameters); - } - - public static String toJson(Object obj, Class model) { - ObjectWriter writer = mapper.writer(); - if(null != obj){ - try{ - if(null != model){ - writer = writer.withView(model); - } - return writer.writeValueAsString(obj); - } catch (JsonProcessingException e) { - String message = "Unable to serialize the object in type: ["+ (null != model ? model.getSimpleName() : "UNKNOWN") + "]"; - throw new RuntimeException(message, e); - } - } - return null; - } - - public static String toJson(Object obj, Class model, boolean beautify){ - ObjectWriter writer = mapper.writer(); - if(null != obj){ - try{ - if(null != model){ - writer = writer.withView(model); - } - if(beautify){ - return writer.withDefaultPrettyPrinter().writeValueAsString(obj); - } - return writer.writeValueAsString(obj); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - return null; - } - - /** - * Convert object using serialization and deserialization - * - * @param simpleObj simpleObj - * @param tClass type class - * @param T - * @return result - */ - @SuppressWarnings("unchecked") - public static T convert(Object simpleObj, Class tClass, Class... parameters) { - try { - if (parameters.length > 0) { - return mapper.convertValue(simpleObj, mapper.getTypeFactory().constructParametricType(tClass, parameters)); - } - return (T) mapper.convertValue(simpleObj, tClass); - } catch (Exception e) { - String message = "Fail to process method 'convert(" + simpleObj + ": " + simpleObj.getClass().getSimpleName() + - ", " + tClass.getSimpleName() + ": "+ Class.class + ", ...: " + Class.class + ")"; - throw new RuntimeException(message, e); - } - } - - public static T convert(Object simpleObj, JavaType javaType) { - try { - return mapper.convertValue(simpleObj, javaType); - } catch (Exception e) { - String message = "Fail to process method 'convert(" + simpleObj + ": " + simpleObj.getClass().getSimpleName() + - ", " + javaType.getTypeName() + ": "+ JavaType.class + ")"; - throw new RuntimeException(message, e); - } - } - - public static ObjectMapper getMapper(){ - return mapper; - } - -} - diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java deleted file mode 100644 index 583483a6f..000000000 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/EntityPostAction.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.ValueNode; -import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.Json; -import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.entities.Resource; -import org.apache.http.HttpEntity; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.http.entity.mime.MultipartEntityBuilder; -import java.nio.charset.StandardCharsets; -import java.util.*; - -/** - * Post action for entity - */ -public class EntityPostAction extends PostAction{ - - private static final String RESOURCE_FIELD_NAME = "resources"; - - private T entity; - - public EntityPostAction(String uri, T entity) { - super(uri); - if (Objects.isNull(entity)){ - throw new IllegalArgumentException("Entity cannot be null in action"); - } - this.entity = entity; - } - - @Override - HttpEntity getHttpEntity() { - try { - if (entity instanceof Resource) { - MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create(); - String entityStr = Json.toJson(entity, null); - JsonNode rootNode = Json.getMapper().readTree(entityStr); - addEntityBody(entityBuilder, "", rootNode); - Optional.ofNullable(((Resource) entity).getResources()).ifPresent(resources -> { - for(int i = 0; i < resources.size(); i ++){ - entityBuilder.addBinaryBody(RESOURCE_FIELD_NAME + "[" + i +"]", resources.get(i)); - } - }); - return entityBuilder.build(); - } else { - StringEntity stringEntity; - stringEntity = new StringEntity(Objects.requireNonNull(Json.toJson(entity, null)), "UTF-8"); - stringEntity.setContentType(ContentType.APPLICATION_JSON.toString()); - return stringEntity; - } - } catch (Exception e) { - throw new IllegalArgumentException("Cannot deserialize the entity: " + entity.getClass().getName(), e); - } - } - - private void addEntityBody(MultipartEntityBuilder builder, String prefix, JsonNode node){ - if (node instanceof ObjectNode){ - if (null != prefix && !prefix.trim().equals("")){ - prefix += "."; - } - Iterator> fields = node.fields(); - while (fields.hasNext()){ - Map.Entry entry = fields.next(); - addEntityBody(builder, prefix + entry.getKey(), entry.getValue()); - } - } else if (node instanceof ArrayNode){ - ArrayNode arrayNode = ((ArrayNode)node); - for (int i = 0 ; i < arrayNode.size(); i ++){ - addEntityBody(builder, prefix + "[" + i + "]", arrayNode.get(i)); - } - } else if (node instanceof ValueNode){ - ContentType strContent = ContentType.create("text/plain", StandardCharsets.UTF_8); - builder.addTextBody(prefix, node.asText(), strContent); - } - } - -} \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/PostAction.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/PostAction.java deleted file mode 100644 index 9b3ea003f..000000000 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/http/request/PostAction.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.request; - -import com.webank.wedatasphere.streamis.jobmanager.log.collector.sender.http.Json; -import org.apache.http.HttpEntity; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; - -import java.util.Objects; - -/** - * Post action - */ -public class PostAction extends AbstractHttpAction { - - public PostAction(String uri) { - super(uri); - } - - /** - * Http entity (use stringEntity and "application/json" default) - * @return http entity - */ - HttpEntity getHttpEntity(){ - StringEntity stringEntity; - try{ - stringEntity = new StringEntity(Objects.requireNonNull(Json.toJson(getRequestPayload(), null)), "UTF-8"); - } catch (Exception e){ - throw new IllegalArgumentException("Cannot deserialize the requestPayLoad: " + getRequestPayload().toString(), e); - } - stringEntity.setContentType(ContentType.APPLICATION_JSON.toString()); - return stringEntity; - } - - @Override - protected HttpPost getRequestMethod() { - HttpPost httpPost = new HttpPost(); - httpPost.setEntity(getHttpEntity()); - return httpPost; - } -} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml index a675aa404..4a171b938 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml @@ -14,15 +14,15 @@ 8 8 - 2.13.2.2 + - - com.fasterxml.jackson.core - jackson-databind - ${jackson-databind.version} - + + + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java index e253d2a32..da3a7054b 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/LogElement.java @@ -30,4 +30,5 @@ public interface LogElement { * @return */ int mark(); + } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java index 3aea21d6f..fe340e066 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java @@ -1,8 +1,10 @@ package com.webank.wedatasphere.streamis.jobmanager.log.entities; -import com.fasterxml.jackson.annotation.JsonIgnore; + +import com.webank.wedatasphere.streamis.jobmanager.log.json.JsonTool; import java.io.Serializable; +import java.util.Objects; /** * Log event for streamis @@ -42,7 +44,6 @@ public long getLogTimeStamp() { } @Override - @JsonIgnore public String[] getContents() { return new String[]{content}; } @@ -71,4 +72,13 @@ public void setMark(int mark) { public void setSequenceId(int sequenceId){ // Ignore } + + public String toJson(){ + return "{" + + "\"logTimeStamp\":" + logTimeInMills + + ",\"content\":" + (Objects.isNull(content)? null : "\"" + JsonTool.encodeStrValue(content) + "\"") + + ",\"sequenceId\":0" + + "}"; + + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java index 8e58aa55d..7e942b675 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java @@ -1,8 +1,9 @@ package com.webank.wedatasphere.streamis.jobmanager.log.entities; -import com.fasterxml.jackson.annotation.JsonIgnore; +import com.webank.wedatasphere.streamis.jobmanager.log.json.JsonTool; import java.io.Serializable; +import java.util.Objects; public class StreamisLogEvents implements LogElement, Serializable { @@ -50,7 +51,6 @@ public long getLogTimeStamp() { @Override - @JsonIgnore public String[] getContents() { String[] contents = new String[events.length]; for(int i = 0 ; i < contents.length; i++){ @@ -87,4 +87,25 @@ public void setEvents(StreamisLogEvent[] events) { public void setSequenceId(int sequenceId){ // Ignore } + + public String toJson(){ + return "{" + + "\"logTimeStamp\":" + logTimeInMills + + ",\"appName\":" + (Objects.isNull(appName)? null : "\"" + JsonTool.encodeStrValue(appName) + "\"") + + ",\"events\":[" + + (Objects.isNull(events) || events.length <=0 ? "" : joinEvents(events, ",") ) + "]" + + ",\"sequenceId\":0" + + "}"; + } + + private String joinEvents(StreamisLogEvent[] events, String separator){ + StringBuilder builder = new StringBuilder(); + for(int i = 0; i < events.length; i ++){ + builder.append(events[i].toJson()); + if (i < events.length - 1){ + builder.append(separator); + } + } + return builder.toString(); + } } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/json/JsonTool.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/json/JsonTool.java new file mode 100644 index 000000000..d59749261 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/json/JsonTool.java @@ -0,0 +1,45 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.json; + +public class JsonTool { + + /** + * Avoid the special char + * @param input input string + * @return output string + */ + public static String encodeStrValue(String input){ + char[] chars = input.toCharArray(); + StringBuilder sb = new StringBuilder(); + for (char c : chars) { + switch (c) { + case '\"': + sb.append("\\\""); + break; + case '\\': + sb.append("\\\\"); + break; + case '/': + sb.append("\\/"); + break; + case '\b': + sb.append("\\b"); + break; + case '\f': + sb.append("\\f"); + break; + case '\n': + sb.append("\\n"); + break; + case '\r': + sb.append("\\r"); + break; + case '\t': + sb.append("\\t"); + break; + default: + sb.append(c); + } + } + return sb.toString(); + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/entities/StreamisLogEvents.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/entities/StreamisLogEvents.java new file mode 100644 index 000000000..8676c5778 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/entities/StreamisLogEvents.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.server.entities; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +public class StreamisLogEvents extends com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvents { + + @Override + @JsonIgnore + public String[] getContents() { + return super.getContents(); + } +} diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java index 7b67c4b2c..595a63e50 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java @@ -1,6 +1,6 @@ package com.webank.wedatasphere.streamis.jobmanager.log.server.restful; -import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvents; +import com.webank.wedatasphere.streamis.jobmanager.log.server.entities.StreamisLogEvents; import com.webank.wedatasphere.streamis.jobmanager.log.server.exception.StreamJobLogException; import com.webank.wedatasphere.streamis.jobmanager.log.server.service.StreamisJobLogService; import org.apache.commons.lang.StringUtils; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java index 04cdfd430..0915e0fa0 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/bucket/Log4j2JobLogBucket.java @@ -274,4 +274,5 @@ private String resolveFilePattern(String fileName, String format){ } return filePattern; } + } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/test/com/webank/wedatasphere/streamis/jobmanager/log/LogStorageTest.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/test/com/webank/wedatasphere/streamis/jobmanager/log/LogStorageTest.java new file mode 100644 index 000000000..6ced214f2 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/test/com/webank/wedatasphere/streamis/jobmanager/log/LogStorageTest.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.streamis.jobmanager.log; + +public class LogStorageTest { +} From 6f36f07f21114613dc7969b163af9cc0446a7eba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?v=5Fboweicai=28=E8=94=A1=E5=8D=9A=E4=BC=9F=29?= Date: Wed, 26 Oct 2022 11:06:07 +0800 Subject: [PATCH 037/240] =?UTF-8?q?fix:=20=E5=85=B3=E9=97=AD=E5=BC=B9?= =?UTF-8?q?=E7=AA=97=E6=97=B6=E6=B8=85=E7=A9=BA=E5=BF=AB=E7=85=A7=E8=B7=AF?= =?UTF-8?q?=E5=BE=84=E6=95=B0=E6=8D=AE=EF=BC=8C=E9=81=BF=E5=85=8D=E5=AF=B9?= =?UTF-8?q?=E4=B8=8B=E6=AC=A1=E5=BC=B9=E7=AA=97=E4=BF=A1=E6=81=AF=E9=80=A0?= =?UTF-8?q?=E6=88=90=E5=B9=B2=E6=89=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/src/apps/streamis/module/jobList/index.vue | 1 + 1 file changed, 1 insertion(+) diff --git a/web/src/apps/streamis/module/jobList/index.vue b/web/src/apps/streamis/module/jobList/index.vue index 64551087a..23ea4bd5d 100644 --- a/web/src/apps/streamis/module/jobList/index.vue +++ b/web/src/apps/streamis/module/jobList/index.vue @@ -783,6 +783,7 @@ export default { onClose(status) { if (status) return; this.processModalVisable = false; + this.snapPaths = [] this.isFinish = false; clearTimeout(this.timer); this.failTasks = []; From 1d4be933afa74b95d4a8fbd65b82718f61955aa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?v=5Fboweicai=28=E8=94=A1=E5=8D=9A=E4=BC=9F=29?= Date: Wed, 26 Oct 2022 14:29:19 +0800 Subject: [PATCH 038/240] =?UTF-8?q?fix:=20=E4=B8=8A=E4=BC=A0=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E8=A1=A8=E5=8D=95=E6=A0=A1=E9=AA=8C=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/src/apps/streamis/i18n/en.json | 4 ++- web/src/apps/streamis/i18n/zh.json | 4 ++- .../apps/streamis/module/uploadFile/index.vue | 31 +++++++++++++++++-- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/web/src/apps/streamis/i18n/en.json b/web/src/apps/streamis/i18n/en.json index 8bf799ba2..fad63b15f 100644 --- a/web/src/apps/streamis/i18n/en.json +++ b/web/src/apps/streamis/i18n/en.json @@ -179,7 +179,8 @@ "selectJar": "Please choose Jar", "upload": "Upload", "choosedJar" : "Choosed Jar", - "jarError": "File must be Jar" + "jarError": "File must be Jar", + "choosedFile": "Choosed File" }, "projectFile": { "file": "File", @@ -187,6 +188,7 @@ "overrideImport": "Override import", "chooseUploadFile": "Please choose the import resource file", "versionPlaceholder": "Only number and ., such as 0.1.0", + "commentLength": "The length of comment text is too long", "versionEmpty": "Version Cannot be empty", "fileEmpty": "File cannot be empty", "delete": "Delete", diff --git a/web/src/apps/streamis/i18n/zh.json b/web/src/apps/streamis/i18n/zh.json index d19676419..13eccbe64 100644 --- a/web/src/apps/streamis/i18n/zh.json +++ b/web/src/apps/streamis/i18n/zh.json @@ -180,7 +180,8 @@ "choosedJar" : "选中的Jar包", "jarError": "文件必须是Jar包", "jarEmpty": "必须选择Jar包", - "jobNameEmpty": "作业名称不能为空" + "jobNameEmpty": "作业名称不能为空", + "choosedFile": "选中的文件" }, "projectFile": { "file": "文件", @@ -188,6 +189,7 @@ "overrideImport": "覆盖式导入", "chooseUploadFile": "请选择您要导入的资源文件", "versionPlaceholder": "只支持数字和.，例如0.1.0", + "commentLength": "文本长度过长,不能超过50个字符", "versionEmpty": "版本不能为空", "fileEmpty": "文件不能为空", "delete": "删除", diff --git a/web/src/apps/streamis/module/uploadFile/index.vue b/web/src/apps/streamis/module/uploadFile/index.vue index a78641a16..20e262d40 100644 --- a/web/src/apps/streamis/module/uploadFile/index.vue +++ b/web/src/apps/streamis/module/uploadFile/index.vue @@ -20,7 +20,7 @@ }}
- {{ $t('message.streamis.uploadJar.choosedJar') }}: {{ file.name }} + {{ $t('message.streamis.uploadJar.choosedFile') }}: {{ file.name }}
50){ + return callback(new Error(_this.$t('message.streamis.projectFile.commentLength'))) + }else{ + callback() + } + }, } } From a5da4b99dbeb64d5acbe07d96be311148d814088 Mon Sep 17 00:00:00 2001 From: davidhua Date: Wed, 26 Oct 2022 16:32:13 +0800 Subject: [PATCH 039/240] Add module to collect spark container log. --- .../xspark-streamis-log-collector/pom.xml | 51 +++++++++++++++++ .../src/main/assembly/package.xml | 19 +++++++ .../spark/SparkStreamisConfigAutowired.java | 56 +++++++++++++++++++ ....jobmanager.plugin.StreamisConfigAutowired | 1 + streamis-jobmanager/streamis-job-log/pom.xml | 1 + .../log/entities/StreamisLogEvents.java | 1 + .../log/server/restful/JobLogRestfulApi.java | 4 ++ .../server/storage/StreamisJobLogStorage.java | 51 +++++++++-------- 8 files changed, 161 insertions(+), 23 deletions(-) create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/assembly/package.xml create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/spark/SparkStreamisConfigAutowired.java create mode 100644 streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml new file mode 100644 index 000000000..6c2122698 --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml @@ -0,0 +1,51 @@ + + + + streamis-job-log + com.webank.wedatasphere.streamis + 0.2.0 + + 4.0.0 + + xspark-streamis-log-collector + + + 8 + 8 + + + + + com.webank.wedatasphere.streamis + streamis-job-log-collector1x + ${streamis.version} + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + + + assemble + + single + + + install + + + + + src/main/assembly/package.xml + + false + + + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/assembly/package.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/assembly/package.xml new file mode 100644 index 000000000..8da27bf2c --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/assembly/package.xml @@ -0,0 +1,19 @@ + + + package + + + jar + + false + + + / + true + runtime + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/spark/SparkStreamisConfigAutowired.java b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/spark/SparkStreamisConfigAutowired.java new file mode 100644 index 000000000..a2a73adbe --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/spark/SparkStreamisConfigAutowired.java @@ -0,0 +1,56 @@ +package com.webank.wedatasphere.streamis.jobmanager.log.collector.spark; + +import com.webank.wedatasphere.streamis.jobmanager.log.collector.config.StreamisLogAppenderConfig; +import com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired; + +import java.util.Optional; + +/** + * Autoconfigure the streamis config in Spark environment + */ +public class SparkStreamisConfigAutowired implements StreamisConfigAutowired { + + private static final String APP_NAME_CONFIG = "app.name"; + + private static final String SERVER_ADDRESS_CONFIG = "streamis.url"; + + private static final String COLLECTOR_URI_CONFIG = "streamis.log.collector.uri"; + + private static final String PROJECT_NAME_CONFIG = "project.name"; + + private static final String DEFAULT_COLLECTOR_URI = "/api/rest_j/v1/streamis/streamJobManager/log/collect/events"; + @Override + public StreamisLogAppenderConfig logAppenderConfig(StreamisLogAppenderConfig.Builder builder) throws Exception { + // Load the config from system properties + Optional.ofNullable(System.getProperty(APP_NAME_CONFIG)).ifPresent(appName -> { + String projectName = System.getProperty(PROJECT_NAME_CONFIG); + if (null != projectName && !projectName.trim().equals("")){ + appName = projectName + "." + appName; + } + System.out.println("Spark env to streamis: application name =>" + appName); + builder.setAppName(appName); + }); + String serverAddress = System.getProperty(SERVER_ADDRESS_CONFIG); + if (null != serverAddress && !serverAddress.trim().equals("")){ + if (serverAddress.endsWith("/")){ + serverAddress = serverAddress.substring(0, serverAddress.length() - 1); + } + String collectorUri = System.getProperty(COLLECTOR_URI_CONFIG, DEFAULT_COLLECTOR_URI); + if (null != collectorUri && !collectorUri.trim().equals("")){ + if (!collectorUri.startsWith("/")){ + collectorUri = "/" + collectorUri; + } + serverAddress += collectorUri; + } + System.out.println("Spark env to streamis: server address =>" + serverAddress); + builder.setRpcAddress(serverAddress); + } + String user = System.getenv("USER"); + if (null == user || user.trim().equals("")){ + user = System.getProperty("user.name", "hadoop"); + } + System.out.println("Spark env to streamis: log user =>" + user); + builder.setRpcAuthTokenUser(user); + return builder.build(); + } +} diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired new file mode 100644 index 000000000..dac2fcaed --- /dev/null +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/src/main/resources/META-INF/services/com.webank.wedatasphere.streamis.jobmanager.plugin.StreamisConfigAutowired @@ -0,0 +1 @@ +com.webank.wedatasphere.streamis.jobmanager.log.collector.spark.SparkStreamisConfigAutowired \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-log/pom.xml b/streamis-jobmanager/streamis-job-log/pom.xml index 0c8b087f1..eb79d84a5 100644 --- a/streamis-jobmanager/streamis-job-log/pom.xml +++ b/streamis-jobmanager/streamis-job-log/pom.xml @@ -16,6 +16,7 @@ job-log-collector/streamis-job-log-collector job-log-collector/streamis-job-log-collector1x job-log-collector/flink-streamis-log-collector + job-log-collector/xspark-streamis-log-collector streamis-job-log-server streamis-job-log-common diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java index 7e942b675..624723a29 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java @@ -108,4 +108,5 @@ private String joinEvents(StreamisLogEvent[] events, String separator){ } return builder.toString(); } + } diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java index 595a63e50..e9d19c651 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/restful/JobLogRestfulApi.java @@ -1,9 +1,12 @@ package com.webank.wedatasphere.streamis.jobmanager.log.server.restful; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.webank.wedatasphere.streamis.jobmanager.log.entities.StreamisLogEvent; import com.webank.wedatasphere.streamis.jobmanager.log.server.entities.StreamisLogEvents; import com.webank.wedatasphere.streamis.jobmanager.log.server.exception.StreamJobLogException; import com.webank.wedatasphere.streamis.jobmanager.log.server.service.StreamisJobLogService; import org.apache.commons.lang.StringUtils; +import org.apache.linkis.common.utils.JsonUtils; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; import org.slf4j.Logger; @@ -15,6 +18,7 @@ import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; +import java.io.IOException; @RestController @RequestMapping(path = "/streamis/streamJobManager/log") diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java index abe312b23..154c8f6f1 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java @@ -79,30 +79,35 @@ public JobLogBucket getOrCreateBucket(String userName, String appName, JobLogBuc public synchronized void init() { if (Objects.isNull(monitorThread)){ monitorThread = Utils.defaultScheduler().scheduleAtFixedRate(() -> { - Thread.currentThread().setName(StreamJobLogConfig.BUCKET_MONITOR_NAME.getValue()); - long maxIdleTime = StreamJobLogConfig.BUCKET_MAX_IDLE_TIME.getValue().toLong(); - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - if (buckets.size() > 0) { - StringBuilder builder = new StringBuilder("Buckets in LogStorage: [\n"); - buckets.forEach((bucketName, bucket) -> { - JobLogBucketState bucketState = bucket.getBucketState(); - builder.append("bucket: [ name: ") - .append(bucketName) - .append(", path: ").append(bucketState.getBucketPath()) - .append(", parts: ").append(bucketState.getBucketParts()) - .append(", write-rate: ").append(bucketState.getBucketWriteRate()).append("/s") - .append(", last-write-time: ").append(dateFormat.format(bucketState.getBucketWriteTime())) - .append(" ]\n"); - if (bucketState.getBucketWriteTime() + maxIdleTime <= System.currentTimeMillis()){ - LOG.info("Close the idle bucket: [ name: {}, last-write-time: {} ]", - bucketName, dateFormat.format(bucketState.getBucketWriteTime())); - bucket.close(); - // Delete the bucket - buckets.remove(bucketName); - } + String threadName = Thread.currentThread().getName(); + try { + Thread.currentThread().setName(StreamJobLogConfig.BUCKET_MONITOR_NAME.getValue()); + long maxIdleTime = StreamJobLogConfig.BUCKET_MAX_IDLE_TIME.getValue().toLong(); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + if (buckets.size() > 0) { + StringBuilder builder = new StringBuilder("Buckets in LogStorage: [\n"); + buckets.forEach((bucketName, bucket) -> { + JobLogBucketState bucketState = bucket.getBucketState(); + builder.append("bucket: [ name: ") + .append(bucketName) + .append(", path: ").append(bucketState.getBucketPath()) + .append(", parts: ").append(bucketState.getBucketParts()) + .append(", write-rate: ").append(bucketState.getBucketWriteRate()).append("/s") + .append(", last-write-time: ").append(dateFormat.format(bucketState.getBucketWriteTime())) + .append(" ]\n"); + if (bucketState.getBucketWriteTime() + maxIdleTime <= System.currentTimeMillis()) { + LOG.info("Close the idle bucket: [ name: {}, last-write-time: {} ]", + bucketName, dateFormat.format(bucketState.getBucketWriteTime())); + bucket.close(); + // Delete the bucket + buckets.remove(bucketName); + } - }); - LOG.info(builder.toString()); + }); + LOG.info(builder.toString()); + } + } finally { + Thread.currentThread().setName(threadName); } },BUCKET_MONITOR_INTERVAL.getValue().toLong(), BUCKET_MONITOR_INTERVAL.getValue().toLong(), TimeUnit.MILLISECONDS); From e226dbd57a041af78cce9c346aefe4b6edbe881d Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Wed, 26 Oct 2022 17:26:04 +0800 Subject: [PATCH 040/240] Add the new requestBody parameter `projectName` for adding task and stop task for existed jobs. --- .../parser/SparkTaskMetricsParser.scala | 5 +- .../jobmanager/restful/api/JobRestfulApi.java | 207 +++++++++--------- 2 files changed, 105 insertions(+), 107 deletions(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala index a8e6bb7c3..c6e00d7ff 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala @@ -25,7 +25,10 @@ class SparkTaskMetricsParser extends AbstractTaskMetricsParser { val addDataNumberDTO: String => Unit = key => { val batch = new JobDetailsVo.DataNumberDTO batch.setDataName(key) - batch.setDataNumber(metricsMap.get(key).toString.toInt) + batch.setDataNumber(metricsMap.get(key) match { + case null => -1 + case num => num.toString.toInt + }) dataNumberDTOS.add(batch) } addDataNumberDTO("waitingBatchs") diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 7f2e396e3..e5796f14a 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -232,78 +232,88 @@ public Message executeHistoryJob(HttpServletRequest req, return Message.ok().data("details", details); } + private Message withStreamJob(HttpServletRequest req, String projectName, + String jobName, String username, + Function streamJobFunction) { + List streamJobs = streamJobService.getByProList(projectName, username, jobName, null, null).getList(); + if(CollectionUtils.isEmpty(streamJobs)) { + return Message.error("Not exits Streamis job " + jobName); + } else if(streamJobs.size() > 1) { + return Message.error("Too many Streamis Job named " + jobName + ", we cannot distinguish between them."); + } else if(!"spark.jar".equals(streamJobs.get(0).getJobType())) { + return Message.error("Only spark.jar Job support to manage task."); + } + StreamJob streamJob = streamJobService.getJobById(streamJobs.get(0).getId()); + if (!streamJobService.hasPermission(streamJob, username) && + !this.privilegeService.hasEditPrivilege(req, streamJob.getProjectName())) { + return Message.error("Have no permission to operate task for StreamJob [" + jobName + "]."); + } + return streamJobFunction.apply(streamJob); + } + @RequestMapping(path = "/addTask", method = RequestMethod.GET) public Message addTask(HttpServletRequest req, + @RequestParam(value = "projectName") String projectName, @RequestParam(value = "jobName") String jobName, @RequestParam(value = "appId") String appId, @RequestParam(value = "appUrl") String appUrl) { String username = SecurityFilter.getLoginUsername(req); LOG.info("User {} try to add a new task for Streamis job {} with appId: {}, appUrl: {}.", username, jobName, appId, appUrl); - List streamJobs = streamJobService.getJobByName(jobName); - if(CollectionUtils.isEmpty(streamJobs)) { - return Message.error("Not exits Streamis job " + jobName); - } else if(streamJobs.size() > 1) { - return Message.error("Too many Streamis Job named " + jobName + ", we cannot distinguish between them."); - } else if(!"spark.jar".equals(streamJobs.get(0).getJobType())) { - return Message.error("Only spark.jar Job support to add new tasks."); - } - if (!streamJobService.hasPermission(streamJobs.get(0), username) && - !this.privilegeService.hasEditPrivilege(req, streamJobs.get(0).getProjectName())) { - return Message.error("Have no permission to add new task for StreamJob [" + jobName + "]."); - } - // 如果存在正在运行的，先将其停止掉 - StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJobs.get(0).getId()); - if(streamTask == null) { - // 这里取个巧，从该工程该用户有权限的Job中找到一个Flink的历史作业，作为这个Spark Streaming作业的jobId和jobInfo - // 替换掉JobInfo中的 yarn 信息，这样我们前端就可以在不修改任何逻辑的情况下正常展示Spark Streaming作业了 - PageInfo jobList = streamJobService.getByProList(streamJobs.get(0).getProjectName(), username, null, null, null); - List copyJobs = jobList.getList().stream().filter(job -> !job.getJobType().startsWith("spark.")) - .collect(Collectors.toList()); - if(copyJobs.isEmpty()) { - return Message.error("no Flink Job has been submitted, the register to Streamis cannot be succeeded."); - } - int index = 0; - while(streamTask == null && index < copyJobs.size()) { - StreamTask copyTask = streamTaskService.getLatestTaskByJobId(copyJobs.get(index).getId()); - if(copyTask == null) { - index ++; - } else { - LOG.warn("Streamis Job {} will bind the linkisJobInfo from history Flink Job {} with linkisJobId: {}, linkisJobInfo: {}.", - jobName, copyJobs.get(index).getName(), copyTask.getLinkisJobId(), copyTask.getLinkisJobInfo()); - streamTask = streamTaskService.createTask(streamJobs.get(0).getId(), (Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue(), username); - streamTask.setLinkisJobId(copyTask.getLinkisJobId()); - streamTask.setLinkisJobInfo(copyTask.getLinkisJobInfo()); - } - } + return withStreamJob(req, projectName, jobName, username, streamJob -> { + // 如果存在正在运行的，先将其停止掉 + StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJob.getId()); if(streamTask == null) { - return Message.error("no Flink task has been executed, the register to Streamis cannot be succeeded."); - } - } else { - if(JobConf.isRunning(streamTask.getStatus())) { - LOG.warn("Streamis Job {} exists running task, update its status from Running to stopped at first.", jobName); - streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); - streamTask.setErrDesc("stopped by App's new task."); - streamTaskService.updateTask(streamTask); + // 这里取个巧，从该工程该用户有权限的Job中找到一个Flink的历史作业，作为这个Spark Streaming作业的jobId和jobInfo + // 替换掉JobInfo中的 yarn 信息，这样我们前端就可以在不修改任何逻辑的情况下正常展示Spark Streaming作业了 + PageInfo jobList = streamJobService.getByProList(streamJob.getProjectName(), username, null, null, null); + List copyJobs = jobList.getList().stream().filter(job -> !job.getJobType().startsWith("spark.")) + .collect(Collectors.toList()); + if(copyJobs.isEmpty()) { + return Message.error("no Flink Job has been submitted, the register to Streamis cannot be succeeded."); + } + int index = 0; + while(streamTask == null && index < copyJobs.size()) { + StreamTask copyTask = streamTaskService.getLatestTaskByJobId(copyJobs.get(index).getId()); + if(copyTask == null) { + index ++; + } else { + LOG.warn("Streamis Job {} will bind the linkisJobInfo from history Flink Job {} with linkisJobId: {}, linkisJobInfo: {}.", + jobName, copyJobs.get(index).getName(), copyTask.getLinkisJobId(), copyTask.getLinkisJobInfo()); + streamTask = streamTaskService.createTask(streamJob.getId(), (Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue(), username); + streamTask.setLinkisJobId(copyTask.getLinkisJobId()); + streamTask.setLinkisJobInfo(copyTask.getLinkisJobInfo()); + } + } + if(streamTask == null) { + return Message.error("no Flink task has been executed, the register to Streamis cannot be succeeded."); + } + } else { + if(JobConf.isRunning(streamTask.getStatus())) { + LOG.warn("Streamis Job {} exists running task, update its status from Running to stopped at first.", jobName); + streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); + streamTask.setErrDesc("stopped by App's new task."); + streamTaskService.updateTask(streamTask); + } + StreamTask newStreamTask = streamTaskService.createTask(streamJob.getId(), (Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue(), username); + streamTask.setId(newStreamTask.getId()); + streamTask.setVersion(newStreamTask.getVersion()); + streamTask.setErrDesc(""); + streamTask.setStatus(newStreamTask.getStatus()); + streamTask.setSubmitUser(username); } - StreamTask newStreamTask = streamTaskService.createTask(streamJobs.get(0).getId(), (Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue(), username); - streamTask.setId(newStreamTask.getId()); - streamTask.setVersion(newStreamTask.getVersion()); - streamTask.setErrDesc(""); - streamTask.setStatus(newStreamTask.getStatus()); - streamTask.setSubmitUser(username); - } - streamTask.setStartTime(new Date()); - streamTask.setLastUpdateTime(new Date()); - StreamTask finalStreamTask = streamTask; - return withFlinkJobInfo(jobName, streamTask.getLinkisJobInfo(), flinkJobInfo -> { - flinkJobInfo.setApplicationId(appId); - flinkJobInfo.setApplicationUrl(appUrl); - flinkJobInfo.setName(jobName); - flinkJobInfo.setStatus(JobConf.getStatusString(finalStreamTask.getStatus())); - StreamTaskUtils.refreshInfo(finalStreamTask, flinkJobInfo); - streamTaskService.updateTask(finalStreamTask); - LOG.info("Streamis Job {} has added a new task successfully.", jobName); - return Message.ok(); + streamTask.setStartTime(new Date()); + streamTask.setLastUpdateTime(new Date()); + StreamTask finalStreamTask = streamTask; + return withFlinkJobInfo(jobName, streamTask.getLinkisJobInfo(), flinkJobInfo -> { + flinkJobInfo.setApplicationId(appId); + flinkJobInfo.setApplicationUrl(appUrl); + flinkJobInfo.setName(jobName); + flinkJobInfo.setStatus(JobConf.getStatusString(finalStreamTask.getStatus())); + StreamTaskUtils.refreshInfo(finalStreamTask, flinkJobInfo); + streamTaskService.updateTask(finalStreamTask); + LOG.info("Streamis Job {} has added a new task successfully.", jobName); + return Message.ok(); + }); }); } @@ -320,67 +330,52 @@ private Message withFlinkJobInfo(String jobName, String flinkJobInfoStr, Functio @RequestMapping(path = "/updateTask", method = RequestMethod.GET) public Message updateTask(HttpServletRequest req, + @RequestParam(value = "projectName") String projectName, @RequestParam(value = "jobName") String jobName, @RequestParam(value = "appId") String appId, @RequestParam(value = "metrics") String metrics) { String username = SecurityFilter.getLoginUsername(req); LOG.info("User {} try to update task for Streamis job {} with appId: {}.", username, jobName, appId); - List streamJobs = streamJobService.getJobByName(jobName); - if(CollectionUtils.isEmpty(streamJobs)) { - return Message.error("Not exits Streamis job " + jobName); - } else if(streamJobs.size() > 1) { - return Message.error("Too many Streamis Job named " + jobName + ", we cannot distinguish between them."); - } else if(!"spark.jar".equals(streamJobs.get(0).getJobType())) { - return Message.error("Only spark.jar Job support to update task."); - } - StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJobs.get(0).getId()); - if(streamTask == null) { - LOG.warn("Job {} is not exists running task, ignore to update its metrics.", jobName); - return Message.ok("not exists running task, ignore it."); - } else if(JobConf.isCompleted(streamTask.getStatus())) { - LOG.warn("The task of job {} is completed, ignore to update its metrics.", jobName); - return Message.ok("Task is completed, ignore to update its metrics."); - } - return withFlinkJobInfo(jobName, streamTask.getLinkisJobInfo(), flinkJobInfo -> { - if(!flinkJobInfo.getApplicationId().equals(appId)) { - LOG.warn("Job {} with running task is not equals to the request appId: {}, ignore to update its metrics.", - jobName, flinkJobInfo.getApplicationId(), appId); - return Message.ok("the request appId is not equals to the running task appId " + flinkJobInfo.getApplicationId()); - } - JobStateInfo jobStateInfo = new JobStateInfo(); - jobStateInfo.setTimestamp(System.currentTimeMillis()); - jobStateInfo.setLocation(metrics); - flinkJobInfo.setJobStates(new JobStateInfo[]{jobStateInfo}); - StreamTaskUtils.refreshInfo(streamTask, flinkJobInfo); - streamTaskService.updateTask(streamTask); - LOG.info("Streamis Job {} has updated the task metrics successfully.", jobName); - return Message.ok(); + return withStreamJob(req, projectName, jobName, username, streamJob -> { + StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJob.getId()); + if (streamTask == null) { + LOG.warn("Job {} is not exists running task, ignore to update its metrics.", jobName); + return Message.ok("not exists running task, ignore it."); + } else if (JobConf.isCompleted(streamTask.getStatus())) { + LOG.warn("The task of job {} is completed, ignore to update its metrics.", jobName); + return Message.ok("Task is completed, ignore to update its metrics."); + } + return withFlinkJobInfo(jobName, streamTask.getLinkisJobInfo(), flinkJobInfo -> { + if (!flinkJobInfo.getApplicationId().equals(appId)) { + LOG.warn("Job {} with running task is not equals to the request appId: {}, ignore to update its metrics.", + jobName, flinkJobInfo.getApplicationId(), appId); + return Message.ok("the request appId is not equals to the running task appId " + flinkJobInfo.getApplicationId()); + } + JobStateInfo jobStateInfo = new JobStateInfo(); + jobStateInfo.setTimestamp(System.currentTimeMillis()); + jobStateInfo.setLocation(metrics); + flinkJobInfo.setJobStates(new JobStateInfo[]{jobStateInfo}); + StreamTaskUtils.refreshInfo(streamTask, flinkJobInfo); + streamTaskService.updateTask(streamTask); + LOG.info("Streamis Job {} has updated the task metrics successfully.", jobName); + return Message.ok(); + }); }); } @RequestMapping(path = "/stopTask", method = RequestMethod.GET) public Message stopTask(HttpServletRequest req, + @RequestParam(value = "projectName") String projectName, @RequestParam(value = "jobName") String jobName, @RequestParam(value = "appId") String appId, @RequestParam(value = "appUrl") String appUrl) { String username = SecurityFilter.getLoginUsername(req); LOG.info("User {} try to stop task for Streamis job {} with appId: {}, appUrl: {}.", username, jobName, appId, appUrl); - List streamJobs = streamJobService.getJobByName(jobName); - if(CollectionUtils.isEmpty(streamJobs)) { - return Message.error("Not exits Streamis job " + jobName); - } else if(streamJobs.size() > 1) { - return Message.error("Too many Streamis Job named " + jobName + ", we cannot distinguish between them."); - } else if(!"spark.jar".equals(streamJobs.get(0).getJobType())) { - return Message.error("Only spark.jar Job support to stop task."); - } - return tryStopTask(req, username, streamJobs.get(0), appId); + return withStreamJob(req, projectName, jobName, username, + streamJob -> tryStopTask(req, username, streamJob, appId)); } private Message tryStopTask(HttpServletRequest req, String username, StreamJob streamJob, String appId) { - if (!streamJobService.hasPermission(streamJob, username) && - !this.privilegeService.hasEditPrivilege(req, streamJob.getProjectName())) { - return Message.error("Have no permission to stop task for StreamJob [" + streamJob.getName() + "]."); - } // 如果存在正在运行的，将其停止掉 StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJob.getId()); if(streamTask != null && JobConf.isRunning(streamTask.getStatus())) { From 8e9b856006ecc2d2eefb1124f68b04ba1fd76ea7 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Wed, 26 Oct 2022 17:28:40 +0800 Subject: [PATCH 041/240] Add the new requestBody parameter `projectName` for adding task and stop task for existed jobs. --- .../streamis/jobmanager/restful/api/JobRestfulApi.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index e5796f14a..1a7cbc593 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -190,7 +190,7 @@ public Message killJob(HttpServletRequest req, } } else { LOG.error("{} try to kill not-supported-management job {} with name {}.", userName, jobId, streamJob.getName()); - return tryStopTask(req, userName, streamJob, null); + return tryStopTask(streamJob, null); } } @@ -372,10 +372,10 @@ public Message stopTask(HttpServletRequest req, String username = SecurityFilter.getLoginUsername(req); LOG.info("User {} try to stop task for Streamis job {} with appId: {}, appUrl: {}.", username, jobName, appId, appUrl); return withStreamJob(req, projectName, jobName, username, - streamJob -> tryStopTask(req, username, streamJob, appId)); + streamJob -> tryStopTask(streamJob, appId)); } - private Message tryStopTask(HttpServletRequest req, String username, StreamJob streamJob, String appId) { + private Message tryStopTask(StreamJob streamJob, String appId) { // 如果存在正在运行的，将其停止掉 StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJob.getId()); if(streamTask != null && JobConf.isRunning(streamTask.getStatus())) { From 85b98129c239597566ec4b4e1c0aa40ef3936ded Mon Sep 17 00:00:00 2001 From: davidhua Date: Thu, 27 Oct 2022 02:48:23 +0800 Subject: [PATCH 042/240] Complete the strategy in JsonTool.escapeStrValue(). --- .../log/entities/StreamisLogEvent.java | 2 +- .../log/entities/StreamisLogEvents.java | 2 +- .../jobmanager/log/json/JsonTool.java | 22 +++++++++++++++++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java index fe340e066..6f8645f77 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvent.java @@ -76,7 +76,7 @@ public void setSequenceId(int sequenceId){ public String toJson(){ return "{" + "\"logTimeStamp\":" + logTimeInMills + - ",\"content\":" + (Objects.isNull(content)? null : "\"" + JsonTool.encodeStrValue(content) + "\"") + + ",\"content\":" + (Objects.isNull(content)? null : "\"" + JsonTool.escapeStrValue(content) + "\"") + ",\"sequenceId\":0" + "}"; diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java index 624723a29..f2843c8af 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/entities/StreamisLogEvents.java @@ -91,7 +91,7 @@ public void setSequenceId(int sequenceId){ public String toJson(){ return "{" + "\"logTimeStamp\":" + logTimeInMills + - ",\"appName\":" + (Objects.isNull(appName)? null : "\"" + JsonTool.encodeStrValue(appName) + "\"") + + ",\"appName\":" + (Objects.isNull(appName)? null : "\"" + JsonTool.escapeStrValue(appName) + "\"") + ",\"events\":[" + (Objects.isNull(events) || events.length <=0 ? "" : joinEvents(events, ",") ) + "]" + ",\"sequenceId\":0" diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/json/JsonTool.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/json/JsonTool.java index d59749261..0822e820d 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/json/JsonTool.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/json/JsonTool.java @@ -1,13 +1,16 @@ package com.webank.wedatasphere.streamis.jobmanager.log.json; +import java.util.Locale; + public class JsonTool { + static final char[] HEX_DIGITS = new char[] {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'}; /** * Avoid the special char * @param input input string * @return output string */ - public static String encodeStrValue(String input){ + public static String escapeStrValue(String input){ char[] chars = input.toCharArray(); StringBuilder sb = new StringBuilder(); for (char c : chars) { @@ -37,9 +40,24 @@ public static String encodeStrValue(String input){ sb.append("\\t"); break; default: - sb.append(c); + sb.append((c < 32) ? escapeUnicode(c) : c); } } return sb.toString(); } + + /** + * Escape unicode + * @param code char code + * @return escaped string + */ + private static String escapeUnicode(int code){ + if (code > 0xffff){ + return "\\u" + Integer.toHexString(code).toUpperCase(Locale.ENGLISH); + } else { + return "\\u" + HEX_DIGITS[(code >> 12) & 15] + + HEX_DIGITS[(code >> 8) & 15] + HEX_DIGITS[(code >> 4) & 15] + HEX_DIGITS[code & 15]; + } + } + } From 843934b2badcd6c0b7240af6434201d79e1369a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?v=5Fboweicai=28=E8=94=A1=E5=8D=9A=E4=BC=9F=29?= Date: Fri, 28 Oct 2022 17:43:49 +0800 Subject: [PATCH 043/240] =?UTF-8?q?upd:=20=E8=BF=9B=E5=85=A5=E9=A6=96?= =?UTF-8?q?=E9=A1=B5=E6=97=B6=E8=8B=A5projectName=E6=9C=AA=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E5=88=99=E4=BB=8Elocation=20search=E4=B8=AD=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E5=90=8E=E8=B7=AF=E7=94=B1=E5=88=B0=E5=AF=B9=E5=BA=94?= =?UTF-8?q?=E7=9A=84=E9=A1=B5=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/src/apps/streamis/view/realTimeJobCenter/index.vue | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web/src/apps/streamis/view/realTimeJobCenter/index.vue b/web/src/apps/streamis/view/realTimeJobCenter/index.vue index 56f859270..31c093739 100644 --- a/web/src/apps/streamis/view/realTimeJobCenter/index.vue +++ b/web/src/apps/streamis/view/realTimeJobCenter/index.vue @@ -16,10 +16,14 @@ export default { data() { return { navHeight: 0, + projectName: this.$route.query.projectName || (new URLSearchParams(window.location.search)).get('projectName') }; }, mounted() { // this.init(); + if(!this.$route.query.projectName){ + this.$router.push(`/realTimeJobCenter?projectName=${this.projectName}`) + } }, methods: { resize(height) { From 865d41fe7b7bf7225b01770c69b6265b0b32990d Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Sat, 29 Oct 2022 15:56:58 +0800 Subject: [PATCH 044/240] resolve 413 response status for update task. --- .../jobmanager/restful/api/JobRestfulApi.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 1a7cbc593..a9f187efe 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -328,14 +328,15 @@ private Message withFlinkJobInfo(String jobName, String flinkJobInfoStr, Functio return flinkJobInfoFunction.apply(flinkJobInfo); } - @RequestMapping(path = "/updateTask", method = RequestMethod.GET) + @RequestMapping(path = "/updateTask", method = RequestMethod.POST) public Message updateTask(HttpServletRequest req, - @RequestParam(value = "projectName") String projectName, - @RequestParam(value = "jobName") String jobName, - @RequestParam(value = "appId") String appId, - @RequestParam(value = "metrics") String metrics) { + @RequestBody Map json) { String username = SecurityFilter.getLoginUsername(req); - LOG.info("User {} try to update task for Streamis job {} with appId: {}.", username, jobName, appId); + String projectName = json.get("projectName"); + String jobName = json.get("jobName"); + String appId = json.get("appId"); + String metrics = json.get("metrics"); + LOG.info("User {} try to update task for Streamis job {} with appId: {}, metrics: {}.", username, jobName, appId, metrics); return withStreamJob(req, projectName, jobName, username, streamJob -> { StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJob.getId()); if (streamTask == null) { From c99097aab4574ec5a99848740ed45b8baa88e629 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Sat, 29 Oct 2022 16:07:51 +0800 Subject: [PATCH 045/240] resolve 413 response status for update task. --- .../streamis/jobmanager/restful/api/JobRestfulApi.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index a9f187efe..5eff10874 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -235,6 +235,11 @@ public Message executeHistoryJob(HttpServletRequest req, private Message withStreamJob(HttpServletRequest req, String projectName, String jobName, String username, Function streamJobFunction) { + if(StringUtils.isBlank(projectName)) { + return Message.error("projectName cannot be empty!"); + } else if(StringUtils.isBlank(jobName)) { + return Message.error("jobName cannot be empty!"); + } List streamJobs = streamJobService.getByProList(projectName, username, jobName, null, null).getList(); if(CollectionUtils.isEmpty(streamJobs)) { return Message.error("Not exits Streamis job " + jobName); @@ -259,6 +264,9 @@ public Message addTask(HttpServletRequest req, @RequestParam(value = "appUrl") String appUrl) { String username = SecurityFilter.getLoginUsername(req); LOG.info("User {} try to add a new task for Streamis job {} with appId: {}, appUrl: {}.", username, jobName, appId, appUrl); + if(StringUtils.isBlank(appId)) { + return Message.error("appId cannot be empty!"); + } return withStreamJob(req, projectName, jobName, username, streamJob -> { // 如果存在正在运行的，先将其停止掉 StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJob.getId()); From 3728a99eee5143d77d0e78b6633b0cf57c9bf5e0 Mon Sep 17 00:00:00 2001 From: davidhua Date: Sat, 29 Oct 2022 16:13:01 +0800 Subject: [PATCH 046/240] Fix the in method restoreInvariants of AbstractRpcLogSender. --- .../jobmanager/log/collector/sender/AbstractRpcLogSender.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java index d90eb1a7f..3afc34147 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/AbstractRpcLogSender.java @@ -435,7 +435,8 @@ private void restoreInvariants(int i, int take, boolean clearItems){ this.count -= i; if (clearItems){ int index = this.takeIndex; - for (; i > 0; i --){ + int j = i; + for (; j > 0; j --){ this.items[index] = null; if (++index == items.length){ index = 0; From 1d0f3140bab15aa279035897cf6f7c3e945e0188 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Mon, 31 Oct 2022 16:11:12 +0800 Subject: [PATCH 047/240] resolve 413 response status for update task. --- .../jobmanager/restful/api/JobRestfulApi.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 5eff10874..44e1bdcc6 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -336,14 +336,13 @@ private Message withFlinkJobInfo(String jobName, String flinkJobInfoStr, Functio return flinkJobInfoFunction.apply(flinkJobInfo); } - @RequestMapping(path = "/updateTask", method = RequestMethod.POST) + @RequestMapping(path = "/updateTask", method = RequestMethod.GET) public Message updateTask(HttpServletRequest req, - @RequestBody Map json) { + @RequestParam(value = "projectName") String projectName, + @RequestParam(value = "jobName") String jobName, + @RequestParam(value = "appId") String appId, + @RequestParam(value = "metrics") String metrics) { String username = SecurityFilter.getLoginUsername(req); - String projectName = json.get("projectName"); - String jobName = json.get("jobName"); - String appId = json.get("appId"); - String metrics = json.get("metrics"); LOG.info("User {} try to update task for Streamis job {} with appId: {}, metrics: {}.", username, jobName, appId, metrics); return withStreamJob(req, projectName, jobName, username, streamJob -> { StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJob.getId()); @@ -372,6 +371,16 @@ public Message updateTask(HttpServletRequest req, }); } + @RequestMapping(path = "/updateTask", method = RequestMethod.POST) + public Message updateTask(HttpServletRequest req, + @RequestBody Map json) { + String projectName = json.get("projectName"); + String jobName = json.get("jobName"); + String appId = json.get("appId"); + String metrics = json.get("metrics"); + return updateTask(req, projectName, jobName, appId, metrics); + } + @RequestMapping(path = "/stopTask", method = RequestMethod.GET) public Message stopTask(HttpServletRequest req, @RequestParam(value = "projectName") String projectName, From 59f0251c3607598457c16a1691667be17364459c Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 7 Nov 2022 11:31:12 +0800 Subject: [PATCH 048/240] adjust the relation path in pom.xml. --- .../job-log-collector/flink-streamis-log-collector/pom.xml | 1 + .../job-log-collector/streamis-job-log-collector-core/pom.xml | 1 + .../job-log-collector/streamis-job-log-collector/pom.xml | 1 + .../job-log-collector/streamis-job-log-collector1x/pom.xml | 1 + .../job-log-collector/xspark-streamis-log-collector/pom.xml | 1 + .../streamis-job-log/streamis-job-log-common/pom.xml | 1 + .../streamis-job-log/streamis-job-log-server/pom.xml | 1 + 7 files changed, 7 insertions(+) diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml index 6bbee8ba8..48625dbcb 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml @@ -6,6 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 + ../../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml index bc68d0796..b8ac1ed54 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml @@ -6,6 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 + ../../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml index e2e556fc8..bca7178c3 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml @@ -6,6 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 + ../../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml index 43792639c..950ec38fb 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml @@ -6,6 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 + ../../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml index 6c2122698..13ef5b0e2 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml @@ -6,6 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 + ../../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml index 4a171b938..1eb4cdb1e 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml @@ -6,6 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 + ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml index ec75cea68..9facffe7d 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml @@ -6,6 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 + ../../pom.xml 4.0.0 From 5c7d3cbc5a7b942913480dc52830fde42f343b14 Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 7 Nov 2022 19:31:11 +0800 Subject: [PATCH 049/240] Fix the problem in pom.xml of stream job log modules. --- .../job-log-collector/flink-streamis-log-collector/pom.xml | 2 +- .../job-log-collector/streamis-job-log-collector-core/pom.xml | 2 +- .../job-log-collector/streamis-job-log-collector/pom.xml | 2 +- .../job-log-collector/streamis-job-log-collector1x/pom.xml | 2 +- .../job-log-collector/xspark-streamis-log-collector/pom.xml | 2 +- .../streamis-job-log/streamis-job-log-common/pom.xml | 2 +- .../streamis-job-log/streamis-job-log-server/pom.xml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml index 48625dbcb..ca49e4399 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml @@ -6,7 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 - ../../../pom.xml + ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml index b8ac1ed54..0cdb47b8d 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml @@ -6,7 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 - ../../../pom.xml + ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml index bca7178c3..55fc60818 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml @@ -6,7 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 - ../../../pom.xml + ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml index 950ec38fb..7632a5718 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml @@ -6,7 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 - ../../../pom.xml + ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml index 13ef5b0e2..904bec4dd 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml @@ -6,7 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 - ../../../pom.xml + ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml index 1eb4cdb1e..fcff3592c 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml @@ -6,7 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 - ../../pom.xml + ../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml index 9facffe7d..165f6a520 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml @@ -6,7 +6,7 @@ streamis-job-log com.webank.wedatasphere.streamis 0.2.0 - ../../pom.xml + ../pom.xml 4.0.0 From ca19836b7fced42919a783c52593267aa5e4c2ac Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Tue, 15 Nov 2022 17:21:13 +0800 Subject: [PATCH 050/240] resolve the LinkisJobInfo is null for adding task. --- .../jobmanager/restful/api/JobRestfulApi.java | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java index 44e1bdcc6..bb84d42f7 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobRestfulApi.java @@ -263,14 +263,20 @@ public Message addTask(HttpServletRequest req, @RequestParam(value = "appId") String appId, @RequestParam(value = "appUrl") String appUrl) { String username = SecurityFilter.getLoginUsername(req); - LOG.info("User {} try to add a new task for Streamis job {} with appId: {}, appUrl: {}.", username, jobName, appId, appUrl); + LOG.info("User {} try to add a new task for Streamis job {}.{} with appId: {}, appUrl: {}.", username, projectName, jobName, appId, appUrl); if(StringUtils.isBlank(appId)) { return Message.error("appId cannot be empty!"); } return withStreamJob(req, projectName, jobName, username, streamJob -> { // 如果存在正在运行的，先将其停止掉 StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJob.getId()); - if(streamTask == null) { + if(streamTask != null && JobConf.isRunning(streamTask.getStatus())) { + LOG.warn("Streamis Job {} exists running task, update its status from Running to stopped at first.", jobName); + streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); + streamTask.setErrDesc("stopped by App's new task."); + streamTaskService.updateTask(streamTask); + } + if(streamTask == null || StringUtils.isBlank(streamTask.getLinkisJobInfo())) { // 这里取个巧，从该工程该用户有权限的Job中找到一个Flink的历史作业，作为这个Spark Streaming作业的jobId和jobInfo // 替换掉JobInfo中的 yarn 信息，这样我们前端就可以在不修改任何逻辑的情况下正常展示Spark Streaming作业了 PageInfo jobList = streamJobService.getByProList(streamJob.getProjectName(), username, null, null, null); @@ -280,9 +286,10 @@ public Message addTask(HttpServletRequest req, return Message.error("no Flink Job has been submitted, the register to Streamis cannot be succeeded."); } int index = 0; + streamTask = null; while(streamTask == null && index < copyJobs.size()) { StreamTask copyTask = streamTaskService.getLatestTaskByJobId(copyJobs.get(index).getId()); - if(copyTask == null) { + if(copyTask == null || StringUtils.isBlank(copyTask.getLinkisJobInfo())) { index ++; } else { LOG.warn("Streamis Job {} will bind the linkisJobInfo from history Flink Job {} with linkisJobId: {}, linkisJobInfo: {}.", @@ -296,12 +303,6 @@ public Message addTask(HttpServletRequest req, return Message.error("no Flink task has been executed, the register to Streamis cannot be succeeded."); } } else { - if(JobConf.isRunning(streamTask.getStatus())) { - LOG.warn("Streamis Job {} exists running task, update its status from Running to stopped at first.", jobName); - streamTask.setStatus((Integer) JobConf.FLINK_JOB_STATUS_STOPPED().getValue()); - streamTask.setErrDesc("stopped by App's new task."); - streamTaskService.updateTask(streamTask); - } StreamTask newStreamTask = streamTaskService.createTask(streamJob.getId(), (Integer) JobConf.FLINK_JOB_STATUS_RUNNING().getValue(), username); streamTask.setId(newStreamTask.getId()); streamTask.setVersion(newStreamTask.getVersion()); @@ -343,7 +344,7 @@ public Message updateTask(HttpServletRequest req, @RequestParam(value = "appId") String appId, @RequestParam(value = "metrics") String metrics) { String username = SecurityFilter.getLoginUsername(req); - LOG.info("User {} try to update task for Streamis job {} with appId: {}, metrics: {}.", username, jobName, appId, metrics); + LOG.info("User {} try to update task for Streamis job {}.{} with appId: {}, metrics: {}.", username, projectName, jobName, appId, metrics); return withStreamJob(req, projectName, jobName, username, streamJob -> { StreamTask streamTask = streamTaskService.getLatestTaskByJobId(streamJob.getId()); if (streamTask == null) { @@ -388,7 +389,7 @@ public Message stopTask(HttpServletRequest req, @RequestParam(value = "appId") String appId, @RequestParam(value = "appUrl") String appUrl) { String username = SecurityFilter.getLoginUsername(req); - LOG.info("User {} try to stop task for Streamis job {} with appId: {}, appUrl: {}.", username, jobName, appId, appUrl); + LOG.info("User {} try to stop task for Streamis job {}.{} with appId: {}, appUrl: {}.", username, projectName, jobName, appId, appUrl); return withStreamJob(req, projectName, jobName, username, streamJob -> tryStopTask(streamJob, appId)); } From 74b118b164df20505a30d81f2f99da9c8627cbce Mon Sep 17 00:00:00 2001 From: jefftlin <57786627+jefftlin@users.noreply.github.com> Date: Tue, 20 Dec 2022 11:30:07 +0800 Subject: [PATCH 051/240] Upgrade the version of pom to 0.2.3 (#55) * Upgrade the version of pom to 0.2.3 * Minor fix Co-authored-by: jefftlin --- assembly/pom.xml | 2 +- pom.xml | 6 +++--- streamis-appconn/pom.xml | 2 +- streamis-jobmanager/pom.xml | 2 +- streamis-jobmanager/streamis-job-launcher/pom.xml | 2 +- .../streamis-job-launcher-base/pom.xml | 2 +- .../streamis-job-launcher-linkis/pom.xml | 2 +- .../streamis-job-launcher-service/pom.xml | 2 +- .../job-log-collector/flink-streamis-log-collector/pom.xml | 2 +- .../streamis-job-log-collector-core/pom.xml | 2 +- .../job-log-collector/streamis-job-log-collector/pom.xml | 2 +- .../job-log-collector/streamis-job-log-collector1x/pom.xml | 2 +- .../job-log-collector/xspark-streamis-log-collector/pom.xml | 2 +- streamis-jobmanager/streamis-job-log/pom.xml | 2 +- .../streamis-job-log/streamis-job-log-common/pom.xml | 2 +- .../streamis-job-log/streamis-job-log-server/pom.xml | 4 ++-- streamis-jobmanager/streamis-job-manager/pom.xml | 2 +- .../streamis-job-manager/streamis-job-manager-base/pom.xml | 2 +- .../streamis-job-manager-service/pom.xml | 2 +- streamis-jobmanager/streamis-jobmanager-common/pom.xml | 2 +- streamis-jobmanager/streamis-jobmanager-server/pom.xml | 2 +- streamis-jobmanager/streamis-projectmanager-server/pom.xml | 2 +- streamis-project/pom.xml | 2 +- streamis-project/streamis-project-common/pom.xml | 2 +- streamis-project/streamis-project-server/pom.xml | 2 +- streamis-server/pom.xml | 2 +- web/package.json | 2 +- 27 files changed, 30 insertions(+), 30 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 8f8038bbc..2b788a362 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -20,7 +20,7 @@ streamis com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/pom.xml b/pom.xml index 0fc4d2fba..93c1a1bbd 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ com.webank.wedatasphere.streamis streamis - 0.2.0 + 0.2.3 pom Streamis Project Parent POM @@ -48,7 +48,7 @@ 1.1.3 4.12 1.1.0 - 0.2.0 + 0.2.3 2.11.12 1.8 3.3.3 @@ -70,7 +70,7 @@ 2.21 1.9.5 1.4.19 - 0.2.0 + 0.2.3 5.1.47 2.0.1.Final diff --git a/streamis-appconn/pom.xml b/streamis-appconn/pom.xml index 54b590f91..4b4a38caa 100644 --- a/streamis-appconn/pom.xml +++ b/streamis-appconn/pom.xml @@ -5,7 +5,7 @@ streamis com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-jobmanager/pom.xml b/streamis-jobmanager/pom.xml index e88713dc2..eec37a4a5 100644 --- a/streamis-jobmanager/pom.xml +++ b/streamis-jobmanager/pom.xml @@ -20,7 +20,7 @@ streamis com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-jobmanager/streamis-job-launcher/pom.xml b/streamis-jobmanager/streamis-job-launcher/pom.xml index 8964504e1..de440be80 100755 --- a/streamis-jobmanager/streamis-job-launcher/pom.xml +++ b/streamis-jobmanager/streamis-job-launcher/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/pom.xml b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/pom.xml index 00f62af34..542447a01 100755 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/pom.xml +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/pom.xml b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/pom.xml index fdd13095d..01d9ec0f2 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/pom.xml +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/pom.xml @@ -20,7 +20,7 @@ streamis-job-launcher com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/pom.xml b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/pom.xml index 545888044..278af5594 100755 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/pom.xml +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml index ca49e4399..1cd05a3a1 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml @@ -5,7 +5,7 @@ streamis-job-log com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml index 0cdb47b8d..fa0785351 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml @@ -5,7 +5,7 @@ streamis-job-log com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml index 55fc60818..098ccf214 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml @@ -5,7 +5,7 @@ streamis-job-log com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml index 7632a5718..2956d8bf9 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml @@ -5,7 +5,7 @@ streamis-job-log com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml index 904bec4dd..cdd5d26ed 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml @@ -5,7 +5,7 @@ streamis-job-log com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/pom.xml b/streamis-jobmanager/streamis-job-log/pom.xml index eb79d84a5..f45e271e1 100644 --- a/streamis-jobmanager/streamis-job-log/pom.xml +++ b/streamis-jobmanager/streamis-job-log/pom.xml @@ -5,7 +5,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml index fcff3592c..b3429ac9e 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml @@ -5,7 +5,7 @@ streamis-job-log com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml index 165f6a520..6a6454c12 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml @@ -5,7 +5,7 @@ streamis-job-log com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../pom.xml 4.0.0 @@ -21,7 +21,7 @@ com.webank.wedatasphere.streamis streamis-job-log-common - 0.2.0 + 0.2.3 org.apache.linkis diff --git a/streamis-jobmanager/streamis-job-manager/pom.xml b/streamis-jobmanager/streamis-job-manager/pom.xml index 8c80a3520..9d2ae3ae0 100755 --- a/streamis-jobmanager/streamis-job-manager/pom.xml +++ b/streamis-jobmanager/streamis-job-manager/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/pom.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/pom.xml index f0ac3a5c2..b4e343e6b 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/pom.xml +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/pom.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/pom.xml index aec49b65b..cd3814726 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/pom.xml +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 ../../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-jobmanager-common/pom.xml b/streamis-jobmanager/streamis-jobmanager-common/pom.xml index b63c59d15..f6c322730 100644 --- a/streamis-jobmanager/streamis-jobmanager-common/pom.xml +++ b/streamis-jobmanager/streamis-jobmanager-common/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-jobmanager/streamis-jobmanager-server/pom.xml b/streamis-jobmanager/streamis-jobmanager-server/pom.xml index 11f353a55..cbf51250f 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/pom.xml +++ b/streamis-jobmanager/streamis-jobmanager-server/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-jobmanager/streamis-projectmanager-server/pom.xml b/streamis-jobmanager/streamis-projectmanager-server/pom.xml index c5066a706..e6ca1ba2e 100644 --- a/streamis-jobmanager/streamis-projectmanager-server/pom.xml +++ b/streamis-jobmanager/streamis-projectmanager-server/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-project/pom.xml b/streamis-project/pom.xml index e5bbcf888..6d5ad7295 100644 --- a/streamis-project/pom.xml +++ b/streamis-project/pom.xml @@ -20,7 +20,7 @@ streamis com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-project/streamis-project-common/pom.xml b/streamis-project/streamis-project-common/pom.xml index 849710b20..9d6d1c9f1 100644 --- a/streamis-project/streamis-project-common/pom.xml +++ b/streamis-project/streamis-project-common/pom.xml @@ -20,7 +20,7 @@ streamis-project com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-project/streamis-project-server/pom.xml b/streamis-project/streamis-project-server/pom.xml index 880d5bf0b..0833f2c6a 100644 --- a/streamis-project/streamis-project-server/pom.xml +++ b/streamis-project/streamis-project-server/pom.xml @@ -20,7 +20,7 @@ streamis-project com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/streamis-server/pom.xml b/streamis-server/pom.xml index 1bf8ecfba..5ab3784ba 100644 --- a/streamis-server/pom.xml +++ b/streamis-server/pom.xml @@ -20,7 +20,7 @@ streamis com.webank.wedatasphere.streamis - 0.2.0 + 0.2.3 4.0.0 diff --git a/web/package.json b/web/package.json index fc5a8bfb2..598101ac9 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "streamis", - "version": "0.2.0", + "version": "0.2.3", "private": true, "scripts": { "serve": "vue-cli-service serve", From 08cf1abac0f38913014ffeaecc8253c21766df2a Mon Sep 17 00:00:00 2001 From: davidhua Date: Tue, 20 Dec 2022 16:56:27 +0800 Subject: [PATCH 052/240] Use the "flink.yarn.ship-directories" instead of "flink.app.user.class.path" fixed #56 --- .../FlinkJarStreamisJobContentTransform.scala | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkJarStreamisJobContentTransform.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkJarStreamisJobContentTransform.scala index 551ae0f98..72206eee0 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkJarStreamisJobContentTransform.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkJarStreamisJobContentTransform.scala @@ -27,6 +27,7 @@ import com.webank.wedatasphere.streamis.jobmanager.manager.utils.JobUtils import scala.collection.JavaConverters._ import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer /** * Created by enjoyyin on 2021/9/23. @@ -50,20 +51,24 @@ class FlinkJarStreamisStartupParamsTransform extends Transform { startupMap.put("flink.app.main.class.jar", transformJobContent.getMainClassJar.getFileName) startupMap.put("flink.app.main.class.jar.bml.json", JsonUtils.jackson.writeValueAsString(getStreamisFileContent(transformJobContent.getMainClassJar))) - val classpathFiles = if(transformJobContent.getDependencyJars != null && transformJobContent.getResources != null) { - startupMap.put("flink.app.user.class.path", transformJobContent.getDependencyJars.asScala.map(_.getFileName).mkString(",")) - transformJobContent.getDependencyJars.asScala ++ transformJobContent.getResources.asScala - } else if(transformJobContent.getDependencyJars != null) { - startupMap.put("flink.app.user.class.path", transformJobContent.getDependencyJars.asScala.map(_.getFileName).mkString(",")) - transformJobContent.getDependencyJars.asScala - } else if(transformJobContent.getResources != null) { - startupMap.put("flink.yarn.ship-directories", transformJobContent.getResources.asScala.map(_.getFileName).mkString(",")) - transformJobContent.getResources.asScala + + /** + * Notice : "flink.app.user.class.path" equals to PipelineOptions.CLASSPATHS in Flink + * paths must specify a protocol (e.g. file://) and be accessible on all nodes + * so we use "flink.yarn.ship-directories" instead + */ + var classPathFiles = Option(transformJobContent.getDependencyJars) match { + case Some(list) => list.asScala + case _ => mutable.Buffer[StreamisFile]() + } + Option(transformJobContent.getResources) match { + case Some(list) => classPathFiles = classPathFiles ++ list.asScala + case _ => // Do nothing } - else mutable.Buffer[StreamisFile]() - if(classpathFiles.nonEmpty) + startupMap.put("flink.yarn.ship-directories", classPathFiles.map(_.getFileName).mkString(",")) + if(classPathFiles.nonEmpty) startupMap.put("flink.app.user.class.path.bml.json", - JsonUtils.jackson.writeValueAsString(classpathFiles.map(getStreamisFileContent).asJava)) + JsonUtils.jackson.writeValueAsString(classPathFiles.map(getStreamisFileContent).asJava)) if(transformJobContent.getHdfsJars != null) startupMap.put("flink.user.lib.path", transformJobContent.getHdfsJars.asScala.mkString(",")) val params = if(job.getParams == null) new util.HashMap[String, Any] else job.getParams From 8fe8da70f303d4ed258d62f62dd01c4b00a3b14f Mon Sep 17 00:00:00 2001 From: liveipooli Date: Tue, 20 Dec 2022 17:55:23 +0800 Subject: [PATCH 053/240] add: features --- web/src/apps/streamis/i18n/en.json | 4 +++ web/src/apps/streamis/i18n/zh.json | 4 +++ .../apps/streamis/module/jobList/index.vue | 27 ++++++++++++++++++- .../apps/streamis/module/jobSummary/index.vue | 16 ++++++++++- .../streamis/module/versionDetail/index.vue | 4 +++ web/vue.config.js | 4 +-- 6 files changed, 55 insertions(+), 4 deletions(-) diff --git a/web/src/apps/streamis/i18n/en.json b/web/src/apps/streamis/i18n/en.json index fad63b15f..0fdd35f53 100644 --- a/web/src/apps/streamis/i18n/en.json +++ b/web/src/apps/streamis/i18n/en.json @@ -56,6 +56,8 @@ "jobListTableColumns":{ "jobName": "Job name", "jobType": "Job type", + "launchMode": "Launch Mode", + "manageMode": "Manage Mode", "taskStatus": "Task status", "lastReleaseTime": "Last release time", "label": "Label", @@ -78,6 +80,8 @@ "modalTitle":"Version detail", "jobId":"Job id", "version":"Version", + "using":"Using", + "versionStatus":"VersionStatus", "description":"Description", "createTime":"Create time", "creator":"Creator", diff --git a/web/src/apps/streamis/i18n/zh.json b/web/src/apps/streamis/i18n/zh.json index 13eccbe64..834a3ff31 100644 --- a/web/src/apps/streamis/i18n/zh.json +++ b/web/src/apps/streamis/i18n/zh.json @@ -55,6 +55,8 @@ "jobListTableColumns":{ "jobName": "作业名称", "jobType": "作业类型", + "launchMode": "运行模式", + "manageMode": "管理模式", "taskStatus": "运行状态", "lastReleaseTime": "最近发布时间", "label": "标签", @@ -77,6 +79,8 @@ "modalTitle":"版本详情", "jobId":"作业id", "version":"版本", + "using":"使用中", + "versionStatus":"版本状态", "description":"描述", "createTime":"创建时间", "creator":"创建者", diff --git a/web/src/apps/streamis/module/jobList/index.vue b/web/src/apps/streamis/module/jobList/index.vue index 23ea4bd5d..eaa7cd9d8 100644 --- a/web/src/apps/streamis/module/jobList/index.vue +++ b/web/src/apps/streamis/module/jobList/index.vue @@ -130,6 +130,7 @@
{{ row.version }}
+
{{row.versionForwards > 0 ? '(+' + row.versionForwards + ')' : ''}}
- + diff --git a/web/src/apps/streamis/module/versionDetail/index.vue b/web/src/apps/streamis/module/versionDetail/index.vue index ba455b640..db678f298 100644 --- a/web/src/apps/streamis/module/versionDetail/index.vue +++ b/web/src/apps/streamis/module/versionDetail/index.vue @@ -5,6 +5,7 @@ v-model="visible" footer-hide width="1200" + class="version-detail-modal" @on-cancel="cancel" > @@ -95,3 +96,11 @@ export default { } + From 4a84e11c8813f4da617bc9af78771886d17252c2 Mon Sep 17 00:00:00 2001 From: Liveipool <123653079@qq.com> Date: Thu, 9 Feb 2023 16:28:04 +0800 Subject: [PATCH 093/240] upd: projectName --- web/src/apps/streamis/module/jobList/index.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/src/apps/streamis/module/jobList/index.vue b/web/src/apps/streamis/module/jobList/index.vue index 4a9244903..2144a8aad 100644 --- a/web/src/apps/streamis/module/jobList/index.vue +++ b/web/src/apps/streamis/module/jobList/index.vue @@ -511,11 +511,11 @@ export default { pageNow: current, pageSize, // 本地开发dev环境用的 - projectName: 'stream_job', + // projectName: 'stream_job', // 本地开发sit环境用的 // projectName: 'streamis025_version', // 正式环境用的 - // projectName: this.projectName + projectName: this.projectName } const { jobName, jobStatus } = this.query if (jobName) { From f45019f3a3dc08093265c3b0b17ccb6f33c4863c Mon Sep 17 00:00:00 2001 From: Liveipool <123653079@qq.com> Date: Thu, 9 Feb 2023 17:25:26 +0800 Subject: [PATCH 094/240] add: pagination --- .../apps/streamis/module/jobList/index.vue | 64 +++++++------------ .../streamis/module/versionDetail/index.vue | 25 +++++++- 2 files changed, 46 insertions(+), 43 deletions(-) diff --git a/web/src/apps/streamis/module/jobList/index.vue b/web/src/apps/streamis/module/jobList/index.vue index 2144a8aad..93cf7d6be 100644 --- a/web/src/apps/streamis/module/jobList/index.vue +++ b/web/src/apps/streamis/module/jobList/index.vue @@ -204,10 +204,12 @@ 0) { - api.fetch(`streamis/streamJobManager/job/${data.id}/versions?pageNow=${1}&pageSize=${100}`, 'get').then(res => { - console.log('versionDetail versions res: ', res); - this.loading = false - this.modalVisible = true - this.versionDatas = res.versions - this.versionDatas.forEach(item => { - item.versionStatus = '--' - if (item.version === data.version) item.versionStatus = this.$t('message.streamis.versionDetail.using') - }) - }).catch(err => { - console.log('versionDetail versions err: ', err); - this.loading = false + api.fetch(`streamis/streamJobManager/job/${data.id}/versions?pageNow=${page ? page : 1}&pageSize=5`, 'get').then(res => { + console.log('versionDetail versions res: ', res); + this.loading = false + this.modalVisible = true + this.$refs.versionDetail.pageData.total = res.totalPage + this.versionDatas = res.versions + this.versionDatas.forEach(item => { + item.versionStatus = '--' + if (item.version === data.version) item.versionStatus = this.$t('message.streamis.versionDetail.using') }) - } else { - api - .fetch( - 'streamis/streamJobManager/job/version?jobId=' + - data.id + - '&version=' + - data.version, - 'get' - ) - .then(res => { - console.log('versionDetail version res: ', res); - if (res) { - this.loading = false - this.modalVisible = true - this.versionDatas = [res.detail] - this.versionDatas.forEach(item => { - item.versionStatus = '--' - if (item.version === data.version) item.versionStatus = this.$t('message.streamis.versionDetail.using') - }) - } - }) - .catch(err => { - console.log('versionDetail version err: ', err); - this.loading = false - }) - } + }).catch(err => { + console.log('versionDetail versions err: ', err); + this.loading = false + }) }, modalCancel() { this.modalVisible = false diff --git a/web/src/apps/streamis/module/versionDetail/index.vue b/web/src/apps/streamis/module/versionDetail/index.vue index db678f298..7340ff4de 100644 --- a/web/src/apps/streamis/module/versionDetail/index.vue +++ b/web/src/apps/streamis/module/versionDetail/index.vue @@ -20,6 +20,14 @@
+ @@ -35,6 +43,10 @@ export default { }, data() { return { + pageData: { + total: 0, + current: 1 + }, columns: [ { title: this.$t('message.streamis.versionDetail.jobId'), @@ -91,11 +103,20 @@ export default { }, cancel() { this.$emit('modalCancel') - } + }, + handlePageChange(page) { + this.pageData.current = page + this.$emit('versionDetailPageChange', page) + }, } } - +