Dev 0.3.2 webank

.gitignore

@@ -20,4 +20,11 @@ streamis-appconn/target
 /logs/streamis-server.log
 /logs/linkis.log
 /test/target
-/test/src
+/test/src
+/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/target/
+/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/target/
+/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/target/
+/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/target/
+/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/target/
+/streamis-jobmanager/streamis-job-log/streamis-job-log-common/target/
+/streamis-jobmanager/streamis-job-log/streamis-job-log-server/target/

README-ZH.md

@@ -46,7 +46,7 @@ Dimension节点、Transform节点、Sink节点 和 [Visualis](https://github.com
 ![流式应用配置](docs/images/stream_job_config_1.png)
 ![流式应用配置](docs/images/stream_job_config_2.png)
 
-       更多功能，请参考：[Streamis 用户手册](docs/zh_CN/0.2.0/使用文档/Streamis用户手册.md)。
+       更多功能，请参考：[Streamis 用户手册](docs/zh_CN/0.2.5-webank/使用文档/Streamis用户手册.md)。
 
 ----
 
@@ -73,14 +73,14 @@ Dimension节点、Transform节点、Sink节点 和 [Visualis](https://github.com
 
 ## 编译和安装部署
 
-       请参考 [Streamis 安装部署文档](docs/zh_CN/0.2.0/Streamis安装文档.md) ，用于安装部署 Streamis 及其依赖环境。
+       请参考 [Streamis 安装部署文档](docs/zh_CN/0.2.5-webank/Streamis安装文档.md) ，用于安装部署 Streamis 及其依赖环境。
 
 
 ----
 
 ## 示例和使用指引
 
-       请到 [用户使用文档](docs/zh_CN/0.2.0/使用文档/Streamis用户手册.md) ，了解如何快速使用 Streamis。
+       请到 [用户使用文档](docs/zh_CN/0.2.5-webank/使用文档/Streamis用户手册.md) ，了解如何快速使用 Streamis。
 
 ----
 

README.md

@@ -71,7 +71,7 @@ The Dimension node, Transform node, Sink node and [Visualis](https://github.com/
 
 ## Compile and install deployment
 
-please refer to [Streamis Installation and Deployment Document](docs/en_US/0.2.0/StreamisDeployment.md) for installing and deploying Streamis.
+please refer to [Streamis Installation and Deployment Document](docs/en_US/0.2.5-webank/StreamisDeployment.md) for installing and deploying Streamis.
 
 ----
 

assembly/pom.xml

@@ -20,7 +20,7 @@
     <parent>
         <artifactId>streamis</artifactId>
         <groupId>com.webank.wedatasphere.streamis</groupId>
-        <version>0.2.0</version>
+        <version>0.3.1-webank</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

bin/upgrade.sh

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-# Use to upgrade from 0.1.0 to 0.2.0
+# Use to upgrade from 0.1.0 to 0.2.5-webank
 
 if [ -f "~/.bashrc" ];then
   echo "Warning! user bashrc file does not exist."
@@ -39,7 +39,7 @@ interact_echo(){
   done
 }
 
-interact_echo "Are you sure the current version of Streamis is 0.1.0 and need to upgrade to 0.2.0 ?"
+interact_echo "Are you sure the current version of Streamis is 0.2.x < 0.2.5 and need to upgrade to 0.2.5 ?"
 if [[ $? == 0 ]]; then
   source ${workDir}/conf/db.sh
   echo "<------ Will connect to [${MYSQL_HOST}:${MYSQL_PORT}] to upgrade the tables in database... ------>"

conf/config.sh

@@ -40,6 +40,6 @@ GATEWAY_PORT=9001
 STREAMIS_SERVER_INSTALL_IP=127.0.0.1
 STREAMIS_SERVER_INSTALL_PORT=9400
 
-STREAMIS_VERSION=0.2.0
+STREAMIS_VERSION=0.3.1-webank
 
 STREAMIS_FILE_NAME="STREAMIS-$STREAMIS_VERSION"

conf/db.sh

@@ -1,8 +1,6 @@
-### for DSS-Server and Eventchecker APPCONN
+### for Streamis-Server
 MYSQL_HOST=
 MYSQL_PORT=
 MYSQL_DB=
 MYSQL_USER=
-MYSQL_PASSWORD=
-
-
+MYSQL_PASSWORD=

db/streamis_ddl.sql

@@ -132,9 +132,10 @@ CREATE TABLE `linkis_stream_job` (
   `create_time` datetime DEFAULT NULL,
   `label` varchar(200) DEFAULT NULL,
   `description` varchar(200) DEFAULT NULL,
-  `job_type` varchar(30) DEFAULT NULL COMMENT '目前只支持flink.sql、flink.jar',
+  `job_type` varchar(30) DEFAULT NULL COMMENT '目前只支持flink.sql、flink.jar、spark.jar',
   `submit_user` varchar(100) DEFAULT NULL,
   `workspace_name` varchar(50) DEFAULT NULL,
+  `current_version` varchar(50) DEFAULT NULL,
   PRIMARY KEY (`id`) USING BTREE,
   UNIQUE KEY(`project_name`, `name`)
 ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT COMMENT='作业表';
@@ -149,6 +150,7 @@ CREATE TABLE `linkis_stream_job_version` (
   `version` varchar(20) DEFAULT NULL,
   `source` varchar(255) DEFAULT NULL COMMENT '这个版本的来源，比如：用户上传，由某个历史版本回退回来的',
   `job_content` text COMMENT '内容为meta.json',
+  `manage_mode` varchar(30) DEFAULT 'EngineConn' COMMENT 'Manage mode',
   `comment` varchar(255) DEFAULT NULL,
   `create_time` datetime DEFAULT NULL,
   `create_by` varchar(32) DEFAULT NULL,

db/streamis_dml.sql

@@ -22,4 +22,5 @@ INSERT INTO `linkis_stream_job_config_def` VALUES (32,'wds.linkis.flink.authorit
 INSERT INTO `linkis_stream_job_config_def` VALUES (33,'wds.linkis.flink.authority.visible','可见人员','INPUT',0,'可见人员','None',NULL,'',1,1,NULL,NULL,'',32,0,0);
 INSERT INTO `linkis_stream_job_config_def` VALUES (34,'wds.linkis.rm.yarnqueue','使用Yarn队列','INPUT',0,'使用Yarn队列','None',NULL,'',1,1,NULL,NULL,'',1,0,0);
 INSERT INTO `linkis_stream_job_config_def` VALUES (35,'wds.linkis.flink.app.fail-restart.switch','作业失败自动拉起开关','SELECT',1,'作业失败自动拉起开关','None',NULL,'',1,1,NULL,'OFF','ON,OFF',8,0,0);
-INSERT INTO `linkis_stream_job_config_def` VALUES (36,'wds.linkis.flink.app.start-auto-restore.switch','作业启动状态自恢复','SELECT',2,'作业启动状态自恢复','None',NULL,'',1,1,NULL,'ON','ON,OFF',8,0,0);
+INSERT INTO `linkis_stream_job_config_def` VALUES (36,'wds.linkis.flink.app.start-auto-restore.switch','作业启动状态自恢复','SELECT',2,'作业启动状态自恢复','None',NULL,'',1,1,NULL,'ON','ON,OFF',8,0,0);
+INSERT INTO `linkis_stream_job_config_def` VALUES (38, 'linkis.ec.app.manage.mode', '管理模式', 'SELECT', 3, 'EngineConn管理模式', 'None', NULL, '', 1, 1, NULL, 'attach', 'detach,attach', 8, 0, 0);

db/upgrade/0.2.5-webank/upgrade_ddl.sql

@@ -0,0 +1,2 @@
+ALTER TABLE linkis_stream_job ADD COLUMN IF NOT EXISTS `current_version` varchar(50);
+ALTER TABLE linkis_stream_job_version ADD COLUMN IF NOT EXISTS `manage_mode` varchar(30) NOT NULL DEFAULT 'EngineConn';

db/upgrade/0.2.5-webank/upgrade_dml.sql

@@ -0,0 +1 @@
+UPDATE linkis_stream_job j SET current_version = (SELECT version FROM linkis_stream_job_version v WHERE v.job_id = j.id ORDER BY id DESC limit 1);

db/upgrade/0.3.1-webank/upgrade_ddl.sql

@@ -0,0 +1,2 @@
+ALTER TABLE linkis_stream_project_files ADD update_time datetime NULL;
+ALTER TABLE linkis_stream_project_files ADD md5 varchar(100) NULL;

db/upgrade/0.3.1-webank/upgrade_dml.sql

@@ -0,0 +1,5 @@
+UPDATE bdp_easy_ide.linkis_stream_job_config_def
+SET `key`='wds.linkis.flink.alert.failure.user', name='失败时告警用户', `type`='INPUT', sort=0, description='失败时告警用户', validate_type='None', validate_rule=NULL, `style`='', visiable=1, `level`=1, unit=NULL, default_value='', ref_values='', parent_ref=8, required=0, is_temp=0
+WHERE id=16;
+
+INSERT INTO `linkis_stream_job_config_def` VALUES (38, 'linkis.ec.app.manage.mode', '管理模式', 'SELECT', 3, 'EngineConn管理模式', 'None', NULL, '', 1, 1, NULL, 'attach', 'detach,attach', 8, 0, 0);

docs/en_US/0.2.0/StreamisDeployment.md → docs/en_US/0.2.5/StreamisDeployment.md

@@ -1,7 +1,7 @@
 # Streamis installation and deployment documentation
 
 ## 1. Component introduction
-Streamis0.2.0 provides the Streamis-JobManager component, the role of the component is <br>
+Streamis0.2.5-webank provides the Streamis-JobManager component, the role of the component is <br>
 1. Publish streaming applications<br>
 2. Set streaming application parameters, such as the number of Flink slots, checkpoint related parameters, etc.<br>
 3. Manage streaming applications (e.g. start and stop)<br>
@@ -156,10 +156,10 @@ sudo nginx -s reload
 
 ## 5. Access to DSS
 
-If you want to use the streamis0.2.0 front end normally, you also need to install the DSS StreamisAppConn plug-in. Please refer to: [StreamisAppConn plug-in installation document](development/StreamisAppConnInstallationDocument.md)
+If you want to use the streamis0.2.5-webank front end normally, you also need to install the DSS StreamisAppConn plug-in. Please refer to: [StreamisAppConn plug-in installation document](development/StreamisAppConnInstallationDocument.md)
 
 ## 6. Linkis Flink engine compilation and installation
-If you want to run streamis0.2.0 normally, you also need to install the linkis Flink engine. Please refer to: [linkis Flink engine installation document](https://linkis.apache.org/zh-CN/docs/1.1.2/engine_usage/flink/)
+If you want to run streamis0.2.5-webank normally, you also need to install the linkis Flink engine. Please refer to: [linkis Flink engine installation document](https://linkis.apache.org/zh-CN/docs/1.1.2/engine_usage/flink/)
 
 ## 7. Streamis component upgrade document / script
-If you want to upgrade from a lower version of streamis to streamis0.2.0, please refer to: [streamis upgrade document](development/StreamisUpgradeDocumentation.md)
+If you want to upgrade from a lower version of streamis to streamis0.2.5-webank, please refer to: [streamis upgrade document](development/StreamisUpgradeDocumentation.md)

docs/en_US/0.2.0/development/StreamisAppConnInstallationDocument.md → docs/en_US/0.2.5/development/StreamisAppConnInstallationDocument.md

@@ -1,7 +1,7 @@
 Streamisappconn installation document this article mainly introduces the deployment, configuration and installation of streamisappconn in DSS (datasphere studio) 1.1.0
 
 # 1. Preparation for deploying streamisappconn
-Before deploying streamisappconn, please complete the installation of streamis0.2.0 and other related components, and ensure that the basic functions of the project are available.
+Before deploying streamisappconn, please complete the installation of streamis0.2.5-webank and other related components, and ensure that the basic functions of the project are available.
 
 # 2. Download and compilation of streamisappconn plug-in
 1) Download binary package

docs/en_US/0.2.0/development/StreamisUpgradeDocumentation.md → docs/en_US/0.2.5/development/StreamisUpgradeDocumentation.md

@@ -1,4 +1,4 @@
-Streamis upgrade document. This article mainly introduces the upgrade steps of adapting DSS1.1.0 and linkis1.1.1 based on the original installation of Streamis service. The biggest difference between Streamis 0.2.0 and Streamis 0.1.0 is that it accesses DSS appconn and optimizes the start and stop of jobs.
+Streamis upgrade document. This article mainly introduces the upgrade steps of adapting DSS1.1.0 and linkis1.1.1 based on the original installation of Streamis service. The biggest difference between Streamis 0.2.5-webank and Streamis 0.1.0 is that it accesses DSS appconn and optimizes the start and stop of jobs.
 
 # 1. Work before upgrading streamis
 Before upgrading Streamis, please install linkis1.1.1 and DSS1.1.0 or above, and ensure that the linkis Flink engine and DSS can be used normally. For the installation of DSS and linkis, please refer to [dss & linkis one click installation and deployment document](https://github.com/WeBankFinTech/DataSphereStudio-Doc/blob/main/zh_CN/%E5%AE%89%E8%A3%85%E9%83%A8%E7%BD%B2/DSS%E5%8D%95%E6%9C%BA%E9%83%A8%E7%BD%B2%E6%96%87%E6%A1%A3.md).

docs/en_US/userManual/StreamisUserManual.md

@@ -2,7 +2,7 @@
 
 ## 1. Preface
 
-         This article is a quick start document for Streamis 0.2.0, which covers the basic usage process of Stremis. More details on operation and usage will be provided in the user documentation.
+         This article is a quick start document for Streamis 0.2.5-webank, which covers the basic usage process of Stremis. More details on operation and usage will be provided in the user documentation.
 
 
 ## 2. Streamis entrance