diff --git a/backend/pom.xml b/backend/pom.xml
index 29fec47f92..177a028f21 100644
--- a/backend/pom.xml
+++ b/backend/pom.xml
@@ -17,6 +17,7 @@
1.8
20.1.0
3.12.1
+ 3.1.1
@@ -333,6 +334,36 @@
test
+
+ org.apache.spark
+ spark-core_2.12
+ ${spark.version}
+
+
+ org.slf4j
+ slf4j-log4j12
+
+
+ log4j
+ log4j
+
+
+ provided
+
+
+
+ org.apache.spark
+ spark-streaming_2.12
+ ${spark.version}
+ provided
+
+
+
+ org.apache.spark
+ spark-sql_2.12
+ ${spark.version}
+
+
diff --git a/backend/src/main/java/io/dataease/config/HbaseConfig.java b/backend/src/main/java/io/dataease/config/CommonConfig.java
similarity index 67%
rename from backend/src/main/java/io/dataease/config/HbaseConfig.java
rename to backend/src/main/java/io/dataease/config/CommonConfig.java
index f7b8bfaece..229ee6a069 100644
--- a/backend/src/main/java/io/dataease/config/HbaseConfig.java
+++ b/backend/src/main/java/io/dataease/config/CommonConfig.java
@@ -1,6 +1,9 @@
package io.dataease.config;
import com.fit2cloud.autoconfigure.QuartzAutoConfiguration;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.sql.SparkSession;
import org.springframework.boot.autoconfigure.AutoConfigureBefore;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.context.annotation.Bean;
@@ -12,7 +15,7 @@ import javax.annotation.Resource;
@Configuration
@AutoConfigureBefore(QuartzAutoConfiguration.class)
-public class HbaseConfig {
+public class CommonConfig {
@Resource
private Environment env; // 保存了配置文件的信息
@@ -27,4 +30,14 @@ public class HbaseConfig {
configuration.set("hbase.client.retries.number", env.getProperty("hbase.client.retries.number", "1"));
return configuration;
}
+
+
+ @Bean
+ @ConditionalOnMissingBean
+ public JavaSparkContext javaSparkContext(){
+ SparkConf conf = new SparkConf().setAppName(env.getProperty("spark.appName", "DataeaseJob") ).setMaster(env.getProperty("spark.master", "local[*]") );
+ SparkSession spark = SparkSession.builder().config(conf).getOrCreate();
+ JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
+ return sc;
+ }
}
diff --git a/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java b/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java
index 3c409114d8..8400020daa 100644
--- a/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java
+++ b/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java
@@ -18,7 +18,6 @@ public abstract class DeScheduleJob implements Job {
this.taskId = jobDataMap.getString("taskId");
LogUtil.info(jobKey.getGroup() + " Running: " + datasetTableId);
- LogUtil.info(jobKey.getName() + " Running: " + datasetTableId);
LogUtil.info("CronExpression: " + expression);
businessExecute(context);
}
diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
index 037fe397d5..248d98f497 100644
--- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
+++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
@@ -6,6 +6,7 @@ import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.DatasetTableTaskLog;
import io.dataease.commons.constants.JobStatus;
import io.dataease.commons.utils.CommonBeanFactory;
+import io.dataease.commons.utils.LogUtil;
import io.dataease.dto.dataset.DataTableInfoDTO;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.TableName;
@@ -76,6 +77,9 @@ public class ExtractDataService {
datasetTableTaskLog.setEndTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
}catch (Exception e){
+ e.printStackTrace();
+ LogUtil.error("ExtractData error, dataaset: " + datasetTableId);
+ LogUtil.error(e.getMessage(), e);
datasetTableTaskLog.setStatus(JobStatus.Error.name());
datasetTableTaskLog.setEndTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);