diff --git a/backend/pom.xml b/backend/pom.xml index 29fec47f92..177a028f21 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -17,6 +17,7 @@ 1.8 20.1.0 3.12.1 + 3.1.1 @@ -333,6 +334,36 @@ test + + org.apache.spark + spark-core_2.12 + ${spark.version} + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + provided + + + + org.apache.spark + spark-streaming_2.12 + ${spark.version} + provided + + + + org.apache.spark + spark-sql_2.12 + ${spark.version} + + diff --git a/backend/src/main/java/io/dataease/config/HbaseConfig.java b/backend/src/main/java/io/dataease/config/CommonConfig.java similarity index 67% rename from backend/src/main/java/io/dataease/config/HbaseConfig.java rename to backend/src/main/java/io/dataease/config/CommonConfig.java index f7b8bfaece..229ee6a069 100644 --- a/backend/src/main/java/io/dataease/config/HbaseConfig.java +++ b/backend/src/main/java/io/dataease/config/CommonConfig.java @@ -1,6 +1,9 @@ package io.dataease.config; import com.fit2cloud.autoconfigure.QuartzAutoConfiguration; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.SparkSession; import org.springframework.boot.autoconfigure.AutoConfigureBefore; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.context.annotation.Bean; @@ -12,7 +15,7 @@ import javax.annotation.Resource; @Configuration @AutoConfigureBefore(QuartzAutoConfiguration.class) -public class HbaseConfig { +public class CommonConfig { @Resource private Environment env; // 保存了配置文件的信息 @@ -27,4 +30,14 @@ public class HbaseConfig { configuration.set("hbase.client.retries.number", env.getProperty("hbase.client.retries.number", "1")); return configuration; } + + + @Bean + @ConditionalOnMissingBean + public JavaSparkContext javaSparkContext(){ + SparkConf conf = new SparkConf().setAppName(env.getProperty("spark.appName", "DataeaseJob") ).setMaster(env.getProperty("spark.master", "local[*]") ); + SparkSession spark = SparkSession.builder().config(conf).getOrCreate(); + JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); + return sc; + } } diff --git a/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java b/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java index 3c409114d8..8400020daa 100644 --- a/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java +++ b/backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java @@ -18,7 +18,6 @@ public abstract class DeScheduleJob implements Job { this.taskId = jobDataMap.getString("taskId"); LogUtil.info(jobKey.getGroup() + " Running: " + datasetTableId); - LogUtil.info(jobKey.getName() + " Running: " + datasetTableId); LogUtil.info("CronExpression: " + expression); businessExecute(context); } diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index 037fe397d5..248d98f497 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -6,6 +6,7 @@ import io.dataease.base.domain.DatasetTableField; import io.dataease.base.domain.DatasetTableTaskLog; import io.dataease.commons.constants.JobStatus; import io.dataease.commons.utils.CommonBeanFactory; +import io.dataease.commons.utils.LogUtil; import io.dataease.dto.dataset.DataTableInfoDTO; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.TableName; @@ -76,6 +77,9 @@ public class ExtractDataService { datasetTableTaskLog.setEndTime(System.currentTimeMillis()); dataSetTableTaskLogService.save(datasetTableTaskLog); }catch (Exception e){ + e.printStackTrace(); + LogUtil.error("ExtractData error, dataaset: " + datasetTableId); + LogUtil.error(e.getMessage(), e); datasetTableTaskLog.setStatus(JobStatus.Error.name()); datasetTableTaskLog.setEndTime(System.currentTimeMillis()); dataSetTableTaskLogService.save(datasetTableTaskLog);