public/dataease
1
0
Fork 0
mirror of https://github.com/dataease/dataease.git synced 2026-05-20 02:58:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:dataease/dataease into main

Browse Source
This commit is contained in:
taojinlong
2021-04-14 14:18:26 +08:00
parent e5226cfab2 6607bb56c5
commit 7c98665ddc
5 changed files with 37 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/src/main/java/io/dataease/listener/AppStartReadHBaseListener.java
View File

@@ -45,14 +45,14 @@ public class AppStartReadHBaseListener implements ApplicationListener<Applicatio
datasetTableExample.createCriteria().andModeEqualTo(1);
List<DatasetTable> datasetTables = datasetTableMapper.selectByExampleWithBLOBs(datasetTableExample);
for (DatasetTable table : datasetTables) {
commonThreadPool.addTask(() -> {
// commonThreadPool.addTask(() -> {
try {
List<DatasetTableField> fields = dataSetTableFieldsService.getFieldsByTableId(table.getId());
sparkCalc.getHBaseDataAndCache(table.getId(), fields);
} catch (Exception e) {
e.printStackTrace();
}
});
// });
}
}
}

3
backend/src/main/java/io/dataease/service/spark/SparkCalc.java
View File

@@ -22,6 +22,7 @@ import org.apache.spark.sql.*;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
import org.apache.spark.storage.StorageLevel;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Service;
import scala.Tuple2;
@@ -144,7 +145,7 @@ public class SparkCalc {
});
StructType structType = DataTypes.createStructType(structFields);
Dataset<Row> dataFrame = sqlContext.createDataFrame(rdd, structType).persist();
Dataset<Row> dataFrame = sqlContext.createDataFrame(rdd, structType).persist(StorageLevel.MEMORY_AND_DISK_SER());
CacheUtil.getInstance().addCacheData(hTable, dataFrame);
dataFrame.count();
return dataFrame;