public/dataease
1
0
Fork 0
mirror of https://github.com/dataease/dataease.git synced 2026-05-21 12:45:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into main

Browse Source 
# Conflicts:
#	backend/src/main/resources/i18n/messages_en_US.properties
#	backend/src/main/resources/i18n/messages_zh_CN.properties
#	backend/src/main/resources/i18n/messages_zh_TW.properties
#	frontend/src/components/canvas/components/Toolbar.vue
This commit is contained in:
wangjiahao
2021-05-24 17:30:13 +08:00
parent b59c4967f7 44dcb71cf1
commit 746e6c49c3
26 changed files with 164 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
backend/src/main/java/io/dataease/datasource/service/DatasourceService.java
View File

@@ -7,6 +7,8 @@ import io.dataease.base.mapper.ext.ExtDataSourceMapper;
import io.dataease.base.mapper.ext.query.GridExample;
import io.dataease.commons.exception.DEException;
import io.dataease.commons.utils.AuthUtils;
import io.dataease.commons.utils.CommonThreadPool;
import io.dataease.commons.utils.LogUtil;
import io.dataease.controller.request.DatasourceUnionRequest;
import io.dataease.controller.sys.base.BaseGridRequest;
import io.dataease.controller.sys.base.ConditionEntity;
@@ -40,6 +42,8 @@ public class DatasourceService {
private DatasetTableMapper datasetTableMapper;
@Resource
private DataSetGroupService dataSetGroupService;
@Resource
private CommonThreadPool commonThreadPool;
public Datasource addDatasource(Datasource datasource) {
DatasourceExample example = new DatasourceExample();
@@ -114,7 +118,6 @@ public class DatasourceService {
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class);
if (StringUtils.equals(name, dataTableInfoDTO.getTable())) {
dbTableDTO.setEnableCheck(false);
List<DatasetGroup> parents = dataSetGroupService.getParents(datasetTable.getSceneId());
StringBuilder stringBuilder = new StringBuilder();
parents.forEach(ele -> stringBuilder.append(ele.getName()).append("/"));
@@ -131,4 +134,25 @@ public class DatasourceService {
public Datasource get(String id) {
return datasourceMapper.selectByPrimaryKey(id);
}
public void initAllDataSourceConnectionPool(){
List<Datasource> datasources = datasourceMapper.selectByExampleWithBLOBs(new DatasourceExample());
datasources.forEach(datasource -> {
try {
commonThreadPool.addTask(() ->{
try {
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(datasource.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(datasource);
datasourceProvider.initDataSource(datasourceRequest);
LogUtil.error("Succsss to init datasource connection pool: " + datasource.getName());
}catch (Exception e){
LogUtil.error("Failed to init datasource connection pool: " + datasource.getName(), e);
}
});
}catch (Exception e){
e.printStackTrace();
}
});
}
}

47
backend/src/main/java/io/dataease/listener/AppStartReadHBaseListener.java
View File

@@ -1,47 +0,0 @@
//package io.dataease.listener;
//
//import io.dataease.base.mapper.DatasetTableMapper;
//import io.dataease.commons.utils.CommonThreadPool;
//import io.dataease.service.dataset.DataSetTableFieldsService;
//import org.springframework.boot.context.event.ApplicationReadyEvent;
//import org.springframework.context.ApplicationListener;
//import org.springframework.core.annotation.Order;
//import org.springframework.core.env.Environment;
//import org.springframework.stereotype.Component;
//
//import javax.annotation.Resource;
//
//@Component
//@Order(value = 2)
//public class AppStartReadHBaseListener implements ApplicationListener<ApplicationReadyEvent> {
// @Resource
// private CommonThreadPool commonThreadPool;
//// @Resource
//// private SparkCalc sparkCalc;
// @Resource
// private Environment env; // 保存了配置文件的信息
//
// @Resource
// private DatasetTableMapper datasetTableMapper;
// @Resource
// private DataSetTableFieldsService dataSetTableFieldsService;
//
// @Override
// public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
//// System.out.println("================= Read HBase start =================");
//// // 项目启动，从数据集中找到定时抽取的表，从HBase中读取放入缓存
//// DatasetTableExample datasetTableExample = new DatasetTableExample();
//// datasetTableExample.createCriteria().andModeEqualTo(1);
//// List<DatasetTable> datasetTables = datasetTableMapper.selectByExampleWithBLOBs(datasetTableExample);
//// for (DatasetTable table : datasetTables) {
////// commonThreadPool.addTask(() -> {
//// try {
//// List<DatasetTableField> fields = dataSetTableFieldsService.getFieldsByTableId(table.getId());
//// sparkCalc.getHBaseDataAndCache(table.getId(), fields);
//// } catch (Exception e) {
//// e.printStackTrace();
//// }
////// });
//// }
// }
//}

25
backend/src/main/java/io/dataease/listener/DataSourceInitStartListener.java Normal file
View File

@@ -0,0 +1,25 @@
package io.dataease.listener;
import io.dataease.base.domain.DatasetTableTask;
import io.dataease.datasource.service.DatasourceService;
import io.dataease.service.ScheduleService;
import io.dataease.service.dataset.DataSetTableTaskService;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.ApplicationListener;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.List;
@Component
@Order(value = 1)
public class DataSourceInitStartListener implements ApplicationListener<ApplicationReadyEvent> {
@Resource
private DatasourceService datasourceService;
@Override
public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
datasourceService.initAllDataSourceConnectionPool();
}
}

3
backend/src/main/java/io/dataease/service/chart/ChartGroupService.java
View File

@@ -9,6 +9,7 @@ import io.dataease.commons.utils.TreeUtils;
import io.dataease.controller.request.chart.ChartGroupRequest;
import io.dataease.controller.request.dataset.DataSetTableRequest;
import io.dataease.dto.chart.ChartGroupDTO;
import io.dataease.i18n.Translator;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
@@ -104,7 +105,7 @@ public class ChartGroupService {
}
List<ChartGroup> list = chartGroupMapper.selectByExample(chartGroupExample);
if (list.size() > 0) {
throw new RuntimeException("Name can't repeat in same group.");
throw new RuntimeException(Translator.get("i18n_name_cant_repeat_same_group"));
}
}
}

3
backend/src/main/java/io/dataease/service/chart/ChartViewService.java
View File

@@ -20,6 +20,7 @@ import io.dataease.dto.chart.ChartViewDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.chart.Series;
import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.i18n.Translator;
import io.dataease.provider.QueryProvider;
import io.dataease.service.dataset.DataSetTableFieldsService;
import io.dataease.service.dataset.DataSetTableService;
@@ -226,7 +227,7 @@ public class ChartViewService {
}
List<ChartViewWithBLOBs> list = chartViewMapper.selectByExampleWithBLOBs(chartViewExample);
if (list.size() > 0) {
throw new RuntimeException("Name can't repeat in same group.");
throw new RuntimeException(Translator.get("i18n_name_cant_repeat_same_group"));
}
}

5
backend/src/main/java/io/dataease/service/dataset/DataSetGroupService.java
View File

@@ -12,8 +12,10 @@ import io.dataease.controller.request.dataset.DataSetGroupRequest;
import io.dataease.controller.request.dataset.DataSetTableRequest;
import io.dataease.dto.dataset.DataSetGroupDTO;
import io.dataease.dto.dataset.DataSetTableDTO;
import io.dataease.i18n.Translator;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
@@ -32,6 +34,7 @@ public class DataSetGroupService {
@Resource
private DatasetGroupMapper datasetGroupMapper;
@Resource
@Lazy
private DataSetTableService dataSetTableService;
@Resource
private ExtDataSetGroupMapper extDataSetGroupMapper;
@@ -115,7 +118,7 @@ public class DataSetGroupService {
}
List<DatasetGroup> list = datasetGroupMapper.selectByExample(datasetGroupExample);
if (list.size() > 0) {
throw new RuntimeException("Name can't repeat in same group.");
throw new RuntimeException(Translator.get("i18n_name_cant_repeat_same_group"));
}
}

21
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@@ -19,6 +19,7 @@ import io.dataease.dto.dataset.DataSetTableDTO;
import io.dataease.dto.dataset.DataSetTableUnionDTO;
import io.dataease.dto.dataset.DataTableInfoCustomUnion;
import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.i18n.Translator;
import io.dataease.provider.DDLProvider;
import io.dataease.provider.QueryProvider;
import org.apache.commons.collections4.CollectionUtils;
@@ -34,6 +35,7 @@ import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
@@ -49,6 +51,7 @@ import java.util.stream.Collectors;
* @Date 2021/2/23 2:54 下午
*/
@Service
@Transactional(rollbackFor = Exception.class)
public class DataSetTableService {
@Resource
private DatasetTableMapper datasetTableMapper;
@@ -177,7 +180,7 @@ public class DataSetTableService {
.id("count")
.tableId(dataSetTableRequest.getId())
.originName("*")
.name("记录数*")
.name(Translator.get("i18n_chart_count"))
.dataeaseName("*")
.type("INT")
.checked(true)
@@ -500,7 +503,7 @@ public class DataSetTableService {
return;
}
QueryProvider qp = null;
if(!ObjectUtils.isEmpty(ds)) {
if (!ObjectUtils.isEmpty(ds)) {
qp = ProviderFactory.getQueryProvider(ds.getType());
}
if (CollectionUtils.isNotEmpty(fields)) {
@@ -611,7 +614,7 @@ public class DataSetTableService {
}
List<DatasetTable> list = datasetTableMapper.selectByExample(datasetTableExample);
if (list.size() > 0) {
throw new RuntimeException("Name can't repeat in same group.");
throw new RuntimeException(Translator.get("i18n_name_cant_repeat_same_group"));
}
}
@@ -647,11 +650,11 @@ public class DataSetTableService {
if (StringUtils.equalsIgnoreCase(suffix, "xls")) {
HSSFWorkbook workbook = new HSSFWorkbook(inputStream);
HSSFSheet sheet0 = workbook.getSheetAt(0);
for (int i=0;i<workbook.getNumberOfSheets();i++){
for (int i = 0; i < workbook.getNumberOfSheets(); i++) {
sheets.add(workbook.getSheetAt(i).getSheetName());
}
if (sheet0.getNumMergedRegions() > 0) {
throw new RuntimeException("Sheet have merged regions.");
throw new RuntimeException(Translator.get("i18n_excel_have_merge_region"));
}
int rows;
if (isPreview) {
@@ -668,7 +671,7 @@ public class DataSetTableService {
tableFiled.setFieldType("TEXT");
tableFiled.setFieldSize(1024);
String columnName = readCell(row.getCell(j));
if(StringUtils.isEmpty(columnName)){
if (StringUtils.isEmpty(columnName)) {
columnName = "NONE_" + String.valueOf(j);
}
tableFiled.setFieldName(columnName);
@@ -686,11 +689,11 @@ public class DataSetTableService {
} else if (StringUtils.equalsIgnoreCase(suffix, "xlsx")) {
XSSFWorkbook xssfWorkbook = new XSSFWorkbook(inputStream);
XSSFSheet sheet0 = xssfWorkbook.getSheetAt(0);
for (int i=0;i<xssfWorkbook.getNumberOfSheets();i++){
for (int i = 0; i < xssfWorkbook.getNumberOfSheets(); i++) {
sheets.add(xssfWorkbook.getSheetAt(i).getSheetName());
}
if (sheet0.getNumMergedRegions() > 0) {
throw new RuntimeException("Sheet have merged regions.");
throw new RuntimeException(Translator.get("i18n_excel_have_merge_region"));
}
int rows;
if (isPreview) {
@@ -707,7 +710,7 @@ public class DataSetTableService {
tableFiled.setFieldType("TEXT");
tableFiled.setFieldSize(1024);
String columnName = readCell(row.getCell(j));
if(StringUtils.isEmpty(columnName)){
if (StringUtils.isEmpty(columnName)) {
columnName = "NONE_" + String.valueOf(j);
}
tableFiled.setFieldName(columnName);

3
backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java
View File

@@ -3,6 +3,7 @@ package io.dataease.service.dataset;
import io.dataease.base.domain.DatasetTableTask;
import io.dataease.base.domain.DatasetTableTaskExample;
import io.dataease.base.mapper.DatasetTableTaskMapper;
import io.dataease.i18n.Translator;
import io.dataease.service.ScheduleService;
import org.apache.commons.lang3.StringUtils;
import org.quartz.CronExpression;
@@ -31,7 +32,7 @@ public class DataSetTableTaskService {
// check
if (StringUtils.isNotEmpty(datasetTableTask.getCron())) {
if (!CronExpression.isValidExpression(datasetTableTask.getCron())) {
throw new RuntimeException("cron expression error.");
throw new RuntimeException(Translator.get("i18n_cron_expression_error"));
}
}
if (StringUtils.isEmpty(datasetTableTask.getId())) {

33
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@@ -63,6 +63,7 @@ import org.pentaho.di.trans.steps.userdefinedjavaclass.UserDefinedJavaClassDef;
import org.pentaho.di.trans.steps.userdefinedjavaclass.UserDefinedJavaClassMeta;
import org.pentaho.di.www.SlaveServerJobStatus;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
@@ -82,6 +83,7 @@ import java.util.stream.Collectors;
public class ExtractDataService {
@Resource
@Lazy
private DataSetTableService dataSetTableService;
@Resource
private DataSetTableFieldsService dataSetTableFieldsService;
@@ -91,7 +93,6 @@ public class ExtractDataService {
private DataSetTableTaskService dataSetTableTaskService;
@Resource
private DatasourceMapper datasourceMapper;
private static ExecutorService pool = Executors.newScheduledThreadPool(50); //设置连接池
private static String lastUpdateTime = "${__last_update_time__}";
private static String currentUpdateTime = "${__current_update_time__}";
@@ -113,7 +114,6 @@ public class ExtractDataService {
"UNIQUE KEY(dataease_uuid)\n" +
"DISTRIBUTED BY HASH(dataease_uuid) BUCKETS 10\n" +
"PROPERTIES(\"replication_num\" = \"1\");";
private static String shellScript = "curl --location-trusted -u %s:%s -H \"label:%s\" -H \"column_separator:%s\" -H \"columns:%s\" -H \"merge_type: %s\" -T %s -XPUT http://%s:%s/api/%s/%s/_stream_load\n" +
"rm -rf %s\n";
@@ -448,6 +448,7 @@ public class ExtractDataService {
selectSQL = qp.createQuerySQL(tableName, datasetTableFields);
}
inputStep = inputStep(transMeta, selectSQL);
udjcStep = udjc(datasetTableFields, false);
break;
case sqlServer:
SqlServerConfigration sqlServerConfigration = new Gson().fromJson(datasource.getConfiguration(), SqlServerConfigration.class);
@@ -459,10 +460,12 @@ public class ExtractDataService {
selectSQL = qp.createQuerySQL(tableName, datasetTableFields);
}
inputStep = inputStep(transMeta, selectSQL);
udjcStep = udjc(datasetTableFields, false);
break;
case excel:
String filePath = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getData();
inputStep = excelInputStep(filePath, datasetTableFields);
udjcStep = udjc(datasetTableFields, true);
default:
break;
}
@@ -487,7 +490,7 @@ public class ExtractDataService {
break;
}
udjcStep = udjc(datasetTableFields);
outputStep = outputStep(dorisOutputTable);
hi1 = new TransHopMeta(inputStep, udjcStep);
hi2 = new TransHopMeta(udjcStep, outputStep);
@@ -569,11 +572,11 @@ public class ExtractDataService {
return outputStep;
}
private StepMeta udjc(List<DatasetTableField> datasetTableFields) {
String needToChangeolumnType = "";
private StepMeta udjc(List<DatasetTableField> datasetTableFields, boolean isExcel) {
String needToChangeColumnType = "";
for (DatasetTableField datasetTableField : datasetTableFields) {
if (datasetTableField.getDeExtractType() != null && datasetTableField.getDeExtractType() == 4) {
needToChangeolumnType = needToChangeolumnType + alterColumnTypeCode.replace("FILED", datasetTableField.getOriginName());
needToChangeColumnType = needToChangeColumnType + alterColumnTypeCode.replace("FILED", datasetTableField.getOriginName());
}
}
@@ -583,8 +586,13 @@ public class ExtractDataService {
fields.add(fieldInfo);
userDefinedJavaClassMeta.setFieldInfo(fields);
List<UserDefinedJavaClassDef> definitions = new ArrayList<UserDefinedJavaClassDef>();
UserDefinedJavaClassDef userDefinedJavaClassDef = new UserDefinedJavaClassDef(UserDefinedJavaClassDef.ClassType.TRANSFORM_CLASS, "Processor",
code.replace("alterColumnTypeCode", needToChangeolumnType).replace("Column_Fields", String.join(",", datasetTableFields.stream().map(DatasetTableField::getOriginName).collect(Collectors.toList()))));
String tmp_code = code.replace("alterColumnTypeCode", needToChangeColumnType).replace("Column_Fields", String.join(",", datasetTableFields.stream().map(DatasetTableField::getOriginName).collect(Collectors.toList())));
if(isExcel){
tmp_code = tmp_code.replace("handleExcelIntColumn", handleExcelIntColumn);
}else {
tmp_code = tmp_code.replace("handleExcelIntColumn", "");
}
UserDefinedJavaClassDef userDefinedJavaClassDef = new UserDefinedJavaClassDef(UserDefinedJavaClassDef.ClassType.TRANSFORM_CLASS, "Processor", tmp_code);
userDefinedJavaClassDef.setActive(true);
definitions.add(userDefinedJavaClassDef);
@@ -629,6 +637,14 @@ public class ExtractDataService {
" }\n" +
" }\n";
private static String handleExcelIntColumn = " \t\tif(tmp != null && tmp.endsWith(\".0\")){\n" +
" try {\n" +
" Integer.valueOf(tmp.substring(0, tmp.length()-2));\n" +
" get(Fields.Out, filed).setValue(r, tmp.substring(0, tmp.length()-2));\n" +
" get(Fields.Out, filed).getValueMeta().setType(2);\n" +
" }catch (Exception e){}\n" +
" }";
private static String code = "import org.pentaho.di.core.row.ValueMetaInterface;\n" +
"import java.util.List;\n" +
"import java.io.File;\n" +
@@ -659,6 +675,7 @@ public class ExtractDataService {
" for (String filed : fileds) {\n" +
" String tmp = get(Fields.In, filed).getString(r);\n" +
"alterColumnTypeCode \n" +
"handleExcelIntColumn \n" +
" str = str + tmp;\n" +
" }\n" +
"\n" +