public/dataease
1
0
Fork 0
mirror of https://github.com/dataease/dataease.git synced 2026-05-21 04:08:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: 支持数据源插件

Browse Source
This commit is contained in:
taojinlong
2022-04-17 15:05:01 +08:00
parent 8f9b355b89
commit ea43644f07
20 changed files with 240 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
backend/pom.xml
View File

@@ -486,45 +486,45 @@
<build>
<plugins>
<!-- <plugin>-->
<!-- <groupId>org.apache.maven.plugins</groupId>-->
<!-- <artifactId>maven-antrun-plugin</artifactId>-->
<!-- <executions>-->
<!-- <execution>-->
<!-- <?m2e execute onConfiguration?>-->
<!-- <id>main-class-placement</id>-->
<!-- <phase>generate-resources</phase>-->
<!-- <configuration>-->
<!-- <target>-->
<!-- <move todir="src/main/resources/static">-->
<!-- <fileset dir="../frontend/dist">-->
<!-- <exclude name="*.html"/>-->
<!-- </fileset>-->
<!-- </move>-->
<!-- <move todir="src/main/resources/templates">-->
<!-- <fileset dir="../frontend/dist">-->
<!-- <include name="*.html"/>-->
<!-- </fileset>-->
<!-- </move>-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<?m2e execute onConfiguration?>
<id>main-class-placement</id>
<phase>generate-resources</phase>
<configuration>
<target>
<move todir="src/main/resources/static">
<fileset dir="../frontend/dist">
<exclude name="*.html"/>
</fileset>
</move>
<move todir="src/main/resources/templates">
<fileset dir="../frontend/dist">
<include name="*.html"/>
</fileset>
</move>
<!-- <copy todir="src/main/resources/static/de-app">-->
<!-- <fileset dir="../mobile/dist">-->
<!-- <exclude name="*.html"/>-->
<!-- </fileset>-->
<!-- </copy>-->
<copy todir="src/main/resources/static/de-app">
<fileset dir="../mobile/dist">
<exclude name="*.html"/>
</fileset>
</copy>
<!-- <copy file="../mobile/dist/index.html" tofile="src/main/resources/templates/app.html" />-->
<copy file="../mobile/dist/index.html" tofile="src/main/resources/templates/app.html" />
<!-- </target>-->
<!-- </configuration>-->
<!-- <goals>-->
<!-- <goal>run</goal>-->
<!-- </goals>-->
<!-- </execution>-->
<!-- </executions>-->
<!-- </plugin>-->
</target>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>

46
backend/src/main/java/io/dataease/commons/utils/DeFileUtils.java
View File

@@ -65,6 +65,52 @@ public class DeFileUtils {
}
return null;
}
public static void copyFolder(String sourcePath,String targetPath) throws Exception{
//源文件夹路径
File sourceFile = new File(sourcePath);
//目标文件夹路径
File targetFile = new File(targetPath);
if(!sourceFile.exists()){
throw new Exception("文件夹不存在");
}
if(!sourceFile.isDirectory()){
throw new Exception("源文件夹不是目录");
}
if(!targetFile.exists()){
targetFile.mkdirs();
}
if(!targetFile.isDirectory()){
throw new Exception("目标文件夹不是目录");
}
File[] files = sourceFile.listFiles();
if(files == null || files.length == 0){
return;
}
for(File file : files){
//文件要移动的路径
String movePath = targetFile+File.separator+file.getName();
if(file.isDirectory()){
//如果是目录则递归调用
copyFolder(file.getAbsolutePath(),movePath);
}else {
//如果是文件则复制文件
BufferedInputStream in = new BufferedInputStream(new FileInputStream(file));
BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(movePath));
byte[] b = new byte[1024];
int temp = 0;
while((temp = in.read(b)) != -1){
out.write(b,0,temp);
}
out.close();
in.close();
}
}
}
public static String copy(File source, String targetDir) throws IOException{
String name = source.getName();

50
backend/src/main/java/io/dataease/commons/utils/ZipUtils.java
View File

@@ -56,38 +56,34 @@ public class ZipUtils {
}
public static void unzip(File source, String out) throws IOException {
try (ZipInputStream zis = new ZipInputStream(new FileInputStream(source))) {
ZipInputStream zis = new ZipInputStream(new FileInputStream(source));
ZipEntry entry = zis.getNextEntry();
while (entry != null) {
File file = protectZipSlip(entry.getName(), out);
ZipEntry entry = zis.getNextEntry();
if (entry.isDirectory()) {
if (!file.mkdirs()) {
}
} else {
File parent = file.getParentFile();
while (entry != null) {
File file = protectZipSlip(entry.getName(), out);
if (entry.isDirectory()) {
if (!file.mkdirs()) {
}
} else {
File parent = file.getParentFile();
if (!parent.exists()) {
if (!parent.mkdirs()) {
}
}
try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(file))) {
byte[] buffer = new byte[Math.toIntExact(entry.getSize())];
int location;
while ((location = zis.read(buffer)) != -1) {
bos.write(buffer, 0, location);
}
if (!parent.exists()) {
if (!parent.mkdirs()) {
}
}
try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(file))) {
byte[] buffer = new byte[Math.toIntExact(entry.getSize())];
int location;
while ((location = zis.read(buffer)) != -1) {
bos.write(buffer, 0, location);
}
}
entry = zis.getNextEntry();
}
entry = zis.getNextEntry();
}
}

2
backend/src/main/java/io/dataease/dto/DatasourceDTO.java
View File

@@ -2,6 +2,7 @@ package io.dataease.dto;
import com.alibaba.fastjson.JSONArray;
import io.dataease.plugins.common.base.domain.Datasource;
import io.dataease.plugins.common.constants.DatasourceCalculationMode;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@@ -17,4 +18,5 @@ public class DatasourceDTO extends Datasource {
private String privileges;
private JSONArray apiConfiguration;
private String typeDesc;
private DatasourceCalculationMode calculationMode;
}

9
backend/src/main/java/io/dataease/dto/MyPluginDTO.java Normal file
View File

@@ -0,0 +1,9 @@
package io.dataease.dto;
import io.dataease.plugins.common.base.domain.MyPlugin;
import lombok.Data;
@Data
public class MyPluginDTO extends MyPlugin {
private String require = "1.9.0";
}

117
backend/src/main/java/io/dataease/provider/ProviderFactory.java
View File

@@ -24,80 +24,75 @@ public class ProviderFactory implements ApplicationContextAware {
this.context = ctx;
for(final DatasourceTypes d: DatasourceTypes.values()) {
final ConfigurableListableBeanFactory beanFactory = ((ConfigurableApplicationContext) context).getBeanFactory();
DataSourceType dataSourceType = new DataSourceType();
dataSourceType.setType(d.getType());
dataSourceType.setName(d.getName());
dataSourceType.setAliasPrefix(d.getAliasPrefix());
dataSourceType.setAliasSuffix(d.getAliasSuffix());
dataSourceType.setKeywordPrefix(d.getKeywordPrefix());
dataSourceType.setKeywordSuffix(d.getKeywordSuffix());
dataSourceType.setPlugin(false);
dataSourceType.setExtraParams(d.getExtraParams());
System.out.println(new Gson().toJson(dataSourceType));
beanFactory.registerSingleton(d.getType(), dataSourceType);
if(d.isDatasource()){
DataSourceType dataSourceType = new DataSourceType(d.getType(), d.getName(), false, d.getExtraParams(), d.getCalculationMode());
beanFactory.registerSingleton(d.getType(), dataSourceType);
}
}
}
public static Provider getProvider(String type) {
Map<String, DataSourceType> dataSourceTypeMap = SpringContextUtil.getApplicationContext().getBeansOfType((DataSourceType.class));
if(dataSourceTypeMap.get(type).isPlugin()){
return context.getBean(type, Provider.class);
if(dataSourceTypeMap.keySet().contains(type)){
DatasourceTypes datasourceType = DatasourceTypes.valueOf(type);
switch (datasourceType) {
case es:
return context.getBean("es", Provider.class);
case api:
return context.getBean("api", Provider.class);
default:
return context.getBean("jdbc", Provider.class);
}
}
DatasourceTypes datasourceType = DatasourceTypes.valueOf(type);
switch (datasourceType) {
case es:
return context.getBean("es", Provider.class);
case api:
return context.getBean("api", Provider.class);
default:
return context.getBean("jdbc", Provider.class);
}
return SpringContextUtil.getApplicationContext().getBean(type + "DsProvider", Provider.class);
}
public static QueryProvider getQueryProvider(String type) {
Map<String, DataSourceType> dataSourceTypeMap = SpringContextUtil.getApplicationContext().getBeansOfType((DataSourceType.class));
if(dataSourceTypeMap.get(type).isPlugin()){
return context.getBean(type, QueryProvider.class);
}
DatasourceTypes datasourceType = DatasourceTypes.valueOf(type);
switch (datasourceType) {
case mysql:
case mariadb:
case ds_doris:
case TiDB:
case StarRocks:
return context.getBean("mysqlQuery", QueryProvider.class);
case sqlServer:
return context.getBean("sqlserverQuery", QueryProvider.class);
case pg:
return context.getBean("pgQuery", QueryProvider.class);
case oracle:
return context.getBean("oracleQuery", QueryProvider.class);
case es:
return context.getBean("esQuery", QueryProvider.class);
case ck:
return context.getBean("ckQuery", QueryProvider.class);
case mongo:
return context.getBean("mongoQuery", QueryProvider.class);
case redshift:
return context.getBean("redshiftQuery", QueryProvider.class);
case hive:
return context.getBean("hiveQuery", QueryProvider.class);
case impala:
return context.getBean("impalaQuery", QueryProvider.class);
case db2:
return context.getBean("db2Query", QueryProvider.class);
case api:
return context.getBean("apiQuery", QueryProvider.class);
case engine_doris:
return context.getBean("dorisEngineQuery", QueryProvider.class);
case engine_mysql:
return context.getBean("mysqlEngineQuery", QueryProvider.class);
default:
return context.getBean("mysqlQuery", QueryProvider.class);
if(dataSourceTypeMap.keySet().contains(type)){
DatasourceTypes datasourceType = DatasourceTypes.valueOf(type);
switch (datasourceType) {
case mysql:
case mariadb:
case ds_doris:
case TiDB:
case StarRocks:
return context.getBean("mysqlQuery", QueryProvider.class);
case sqlServer:
return context.getBean("sqlserverQuery", QueryProvider.class);
case pg:
return context.getBean("pgQuery", QueryProvider.class);
case oracle:
return context.getBean("oracleQuery", QueryProvider.class);
case es:
return context.getBean("esQuery", QueryProvider.class);
case ck:
return context.getBean("ckQuery", QueryProvider.class);
case mongo:
return context.getBean("mongoQuery", QueryProvider.class);
case redshift:
return context.getBean("redshiftQuery", QueryProvider.class);
case hive:
return context.getBean("hiveQuery", QueryProvider.class);
case impala:
return context.getBean("impalaQuery", QueryProvider.class);
case db2:
return context.getBean("db2Query", QueryProvider.class);
case api:
return context.getBean("apiQuery", QueryProvider.class);
case engine_doris:
return context.getBean("dorisEngineQuery", QueryProvider.class);
case engine_mysql:
return context.getBean("mysqlEngineQuery", QueryProvider.class);
default:
return context.getBean("mysqlQuery", QueryProvider.class);
}
}
return SpringContextUtil.getApplicationContext().getBean(type + "QueryProvider", QueryProvider.class);
}
public static DDLProvider getDDLProvider(String type) {

2
backend/src/main/java/io/dataease/provider/datasource/JdbcProvider.java
View File

@@ -34,7 +34,7 @@ public class JdbcProvider extends DefaultJdbcProvider {
return false;
}
@Override
public String getName(){
public String getType(){
return "built-in";
}
/**

83
backend/src/main/java/io/dataease/provider/datasource/MaxCompute.java
View File

@@ -1,83 +0,0 @@
package io.dataease.provider.datasource;
import com.alibaba.druid.pool.DruidDataSource;
import com.google.gson.Gson;
import io.dataease.plugins.common.constants.DatasourceTypes;
import io.dataease.plugins.common.dto.datasource.TableDesc;
import io.dataease.plugins.common.dto.datasource.TableField;
import io.dataease.plugins.common.exception.DataEaseException;
import io.dataease.plugins.common.request.datasource.DatasourceRequest;
import io.dataease.plugins.datasource.entity.JdbcConfiguration;
import io.dataease.plugins.datasource.provider.DefaultJdbcProvider;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.sql.*;
import java.util.*;
@Component("maxcompute")
public class MaxCompute extends DefaultJdbcProvider {
@Override
public String getName(){
return "maxcompute";
}
@Override
public boolean isUseDatasourcePool(){
return false;
}
@Override
public Connection getConnection(DatasourceRequest datasourceRequest) throws Exception {
Properties props = new Properties();
MaxcomputeConfig maxcomputeConfig = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), MaxcomputeConfig.class);
String username = maxcomputeConfig.getUsername();
String password = maxcomputeConfig.getPassword();
String driver = maxcomputeConfig.getDriver();
String url = maxcomputeConfig.getUrl();
Driver driverClass = (Driver) extendedJdbcClassLoader.loadClass(driver).newInstance();
if (StringUtils.isNotBlank(username)) {
props.setProperty("user", username);
if (StringUtils.isNotBlank(password)) {
props.setProperty("password", password);
}
}
Connection conn = driverClass.connect(url, props);
return conn;
}
@Override
public List<TableDesc> getTables(DatasourceRequest datasourceRequest) throws Exception {
List<TableDesc> tables = new ArrayList<>();
String queryStr = getTablesSql(datasourceRequest);
try (Connection con = getConnectionFromPool(datasourceRequest); Statement statement = con.createStatement(); ResultSet resultSet = statement.executeQuery(queryStr)) {
while (resultSet.next()) {
tables.add(getTableDesc(datasourceRequest, resultSet));
}
} catch (Exception e) {
DataEaseException.throwException(e);
}
return tables;
}
private TableDesc getTableDesc(DatasourceRequest datasourceRequest, ResultSet resultSet) throws SQLException {
TableDesc tableDesc = new TableDesc();
tableDesc.setName(resultSet.getString(1));
return tableDesc;
}
@Override
public List<TableField> getTableFileds(DatasourceRequest datasourceRequest) throws Exception {
datasourceRequest.setQuery("select * from " + datasourceRequest.getTable() + " limit 0");
return fetchResultField(datasourceRequest);
}
}

14
backend/src/main/java/io/dataease/provider/datasource/MaxcomputeConfig.java
View File

@@ -1,14 +0,0 @@
package io.dataease.provider.datasource;
import io.dataease.plugins.datasource.entity.JdbcConfiguration;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.StringUtils;
@Getter
@Setter
public class MaxcomputeConfig extends JdbcConfiguration {
private String driver = "com.aliyun.odps.jdbc.OdpsDriver";
private String url;
}

20
backend/src/main/java/io/dataease/service/datasource/DatasourceService.java
View File

@@ -45,6 +45,7 @@ import org.springframework.transaction.annotation.Transactional;
import javax.annotation.Resource;
import java.util.*;
import java.util.stream.Collectors;
@Service
@Transactional(rollbackFor = Exception.class)
@@ -62,15 +63,19 @@ public class DatasourceService {
private CommonThreadPool commonThreadPool;
@Resource
private SysAuthService sysAuthService;
// private static List<String> dsTypes = Arrays.asList("TiDB", "StarRocks", "excel", "mysql", "hive", "impala", "mariadb", "ds_doris", "pg", "sqlServer", "oracle", "mongo", "ck", "db2", "es", "redshift", "api");
public Collection<DataSourceType>types(){
return SpringContextUtil.getApplicationContext().getBeansOfType((DataSourceType.class)).values();
Collection<DataSourceType> types = new ArrayList<>();
types.addAll(SpringContextUtil.getApplicationContext().getBeansOfType(DataSourceType.class).values());
SpringContextUtil.getApplicationContext().getBeansOfType(io.dataease.plugins.datasource.service.DatasourceService.class).values().forEach(datasourceService -> {
types.add(datasourceService.getDataSourceType());
});
return types;
}
@DeCleaner(DePermissionType.DATASOURCE)
public Datasource addDatasource(Datasource datasource) throws Exception{
if(!SpringContextUtil.getApplicationContext().getBeansOfType(DataSourceType.class).keySet().contains(datasource.getType())){
if(!types().stream().map(DataSourceType::getType).collect(Collectors.toList()).contains(datasource.getType())){
throw new Exception("Datasource type not supported.");
}
checkName(datasource.getName(),datasource.getType(), datasource.getId());
@@ -104,7 +109,12 @@ public class DatasourceService {
request.setSort("update_time desc");
List<DatasourceDTO> datasourceDTOS = extDataSourceMapper.queryUnion(request);
datasourceDTOS.forEach(datasourceDTO -> {
datasourceDTO.setTypeDesc(SpringContextUtil.getApplicationContext().getBean(datasourceDTO.getType(), DataSourceType.class).getName());
types().forEach(dataSourceType -> {
if(dataSourceType.getType().equalsIgnoreCase(datasourceDTO.getType())){
datasourceDTO.setTypeDesc(dataSourceType.getName());
datasourceDTO.setCalculationMode(dataSourceType.getCalculationMode());
}
});
if(datasourceDTO.getType().equalsIgnoreCase(DatasourceTypes.api.toString())){
JSONArray apiDefinitionList = JSONObject.parseArray(datasourceDTO.getConfiguration());
JSONArray apiDefinitionListWithStatus = new JSONArray();
@@ -165,7 +175,7 @@ public class DatasourceService {
}
public void updateDatasource(UpdataDsRequest updataDsRequest)throws Exception{
if(!SpringContextUtil.getApplicationContext().getBeansOfType(DataSourceType.class).keySet().contains(updataDsRequest.getType())){
if(!types().stream().map(DataSourceType::getType).collect(Collectors.toList()).contains(updataDsRequest.getType())){
throw new Exception("Datasource type not supported.");
}
checkName(updataDsRequest.getName(),updataDsRequest.getType(),updataDsRequest.getId());

25
backend/src/main/java/io/dataease/service/sys/PluginService.java
View File

@@ -2,6 +2,7 @@ package io.dataease.service.sys;
import cn.hutool.core.io.FileUtil;
import com.google.gson.Gson;
import io.dataease.dto.MyPluginDTO;
import io.dataease.ext.ExtSysPluginMapper;
import io.dataease.ext.query.GridExample;
import io.dataease.commons.constants.AuthConstants;
@@ -65,13 +66,13 @@ public class PluginService {
* @param file
* @return
*/
public Map<String, Object> localInstall(MultipartFile file) {
public Map<String, Object> localInstall(MultipartFile file) throws Exception{
//1.上传文件到服务器pluginDir目录下
File dest = DeFileUtils.upload(file, pluginDir + "temp/");
//2.解压目标文件dest 得到plugin.json和jar
String folder = pluginDir + "folder/";
try {
ZipUtils.upZipFile(dest, folder);
ZipUtils.unzip(dest, folder);
} catch (IOException e) {
DeFileUtils.deleteFile(pluginDir + "temp/");
DeFileUtils.deleteFile(folder);
@@ -90,7 +91,7 @@ public class PluginService {
LogUtil.error(msg);
DEException.throwException(new RuntimeException(msg));
}
MyPlugin myPlugin = formatJsonFile(jsonFiles[0]);
MyPluginDTO myPlugin = formatJsonFile(jsonFiles[0]);
if (!versionMatch(myPlugin.getRequire())) {
String msg = "当前插件要求系统版本最低为：" + myPlugin.getRequire();
@@ -118,6 +119,9 @@ public class PluginService {
targetDir = makeTargetDir(myPlugin);
String jarPath;
jarPath = DeFileUtils.copy(jarFile, targetDir);
if(myPlugin.getCategory().equalsIgnoreCase("datasource")){
DeFileUtils.copyFolder(folder + "/" + myPlugin.getDsType() + "Driver", targetDir + myPlugin.getDsType() + "Driver");
}
loadJar(jarPath, myPlugin);
myPluginMapper.insert(myPlugin);
@@ -192,6 +196,11 @@ public class PluginService {
String path = pluginDir + plugin.getStore() + "/" + fileName;
File jarFile = new File(path);
FileUtil.del(jarFile);
if(plugin.getCategory().equalsIgnoreCase("datasource")){
File driverFile = new File(pluginDir + plugin.getStore() + "/" + plugin.getDsType() + "Driver");
FileUtil.del(driverFile);
}
}
/**
@@ -224,13 +233,13 @@ public class PluginService {
*
* @return
*/
private MyPlugin formatJsonFile(File file) {
private MyPluginDTO formatJsonFile(File file) {
String str = DeFileUtils.readJson(file);
Gson gson = new Gson();
Map<String, Object> myPlugin = gson.fromJson(str, Map.class);
myPlugin.put("free", (Double) myPlugin.get("free") > 0.0);
myPlugin.put("loadMybatis", (Double) myPlugin.get("loadMybatis") > 0.0);
MyPlugin result = new MyPlugin();
myPlugin.put("loadMybatis", myPlugin.get("loadMybatis") == null ? false : (Double) myPlugin.get("loadMybatis") > 0.0);
MyPluginDTO result = new MyPluginDTO();
try {
org.apache.commons.beanutils.BeanUtils.populate(result, myPlugin);
result.setInstallTime(System.currentTimeMillis());
@@ -240,6 +249,10 @@ public class PluginService {
e.printStackTrace();
}
//BeanUtils.copyBean(result, myPlugin);
if(result.getCategory().equalsIgnoreCase("datasource")){
result.setStore("thirdpart");
}
return result;
}