diff --git a/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java b/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java index fcaf060a42..3724c7f8ea 100644 --- a/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java +++ b/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java @@ -3,6 +3,7 @@ package io.dataease.datasource.constants; public enum DatasourceTypes { excel("excel", "excel", "", "", "", "", ""), mysql("mysql", "mysql", "com.mysql.jdbc.Driver", "`", "`", "'", "'"), + pg("pg", "pg", "org.postgresql.Driver", "\"", "\"", "\"", "\""), sqlServer("sqlServer", "sqlServer", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "\"", "\"", "\"", "\""), doris("doris", "doris", "com.mysql.jdbc.Driver", "`", "`", "", ""), oracle("oracle", "oracle", "oracle.jdbc.driver.OracleDriver", "\"", "\"", "\"", "\""); diff --git a/backend/src/main/java/io/dataease/datasource/dto/JdbcDTO.java b/backend/src/main/java/io/dataease/datasource/dto/JdbcDTO.java index 4f5c6b822a..e22657cdf0 100644 --- a/backend/src/main/java/io/dataease/datasource/dto/JdbcDTO.java +++ b/backend/src/main/java/io/dataease/datasource/dto/JdbcDTO.java @@ -12,5 +12,6 @@ public class JdbcDTO { private String username; private String password; private String dataBase; + private String schema; private String dataSourceType = "jdbc"; } diff --git a/backend/src/main/java/io/dataease/datasource/dto/OracleConfigration.java b/backend/src/main/java/io/dataease/datasource/dto/OracleConfigration.java index 6cebb06f13..6323791f90 100644 --- a/backend/src/main/java/io/dataease/datasource/dto/OracleConfigration.java +++ b/backend/src/main/java/io/dataease/datasource/dto/OracleConfigration.java @@ -9,7 +9,6 @@ public class OracleConfigration extends JdbcDTO { private String driver = "oracle.jdbc.driver.OracleDriver"; private String connectionType; - private String schema; public String getJdbc() { // 连接参数先写死，后边要把编码、时区等参数放到数据源的设置中 diff --git a/backend/src/main/java/io/dataease/datasource/dto/PgConfigration.java b/backend/src/main/java/io/dataease/datasource/dto/PgConfigration.java new file mode 100644 index 0000000000..3258c575c0 --- /dev/null +++ b/backend/src/main/java/io/dataease/datasource/dto/PgConfigration.java @@ -0,0 +1,19 @@ +package io.dataease.datasource.dto; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class PgConfigration extends JdbcDTO { + + private String driver = "org.postgresql.Driver"; + + public String getJdbc() { + // 连接参数先写死，后边要把编码、时区等参数放到数据源的设置中 + return "jdbc:postgresql://HOSTNAME:PORT/DATABASE" + .replace("HOSTNAME", getHost()) + .replace("PORT", getPort().toString()) + .replace("DATABASE", getDataBase()); + } +} \ No newline at end of file diff --git a/backend/src/main/java/io/dataease/datasource/dto/SqlServerConfigration.java b/backend/src/main/java/io/dataease/datasource/dto/SqlServerConfigration.java index 1cbe90c83a..7282704586 100644 --- a/backend/src/main/java/io/dataease/datasource/dto/SqlServerConfigration.java +++ b/backend/src/main/java/io/dataease/datasource/dto/SqlServerConfigration.java @@ -8,7 +8,6 @@ import lombok.Setter; @Setter public class SqlServerConfigration extends JdbcDTO { private String driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; - private String schema; public String getJdbc(){ return "jdbc:sqlserver://HOSTNAME:PORT;DatabaseName=DATABASE".replace("HOSTNAME", getHost()).replace("PORT", getPort().toString()).replace("DATABASE", getDataBase()); diff --git a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java index 12e30186df..e53caacbc9 100644 --- a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java +++ b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java @@ -3,10 +3,7 @@ package io.dataease.datasource.provider; import com.google.gson.Gson; import com.mchange.v2.c3p0.ComboPooledDataSource; import io.dataease.datasource.constants.DatasourceTypes; -import io.dataease.datasource.dto.MysqlConfigration; -import io.dataease.datasource.dto.OracleConfigration; -import io.dataease.datasource.dto.SqlServerConfigration; -import io.dataease.datasource.dto.TableFiled; +import io.dataease.datasource.dto.*; import io.dataease.datasource.request.DatasourceRequest; import io.dataease.exception.DataEaseException; import io.dataease.i18n.Translator; @@ -309,7 +306,7 @@ public class JdbcProvider extends DatasourceProvider { resultSet.close(); ps.close(); } catch (Exception e) { - DataEaseException.throwException(Translator.get("i18n_datasource_connect_error") + e.getMessage()); + DataEaseException.throwException(e.getMessage()); } finally { if(con != null){con.close();} } @@ -433,6 +430,12 @@ public class JdbcProvider extends DatasourceProvider { props.put( "oracle.net.CONNECT_TIMEOUT" , "5000") ; // props.put( "oracle.jdbc.ReadTimeout" , "5000" ) ; break; + case pg: + PgConfigration pgConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), PgConfigration.class); + username = pgConfigration.getUsername(); + password = pgConfigration.getPassword(); + driver = pgConfigration.getDriver(); + jdbcurl = pgConfigration.getJdbc(); default: break; } @@ -477,6 +480,13 @@ public class JdbcProvider extends DatasourceProvider { dataSource.setPassword(oracleConfigration.getPassword()); dataSource.setJdbcUrl(oracleConfigration.getJdbc()); break; + case pg: + PgConfigration pgConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), PgConfigration.class); + dataSource.setUser(pgConfigration.getUsername()); + dataSource.setDriverClass(pgConfigration.getDriver()); + dataSource.setPassword(pgConfigration.getPassword()); + dataSource.setJdbcUrl(pgConfigration.getJdbc()); + break; default: break; } @@ -494,6 +504,9 @@ public class JdbcProvider extends DatasourceProvider { case sqlServer: SqlServerConfigration sqlServerConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), SqlServerConfigration.class); return sqlServerConfigration.getDataBase(); + case pg: + PgConfigration pgConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), PgConfigration.class); + return pgConfigration.getDataBase(); default: return null; } @@ -517,6 +530,12 @@ public class JdbcProvider extends DatasourceProvider { throw new Exception(Translator.get("i18n_schema_is_empty")); } return "select table_name, owner from all_tables where owner='" + oracleConfigration.getSchema() + "'"; + case pg: + PgConfigration pgConfigration = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), PgConfigration.class); + if(StringUtils.isEmpty(pgConfigration.getSchema())){ + throw new Exception(Translator.get("i18n_schema_is_empty")); + } + return "SELECT tablename FROM pg_tables WHERE tablename NOT LIKE 'pg%' AND tablename NOT LIKE 'sql_%' AND schemaname='SCHEMA' ;".replace("SCHEMA", pgConfigration.getSchema()); default: return "show tables;"; } @@ -529,6 +548,8 @@ public class JdbcProvider extends DatasourceProvider { return "select * from all_users"; case sqlServer: return "select name from sys.schemas;"; + case pg: + return "SELECT nspname FROM pg_namespace;"; default: return "show tables;"; } diff --git a/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java b/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java index 88a720571a..d780399648 100644 --- a/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java +++ b/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java @@ -28,6 +28,8 @@ public class ProviderFactory implements ApplicationContextAware { return context.getBean("jdbc", DatasourceProvider.class); case sqlServer: return context.getBean("jdbc", DatasourceProvider.class); + case pg: + return context.getBean("jdbc", DatasourceProvider.class); default: return context.getBean("jdbc", DatasourceProvider.class); } @@ -42,6 +44,8 @@ public class ProviderFactory implements ApplicationContextAware { return context.getBean("dorisQuery", QueryProvider.class); case sqlServer: return context.getBean("sqlserverQuery", QueryProvider.class); + case pg: + return context.getBean("pgQuery", QueryProvider.class); case oracle: return context.getBean("oracleQuery", QueryProvider.class); default: diff --git a/backend/src/main/java/io/dataease/dto/chart/ChartViewDTO.java b/backend/src/main/java/io/dataease/dto/chart/ChartViewDTO.java index 035e916527..81629da981 100644 --- a/backend/src/main/java/io/dataease/dto/chart/ChartViewDTO.java +++ b/backend/src/main/java/io/dataease/dto/chart/ChartViewDTO.java @@ -1,11 +1,9 @@ package io.dataease.dto.chart; import io.dataease.base.domain.ChartViewWithBLOBs; -import io.dataease.controller.request.chart.ChartDrillRequest; import lombok.Getter; import lombok.Setter; -import java.util.List; import java.util.Map; /** diff --git a/backend/src/main/java/io/dataease/provider/QueryProvider.java b/backend/src/main/java/io/dataease/provider/QueryProvider.java index 4c56399036..b3f4ad5a8a 100644 --- a/backend/src/main/java/io/dataease/provider/QueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/QueryProvider.java @@ -1,10 +1,15 @@ package io.dataease.provider; +import com.google.gson.Gson; import io.dataease.base.domain.DatasetTableField; import io.dataease.base.domain.Datasource; import io.dataease.controller.request.chart.ChartExtFilterRequest; +import io.dataease.datasource.dto.JdbcDTO; +import io.dataease.datasource.dto.SqlServerConfigration; import io.dataease.dto.chart.ChartCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; +import io.dataease.dto.sqlObj.SQLObj; +import io.dataease.provider.pg.PgConstants; import java.util.List; @@ -30,15 +35,15 @@ public abstract class QueryProvider { public abstract String createQuerySQLAsTmpWithPage(String sql, List fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup); - public abstract String getSQL(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList); + public abstract String getSQL(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, Datasource ds); public abstract String getSQLAsTmp(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList); - public abstract String getSQLStack(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extStack); + public abstract String getSQLStack(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extStack, Datasource ds); public abstract String getSQLAsTmpStack(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extStack); - public abstract String getSQLScatter(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extBubble); + public abstract String getSQLScatter(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extBubble, Datasource ds); public abstract String getSQLAsTmpScatter(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extBubble); @@ -59,7 +64,15 @@ public abstract class QueryProvider { public abstract String wrapSql(String sql); - public abstract String createRawQuerySQL(String table, List fields); + public abstract String createRawQuerySQL(String table, List fields, Datasource ds); public abstract String createRawQuerySQLAsTmp(String sql, List fields); + + public void setSchema(SQLObj tableObj, Datasource ds){ + if(ds != null){ + String schema = new Gson().fromJson(ds.getConfiguration(), JdbcDTO.class).getSchema(); + schema = String.format( PgConstants.KEYWORD_TABLE, schema); + tableObj.setTableName(schema + "." + tableObj.getTableName()); + } + } } diff --git a/backend/src/main/java/io/dataease/provider/doris/DorisQueryProvider.java b/backend/src/main/java/io/dataease/provider/doris/DorisQueryProvider.java index 8a411a6de6..f7638fe8c9 100644 --- a/backend/src/main/java/io/dataease/provider/doris/DorisQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/doris/DorisQueryProvider.java @@ -174,7 +174,7 @@ public class DorisQueryProvider extends QueryProvider { } @Override - public String getSQL(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList) { + public String getSQL(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, Datasource ds) { SQLObj tableObj = SQLObj.builder() .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(DorisConstants.KEYWORD_TABLE, table)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) @@ -281,11 +281,11 @@ public class DorisQueryProvider extends QueryProvider { @Override public String getSQLAsTmp(String sql, List xAxis, List yAxis, List customFilter, List extFilterRequestList) { - return getSQL("(" + sql + ")", xAxis, yAxis, customFilter, extFilterRequestList); + return getSQL("(" + sql + ")", xAxis, yAxis, customFilter, extFilterRequestList, null); } @Override - public String getSQLStack(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extStack) { + public String getSQLStack(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extStack, Datasource ds) { SQLObj tableObj = SQLObj.builder() .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(DorisConstants.KEYWORD_TABLE, table)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) @@ -395,11 +395,11 @@ public class DorisQueryProvider extends QueryProvider { @Override public String getSQLAsTmpStack(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extStack) { - return getSQLStack("(" + table + ")", xAxis, yAxis, customFilter, extFilterRequestList, extStack); + return getSQLStack("(" + table + ")", xAxis, yAxis, customFilter, extFilterRequestList, extStack, null); } @Override - public String getSQLScatter(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extBubble) { + public String getSQLScatter(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extBubble, Datasource ds) { SQLObj tableObj = SQLObj.builder() .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(DorisConstants.KEYWORD_TABLE, table)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) @@ -509,7 +509,7 @@ public class DorisQueryProvider extends QueryProvider { @Override public String getSQLAsTmpScatter(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extBubble) { - return getSQLScatter("(" + table + ")", xAxis, yAxis, customFilter, extFilterRequestList, extBubble); + return getSQLScatter("(" + table + ")", xAxis, yAxis, customFilter, extFilterRequestList, extBubble, null); } @Override @@ -605,7 +605,7 @@ public class DorisQueryProvider extends QueryProvider { } @Override - public String createRawQuerySQL(String table, List fields) { + public String createRawQuerySQL(String table, List fields, Datasource ds) { String[] array = fields.stream().map(f -> { StringBuilder stringBuilder = new StringBuilder(); stringBuilder.append("`").append(f.getDataeaseName()).append("` AS ").append(f.getDataeaseName()); @@ -616,7 +616,7 @@ public class DorisQueryProvider extends QueryProvider { @Override public String createRawQuerySQLAsTmp(String sql, List fields) { - return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields); + return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields, null); } private String sqlFix(String sql) { diff --git a/backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java b/backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java index 5702a9a68d..7d95965a19 100644 --- a/backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java @@ -165,7 +165,7 @@ public class MysqlQueryProvider extends QueryProvider { } @Override - public String getSQL(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList) { + public String getSQL(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, Datasource ds) { SQLObj tableObj = SQLObj.builder() .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MySQLConstants.KEYWORD_TABLE, table)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) @@ -273,11 +273,11 @@ public class MysqlQueryProvider extends QueryProvider { @Override public String getSQLAsTmp(String sql, List xAxis, List yAxis, List customFilter, List extFilterRequestList) { - return getSQL("(" + sqlFix(sql) + ")", xAxis, yAxis, customFilter, extFilterRequestList); + return getSQL("(" + sqlFix(sql) + ")", xAxis, yAxis, customFilter, extFilterRequestList, null); } @Override - public String getSQLStack(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extStack) { + public String getSQLStack(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extStack, Datasource ds) { SQLObj tableObj = SQLObj.builder() .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MySQLConstants.KEYWORD_TABLE, table)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) @@ -387,11 +387,11 @@ public class MysqlQueryProvider extends QueryProvider { @Override public String getSQLAsTmpStack(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extStack) { - return getSQLStack("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extStack); + return getSQLStack("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extStack, null); } @Override - public String getSQLScatter(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extBubble) { + public String getSQLScatter(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extBubble, Datasource ds) { SQLObj tableObj = SQLObj.builder() .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MySQLConstants.KEYWORD_TABLE, table)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) @@ -501,7 +501,7 @@ public class MysqlQueryProvider extends QueryProvider { @Override public String getSQLAsTmpScatter(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extBubble) { - return getSQLScatter("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extBubble); + return getSQLScatter("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extBubble, null); } @Override @@ -597,7 +597,7 @@ public class MysqlQueryProvider extends QueryProvider { } @Override - public String createRawQuerySQL(String table, List fields) { + public String createRawQuerySQL(String table, List fields, Datasource ds) { String[] array = fields.stream().map(f -> { StringBuilder stringBuilder = new StringBuilder(); if (f.getDeExtractType() == 4) { // 处理 tinyint @@ -612,7 +612,7 @@ public class MysqlQueryProvider extends QueryProvider { @Override public String createRawQuerySQLAsTmp(String sql, List fields) { - return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields); + return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields, null); } public String transMysqlFilterTerm(String term) { diff --git a/backend/src/main/java/io/dataease/provider/oracle/OracleQueryProvider.java b/backend/src/main/java/io/dataease/provider/oracle/OracleQueryProvider.java index 02af808450..138a2cac52 100644 --- a/backend/src/main/java/io/dataease/provider/oracle/OracleQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/oracle/OracleQueryProvider.java @@ -199,7 +199,7 @@ public class OracleQueryProvider extends QueryProvider { } @Override - public String getSQL(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList) { + public String getSQL(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, Datasource ds) { SQLObj tableObj = SQLObj.builder() .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(OracleConstants.KEYWORD_TABLE, table)) .tableAlias(String.format(OracleConstants.ALIAS_FIX, String.format(TABLE_ALIAS_PREFIX, 0))) @@ -306,11 +306,11 @@ public class OracleQueryProvider extends QueryProvider { @Override public String getSQLAsTmp(String sql, List xAxis, List yAxis, List customFilter, List extFilterRequestList) { - return getSQL("(" + sqlFix(sql) + ")", xAxis, yAxis, customFilter, extFilterRequestList); + return getSQL("(" + sqlFix(sql) + ")", xAxis, yAxis, customFilter, extFilterRequestList, null); } @Override - public String getSQLStack(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extStack) { + public String getSQLStack(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extStack, Datasource ds) { SQLObj tableObj = SQLObj.builder() .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(OracleConstants.KEYWORD_TABLE, table)) .tableAlias(String.format(OracleConstants.ALIAS_FIX, String.format(TABLE_ALIAS_PREFIX, 0))) @@ -420,11 +420,11 @@ public class OracleQueryProvider extends QueryProvider { @Override public String getSQLAsTmpStack(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extStack) { - return getSQLStack("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extStack); + return getSQLStack("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extStack, null); } @Override - public String getSQLScatter(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extBubble) { + public String getSQLScatter(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extBubble, Datasource ds) { SQLObj tableObj = SQLObj.builder() .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(OracleConstants.KEYWORD_TABLE, table)) .tableAlias(String.format(OracleConstants.ALIAS_FIX, String.format(TABLE_ALIAS_PREFIX, 0))) @@ -534,7 +534,7 @@ public class OracleQueryProvider extends QueryProvider { @Override public String getSQLAsTmpScatter(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extBubble) { - return getSQLScatter("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extBubble); + return getSQLScatter("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extBubble, null); } @Override @@ -629,7 +629,7 @@ public class OracleQueryProvider extends QueryProvider { } @Override - public String createRawQuerySQL(String table, List fields) { + public String createRawQuerySQL(String table, List fields, Datasource ds) { String[] array = fields.stream().map(f -> { StringBuilder stringBuilder = new StringBuilder(); stringBuilder.append(" \"").append(f.getOriginName()).append("\""); @@ -640,7 +640,7 @@ public class OracleQueryProvider extends QueryProvider { @Override public String createRawQuerySQLAsTmp(String sql, List fields) { - return createRawQuerySQL(" (" + sqlFix(sql) + ") DE_TMP ", fields); + return createRawQuerySQL(" (" + sqlFix(sql) + ") DE_TMP ", fields, null); } public String transMysqlFilterTerm(String term) { diff --git a/backend/src/main/java/io/dataease/provider/pg/PgConstants.java b/backend/src/main/java/io/dataease/provider/pg/PgConstants.java new file mode 100644 index 0000000000..e9f9555ae3 --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/pg/PgConstants.java @@ -0,0 +1,43 @@ +package io.dataease.provider.pg; + +import io.dataease.provider.SQLConstants; + +import static io.dataease.datasource.constants.DatasourceTypes.pg; + +/** + * @Author gin + * @Date 2021/7/8 7:22 下午 + */ +public class PgConstants extends SQLConstants { + public static final String KEYWORD_TABLE = pg.getKeywordPrefix() + "%s" + pg.getKeywordSuffix(); + + public static final String KEYWORD_FIX = "%s." + pg.getKeywordPrefix() + "%s" + pg.getKeywordSuffix(); + + public static final String UNIX_TIMESTAMP = "floor(extract(epoch from(( %s - timestamp '1970-01-01 00:00:00')*1000))) "; + + public static final String DATE_FORMAT = "to_char(%s, %s)"; + + public static final String FROM_UNIXTIME = "to_timestamp(%s)"; + + public static final String CAST = "CAST(%s AS %s)"; + + public static final String DEFAULT_DATE_FORMAT = "'YYYY-MM-DD HH24:MI:SS'"; + + public static final String DEFAULT_INT_FORMAT = "numeric(18,0)"; + + public static final String DEFAULT_FLOAT_FORMAT = "numeric(18,2)"; + + public static final String WHERE_VALUE_NULL = "(NULL,'')"; + + public static final String WHERE_VALUE_VALUE = "'%s'"; + + public static final String AGG_COUNT = "COUNT(*)"; + + public static final String AGG_FIELD = "%s(%s)"; + + public static final String WHERE_BETWEEN = "'%s' AND '%s'"; + + public static final String BRACKETS = "(%s)"; + + +} diff --git a/backend/src/main/java/io/dataease/provider/pg/PgQueryProvider.java b/backend/src/main/java/io/dataease/provider/pg/PgQueryProvider.java new file mode 100644 index 0000000000..eecb57b2d2 --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/pg/PgQueryProvider.java @@ -0,0 +1,922 @@ +package io.dataease.provider.pg; + +import com.google.gson.Gson; +import io.dataease.base.domain.DatasetTableField; +import io.dataease.base.domain.DatasetTableFieldExample; +import io.dataease.base.domain.Datasource; +import io.dataease.base.mapper.DatasetTableFieldMapper; +import io.dataease.commons.constants.DeTypeConstants; +import io.dataease.controller.request.chart.ChartExtFilterRequest; +import io.dataease.datasource.dto.JdbcDTO; +import io.dataease.datasource.dto.PgConfigration; +import io.dataease.datasource.dto.SqlServerConfigration; +import io.dataease.dto.chart.ChartCustomFilterDTO; +import io.dataease.dto.chart.ChartViewFieldDTO; +import io.dataease.dto.sqlObj.SQLObj; +import io.dataease.provider.QueryProvider; +import io.dataease.provider.SQLConstants; +import io.dataease.provider.sqlserver.SqlServerSQLConstants; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; +import org.stringtemplate.v4.ST; +import org.stringtemplate.v4.STGroup; +import org.stringtemplate.v4.STGroupFile; + +import javax.annotation.Resource; +import java.text.MessageFormat; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; + + + +@Service("pgQuery") +public class PgQueryProvider extends QueryProvider { + @Resource + private DatasetTableFieldMapper datasetTableFieldMapper; + + @Override + public Integer transFieldType(String field) { + field = field.toLowerCase(); + switch (field) { + case "bpchar": + case "varchar": + case "text": + case "tsquery": + case "tsvector": + case "uuid": + case "xml": + case "json": + case "bit": + case "jsonb": + case "cidr": + case "inet": + case "macaddr": + case "txid_snapshot": + case "box": + case "circle": + case "line": + case "lseg": + case "path": + case "point": + case "polygon": + case "bool": + case "interval": + return DeTypeConstants.DE_STRING;// 文本 + case "date": + case "time": + case "timestamp": + case "timestamptz": + return DeTypeConstants.DE_TIME;// 时间 + case "int2": + case "int4": + case "int8": + case "INTEGER": + case "BIGINT": + return DeTypeConstants.DE_INT;// 整型 + case "numeric": + case "float4": + case "float8": + case "money": + return DeTypeConstants.DE_FLOAT;// 浮点 +// case "bool": + case "TINYINT": + return DeTypeConstants.DE_BOOL;// 布尔 + case "bytea": + return DeTypeConstants.DE_Binary;// 二进制 + default: + return DeTypeConstants.DE_STRING; + } + } + + @Override + public String createSQLPreview(String sql, String orderBy) { + return "SELECT * FROM (" + sqlFix(sql) + ") AS tmp " + " LIMIT 1000 offset 0"; + } + + @Override + public String createQuerySQL(String table, List fields, boolean isGroup, Datasource ds) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format( PgConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + + setSchema(tableObj,ds); + List xFields = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(fields)) { + for (int i = 0; i < fields.size(); i++) { + DatasetTableField f = fields.get(i); + String originField; + if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(f.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == 1) { + originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName()); + } else { + originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + String fieldName = ""; + // 处理横轴字段 + if (f.getDeExtractType() == DeTypeConstants.DE_TIME) { + if (f.getDeType() == DeTypeConstants.DE_INT || f.getDeType() == DeTypeConstants.DE_FLOAT) { + fieldName = String.format( PgConstants.UNIX_TIMESTAMP, originField); + } else { + fieldName = originField; + } + } else if (f.getDeExtractType() == DeTypeConstants.DE_STRING) { + if (f.getDeType() == DeTypeConstants.DE_INT) { + fieldName = String.format( PgConstants.CAST, originField, PgConstants.DEFAULT_INT_FORMAT); + } else if (f.getDeType() == DeTypeConstants.DE_FLOAT) { + fieldName = String.format( PgConstants.CAST, originField, PgConstants.DEFAULT_FLOAT_FORMAT); + } else if (f.getDeType() == DeTypeConstants.DE_TIME) { + fieldName = String.format( PgConstants.CAST, originField, "timestamp"); + } else { + fieldName = originField; + } + } else { + if (f.getDeType() == DeTypeConstants.DE_TIME) { + String cast = String.format( PgConstants.CAST, originField, "bigint"); + fieldName = String.format( PgConstants.FROM_UNIXTIME, cast ); + } else if (f.getDeType() == DeTypeConstants.DE_INT) { + fieldName = String.format( PgConstants.CAST, originField, PgConstants.DEFAULT_INT_FORMAT); + } else { + fieldName = originField; + } + } + xFields.add(SQLObj.builder() + .fieldName(fieldName) + .fieldAlias(fieldAlias) + .build()); + } + } + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("previewSql"); + st_sql.add("isGroup", isGroup); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + return st_sql.render(); + } + + @Override + public String createQuerySQLAsTmp(String sql, List fields, boolean isGroup) { + return createQuerySQL("(" + sqlFix(sql) + ")", fields, isGroup, null); + } + + @Override + public String createQuerySQLWithPage(String table, List fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup, Datasource ds) { + return createQuerySQL(table, fields, isGroup, null) + " LIMIT " + realSize + " offset " + (page - 1) * pageSize; + } + + @Override + public String createQueryTableWithLimit(String table, List fields, Integer limit, boolean isGroup, Datasource ds) { + return createQuerySQL(table, fields, isGroup, null) + " LIMIT " + limit + " offset 0"; + } + + @Override + public String createQuerySqlWithLimit(String sql, List fields, Integer limit, boolean isGroup) { + return createQuerySQLAsTmp(sql, fields, isGroup) + " LIMIT " + limit + " offset 0"; + } + + @Override + public String createQuerySQLAsTmpWithPage(String sql, List fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup) { + return createQuerySQLAsTmp(sql, fields, isGroup) + " LIMIT " + realSize + " offset " + (page - 1) * pageSize; + } + + @Override + public String getSQL(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, Datasource ds) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format( PgConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + setSchema(tableObj,ds); + List xFields = new ArrayList<>(); + List xWheres = new ArrayList<>(); + List xOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(xAxis)) { + for (int i = 0; i < xAxis.size(); i++) { + ChartViewFieldDTO x = xAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(x.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) { + originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } else { + originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + // 处理横轴字段 + xFields.add(getXFields(x, originField, fieldAlias)); + // 处理横轴过滤 +// xWheres.addAll(getXWheres(x, originField, fieldAlias)); + // 处理横轴排序 + if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) { + xOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(x.getSort()) + .build()); + } + } + } + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(yAxis)) { + for (int i = 0; i < yAxis.size(); i++) { + ChartViewFieldDTO y = yAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(y.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) { + originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } else { + originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.addAll(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + List customWheres = transCustomFilterList(tableObj, customFilter); + // 处理仪表板字段过滤 + List extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(xFields); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + wheres.addAll(xWheres); + if (customWheres != null) wheres.addAll(customWheres); + if (extWheres != null) wheres.addAll(extWheres); + List groups = new ArrayList<>(); + groups.addAll(xFields); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(xOrders); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format( PgConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return st.render(); + } + + + @Override + public String getSQLAsTmp(String sql, List xAxis, List yAxis, List customFilter, List extFilterRequestList) { + return getSQL("(" + sqlFix(sql) + ")", xAxis, yAxis, customFilter, extFilterRequestList, null); + } + + @Override + public String getSQLStack(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extStack, Datasource ds) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format( PgConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + setSchema(tableObj,ds); + List xFields = new ArrayList<>(); + List xWheres = new ArrayList<>(); + List xOrders = new ArrayList<>(); + List xList = new ArrayList<>(); + xList.addAll(xAxis); + xList.addAll(extStack); + if (CollectionUtils.isNotEmpty(xList)) { + for (int i = 0; i < xList.size(); i++) { + ChartViewFieldDTO x = xList.get(i); + String originField; + if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(x.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) { + originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } else { + originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + // 处理横轴字段 + xFields.add(getXFields(x, originField, fieldAlias)); + // 处理横轴过滤 +// xWheres.addAll(getXWheres(x, originField, fieldAlias)); + // 处理横轴排序 + if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) { + xOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(x.getSort()) + .build()); + } + } + } + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(yAxis)) { + for (int i = 0; i < yAxis.size(); i++) { + ChartViewFieldDTO y = yAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(y.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) { + originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } else { + originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.addAll(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + List customWheres = transCustomFilterList(tableObj, customFilter); + // 处理仪表板字段过滤 + List extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(xFields); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + wheres.addAll(xWheres); + if (customWheres != null) wheres.addAll(customWheres); + if (extWheres != null) wheres.addAll(extWheres); + List groups = new ArrayList<>(); + groups.addAll(xFields); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(xOrders); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format( PgConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return st.render(); + } + + @Override + public String getSQLAsTmpStack(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extStack) { + return getSQLStack("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extStack, null); + } + + @Override + public String getSQLScatter(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extBubble, Datasource ds) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format( PgConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + setSchema(tableObj,ds); + List xFields = new ArrayList<>(); + List xWheres = new ArrayList<>(); + List xOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(xAxis)) { + for (int i = 0; i < xAxis.size(); i++) { + ChartViewFieldDTO x = xAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(x.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) { + originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } else { + originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + // 处理横轴字段 + xFields.add(getXFields(x, originField, fieldAlias)); + // 处理横轴过滤 +// xWheres.addAll(getXWheres(x, originField, fieldAlias)); + // 处理横轴排序 + if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) { + xOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(x.getSort()) + .build()); + } + } + } + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + List yList = new ArrayList<>(); + yList.addAll(yAxis); + yList.addAll(extBubble); + if (CollectionUtils.isNotEmpty(yList)) { + for (int i = 0; i < yList.size(); i++) { + ChartViewFieldDTO y = yList.get(i); + String originField; + if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(y.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) { + originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } else { + originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.addAll(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + List customWheres = transCustomFilterList(tableObj, customFilter); + // 处理仪表板字段过滤 + List extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(xFields); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + wheres.addAll(xWheres); + if (customWheres != null) wheres.addAll(customWheres); + if (extWheres != null) wheres.addAll(extWheres); + List groups = new ArrayList<>(); + groups.addAll(xFields); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(xOrders); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format( PgConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return st.render(); + } + + @Override + public String getSQLAsTmpScatter(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extBubble) { + return getSQLScatter("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extBubble, null); + } + + @Override + public String searchTable(String table) { + return "SELECT table_name FROM information_schema.TABLES WHERE table_name ='" + table + "'"; + } + + @Override + public String getSQLSummary(String table, List yAxis, List customFilter, List extFilterRequestList) { + // 字段汇总 排序等 + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format( PgConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(yAxis)) { + for (int i = 0; i < yAxis.size(); i++) { + ChartViewFieldDTO y = yAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(y.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) { + originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } else { + originField = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.addAll(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + List customWheres = transCustomFilterList(tableObj, customFilter); + // 处理仪表板字段过滤 + List extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + if (customWheres != null) wheres.addAll(customWheres); + if (extWheres != null) wheres.addAll(extWheres); + List groups = new ArrayList<>(); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format( PgConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return st.render(); + } + + @Override + public String getSQLSummaryAsTmp(String sql, List yAxis, List customFilter, List extFilterRequestList) { + return getSQLSummary("(" + sqlFix(sql) + ")", yAxis, customFilter, extFilterRequestList); + } + + @Override + public String wrapSql(String sql) { + sql = sql.trim(); + if (sql.lastIndexOf(";") == (sql.length() - 1)) { + sql = sql.substring(0, sql.length() - 1); + } + String tmpSql = "SELECT * FROM (" + sql + ") AS tmp " + " LIMIT 0 "; + return tmpSql; + } + + @Override + public String createRawQuerySQL(String table, List fields, Datasource ds) { + String[] array = fields.stream().map(f -> { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("\"").append(f.getOriginName()).append("\" AS ").append(f.getDataeaseName()); + return stringBuilder.toString(); + }).toArray(String[]::new); + if(ds != null){ + String schema = new Gson().fromJson(ds.getConfiguration(), JdbcDTO.class).getSchema(); + String tableWithSchema = String.format(SqlServerSQLConstants.KEYWORD_TABLE, schema) + "." + String.format(SqlServerSQLConstants.KEYWORD_TABLE, table); + return MessageFormat.format("SELECT {0} FROM {1} ", StringUtils.join(array, ","), tableWithSchema); + }else { + return MessageFormat.format("SELECT {0} FROM {1} ", StringUtils.join(array, ","), table); + } + } + + @Override + public String createRawQuerySQLAsTmp(String sql, List fields) { + return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields, null); + } + + public String transMysqlFilterTerm(String term) { + switch (term) { + case "eq": + return " = "; + case "not_eq": + return " <> "; + case "lt": + return " < "; + case "le": + return " <= "; + case "gt": + return " > "; + case "ge": + return " >= "; + case "in": + return " IN "; + case "not in": + return " NOT IN "; + case "like": + return " LIKE "; + case "not like": + return " NOT LIKE "; + case "null": + return " IN "; + case "not_null": + return " IS NOT NULL AND %s <> ''"; + case "between": + return " BETWEEN "; + default: + return ""; + } + } + + public List transCustomFilterList(SQLObj tableObj, List requestList) { + if (CollectionUtils.isEmpty(requestList)) { + return null; + } + List list = new ArrayList<>(); + for (ChartCustomFilterDTO request : requestList) { + DatasetTableField field = request.getField(); + if (ObjectUtils.isEmpty(field)) { + continue; + } + String value = request.getValue(); + String whereName = ""; + String whereTerm = transMysqlFilterTerm(request.getTerm()); + String whereValue = ""; + String originName; + if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originName = calcFieldRegex(field.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) { + originName = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); + } else { + originName = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); + } + if (field.getDeType() == 1 && field.getDeExtractType() != 1) { + String cast = String.format( PgConstants.CAST, originName, "bigint"); + whereName = String.format( PgConstants.FROM_UNIXTIME, cast); + } else { + whereName = originName; + } + if (StringUtils.equalsIgnoreCase(request.getTerm(), "null")) { + whereValue = PgConstants.WHERE_VALUE_NULL; + } else if (StringUtils.equalsIgnoreCase(request.getTerm(), "not_null")) { + whereTerm = String.format(whereTerm, originName); + } else if (StringUtils.containsIgnoreCase(request.getTerm(), "in")) { + whereValue = "('" + StringUtils.join(value, "','") + "')"; + } else if (StringUtils.containsIgnoreCase(request.getTerm(), "like")) { + whereValue = "'%" + value + "%'"; + } else { + whereValue = String.format( PgConstants.WHERE_VALUE_VALUE, value); + } + list.add(SQLObj.builder() + .whereField(whereName) + .whereTermAndValue(whereTerm + whereValue) + .build()); + } + return list; + } + + public List transExtFilterList(SQLObj tableObj, List requestList) { + if (CollectionUtils.isEmpty(requestList)) { + return null; + } + List list = new ArrayList<>(); + for (ChartExtFilterRequest request : requestList) { + List value = request.getValue(); + DatasetTableField field = request.getDatasetTableField(); + if (CollectionUtils.isEmpty(value) || ObjectUtils.isEmpty(field)) { + continue; + } + String whereName = ""; + String whereTerm = transMysqlFilterTerm(request.getOperator()); + String whereValue = ""; + + String originName; + if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originName = calcFieldRegex(field.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) { + originName = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); + } else { + originName = String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); + } + + if (field.getDeType() == 1 && field.getDeExtractType() != 1) { + String cast = String.format( PgConstants.CAST, originName, "bigint"); + whereName = String.format( PgConstants.FROM_UNIXTIME, cast); + } else { + whereName = originName; + } + + if (StringUtils.containsIgnoreCase(request.getOperator(), "in")) { + whereValue = "('" + StringUtils.join(value, "','") + "')"; + } else if (StringUtils.containsIgnoreCase(request.getOperator(), "like")) { + whereValue = "'%" + value.get(0) + "%'"; + } else if (StringUtils.containsIgnoreCase(request.getOperator(), "between")) { + if (request.getDatasetTableField().getDeType() == 1) { + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String startTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(0)))); + String endTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(1)))); + whereValue = String.format( PgConstants.WHERE_BETWEEN, startTime, endTime); + } else { + whereValue = String.format( PgConstants.WHERE_BETWEEN, value.get(0), value.get(1)); + } + } else { + whereValue = String.format( PgConstants.WHERE_VALUE_VALUE, value.get(0)); + } + list.add(SQLObj.builder() + .whereField(whereName) + .whereTermAndValue(whereTerm + whereValue) + .build()); + } + return list; + } + + private String sqlFix(String sql) { + if (sql.lastIndexOf(";") == (sql.length() - 1)) { + sql = sql.substring(0, sql.length() - 1); + } + return sql; + } + + private String transDateFormat(String dateStyle, String datePattern) { + String split = "-"; + if (StringUtils.equalsIgnoreCase(datePattern, "date_sub")) { + split = "-"; + } else if (StringUtils.equalsIgnoreCase(datePattern, "date_split")) { + split = "/"; + } + + switch (dateStyle) { + case "y": + return "'YYYY'"; + case "y_M": + return "'YYYY" + split + "MM'"; + case "y_M_d": + return "'YYYY" + split + "MM" + split + "DD'"; + case "H_m_s": + return "'HH24:MI:SS'"; + case "y_M_d_H_m": + return "'YYYY" + split + "MM" + split + "DD" + " HH24:MI'"; + case "y_M_d_H_m_s": + return "'YYYY" + split + "MM" + split + "DD" + " HH24:MI:SS'"; + default: + return "'YYYY-MM-DD HH24:MI:SS'"; + } + } + + private SQLObj getXFields(ChartViewFieldDTO x, String originField, String fieldAlias) { + String fieldName = ""; + if (x.getDeExtractType() == DeTypeConstants.DE_TIME) { + if (x.getDeType() == 2 || x.getDeType() == 3) { + fieldName = String.format( PgConstants.UNIX_TIMESTAMP, originField); + } else if (x.getDeType() == DeTypeConstants.DE_TIME) { + String format = transDateFormat(x.getDateStyle(), x.getDatePattern()); + fieldName = String.format( PgConstants.DATE_FORMAT, originField, format); + } else { + fieldName = originField; + } + } else { + if (x.getDeType() == DeTypeConstants.DE_TIME) { + String format = transDateFormat(x.getDateStyle(), x.getDatePattern()); + if (x.getDeExtractType() == DeTypeConstants.DE_STRING) { + fieldName = String.format( PgConstants.DATE_FORMAT, originField, format); + } else { + String cast = String.format( PgConstants.CAST, originField, "bigint"); + String from_unixtime = String.format( PgConstants.FROM_UNIXTIME, cast); + fieldName = String.format( PgConstants.DATE_FORMAT, from_unixtime, format); + } + } else { + fieldName = originField; + } + } + return SQLObj.builder() + .fieldName(fieldName) + .fieldAlias(fieldAlias) + .build(); + } + + private SQLObj getYFields(ChartViewFieldDTO y, String originField, String fieldAlias) { + String fieldName = ""; + if (StringUtils.equalsIgnoreCase(y.getOriginName(), "*")) { + fieldName = PgConstants.AGG_COUNT; + } else if (SQLConstants.DIMENSION_TYPE.contains(y.getDeType())) { + fieldName = String.format( PgConstants.AGG_FIELD, y.getSummary(), originField); + } else { + if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) { + String cast = String.format( PgConstants.CAST, originField, y.getDeType() == DeTypeConstants.DE_INT ? PgConstants.DEFAULT_INT_FORMAT : PgConstants.DEFAULT_FLOAT_FORMAT); + String agg = String.format( PgConstants.AGG_FIELD, y.getSummary(), cast); + fieldName = String.format( PgConstants.CAST, agg, PgConstants.DEFAULT_FLOAT_FORMAT); + } else { + String cast = String.format( PgConstants.CAST, originField, y.getDeType() == DeTypeConstants.DE_INT ? PgConstants.DEFAULT_INT_FORMAT : PgConstants.DEFAULT_FLOAT_FORMAT); + fieldName = String.format( PgConstants.AGG_FIELD, y.getSummary(), cast); + } + } + return SQLObj.builder() + .fieldName(fieldName) + .fieldAlias(fieldAlias) + .build(); + } + + private List getYWheres(ChartViewFieldDTO y, String originField, String fieldAlias) { + List list = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0) { + y.getFilter().forEach(f -> { + String whereTerm = transMysqlFilterTerm(f.getTerm()); + String whereValue = ""; + // 原始类型不是时间，在de中被转成时间的字段做处理 + if (StringUtils.equalsIgnoreCase(f.getTerm(), "null")) { + whereValue = PgConstants.WHERE_VALUE_NULL; + } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_null")) { + whereTerm = String.format(whereTerm, originField); + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { + whereValue = "('" + StringUtils.join(f.getValue(), "','") + "')"; + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { + whereValue = "'%" + f.getValue() + "%'"; + } else { + whereValue = String.format( PgConstants.WHERE_VALUE_VALUE, f.getValue()); + } + list.add(SQLObj.builder() + .whereField(fieldAlias) + .whereAlias(fieldAlias) + .whereTermAndValue(whereTerm + whereValue) + .build()); + }); + } + return list; + } + + private String calcFieldRegex(String originField, SQLObj tableObj) { + originField = originField.replaceAll("[\\t\\n\\r]]", ""); + // 正则提取[xxx] + String regex = "\\[(.*?)]"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(originField); + Set ids = new HashSet<>(); + while (matcher.find()) { + String id = matcher.group(1); + ids.add(id); + } + if (CollectionUtils.isEmpty(ids)) { + return originField; + } + DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); + datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); + List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); + for (DatasetTableField ele : calcFields) { + originField = originField.replaceAll("\\[" + ele.getId() + "]", + String.format( PgConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); + } + return originField; + } +} diff --git a/backend/src/main/java/io/dataease/provider/sqlserver/SqlserverQueryProvider.java b/backend/src/main/java/io/dataease/provider/sqlserver/SqlserverQueryProvider.java index 2c8d0254ba..6820889473 100644 --- a/backend/src/main/java/io/dataease/provider/sqlserver/SqlserverQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/sqlserver/SqlserverQueryProvider.java @@ -7,13 +7,13 @@ import io.dataease.base.domain.Datasource; import io.dataease.base.mapper.DatasetTableFieldMapper; import io.dataease.commons.constants.DeTypeConstants; import io.dataease.controller.request.chart.ChartExtFilterRequest; +import io.dataease.datasource.dto.JdbcDTO; import io.dataease.datasource.dto.SqlServerConfigration; import io.dataease.dto.chart.ChartCustomFilterDTO; import io.dataease.dto.chart.ChartViewFieldDTO; import io.dataease.dto.sqlObj.SQLObj; import io.dataease.provider.QueryProvider; import io.dataease.provider.SQLConstants; -import io.dataease.provider.mysql.MySQLConstants; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -91,10 +91,7 @@ public class SqlserverQueryProvider extends QueryProvider { .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(SqlServerSQLConstants.KEYWORD_TABLE, table)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) .build(); - if(ds != null){ - String schema = new Gson().fromJson(ds.getConfiguration(), SqlServerConfigration.class).getSchema(); - tableObj.setTableName(schema + "." + tableObj.getTableName()); - } + setSchema(tableObj,ds); List xFields = new ArrayList<>(); if (CollectionUtils.isNotEmpty(fields)) { @@ -171,11 +168,12 @@ public class SqlserverQueryProvider extends QueryProvider { } @Override - public String getSQL(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList) { + public String getSQL(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, Datasource ds) { SQLObj tableObj = SQLObj.builder() .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(SqlServerSQLConstants.KEYWORD_TABLE, table)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) .build(); + setSchema(tableObj,ds); List xFields = new ArrayList<>(); List xWheres = new ArrayList<>(); List xOrders = new ArrayList<>(); @@ -263,15 +261,16 @@ public class SqlserverQueryProvider extends QueryProvider { @Override public String getSQLAsTmp(String sql, List xAxis, List yAxis, List customFilter, List extFilterRequestList) { - return getSQL("(" + sqlFix(sql) + ")", xAxis, yAxis, customFilter, extFilterRequestList); + return getSQL("(" + sqlFix(sql) + ")", xAxis, yAxis, customFilter, extFilterRequestList, null); } @Override - public String getSQLStack(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extStack) { + public String getSQLStack(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extStack, Datasource ds) { SQLObj tableObj = SQLObj.builder() .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(SqlServerSQLConstants.KEYWORD_TABLE, table)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) .build(); + setSchema(tableObj,ds); List xFields = new ArrayList<>(); List xWheres = new ArrayList<>(); List xOrders = new ArrayList<>(); @@ -380,15 +379,16 @@ public class SqlserverQueryProvider extends QueryProvider { @Override public String getSQLAsTmpStack(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extStack) { - return getSQLStack("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extStack); + return getSQLStack("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extStack, null); } @Override - public String getSQLScatter(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extBubble) { + public String getSQLScatter(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extBubble, Datasource ds) { SQLObj tableObj = SQLObj.builder() - .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(MySQLConstants.KEYWORD_TABLE, table)) + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(SqlServerSQLConstants.KEYWORD_TABLE, table)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) .build(); + setSchema(tableObj,ds); List xFields = new ArrayList<>(); List xWheres = new ArrayList<>(); List xOrders = new ArrayList<>(); @@ -400,9 +400,9 @@ public class SqlserverQueryProvider extends QueryProvider { // 解析origin name中有关联的字段生成sql表达式 originField = calcFieldRegex(x.getOriginName(), tableObj); } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) { - originField = String.format(MySQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + originField = String.format(SqlServerSQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); } else { - originField = String.format(MySQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + originField = String.format(SqlServerSQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); } String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); // 处理横轴字段 @@ -433,9 +433,9 @@ public class SqlserverQueryProvider extends QueryProvider { // 解析origin name中有关联的字段生成sql表达式 originField = calcFieldRegex(y.getOriginName(), tableObj); } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) { - originField = String.format(MySQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + originField = String.format(SqlServerSQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); } else { - originField = String.format(MySQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + originField = String.format(SqlServerSQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); } String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); // 处理纵轴字段 @@ -483,7 +483,7 @@ public class SqlserverQueryProvider extends QueryProvider { ST st = stg.getInstanceOf("querySql"); SQLObj tableSQL = SQLObj.builder() - .tableName(String.format(MySQLConstants.BRACKETS, sql)) + .tableName(String.format(SqlServerSQLConstants.BRACKETS, sql)) .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) .build(); if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); @@ -494,7 +494,7 @@ public class SqlserverQueryProvider extends QueryProvider { @Override public String getSQLAsTmpScatter(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extBubble) { - return getSQLScatter("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extBubble); + return getSQLScatter("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extBubble, null); } @Override @@ -582,23 +582,24 @@ public class SqlserverQueryProvider extends QueryProvider { } @Override - public String createRawQuerySQL(String table, List fields) { + public String createRawQuerySQL(String table, List fields, Datasource ds) { String[] array = fields.stream().map(f -> { StringBuilder stringBuilder = new StringBuilder(); -// if (f.getDeExtractType() == 4) { // 处理 tinyint -// stringBuilder.append("concat(`").append(f.getOriginName()).append("`,'') AS ").append(f.getDataeaseName()); -// } else { -// stringBuilder.append("`").append(f.getOriginName()).append("` AS ").append(f.getDataeaseName()); -// } stringBuilder.append("\"").append(f.getOriginName()).append("\" AS ").append(f.getDataeaseName()); return stringBuilder.toString(); }).toArray(String[]::new); - return MessageFormat.format("SELECT {0} FROM {1}", StringUtils.join(array, ","), table); + if(ds != null){ + String schema = new Gson().fromJson(ds.getConfiguration(), JdbcDTO.class).getSchema(); + String tableWithSchema = String.format(SqlServerSQLConstants.KEYWORD_TABLE, schema) + "." + String.format(SqlServerSQLConstants.KEYWORD_TABLE, table); + return MessageFormat.format("SELECT {0} FROM {1} ", StringUtils.join(array, ","), tableWithSchema); + }else { + return MessageFormat.format("SELECT {0} FROM {1} ", StringUtils.join(array, ","), table); + } } @Override public String createRawQuerySQLAsTmp(String sql, List fields) { - return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields); + return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields, null); } public String transMysqlFilterTerm(String term) { @@ -773,47 +774,6 @@ public class SqlserverQueryProvider extends QueryProvider { } } - private String transStringToDateFormat(String dateStyle, String datePattern, String originField) { - String split = "-"; - if (StringUtils.equalsIgnoreCase(datePattern, "date_sub")) { - split = "-"; - } else if (StringUtils.equalsIgnoreCase(datePattern, "date_split")) { - split = "/"; - } - switch (dateStyle) { - case "y": - return "CONVERT(varchar(100), datepart(yy, " + "SELECT CONVERT(datetime, " + originField + " ,120)" + "))"; - case "y_M": - if(split.equalsIgnoreCase("-")){ - return "substring( convert(varchar," + "SELECT CONVERT(datetime, " + originField + " ,120)" + ",120),1,7)"; - }else { - return "replace("+ "substring( convert(varchar," + "SELECT CONVERT(datetime, " + originField + " ,120)" + ",120),1,7), '-','/')"; - } - case "y_M_d": - if(split.equalsIgnoreCase("-")){ - return "CONVERT(varchar(100), " + "SELECT CONVERT(datetime, " + originField + " ,120)" + ", 23)"; - }else { - return "CONVERT(varchar(100), " + "SELECT CONVERT(datetime, " + originField + " ,120)" + ", 111)"; - } - case "H_m_s": - return "CONVERT(varchar(100), " + "SELECT CONVERT(datetime, " + originField + " ,120)" + ", 24)"; - case "y_M_d_H_m": - if(split.equalsIgnoreCase("-")){ - return "substring( convert(varchar," + "SELECT CONVERT(datetime, " + originField + " ,120)" + ",120),1,16)"; - }else { - return "replace("+ "substring( convert(varchar," + "SELECT CONVERT(datetime, " + originField + " ,120)" + ",120),1,16), '-','/')"; - } - case "y_M_d_H_m_s": - if(split.equalsIgnoreCase("-")){ - return "convert(varchar," + "SELECT CONVERT(datetime, " + originField + " ,120)" + ",120)"; - }else { - return "replace("+ "convert(varchar," + "SELECT CONVERT(datetime, " + originField + " ,120)" + ",120), '-','/')"; - } - default: - return "convert(varchar," + "SELECT CONVERT(datetime, " + originField + " ,120)" + ",120)"; - } - } - private SQLObj getXFields(ChartViewFieldDTO x, String originField, String fieldAlias) { String fieldName = ""; if (x.getDeExtractType() == DeTypeConstants.DE_TIME) { @@ -912,7 +872,7 @@ public class SqlserverQueryProvider extends QueryProvider { List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); for (DatasetTableField ele : calcFields) { originField = originField.replaceAll("\\[" + ele.getId() + "]", - String.format(MySQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); + String.format(SqlServerSQLConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); } return originField; } diff --git a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java index cc80f9d81e..69d632ecf1 100644 --- a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java +++ b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java @@ -309,11 +309,11 @@ public class ChartViewService { if (StringUtils.equalsIgnoreCase("text", view.getType()) || StringUtils.equalsIgnoreCase("gauge", view.getType())) { datasourceRequest.setQuery(qp.getSQLSummary(dataTableInfoDTO.getTable(), yAxis, customFilter, extFilterList)); } else if (StringUtils.containsIgnoreCase(view.getType(), "stack")) { - datasourceRequest.setQuery(qp.getSQLStack(dataTableInfoDTO.getTable(), xAxis, yAxis, customFilter, extFilterList, extStack)); + datasourceRequest.setQuery(qp.getSQLStack(dataTableInfoDTO.getTable(), xAxis, yAxis, customFilter, extFilterList, extStack, ds)); } else if (StringUtils.containsIgnoreCase(view.getType(), "scatter")) { - datasourceRequest.setQuery(qp.getSQLScatter(dataTableInfoDTO.getTable(), xAxis, yAxis, customFilter, extFilterList, extBubble)); + datasourceRequest.setQuery(qp.getSQLScatter(dataTableInfoDTO.getTable(), xAxis, yAxis, customFilter, extFilterList, extBubble, ds)); } else { - datasourceRequest.setQuery(qp.getSQL(dataTableInfoDTO.getTable(), xAxis, yAxis, customFilter, extFilterList)); + datasourceRequest.setQuery(qp.getSQL(dataTableInfoDTO.getTable(), xAxis, yAxis, customFilter, extFilterList, ds)); } } else if (StringUtils.equalsIgnoreCase(table.getType(), "sql")) { if (StringUtils.equalsIgnoreCase("text", view.getType()) || StringUtils.equalsIgnoreCase("gauge", view.getType())) { @@ -362,11 +362,11 @@ public class ChartViewService { if (StringUtils.equalsIgnoreCase("text", view.getType()) || StringUtils.equalsIgnoreCase("gauge", view.getType())) { datasourceRequest.setQuery(qp.getSQLSummary(tableName, yAxis, customFilter, extFilterList)); } else if (StringUtils.containsIgnoreCase(view.getType(), "stack")) { - datasourceRequest.setQuery(qp.getSQLStack(tableName, xAxis, yAxis, customFilter, extFilterList, extStack)); + datasourceRequest.setQuery(qp.getSQLStack(tableName, xAxis, yAxis, customFilter, extFilterList, extStack, ds)); } else if (StringUtils.containsIgnoreCase(view.getType(), "scatter")) { - datasourceRequest.setQuery(qp.getSQLScatter(tableName, xAxis, yAxis, customFilter, extFilterList, extBubble)); + datasourceRequest.setQuery(qp.getSQLScatter(tableName, xAxis, yAxis, customFilter, extFilterList, extBubble, ds)); } else { - datasourceRequest.setQuery(qp.getSQL(tableName, xAxis, yAxis, customFilter, extFilterList)); + datasourceRequest.setQuery(qp.getSQL(tableName, xAxis, yAxis, customFilter, extFilterList, ds)); } /*// 定时抽取使用缓存 Object cache; diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index 6c90cb0491..c2bb860bb3 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -192,7 +192,7 @@ public class ExtractDataService { dropDorisTable(DorisTableUtils.dorisTmpName(DorisTableUtils.dorisName(datasetTableId))); } finally { deleteFile("all_scope", datasetTableId); - deleteFile(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getData()); +// deleteFile(new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getData()); } break; @@ -437,28 +437,28 @@ public class ExtractDataService { String Column_Fields = "dataease_uuid varchar(50), `"; for (DatasetTableField datasetTableField : datasetTableFields) { Column_Fields = Column_Fields + datasetTableField.getDataeaseName() + "` "; + Integer size = datasetTableField.getSize() * 3; + if (datasetTableField.getSize() > 65533 || datasetTableField.getSize() * 3 > 65533) { + size = 65533; + } switch (datasetTableField.getDeExtractType()) { case 0: - if (datasetTableField.getSize() > 65533 || datasetTableField.getSize() * 3 > 65533) { - Column_Fields = Column_Fields + "varchar(65533)" + ",`"; - } else { - Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize() * 3)) + ",`"; - } + Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(size)) + ",`"; break; case 1: - Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`"; + Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(size)) + ",`"; break; case 2: - Column_Fields = Column_Fields + "bigint(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`"; + Column_Fields = Column_Fields + "bigint(lenth)".replace("lenth", String.valueOf(size)) + ",`"; break; case 3: Column_Fields = Column_Fields + "DOUBLE" + ",`"; break; case 4: - Column_Fields = Column_Fields + "TINYINT(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`"; + Column_Fields = Column_Fields + "TINYINT(lenth)".replace("lenth", String.valueOf(size)) + ",`"; break; default: - Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(datasetTableField.getSize())) + ",`"; + Column_Fields = Column_Fields + "varchar(lenth)".replace("lenth", String.valueOf(size)) + ",`"; break; } } @@ -735,6 +735,14 @@ public class ExtractDataService { inputStep = inputStep(transMeta, selectSQL); udjcStep = udjc(datasetTableFields, DatasourceTypes.sqlServer); break; + case pg: + PgConfigration pgConfigration = new Gson().fromJson(datasource.getConfiguration(), PgConfigration.class); + dataMeta = new DatabaseMeta("db", "POSTGRESQL", "Native", pgConfigration.getHost(), pgConfigration.getDataBase(), pgConfigration.getPort().toString(), pgConfigration.getUsername(), pgConfigration.getPassword()); + transMeta.addDatabase(dataMeta); + selectSQL = getSelectSQL(extractType, datasetTable, datasource, datasetTableFields, selectSQL); + inputStep = inputStep(transMeta, selectSQL); + udjcStep = udjc(datasetTableFields, DatasourceTypes.pg); + break; case oracle: OracleConfigration oracleConfigration = new Gson().fromJson(datasource.getConfiguration(), OracleConfigration.class); if (oracleConfigration.getConnectionType().equalsIgnoreCase("serviceName")) { @@ -795,7 +803,7 @@ public class ExtractDataService { if (extractType.equalsIgnoreCase("all_scope") && datasetTable.getType().equalsIgnoreCase("db")) { String tableName = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable(); QueryProvider qp = ProviderFactory.getQueryProvider(datasource.getType()); - selectSQL = qp.createRawQuerySQL(tableName, datasetTableFields); + selectSQL = qp.createRawQuerySQL(tableName, datasetTableFields, datasource); } if (extractType.equalsIgnoreCase("all_scope") && datasetTable.getType().equalsIgnoreCase("sql")) { @@ -879,12 +887,18 @@ public class ExtractDataService { outputFields[datasetTableFields.size()] = textFileField; textFileOutputMeta.setOutputFields(outputFields); - }else if (datasource.getType().equalsIgnoreCase(DatasourceTypes.sqlServer.name())){ + }else if (datasource.getType().equalsIgnoreCase(DatasourceTypes.sqlServer.name()) || datasource.getType().equalsIgnoreCase(DatasourceTypes.pg.name())){ TextFileField[] outputFields = new TextFileField[datasetTableFields.size() + 1]; for(int i=0;i< datasetTableFields.size();i++){ TextFileField textFileField = new TextFileField(); textFileField.setName(datasetTableFields.get(i).getDataeaseName()); - textFileField.setType("String"); + if (datasetTableFields.get(i).getDeExtractType() == 1) { + textFileField.setType("String"); + textFileField.setFormat("yyyy-MM-dd HH:mm:ss"); + } else { + textFileField.setType("String"); + } + outputFields[i] = textFileField; } TextFileField textFileField = new TextFileField(); @@ -936,7 +950,6 @@ public class ExtractDataService { } tmp_code = tmp_code.replace("handleBinaryType", handleBinaryTypeCode); - UserDefinedJavaClassDef userDefinedJavaClassDef = new UserDefinedJavaClassDef(UserDefinedJavaClassDef.ClassType.TRANSFORM_CLASS, "Processor", tmp_code); userDefinedJavaClassDef.setActive(true); @@ -1019,13 +1032,13 @@ public class ExtractDataService { } } - private static String handleBinaryType = " \t\tif(\"FEILD\".equalsIgnoreCase(filed)){\n" + + private final static String handleBinaryType = " \t\tif(\"FEILD\".equalsIgnoreCase(filed)){\n" + " get(Fields.Out, filed).setValue(r, \"\");\n" + " get(Fields.Out, filed).getValueMeta().setType(2);\n" + " \t}"; - private static String alterColumnTypeCode = " if(\"FILED\".equalsIgnoreCase(filed)){\n" + + private final static String alterColumnTypeCode = " if(\"FILED\".equalsIgnoreCase(filed)){\n" + "\t if(tmp != null && tmp.equalsIgnoreCase(\"Y\")){\n" + " get(Fields.Out, filed).setValue(r, 1);\n" + " get(Fields.Out, filed).getValueMeta().setType(2);\n" + @@ -1035,7 +1048,7 @@ public class ExtractDataService { " }\n" + " }\n"; - private static String handleExcelIntColumn = " \t\tif(tmp != null && tmp.endsWith(\".0\")){\n" + + private final static String handleExcelIntColumn = " \t\tif(tmp != null && tmp.endsWith(\".0\")){\n" + " try {\n" + " Long.valueOf(tmp.substring(0, tmp.length()-2));\n" + " get(Fields.Out, filed).setValue(r, tmp.substring(0, tmp.length()-2));\n" + @@ -1043,14 +1056,14 @@ public class ExtractDataService { " }catch (Exception e){}\n" + " }"; - private static String handleWraps = " if(tmp != null && ( tmp.contains(\"\\r\") || tmp.contains(\"\\n\"))){\n" + + private final static String handleWraps = " if(tmp != null && ( tmp.contains(\"\\r\") || tmp.contains(\"\\n\"))){\n" + "\t\t\ttmp = tmp.trim();\n" + " tmp = tmp.replaceAll(\"\\r\",\" \");\n" + " tmp = tmp.replaceAll(\"\\n\",\" \");\n" + " get(Fields.Out, filed).setValue(r, tmp);\n" + " } "; - private static String code = "import org.pentaho.di.core.row.ValueMetaInterface;\n" + + private final static String code = "import org.pentaho.di.core.row.ValueMetaInterface;\n" + "import java.util.List;\n" + "import java.io.File;\n" + "import java.security.MessageDigest;\n" + diff --git a/frontend/src/views/system/datasource/DsTree.vue b/frontend/src/views/system/datasource/DsTree.vue index 6b4e026daa..ae60457ecb 100644 --- a/frontend/src/views/system/datasource/DsTree.vue +++ b/frontend/src/views/system/datasource/DsTree.vue @@ -156,6 +156,8 @@ export default { return 'SQL Server' } else if (type === 'oracle') { return 'Oracle' + }else if (type === 'pg') { + return 'PostgreSQL' } }, diff --git a/frontend/src/views/system/datasource/form.vue b/frontend/src/views/system/datasource/form.vue index e2ee45c109..1916bbe5a5 100644 --- a/frontend/src/views/system/datasource/form.vue +++ b/frontend/src/views/system/datasource/form.vue @@ -46,30 +46,13 @@ - + {{ $t('datasource.get_schema') }} - - - - - - - - - - {{ $t('datasource.get_schema') }} - - - +