diff --git a/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java b/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java index 3724c7f8ea..340715ce9b 100644 --- a/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java +++ b/backend/src/main/java/io/dataease/datasource/constants/DatasourceTypes.java @@ -6,7 +6,8 @@ public enum DatasourceTypes { pg("pg", "pg", "org.postgresql.Driver", "\"", "\"", "\"", "\""), sqlServer("sqlServer", "sqlServer", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "\"", "\"", "\"", "\""), doris("doris", "doris", "com.mysql.jdbc.Driver", "`", "`", "", ""), - oracle("oracle", "oracle", "oracle.jdbc.driver.OracleDriver", "\"", "\"", "\"", "\""); + oracle("oracle", "oracle", "oracle.jdbc.driver.OracleDriver", "\"", "\"", "\"", "\""), + es("es", "es", "", "\"", "\"", "\"", "\""); private String feature; private String desc; diff --git a/backend/src/main/java/io/dataease/datasource/dto/EsConfigDTO.java b/backend/src/main/java/io/dataease/datasource/dto/EsConfigDTO.java new file mode 100644 index 0000000000..4fb9c2295b --- /dev/null +++ b/backend/src/main/java/io/dataease/datasource/dto/EsConfigDTO.java @@ -0,0 +1,14 @@ +package io.dataease.datasource.dto; + + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class EsConfigDTO { + private String url; + private String username; + private String password; + private String dataSourceType = "es"; +} diff --git a/backend/src/main/java/io/dataease/datasource/dto/es/EsReponse.java b/backend/src/main/java/io/dataease/datasource/dto/es/EsReponse.java new file mode 100644 index 0000000000..cea73d9792 --- /dev/null +++ b/backend/src/main/java/io/dataease/datasource/dto/es/EsReponse.java @@ -0,0 +1,28 @@ +package io.dataease.datasource.dto.es; + +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +public class EsReponse { + private Listcolumns = new ArrayList<>(); + private Listrows = new ArrayList<>(); + private String cursor; + private Integer status; + private Error error; + + @Data + public class Error{ + private String type; + private String reason; + } + + @Data + public class Column { + private String name; + private String type; + } + +} diff --git a/backend/src/main/java/io/dataease/datasource/dto/es/Requst.java b/backend/src/main/java/io/dataease/datasource/dto/es/Requst.java new file mode 100644 index 0000000000..1e0bb8a608 --- /dev/null +++ b/backend/src/main/java/io/dataease/datasource/dto/es/Requst.java @@ -0,0 +1,9 @@ +package io.dataease.datasource.dto.es; + +import lombok.Data; + +@Data +public class Requst { + private String query; + private Integer fetch_size = 10000; +} diff --git a/backend/src/main/java/io/dataease/datasource/dto/es/RequstWithCursor.java b/backend/src/main/java/io/dataease/datasource/dto/es/RequstWithCursor.java new file mode 100644 index 0000000000..cb79c3460e --- /dev/null +++ b/backend/src/main/java/io/dataease/datasource/dto/es/RequstWithCursor.java @@ -0,0 +1,8 @@ +package io.dataease.datasource.dto.es; + +import lombok.Data; + +@Data +public class RequstWithCursor extends Requst{ + private String cursor; +} diff --git a/backend/src/main/java/io/dataease/datasource/provider/EsProvider.java b/backend/src/main/java/io/dataease/datasource/provider/EsProvider.java new file mode 100644 index 0000000000..72c46eb0bc --- /dev/null +++ b/backend/src/main/java/io/dataease/datasource/provider/EsProvider.java @@ -0,0 +1,221 @@ +package io.dataease.datasource.provider; + +import com.google.gson.Gson; +import com.mchange.v2.c3p0.ComboPooledDataSource; +import io.dataease.commons.utils.HttpClientConfig; +import io.dataease.commons.utils.HttpClientUtil; +import io.dataease.datasource.dto.*; +import io.dataease.datasource.dto.es.EsReponse; +import io.dataease.datasource.dto.es.Requst; +import io.dataease.datasource.dto.es.RequstWithCursor; +import io.dataease.datasource.request.DatasourceRequest; +import io.dataease.exception.DataEaseException; +import io.dataease.provider.es.EsQueryProvider; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpHeaders; +import org.springframework.stereotype.Service; + +import java.nio.charset.StandardCharsets; +import java.util.*; + +@Service("es") +public class EsProvider extends DatasourceProvider { + + private static Map jdbcConnection = new HashMap<>(); + private static int initPoolSize = 5; + private static int maxConnections = 200; + + /** + * 增加缓存机制 key 由 'provider_sql_' dsr.datasource.id dsr.table dsr.query共4部分组成，命中则使用缓存直接返回不再执行sql逻辑 + * @param dsr + * @return + * @throws Exception + */ + /** + * 这里使用声明式缓存不是很妥当 + * 改为chartViewService中使用编程式缓存 + @Cacheable( + value = JdbcConstants.JDBC_PROVIDER_KEY, + key = "'provider_sql_' + #dsr.datasource.id + '_' + #dsr.table + '_' + #dsr.query", + condition = "#dsr.pageSize == null || #dsr.pageSize == 0L" + ) + */ + @Override + public List getData(DatasourceRequest dsr) throws Exception { + List list = new LinkedList<>(); + try { + EsConfigDTO esConfigDTO = new Gson().fromJson(dsr.getDatasource().getConfiguration(), EsConfigDTO.class); + + HttpClientConfig httpClientConfig = new HttpClientConfig(); + String auth = esConfigDTO.getUsername() + ":" + esConfigDTO.getPassword(); + byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8)); + httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth)); + Requst requst = new Requst(); + requst.setQuery(dsr.getQuery()); + requst.setFetch_size(dsr.getFetchSize()); + String url = esConfigDTO.getUrl().endsWith("/") ? esConfigDTO.getUrl() + "_sql?format=json" : esConfigDTO.getUrl() + "/" + "_sql?format=json"; + String response = HttpClientUtil.post(url, new Gson().toJson(requst), httpClientConfig); + EsReponse esReponse = new Gson().fromJson(response, EsReponse.class); + + list.addAll(fetchResult(esReponse)); + if(dsr.isPageable()){ + Integer realSize = dsr.getPage() * dsr.getPageSize() < list.size() ? dsr.getPage() * dsr.getPageSize(): list.size(); + list = list.subList((dsr.getPage() - 1) * dsr.getPageSize(), realSize); + } + if(!dsr.isPreviewData()){ + while (StringUtils.isNotEmpty(esReponse.getCursor())) { + RequstWithCursor requstWithCursor = new RequstWithCursor(); + requstWithCursor.setQuery(dsr.getQuery()); + requstWithCursor.setFetch_size(dsr.getFetchSize()); + requstWithCursor.setCursor(esReponse.getCursor()); + response = HttpClientUtil.post(url, new Gson().toJson(requstWithCursor), httpClientConfig); + esReponse = new Gson().fromJson(response, EsReponse.class); + list.addAll(fetchResult(esReponse)); + } + } + } catch (Exception e) { + DataEaseException.throwException(e); + } + return list; + } + + @Override + public List fetchResult(DatasourceRequest datasourceRequest) throws Exception { + List list = new LinkedList<>(); + try { + String response = exexQuery(datasourceRequest, datasourceRequest.getQuery(), "_sql?format=json"); + list = fetchResult(response); + } catch (Exception e) { + DataEaseException.throwException(e); + } + return list; + } + + private List fetchResult(String response) throws Exception { + EsReponse esReponse = new Gson().fromJson(response, EsReponse.class); + return fetchResult(esReponse); + } + + private List fetchResult(EsReponse esReponse) throws Exception { + List list = new LinkedList<>(); + if(esReponse.getError() != null){ + throw new Exception(esReponse.getError().getReason()); + } + list.addAll(esReponse.getRows()); + return list; + } + + @Override + public List fetchResultField(DatasourceRequest datasourceRequest) throws Exception { + List tableFileds = new ArrayList<>(); + try { + String response = exexQuery(datasourceRequest, datasourceRequest.getQuery(), "_sql?format=json"); + tableFileds = fetchResultField(response); + } catch (Exception e) { + DataEaseException.throwException(e); + } + return tableFileds; + } + + private List fetchResultField(String response) throws Exception { + List fieldList = new ArrayList<>(); + EsReponse esReponse = new Gson().fromJson(response, EsReponse.class); + if(esReponse.getError() != null){ + throw new Exception(esReponse.getError().getReason()); + } + + for (String[] row : esReponse.getRows()) { + TableFiled field = new TableFiled(); + field.setFieldName(row[0]); + field.setRemarks(row[0]); + field.setFieldType(row[2]); + field.setFieldSize(EsQueryProvider.transFieldTypeSize(row[2])); + fieldList.add(field); + } + return fieldList; + } + + @Override + public Map fetchResultAndField(DatasourceRequest datasourceRequest) throws Exception { + Map result = new HashMap<>(); + try { + String response = exexQuery(datasourceRequest, datasourceRequest.getQuery(), "_sql?format=json"); + result.put("dataList", fetchResult(response)); + result.put("fieldList", fetchResultField(response)); + } catch (Exception e) { + DataEaseException.throwException(e); + } + return result; + } + + @Override + public void handleDatasource(DatasourceRequest datasourceRequest, String type) throws Exception { + } + + @Override + public List getTables(DatasourceRequest datasourceRequest) throws Exception { + List tables = new ArrayList<>(); + try { + String response = exexQuery(datasourceRequest, "show tables", "_sql?format=json"); + tables = fetchTables(response); + } catch (Exception e) { + DataEaseException.throwException(e); + } + return tables; + } + + private List fetchTables(String response) throws Exception { + List tables = new ArrayList<>(); + EsReponse esReponse = new Gson().fromJson(response, EsReponse.class); + if(esReponse.getError() != null){ + throw new Exception(esReponse.getError().getReason()); + } + + for (String[] row : esReponse.getRows()) { + if(row.length == 3 && row[1].equalsIgnoreCase("TABLE") && row[2].equalsIgnoreCase("INDEX")){ + tables.add(row[0]); + } + } + return tables; + } + + @Override + public List getSchema(DatasourceRequest datasourceRequest) throws Exception { + return new ArrayList<>(); + } + + @Override + public List getTableFileds(DatasourceRequest datasourceRequest) throws Exception { + List tableFileds = new ArrayList<>(); + try { + String response = exexQuery(datasourceRequest, "desc " + datasourceRequest.getTable(), "_sql?format=json"); + tableFileds = fetchResultField(response); + } catch (Exception e) { + DataEaseException.throwException(e); + } + return tableFileds; + } + + @Override + public void checkStatus(DatasourceRequest datasourceRequest) throws Exception { + getTables(datasourceRequest); + } + + private String exexQuery(DatasourceRequest datasourceRequest, String sql, String uri){ + EsConfigDTO esConfigDTO = new Gson().fromJson(datasourceRequest.getDatasource().getConfiguration(), EsConfigDTO.class); + + HttpClientConfig httpClientConfig = new HttpClientConfig(); + String auth = esConfigDTO.getUsername() + ":" + esConfigDTO.getPassword(); + byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.UTF_8)); + httpClientConfig.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + new String(encodedAuth)); + + Requst requst = new Requst(); + requst.setQuery(sql); + requst.setFetch_size(datasourceRequest.getFetchSize()); + String url = esConfigDTO.getUrl().endsWith("/") ? esConfigDTO.getUrl() + uri : esConfigDTO.getUrl() + "/" + uri; + String response = HttpClientUtil.post(url, new Gson().toJson(requst), httpClientConfig); + return response; + } + +} diff --git a/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java b/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java index d780399648..bec2034146 100644 --- a/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java +++ b/backend/src/main/java/io/dataease/datasource/provider/ProviderFactory.java @@ -30,6 +30,8 @@ public class ProviderFactory implements ApplicationContextAware { return context.getBean("jdbc", DatasourceProvider.class); case pg: return context.getBean("jdbc", DatasourceProvider.class); + case es: + return context.getBean("es", DatasourceProvider.class); default: return context.getBean("jdbc", DatasourceProvider.class); } @@ -48,6 +50,8 @@ public class ProviderFactory implements ApplicationContextAware { return context.getBean("pgQuery", QueryProvider.class); case oracle: return context.getBean("oracleQuery", QueryProvider.class); + case es: + return context.getBean("esQuery", QueryProvider.class); default: return context.getBean("mysqlQuery", QueryProvider.class); } diff --git a/backend/src/main/java/io/dataease/datasource/request/DatasourceRequest.java b/backend/src/main/java/io/dataease/datasource/request/DatasourceRequest.java index 9521b50af5..ebf237e3fd 100644 --- a/backend/src/main/java/io/dataease/datasource/request/DatasourceRequest.java +++ b/backend/src/main/java/io/dataease/datasource/request/DatasourceRequest.java @@ -11,8 +11,12 @@ public class DatasourceRequest { protected String query; protected String table; protected Datasource datasource; - private Long pageSize; - private Long startPage; + private Integer pageSize; + private Integer page; + private Integer realSize; + private Integer fetchSize = 10000; + private boolean pageable = false; + private boolean previewData = false; } diff --git a/backend/src/main/java/io/dataease/provider/es/EsQueryProvider.java b/backend/src/main/java/io/dataease/provider/es/EsQueryProvider.java new file mode 100644 index 0000000000..d36cb453d1 --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/es/EsQueryProvider.java @@ -0,0 +1,933 @@ +package io.dataease.provider.es; + +import io.dataease.base.domain.DatasetTableField; +import io.dataease.base.domain.DatasetTableFieldExample; +import io.dataease.base.domain.Datasource; +import io.dataease.base.mapper.DatasetTableFieldMapper; +import io.dataease.commons.constants.DeTypeConstants; +import io.dataease.controller.request.chart.ChartExtFilterRequest; +import io.dataease.dto.chart.ChartCustomFilterDTO; +import io.dataease.dto.chart.ChartViewFieldDTO; +import io.dataease.dto.sqlObj.SQLObj; +import io.dataease.provider.QueryProvider; +import io.dataease.provider.SQLConstants; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; +import org.stringtemplate.v4.ST; +import org.stringtemplate.v4.STGroup; +import org.stringtemplate.v4.STGroupFile; + +import javax.annotation.Resource; +import java.text.MessageFormat; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static io.dataease.provider.SQLConstants.TABLE_ALIAS_PREFIX; + +@Service("esQuery") +public class EsQueryProvider extends QueryProvider { + @Resource + private DatasetTableFieldMapper datasetTableFieldMapper; + + @Override + public Integer transFieldType(String field) { + switch (field) { + case "keyword": + case "text": + case "binary": + case "object": + case "nested": + case "null": + case "ip": + case "unsupported": + return DeTypeConstants.DE_STRING; + case "boolean": + case "byte": + case "short": + case "integer": + case "long": + return DeTypeConstants.DE_INT; + case "double": + case "float": + case "half_float": + case "scaled_float": + return DeTypeConstants.DE_FLOAT; + case "datetime": + return DeTypeConstants.DE_TIME; + default: + return 0; + } + } + + public static Integer transFieldTypeSize(String field) { + switch (field) { + case "null": + return 1; + case "boolean": + return 1; + case "byte": + return 3; + case "short": + return 5; + case "integer": + return 10; + case "long": + return 19; + case "double": + case "float": + case "half_float": + case "scaled_float": + return 15; + case "keyword": + case "text": + case "binary": + case "object": + case "nested": + return 65532; + case "datetime": + return 50; + case "ip": + return 50; + case "unsupported": + return 50; + default: + return 0; + } + } + + @Override + public String createSQLPreview(String sql, String orderBy) { + return "SELECT * FROM (" + sqlFix(sql) + ") AS tmp ORDER BY null " + " LIMIT 1000"; + } + + @Override + public String createQuerySQL(String table, List fields, boolean isGroup, Datasource ds) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(EsSqlLConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List xFields = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(fields)) { + for (int i = 0; i < fields.size(); i++) { + DatasetTableField f = fields.get(i); + String originField; + if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == DeTypeConstants.DE_INT) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(f.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(f.getExtField()) && f.getExtField() == DeTypeConstants.DE_TIME) { + originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName()); + } else { + originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), f.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + String fieldName = ""; + // 处理横轴字段 + if (f.getDeExtractType() == DeTypeConstants.DE_TIME) { + if (f.getDeType() == DeTypeConstants.DE_INT || f.getDeType() == DeTypeConstants.DE_FLOAT) { + fieldName = String.format(EsSqlLConstants.CAST, originField, "bigint"); + } else { + fieldName = originField; + } + } else if (f.getDeExtractType() == DeTypeConstants.DE_STRING) { + if (f.getDeType() == DeTypeConstants.DE_INT) { + fieldName = String.format(EsSqlLConstants.CAST, originField, "bigint"); + } else if (f.getDeType() == DeTypeConstants.DE_FLOAT) { + fieldName = String.format(EsSqlLConstants.CAST, originField, "double"); + } else if (f.getDeType() == DeTypeConstants.DE_TIME) { + fieldName = String.format(EsSqlLConstants.CAST, originField, "timestamp"); + } else { + fieldName = originField; + } + } else { + if (f.getDeType() == DeTypeConstants.DE_TIME) { + String cast = String.format(EsSqlLConstants.CAST, originField, "timestamp"); + fieldName = String.format(EsSqlLConstants.DATETIME_FORMAT, cast, EsSqlLConstants.DEFAULT_DATE_FORMAT); + } else if (f.getDeType() == DeTypeConstants.DE_INT) { + fieldName = String.format(EsSqlLConstants.CAST, originField, "timestamp"); + } else { + fieldName = originField; + } + } + xFields.add(SQLObj.builder() + .fieldName(fieldName) + .fieldAlias(fieldAlias) + .build()); + } + } + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("previewSql"); + st_sql.add("isGroup", isGroup); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + return st_sql.render(); + } + + @Override + public String createQuerySQLAsTmp(String sql, List fields, boolean isGroup) { + return createQuerySQL("(" + sqlFix(sql) + ")", fields, isGroup, null); + } + + @Override + public String createQuerySQLWithPage(String table, List fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup, Datasource ds) { + return createQuerySQL(table, fields, isGroup, null); + } + + @Override + public String createQueryTableWithLimit(String table, List fields, Integer limit, boolean isGroup, Datasource ds) { + return createQuerySQL(table, fields, isGroup, null); + } + + @Override + public String createQuerySqlWithLimit(String sql, List fields, Integer limit, boolean isGroup) { + return createQuerySQLAsTmp(sql, fields, isGroup); + } + + @Override + public String createQuerySQLAsTmpWithPage(String sql, List fields, Integer page, Integer pageSize, Integer realSize, boolean isGroup) { + return createQuerySQLAsTmp(sql, fields, isGroup); + } + + @Override + public String getSQL(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, Datasource ds) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(EsSqlLConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List xFields = new ArrayList<>(); + List xWheres = new ArrayList<>(); + List xOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(xAxis)) { + for (int i = 0; i < xAxis.size(); i++) { + ChartViewFieldDTO x = xAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == DeTypeConstants.DE_INT) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(x.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == DeTypeConstants.DE_TIME) { + originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } else { + originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + // 处理横轴字段 + xFields.add(getXFields(x, originField, fieldAlias)); + // 处理横轴过滤 +// xWheres.addAll(getXWheres(x, originField, fieldAlias)); + // 处理横轴排序 + if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) { + xOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(x.getSort()) + .build()); + } + } + } + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(yAxis)) { + for (int i = 0; i < yAxis.size(); i++) { + ChartViewFieldDTO y = yAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == DeTypeConstants.DE_INT) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(y.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == DeTypeConstants.DE_TIME) { + originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } else { + originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.addAll(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + List customWheres = transCustomFilterList(tableObj, customFilter); + // 处理仪表板字段过滤 + List extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(xFields); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + wheres.addAll(xWheres); + if (customWheres != null) wheres.addAll(customWheres); + if (extWheres != null) wheres.addAll(extWheres); + List groups = new ArrayList<>(); + groups.addAll(xFields); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(xOrders); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(EsSqlLConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return st.render(); + } + + + @Override + public String getSQLAsTmp(String sql, List xAxis, List yAxis, List customFilter, List extFilterRequestList) { + return getSQL("(" + sqlFix(sql) + ")", xAxis, yAxis, customFilter, extFilterRequestList, null); + } + + @Override + public String getSQLStack(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extStack, Datasource ds) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(EsSqlLConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List xFields = new ArrayList<>(); + List xWheres = new ArrayList<>(); + List xOrders = new ArrayList<>(); + List xList = new ArrayList<>(); + xList.addAll(xAxis); + xList.addAll(extStack); + if (CollectionUtils.isNotEmpty(xList)) { + for (int i = 0; i < xList.size(); i++) { + ChartViewFieldDTO x = xList.get(i); + String originField; + if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == DeTypeConstants.DE_INT) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(x.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == DeTypeConstants.DE_TIME) { + originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } else { + originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + // 处理横轴字段 + xFields.add(getXFields(x, originField, fieldAlias)); + // 处理横轴过滤 +// xWheres.addAll(getXWheres(x, originField, fieldAlias)); + // 处理横轴排序 + if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) { + xOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(x.getSort()) + .build()); + } + } + } + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(yAxis)) { + for (int i = 0; i < yAxis.size(); i++) { + ChartViewFieldDTO y = yAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == DeTypeConstants.DE_INT) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(y.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == DeTypeConstants.DE_TIME) { + originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } else { + originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.addAll(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + List customWheres = transCustomFilterList(tableObj, customFilter); + // 处理仪表板字段过滤 + List extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(xFields); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + wheres.addAll(xWheres); + if (customWheres != null) wheres.addAll(customWheres); + if (extWheres != null) wheres.addAll(extWheres); + List groups = new ArrayList<>(); + groups.addAll(xFields); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(xOrders); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(EsSqlLConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return st.render(); + } + + @Override + public String getSQLAsTmpStack(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extStack) { + return getSQLStack("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extStack, null); + } + + @Override + public String getSQLScatter(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extBubble, Datasource ds) { + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(EsSqlLConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List xFields = new ArrayList<>(); + List xWheres = new ArrayList<>(); + List xOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(xAxis)) { + for (int i = 0; i < xAxis.size(); i++) { + ChartViewFieldDTO x = xAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(x.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(x.getExtField()) && x.getExtField() == 1) { + originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } else { + originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), x.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + // 处理横轴字段 + xFields.add(getXFields(x, originField, fieldAlias)); + // 处理横轴过滤 +// xWheres.addAll(getXWheres(x, originField, fieldAlias)); + // 处理横轴排序 + if (StringUtils.isNotEmpty(x.getSort()) && !StringUtils.equalsIgnoreCase(x.getSort(), "none")) { + xOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(x.getSort()) + .build()); + } + } + } + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + List yList = new ArrayList<>(); + yList.addAll(yAxis); + yList.addAll(extBubble); + if (CollectionUtils.isNotEmpty(yList)) { + for (int i = 0; i < yList.size(); i++) { + ChartViewFieldDTO y = yList.get(i); + String originField; + if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(y.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) { + originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } else { + originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.addAll(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + List customWheres = transCustomFilterList(tableObj, customFilter); + // 处理仪表板字段过滤 + List extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(xFields); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + wheres.addAll(xWheres); + if (customWheres != null) wheres.addAll(customWheres); + if (extWheres != null) wheres.addAll(extWheres); + List groups = new ArrayList<>(); + groups.addAll(xFields); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(xOrders); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(xFields)) st_sql.add("groups", xFields); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(EsSqlLConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return st.render(); + } + + @Override + public String getSQLAsTmpScatter(String table, List xAxis, List yAxis, List customFilter, List extFilterRequestList, List extBubble) { + return getSQLScatter("(" + sqlFix(table) + ")", xAxis, yAxis, customFilter, extFilterRequestList, extBubble, null); + } + + @Override + public String searchTable(String table) { + return ""; + } + + @Override + public String getSQLSummary(String table, List yAxis, List customFilter, List extFilterRequestList) { + // 字段汇总 排序等 + SQLObj tableObj = SQLObj.builder() + .tableName((table.startsWith("(") && table.endsWith(")")) ? table : String.format(EsSqlLConstants.KEYWORD_TABLE, table)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 0)) + .build(); + List yFields = new ArrayList<>(); + List yWheres = new ArrayList<>(); + List yOrders = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(yAxis)) { + for (int i = 0; i < yAxis.size(); i++) { + ChartViewFieldDTO y = yAxis.get(i); + String originField; + if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originField = calcFieldRegex(y.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(y.getExtField()) && y.getExtField() == 1) { + originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } else { + originField = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), y.getOriginName()); + } + String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); + // 处理纵轴字段 + yFields.add(getYFields(y, originField, fieldAlias)); + // 处理纵轴过滤 + yWheres.addAll(getYWheres(y, originField, fieldAlias)); + // 处理纵轴排序 + if (StringUtils.isNotEmpty(y.getSort()) && !StringUtils.equalsIgnoreCase(y.getSort(), "none")) { + yOrders.add(SQLObj.builder() + .orderField(originField) + .orderAlias(fieldAlias) + .orderDirection(y.getSort()) + .build()); + } + } + } + // 处理视图中字段过滤 + List customWheres = transCustomFilterList(tableObj, customFilter); + // 处理仪表板字段过滤 + List extWheres = transExtFilterList(tableObj, extFilterRequestList); + // 构建sql所有参数 + List fields = new ArrayList<>(); + fields.addAll(yFields); + List wheres = new ArrayList<>(); + if (customWheres != null) wheres.addAll(customWheres); + if (extWheres != null) wheres.addAll(extWheres); + List groups = new ArrayList<>(); + // 外层再次套sql + List orders = new ArrayList<>(); + orders.addAll(yOrders); + List aggWheres = new ArrayList<>(); + aggWheres.addAll(yWheres); + + STGroup stg = new STGroupFile(SQLConstants.SQL_TEMPLATE); + ST st_sql = stg.getInstanceOf("querySql"); + if (CollectionUtils.isNotEmpty(yFields)) st_sql.add("aggregators", yFields); + if (CollectionUtils.isNotEmpty(wheres)) st_sql.add("filters", wheres); + if (ObjectUtils.isNotEmpty(tableObj)) st_sql.add("table", tableObj); + String sql = st_sql.render(); + + ST st = stg.getInstanceOf("querySql"); + SQLObj tableSQL = SQLObj.builder() + .tableName(String.format(EsSqlLConstants.BRACKETS, sql)) + .tableAlias(String.format(TABLE_ALIAS_PREFIX, 1)) + .build(); + if (CollectionUtils.isNotEmpty(aggWheres)) st.add("filters", aggWheres); + if (CollectionUtils.isNotEmpty(orders)) st.add("orders", orders); + if (ObjectUtils.isNotEmpty(tableSQL)) st.add("table", tableSQL); + return st.render(); + } + + @Override + public String getSQLSummaryAsTmp(String sql, List yAxis, List customFilter, List extFilterRequestList) { + return getSQLSummary("(" + sqlFix(sql) + ")", yAxis, customFilter, extFilterRequestList); + } + + @Override + public String wrapSql(String sql) { + sql = sql.trim(); + if (sql.lastIndexOf(";") == (sql.length() - 1)) { + sql = sql.substring(0, sql.length() - 1); + } + String tmpSql = "SELECT * FROM (" + sql + ") AS tmp " + " LIMIT 0"; + return tmpSql; + } + + @Override + public String createRawQuerySQL(String table, List fields, Datasource ds) { + String[] array = fields.stream().map(f -> { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("\"").append(f.getOriginName()).append("\" AS ").append(f.getDataeaseName()); + return stringBuilder.toString(); + }).toArray(String[]::new); + return MessageFormat.format("SELECT {0} FROM {1} ORDER BY null", StringUtils.join(array, ","), table); + } + + @Override + public String createRawQuerySQLAsTmp(String sql, List fields) { + return createRawQuerySQL(" (" + sqlFix(sql) + ") AS tmp ", fields, null); + } + + public String transMysqlFilterTerm(String term) { + switch (term) { + case "eq": + return " = "; + case "not_eq": + return " <> "; + case "lt": + return " < "; + case "le": + return " <= "; + case "gt": + return " > "; + case "ge": + return " >= "; + case "in": + return " IN "; + case "not in": + return " NOT IN "; + case "like": + return " LIKE "; + case "not like": + return " NOT LIKE "; + case "null": + return " IN "; + case "not_null": + return " IS NOT NULL AND %s <> ''"; + case "between": + return " BETWEEN "; + default: + return ""; + } + } + + public List transCustomFilterList(SQLObj tableObj, List requestList) { + if (CollectionUtils.isEmpty(requestList)) { + return null; + } + List list = new ArrayList<>(); + for (ChartCustomFilterDTO request : requestList) { + DatasetTableField field = request.getField(); + if (ObjectUtils.isEmpty(field)) { + continue; + } + String value = request.getValue(); + String whereName = ""; + String whereTerm = transMysqlFilterTerm(request.getTerm()); + String whereValue = ""; + String originName; + if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originName = calcFieldRegex(field.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) { + originName = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); + } else { + originName = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); + } + if (field.getDeType() == DeTypeConstants.DE_TIME) { + if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) { + whereName = String.format(EsSqlLConstants.CAST, originName, "timestamp"); + } + if (field.getDeExtractType() == DeTypeConstants.DE_INT || field.getDeExtractType() == DeTypeConstants.DE_FLOAT || field.getDeExtractType() == DeTypeConstants.DE_BOOL) { + String cast = String.format(EsSqlLConstants.CAST, originName, "timestamp"); + whereName = String.format(EsSqlLConstants.DATETIME_FORMAT, cast, EsSqlLConstants.DEFAULT_DATE_FORMAT); + } + if (field.getDeExtractType() == 1) { + whereName = originName; + } + } else { + whereName = originName; + } + if (StringUtils.equalsIgnoreCase(request.getTerm(), "null")) { + whereValue = EsSqlLConstants.WHERE_VALUE_NULL; + } else if (StringUtils.equalsIgnoreCase(request.getTerm(), "not_null")) { + whereTerm = String.format(whereTerm, originName); + } else if (StringUtils.containsIgnoreCase(request.getTerm(), "in")) { + whereValue = "('" + StringUtils.join(value, "','") + "')"; + } else if (StringUtils.containsIgnoreCase(request.getTerm(), "like")) { + whereValue = "'%" + value + "%'"; + } else { + whereValue = String.format(EsSqlLConstants.WHERE_VALUE_VALUE, value); + } + list.add(SQLObj.builder() + .whereField(whereName) + .whereTermAndValue(whereTerm + whereValue) + .build()); + } + return list; + } + + public List transExtFilterList(SQLObj tableObj, List requestList) { + if (CollectionUtils.isEmpty(requestList)) { + return null; + } + List list = new ArrayList<>(); + for (ChartExtFilterRequest request : requestList) { + List value = request.getValue(); + DatasetTableField field = request.getDatasetTableField(); + if (CollectionUtils.isEmpty(value) || ObjectUtils.isEmpty(field)) { + continue; + } + String whereName = ""; + String whereTerm = transMysqlFilterTerm(request.getOperator()); + String whereValue = ""; + + String originName; + if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 2) { + // 解析origin name中有关联的字段生成sql表达式 + originName = calcFieldRegex(field.getOriginName(), tableObj); + } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 1) { + originName = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); + } else { + originName = String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), field.getOriginName()); + } + + if (field.getDeType() == 1) { + if (field.getDeExtractType() == 0 || field.getDeExtractType() == 5) { + whereName = String.format(EsSqlLConstants.CAST, originName, "timestamp"); + } + if (field.getDeExtractType() == 2 || field.getDeExtractType() == 3 || field.getDeExtractType() == 4) { + String cast = String.format(EsSqlLConstants.CAST, originName, "timestamp"); + whereName = String.format(EsSqlLConstants.DATETIME_FORMAT, cast, EsSqlLConstants.DEFAULT_DATE_FORMAT); + } + if (field.getDeExtractType() == 1) { + whereName = originName; + } + } else { + whereName = originName; + } + + + if (StringUtils.containsIgnoreCase(request.getOperator(), "in")) { + whereValue = "('" + StringUtils.join(value, "','") + "')"; + } else if (StringUtils.containsIgnoreCase(request.getOperator(), "like")) { + whereValue = "'%" + value.get(0) + "%'"; + } else if (StringUtils.containsIgnoreCase(request.getOperator(), "between")) { + if (request.getDatasetTableField().getDeType() == 1) { + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String startTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(0)))); + String endTime = simpleDateFormat.format(new Date(Long.parseLong(value.get(1)))); + whereValue = String.format(EsSqlLConstants.WHERE_BETWEEN, startTime, endTime); + } else { + whereValue = String.format(EsSqlLConstants.WHERE_BETWEEN, value.get(0), value.get(1)); + } + } else { + whereValue = String.format(EsSqlLConstants.WHERE_VALUE_VALUE, value.get(0)); + } + list.add(SQLObj.builder() + .whereField(whereName) + .whereTermAndValue(whereTerm + whereValue) + .build()); + } + return list; + } + + private String sqlFix(String sql) { + if (sql.lastIndexOf(";") == (sql.length() - 1)) { + sql = sql.substring(0, sql.length() - 1); + } + return sql; + } + + private String transDateFormat(String dateStyle, String datePattern) { + String split = "-"; + if (StringUtils.equalsIgnoreCase(datePattern, "date_sub")) { + split = "-"; + } else if (StringUtils.equalsIgnoreCase(datePattern, "date_split")) { + split = "/"; + } + + switch (dateStyle) { + case "y": + return "YYYY"; + case "y_M": + return "YYYY" + split + "MM"; + case "y_M_d": + return "YYYY" + split + "MM" + split + "dd"; + case "H_m_s": + return "HH:mm:ss"; + case "y_M_d_H_m": + return "YYYY" + split + "MM" + split + "dd" + " HH:mm"; + case "y_M_d_H_m_s": + return "YYYY" + split + "MM" + split + "dd" + " HH:mm:ss"; + default: + return "YYYY-MM-dd HH:mm:ss"; + } + } + + private SQLObj getXFields(ChartViewFieldDTO x, String originField, String fieldAlias) { + String fieldName = ""; + if (x.getDeExtractType() == DeTypeConstants.DE_TIME) { + if (x.getDeType() == DeTypeConstants.DE_INT || x.getDeType() == DeTypeConstants.DE_FLOAT) { + fieldName = String.format(EsSqlLConstants.CAST, originField, "bigint"); + } else if (x.getDeType() == DeTypeConstants.DE_TIME) { + String format = transDateFormat(x.getDateStyle(), x.getDatePattern()); + fieldName = String.format(EsSqlLConstants.DATETIME_FORMAT, originField, format); + } else { + fieldName = originField; + } + } else { + if (x.getDeType() == DeTypeConstants.DE_TIME) { + String format = transDateFormat(x.getDateStyle(), x.getDatePattern()); + if (x.getDeExtractType() == DeTypeConstants.DE_STRING) { + String cast = String.format(EsSqlLConstants.CAST, originField, "timestamp"); + fieldName = String.format(EsSqlLConstants.DATETIME_FORMAT, cast, format); + } else { + String cast = String.format(EsSqlLConstants.CAST, originField, "timestamp"); + fieldName = String.format(EsSqlLConstants.DATETIME_FORMAT, cast, format); + } + } else { + fieldName = originField; + } + } + return SQLObj.builder() + .fieldName(fieldName) + .fieldAlias(fieldAlias) + .build(); + } + + private SQLObj getYFields(ChartViewFieldDTO y, String originField, String fieldAlias) { + String fieldName = ""; + if (StringUtils.equalsIgnoreCase(y.getOriginName(), "*")) { + fieldName = EsSqlLConstants.AGG_COUNT; + } else if (SQLConstants.DIMENSION_TYPE.contains(y.getDeType())) { + fieldName = String.format(EsSqlLConstants.AGG_FIELD, y.getSummary(), originField); + } else { + if (StringUtils.equalsIgnoreCase(y.getSummary(), "avg") || StringUtils.containsIgnoreCase(y.getSummary(), "pop")) { + String cast = String.format(EsSqlLConstants.CAST, originField, y.getDeType() == DeTypeConstants.DE_INT ? "bigint" : "double"); + String agg = String.format(EsSqlLConstants.AGG_FIELD, y.getSummary(), cast); + fieldName = String.format(EsSqlLConstants.CAST, agg, EsSqlLConstants.DEFAULT_FLOAT_FORMAT); + } else { + String cast = String.format(EsSqlLConstants.CAST, originField, y.getDeType() == DeTypeConstants.DE_INT ? "bigint" : "double"); + fieldName = String.format(EsSqlLConstants.AGG_FIELD, y.getSummary(), cast); + } + } + return SQLObj.builder() + .fieldName(fieldName) + .fieldAlias(fieldAlias) + .build(); + } + + private List getYWheres(ChartViewFieldDTO y, String originField, String fieldAlias) { + List list = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(y.getFilter()) && y.getFilter().size() > 0) { + y.getFilter().forEach(f -> { + String whereTerm = transMysqlFilterTerm(f.getTerm()); + String whereValue = ""; + // 原始类型不是时间，在de中被转成时间的字段做处理 + if (StringUtils.equalsIgnoreCase(f.getTerm(), "null")) { + whereValue = EsSqlLConstants.WHERE_VALUE_NULL; + } else if (StringUtils.equalsIgnoreCase(f.getTerm(), "not_null")) { + whereTerm = String.format(whereTerm, originField); + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "in")) { + whereValue = "('" + StringUtils.join(f.getValue(), "','") + "')"; + } else if (StringUtils.containsIgnoreCase(f.getTerm(), "like")) { + whereValue = "'%" + f.getValue() + "%'"; + } else { + whereValue = String.format(EsSqlLConstants.WHERE_VALUE_VALUE, f.getValue()); + } + list.add(SQLObj.builder() + .whereField(fieldAlias) + .whereAlias(fieldAlias) + .whereTermAndValue(whereTerm + whereValue) + .build()); + }); + } + return list; + } + + private String calcFieldRegex(String originField, SQLObj tableObj) { + originField = originField.replaceAll("[\\t\\n\\r]]", ""); + // 正则提取[xxx] + String regex = "\\[(.*?)]"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(originField); + Set ids = new HashSet<>(); + while (matcher.find()) { + String id = matcher.group(1); + ids.add(id); + } + if (CollectionUtils.isEmpty(ids)) { + return originField; + } + DatasetTableFieldExample datasetTableFieldExample = new DatasetTableFieldExample(); + datasetTableFieldExample.createCriteria().andIdIn(new ArrayList<>(ids)); + List calcFields = datasetTableFieldMapper.selectByExample(datasetTableFieldExample); + for (DatasetTableField ele : calcFields) { + originField = originField.replaceAll("\\[" + ele.getId() + "]", + String.format(EsSqlLConstants.KEYWORD_FIX, tableObj.getTableAlias(), ele.getOriginName())); + } + return originField; + } +} diff --git a/backend/src/main/java/io/dataease/provider/es/EsSqlLConstants.java b/backend/src/main/java/io/dataease/provider/es/EsSqlLConstants.java new file mode 100644 index 0000000000..042ab0b1a1 --- /dev/null +++ b/backend/src/main/java/io/dataease/provider/es/EsSqlLConstants.java @@ -0,0 +1,39 @@ +package io.dataease.provider.es; + +import io.dataease.provider.SQLConstants; + +import static io.dataease.datasource.constants.DatasourceTypes.es; + +/** + * @Author gin + * @Date 2021/7/8 7:22 下午 + */ +public class EsSqlLConstants extends SQLConstants { + public static final String KEYWORD_TABLE = es.getKeywordPrefix() + "%s" + es.getKeywordSuffix(); + + public static final String KEYWORD_FIX = "%s." + es.getKeywordPrefix() + "%s" + es.getKeywordSuffix(); + + public static final String UNIX_TIMESTAMP = "UNIX_TIMESTAMP(%s)"; + + public static final String DATETIME_FORMAT = "DATETIME_FORMAT(%s,'%s')"; + + public static final String CAST = "CAST(%s AS %s)"; + + public static final String DEFAULT_DATE_FORMAT = "YYYY-MM-dd HH:mm:ss"; + + public static final String DEFAULT_INT_FORMAT = "DECIMAL(20,0)"; + + public static final String DEFAULT_FLOAT_FORMAT = "DECIMAL(20,2)"; + + public static final String WHERE_VALUE_NULL = "(NULL,'')"; + + public static final String WHERE_VALUE_VALUE = "'%s'"; + + public static final String AGG_COUNT = "COUNT(*)"; + + public static final String AGG_FIELD = "%s(%s)"; + + public static final String WHERE_BETWEEN = "'%s' AND '%s'"; + + public static final String BRACKETS = "(%s)"; +} diff --git a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java index 93e1158b4a..1f3536e601 100644 --- a/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java +++ b/backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java @@ -470,14 +470,22 @@ public class DataSetTableService { QueryProvider qp = ProviderFactory.getQueryProvider(ds.getType()); datasourceRequest.setQuery(qp.createQuerySQLWithPage(table, fields, page, pageSize, realSize, false, ds)); map.put("sql", datasourceRequest.getQuery()); + datasourceRequest.setPage(page); + datasourceRequest.setFetchSize(Integer.parseInt(dataSetTableRequest.getRow())); + datasourceRequest.setPageSize(pageSize); + datasourceRequest.setRealSize(realSize); + datasourceRequest.setPreviewData(true); try { + datasourceRequest.setPageable(true); data.addAll(datasourceProvider.getData(datasourceRequest)); } catch (Exception e) { e.printStackTrace(); DEException.throwException(e.getMessage()); } + try { datasourceRequest.setQuery(qp.createQueryTableWithLimit(table, fields, Integer.valueOf(dataSetTableRequest.getRow()), false, ds)); + datasourceRequest.setPageable(false); dataSetPreviewPage.setTotal(datasourceProvider.getData(datasourceRequest).size()); } catch (Exception e) { e.printStackTrace(); diff --git a/frontend/src/lang/en.js b/frontend/src/lang/en.js index 9edbd7716b..13c6e15940 100644 --- a/frontend/src/lang/en.js +++ b/frontend/src/lang/en.js @@ -1044,6 +1044,7 @@ export default { please_input_user_name: 'Please enter user name', please_input_password: 'Please enter Password', please_input_host: 'Please enter host', + please_input_url: 'Please enter url adress', please_input_port: 'Please enter port', modify: 'Edit data Source', validate_success: 'Verification successful', diff --git a/frontend/src/lang/tw.js b/frontend/src/lang/tw.js index 064e9415dd..762f420bae 100644 --- a/frontend/src/lang/tw.js +++ b/frontend/src/lang/tw.js @@ -1043,6 +1043,7 @@ export default { please_input_user_name: '請輸入用戶名', please_input_password: '請輸入密碼', please_input_host: '請輸入主機', + please_input_url: '請輸入URL地址', please_input_port: '請輸入', modify: '編輯數據源', validate_success: '校驗成功', diff --git a/frontend/src/lang/zh.js b/frontend/src/lang/zh.js index 3f07babb7c..6fe70f05ab 100644 --- a/frontend/src/lang/zh.js +++ b/frontend/src/lang/zh.js @@ -1044,6 +1044,7 @@ export default { please_input_user_name: '请输入用户名', please_input_password: '请输入密码', please_input_host: '请输入主机', + please_input_url: '请输入URL地址', please_input_port: '请输入端口', modify: '编辑数据源', validate_success: '校验成功', diff --git a/frontend/src/views/system/datasource/form.vue b/frontend/src/views/system/datasource/form.vue index a417b7e5a5..f706fbded0 100644 --- a/frontend/src/views/system/datasource/form.vue +++ b/frontend/src/views/system/datasource/form.vue @@ -27,6 +27,9 @@ + + + @@ -36,10 +39,10 @@ {{ $t('datasource.oracle_service_name') }} - + - + @@ -61,25 +64,25 @@ /> - + - + - + - + - + - + - + @@ -135,6 +138,7 @@ export default { 'configuration.username': [{ required: true, message: this.$t('datasource.please_input_user_name'), trigger: 'blur' }], 'configuration.password': [{ required: true, message: this.$t('datasource.please_input_password'), trigger: 'change' }], 'configuration.host': [{ required: true, message: this.$t('datasource.please_input_host'), trigger: 'change' }], + 'configuration.url': [{ required: true, message: this.$t('datasource.please_input_url'), trigger: 'change' }], 'configuration.port': [{ required: true, message: this.$t('datasource.please_input_port'), trigger: 'change' }], 'configuration.initialPoolSize': [{ required: true, message: this.$t('datasource.please_input_initial_pool_size'), trigger: 'change' }], 'configuration.minPoolSize': [{ required: true, message: this.$t('datasource.please_input_min_pool_size'), trigger: 'change' }], @@ -146,7 +150,8 @@ export default { allTypes: [{ name: 'mysql', label: 'MySQL', type: 'jdbc' }, { name: 'oracle', label: 'Oracle', type: 'jdbc' }, { name: 'sqlServer', label: 'SQL Server', type: 'jdbc' }, - { name: 'pg', label: 'PostgreSQL', type: 'jdbc' }], + { name: 'pg', label: 'PostgreSQL', type: 'jdbc' }, + { name: 'es', label: 'Elasticsearch', type: 'es' }], schemas: [], canEdit: false, originConfiguration: {}