diff --git a/core/core-backend/src/main/java/io/dataease/commons/utils/DeSqlparserUtils.java b/core/core-backend/src/main/java/io/dataease/commons/utils/DeSqlparserUtils.java new file mode 100644 index 0000000000..3e4a531263 --- /dev/null +++ b/core/core-backend/src/main/java/io/dataease/commons/utils/DeSqlparserUtils.java @@ -0,0 +1,284 @@ +package io.dataease.commons.utils; + +import com.fasterxml.jackson.core.type.TypeReference; +import io.dataease.api.permissions.user.vo.UserFormVO; +import io.dataease.api.permissions.variable.dto.SysVariableValueDto; +import io.dataease.api.permissions.variable.dto.SysVariableValueItem; +import io.dataease.exception.DEException; +import io.dataease.extensions.datasource.api.PluginManageApi; +import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO; +import io.dataease.extensions.datasource.vo.DatasourceConfiguration; +import io.dataease.extensions.datasource.vo.XpackPluginsDatasourceVO; +import io.dataease.extensions.view.dto.SqlVariableDetails; +import io.dataease.i18n.Translator; +import io.dataease.license.utils.LicenseUtil; +import io.dataease.utils.JsonUtil; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.operators.relational.InExpression; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; + +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +import static io.dataease.chart.manage.ChartDataManage.START_END_SEPARATOR; + +public class DeSqlparserUtils { + private static final String deVariablePattern = "\\$DE_PARAM\\{(.*?)\\}"; + + public static final String sqlParamsRegex = "\\$\\[(.*?)\\]"; + public static final String sysVariableRegex = "\\$f2cde\\[(.*?)\\]"; + private static final String SysParamsSubstitutedParams = "DeSysParams_"; + private UserFormVO userEntity; + private static final String SubstitutedSql = " 'DE-BI' = 'DE-BI' "; + private final List> sysParams = new ArrayList<>(); + TypeReference> listTypeReference = new TypeReference>() { + }; + private List defaultsSqlVariableDetails = new ArrayList<>(); + + public String handleVariableDefaultValue(String sql, String sqlVariableDetails, boolean isEdit, boolean isFromDataSet, List parameters, boolean isCross, Map dsMap, PluginManageApi pluginManage, UserFormVO userEntity) { + DatasourceSchemaDTO ds = dsMap.entrySet().iterator().next().getValue(); + if (StringUtils.isEmpty(sql)) { + DEException.throwException(Translator.get("i18n_sql_not_empty")); + } + this.userEntity = userEntity; + sql = sql.trim(); + if (sql.endsWith(";")) { + sql = sql.substring(0, sql.length() - 1); + } + if (StringUtils.isNotEmpty(sqlVariableDetails)) { + defaultsSqlVariableDetails = JsonUtil.parseList(sqlVariableDetails, listTypeReference); + } + Pattern pattern = Pattern.compile(deVariablePattern); + Matcher matcher = pattern.matcher(sql); + while (matcher.find()) { + String sqlItemWithParam = matcher.group(); + String sqlItem = sqlItemWithParam.substring(10, sqlItemWithParam.length() - 1); + boolean replaceParam = false; + Pattern p = Pattern.compile(sqlParamsRegex); + Matcher m = p.matcher(sqlItemWithParam); + if (m.find()) { // 替换sql参数 + String sqlVariable = m.group(); + SqlVariableDetails defaultsSqlVariableDetail = null; + for (SqlVariableDetails sqlVariableDetail : defaultsSqlVariableDetails) { + if (sqlVariable.substring(2, sqlVariable.length() - 1).equalsIgnoreCase(sqlVariableDetail.getVariableName())) { + defaultsSqlVariableDetail = sqlVariableDetail; + break; + } + } + SqlVariableDetails filterParameter = null; + if (ObjectUtils.isNotEmpty(parameters)) { + for (SqlVariableDetails parameter : parameters) { + if (parameter.getVariableName().equalsIgnoreCase(defaultsSqlVariableDetail.getVariableName())) { + filterParameter = parameter; + } + } + } + if (filterParameter != null) { + sqlItem = sqlItem.replace(sqlVariable, transFilter(filterParameter, dsMap)); + replaceParam = true; + } else { + if (defaultsSqlVariableDetail != null && StringUtils.isNotEmpty(defaultsSqlVariableDetail.getDefaultValue())) { + if (!isEdit && isFromDataSet && defaultsSqlVariableDetail.getDefaultValueScope().equals(SqlVariableDetails.DefaultValueScope.ALLSCOPE)) { + sqlItem = sqlItem.replace(sqlVariable, defaultsSqlVariableDetail.getDefaultValue()); + replaceParam = true; + } + if (isEdit) { + sqlItem = sqlItem.replace(sqlVariable, defaultsSqlVariableDetail.getDefaultValue()); + replaceParam = true; + } + } + } + + } else { //替换系统变量 + p = Pattern.compile(sysVariableRegex); + m = p.matcher(sqlItemWithParam); + if (m.find()) { + String sysVariableId = m.group().substring(7, m.group().length() - 1); + if (!isParams(sysVariableId)) { + continue; + } + sqlItem = sqlItem.replace(m.group(), SysParamsSubstitutedParams + sysVariableId); + try { + Expression expression = CCJSqlParserUtil.parseCondExpression(sqlItem); + String value = null; + if (expression instanceof InExpression) { + value = handleSubstitutedSqlForIn(sysVariableId); + } else { + value = handleSubstitutedSql(sysVariableId); + } + if (value != null) { + sqlItem = sqlItem.replace(SysParamsSubstitutedParams + sysVariableId, value); + replaceParam = true; + } + } catch (Exception e) { + e.printStackTrace(); + } + } + } + if (!replaceParam) { + sql = sql.replace(sqlItemWithParam, SubstitutedSql); + } else { + sql = sql.replace(sqlItemWithParam, sqlItem); + } + } + + try { + if (!isCross) { + Map.Entry next = dsMap.entrySet().iterator().next(); + DatasourceSchemaDTO value = next.getValue(); + + String prefix = ""; + String suffix = ""; + if (Arrays.stream(DatasourceConfiguration.DatasourceType.values()).map(DatasourceConfiguration.DatasourceType::getType).toList().contains(value.getType())) { + DatasourceConfiguration.DatasourceType datasourceType = DatasourceConfiguration.DatasourceType.valueOf(value.getType()); + prefix = datasourceType.getPrefix(); + suffix = datasourceType.getSuffix(); + } else { + if (LicenseUtil.licenseValid()) { + List xpackPluginsDatasourceVOS = pluginManage.queryPluginDs(); + List list = xpackPluginsDatasourceVOS.stream().filter(ele -> StringUtils.equals(ele.getType(), value.getType())).toList(); + if (ObjectUtils.isNotEmpty(list)) { + XpackPluginsDatasourceVO first = list.getFirst(); + prefix = first.getPrefix(); + suffix = first.getSuffix(); + } else { + DEException.throwException("当前数据源插件不存在"); + } + } + } + + Pattern patternCross = Pattern.compile("(`.*?`)"); + Matcher matcherCross = patternCross.matcher(sql); + while (matcherCross.find()) { + String group = matcherCross.group(); + String info = group.substring(1, group.length() - 1); + sql = sql.replaceAll(group, prefix + info + suffix); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + return sql; + } + + private static boolean isParams(String paramId) { + if (Arrays.asList("sysParams.userId", "sysParams.userEmail", "sysParams.userName").contains(paramId)) { + return true; + } + boolean isLong = false; + try { + Long.valueOf(paramId); + isLong = true; + } catch (Exception e) { + isLong = false; + } + if (paramId.length() >= 18 && isLong) { + return true; + } + return false; + } + + + private String transFilter(SqlVariableDetails sqlVariableDetails, Map dsMap) { + if (sqlVariableDetails.getOperator().equals("in")) { + if (StringUtils.equalsIgnoreCase(dsMap.entrySet().iterator().next().getValue().getType(), DatasourceConfiguration.DatasourceType.sqlServer.getType()) && sqlVariableDetails.getDeType() == 0) { + return "N'" + String.join("', N'", sqlVariableDetails.getValue()) + "'"; + } else { + if (sqlVariableDetails.getDeType() == 2 || sqlVariableDetails.getDeType() == 3) { + return String.join(",", sqlVariableDetails.getValue()); + } else { + return "'" + String.join("','", sqlVariableDetails.getValue()) + "'"; + } + } + } else if (sqlVariableDetails.getOperator().equals("between")) { + if (sqlVariableDetails.getDeType() == 1) { + SimpleDateFormat simpleDateFormat = new SimpleDateFormat(sqlVariableDetails.getType().size() > 1 ? (String) sqlVariableDetails.getType().get(1).replace("DD", "dd").replace("YYYY", "yyyy") : "yyyy"); + if (StringUtils.endsWith(sqlVariableDetails.getId(), START_END_SEPARATOR)) { + return simpleDateFormat.format(new Date(Long.parseLong((String) sqlVariableDetails.getValue().get(1)))); + } else { + return simpleDateFormat.format(new Date(Long.parseLong((String) sqlVariableDetails.getValue().get(0)))); + } + } else { + if (StringUtils.endsWith(sqlVariableDetails.getId(), START_END_SEPARATOR)) { + return sqlVariableDetails.getValue().get(1); + } else { + return sqlVariableDetails.getValue().get(0); + } + } + } else { + return (String) sqlVariableDetails.getValue().get(0); + } + + } + + private String handleSubstitutedSql(String sysVariableId) { + if (userEntity != null) { + if (sysVariableId.equalsIgnoreCase("sysParams.userId")) { + return userEntity.getAccount(); + } + if (sysVariableId.equalsIgnoreCase("sysParams.userEmail")) { + return userEntity.getEmail(); + } + if (sysVariableId.equalsIgnoreCase("sysParams.userName")) { + return userEntity.getName(); + } + for (SysVariableValueItem variable : userEntity.getVariables()) { + if (!variable.isValid()) { + continue; + } + if (!sysVariableId.equalsIgnoreCase(variable.getVariableId().toString())) { + continue; + } + if (variable.getSysVariableDto().getType().equalsIgnoreCase("text")) { + for (SysVariableValueDto sysVariableValueDto : variable.getValueList()) { + if (variable.getVariableValueIds().contains(sysVariableValueDto.getId().toString())) { + return sysVariableValueDto.getValue(); + } + } + } else { + return variable.getVariableValue(); + } + } + return null; + } else { + return null; + } + } + + + private String handleSubstitutedSqlForIn(String sysVariableId) { + if (userEntity != null) { + for (SysVariableValueItem variable : userEntity.getVariables()) { + List values = new ArrayList<>(); + if (!variable.isValid()) { + continue; + } + if (!sysVariableId.equalsIgnoreCase(variable.getVariableId().toString())) { + continue; + } + if (variable.getSysVariableDto().getType().equalsIgnoreCase("text")) { + for (SysVariableValueDto sysVariableValueDto : variable.getValueList()) { + if (variable.getVariableValueIds().contains(sysVariableValueDto.getId().toString())) { + values.add(sysVariableValueDto.getValue()); + } + } + } + if (CollectionUtils.isNotEmpty(values)) { + return "'" + String.join("','", values) + "'"; + } + } + return null; + } else { + return null; + } + } +} + + + diff --git a/core/core-backend/src/main/java/io/dataease/commons/utils/SqlparserUtils.java b/core/core-backend/src/main/java/io/dataease/commons/utils/SqlparserUtils.java index 9a35e303db..7aed2887ff 100644 --- a/core/core-backend/src/main/java/io/dataease/commons/utils/SqlparserUtils.java +++ b/core/core-backend/src/main/java/io/dataease/commons/utils/SqlparserUtils.java @@ -22,7 +22,6 @@ import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.statement.Statement; import net.sf.jsqlparser.statement.select.*; import net.sf.jsqlparser.util.deparser.ExpressionDeParser; -import net.sf.jsqlparser.util.deparser.SelectDeParser; import org.apache.calcite.sql.*; import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.util.SqlShuttle; @@ -30,7 +29,6 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.checkerframework.checker.nullness.qual.Nullable; -import org.junit.jupiter.params.provider.CsvSource; import java.text.SimpleDateFormat; import java.util.*; @@ -50,8 +48,15 @@ public class SqlparserUtils { boolean hasVariables = false; private UserFormVO userEntity; private final List> sysParams = new ArrayList<>(); + private static final String deVariablePattern = "\\$DE_PARAM\\{(.*?)\\}"; public String handleVariableDefaultValue(String sql, String sqlVariableDetails, boolean isEdit, boolean isFromDataSet, List parameters, boolean isCross, Map dsMap, PluginManageApi pluginManage, UserFormVO userEntity) { + Pattern r = Pattern.compile(deVariablePattern); + Matcher m = r.matcher(sql); + if (m.find()) { + return new DeSqlparserUtils().handleVariableDefaultValue(sql, sqlVariableDetails, isEdit, isFromDataSet, parameters, isCross, dsMap, pluginManage, userEntity); + } + DatasourceSchemaDTO ds = dsMap.entrySet().iterator().next().getValue(); if (StringUtils.isEmpty(sql)) { DEException.throwException(Translator.get("i18n_sql_not_empty")); @@ -160,6 +165,7 @@ public class SqlparserUtils { } return false; } + private String removeVariables(final String sql, String dsType) throws Exception { String tmpSql = sql.replaceAll("(?m)^\\s*$[\n\r]{0,}", ""); Pattern pattern = Pattern.compile(regex); diff --git a/core/core-frontend/src/custom-component/v-query/Component.vue b/core/core-frontend/src/custom-component/v-query/Component.vue index 5bcf8eaf10..7d09d314b1 100644 --- a/core/core-frontend/src/custom-component/v-query/Component.vue +++ b/core/core-frontend/src/custom-component/v-query/Component.vue @@ -64,7 +64,8 @@ const { element, view, scale } = toRefs(props) const { t } = useI18n() const vQueryRef = ref() const dvMainStore = dvMainStoreWithOut() -const { curComponent, canvasViewInfo, mobileInPc, firstLoadMap, editMode } = storeToRefs(dvMainStore) +const { curComponent, canvasViewInfo, mobileInPc, firstLoadMap, editMode } = + storeToRefs(dvMainStore) const canEdit = ref(false) const queryConfig = ref() const defaultStyle = { @@ -865,7 +866,9 @@ const autoStyle = computed(() => {
{ const parseVariable = () => { state.variablesTmp = [] - const reg = new RegExp('\\${(.*?)}', 'gim') - const match = codeCom.value.state.doc.toString().match(reg) - const names = [] - if (match !== null) { - for (let index = 0; index < match.length; index++) { - let name = match[index].substring(2, match[index].length - 1) - if (names.indexOf(name) < 0) { - names.push(name) - // eslint-disable-next-line - let obj = undefined - for (let i = 0; i < state.variables?.length; i++) { - if (state.variables[i].variableName === name) { - obj = state.variables[i] - if (!obj.hasOwnProperty('defaultValueScope')) { - obj.defaultValueScope = 'EDIT' + const variableReg = new RegExp('\\$DE_PARAM{(.*?)}', 'gim') + const variableMatch = codeCom.value.state.doc.toString().match(variableReg) + if (variableMatch !== null) { + const names = [] + const reg = new RegExp('\\$\\[[^\\]]+\\]', 'gim') + for (let index = 0; index < variableMatch.length; index++) { + let sqlItem = variableMatch[index].substring(10, variableMatch[index].length - 1) + const match = sqlItem.match(reg) + if (match !== null) { + let name = match[0].substring(2, match[0].length - 1) + if (names.indexOf(name) < 0) { + names.push(name) + let obj = undefined + for (let i = 0; i < state.variables?.length; i++) { + if (state.variables[i].variableName === name) { + obj = state.variables[i] + if (!obj.hasOwnProperty('defaultValueScope')) { + obj.defaultValueScope = 'EDIT' + } } } - } - if (obj === undefined) { - obj = { - variableName: name, - alias: '', - type: [], - required: false, - defaultValue: '', - details: '', - defaultValueScope: 'EDIT' + if (obj === undefined) { + obj = { + variableName: name, + alias: '', + type: [], + required: false, + defaultValue: '', + details: '', + defaultValueScope: 'EDIT' + } + obj.type.push('TEXT') } - obj.type.push('TEXT') + state.variablesTmp.push(obj) + } + } + } + } else { + const reg = new RegExp('\\${(.*?)}', 'gim') + const match = codeCom.value.state.doc.toString().match(reg) + const names = [] + if (match !== null) { + for (let index = 0; index < match.length; index++) { + let name = match[index].substring(2, match[index].length - 1) + if (names.indexOf(name) < 0) { + names.push(name) + let obj = undefined + for (let i = 0; i < state.variables?.length; i++) { + if (state.variables[i].variableName === name) { + obj = state.variables[i] + if (!obj.hasOwnProperty('defaultValueScope')) { + obj.defaultValueScope = 'EDIT' + } + } + } + if (obj === undefined) { + obj = { + variableName: name, + alias: '', + type: [], + required: false, + defaultValue: '', + details: '', + defaultValueScope: 'EDIT' + } + obj.type.push('TEXT') + } + state.variablesTmp.push(obj) } - state.variablesTmp.push(obj) } } }