public/dataease
1
0
Fork 0
mirror of https://github.com/dataease/dataease.git synced 2026-06-15 02:21:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into main

Browse Source 
# Conflicts:
#	backend/src/main/java/io/dataease/auth/service/impl/ShiroServiceImpl.java
This commit is contained in:
wangjiahao
2021-03-12 16:10:18 +08:00
parent 8b80d8fa72 086ab38806
commit 13bb7663f8
97 changed files with 2758 additions and 1089 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
backend/src/main/java/io/dataease/auth/config/F2CRealm.java
View File

@@ -13,8 +13,9 @@ import org.apache.shiro.authz.AuthorizationInfo;
import org.apache.shiro.authz.SimpleAuthorizationInfo;
import org.apache.shiro.realm.AuthorizingRealm;
import org.apache.shiro.subject.PrincipalCollection;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.Set;
import java.util.stream.Collectors;
@@ -23,7 +24,8 @@ import java.util.stream.Collectors;
@Component
public class F2CRealm extends AuthorizingRealm {
@Resource
@Autowired
@Lazy //shiro组件加载过早 让authUserService等一等再注入 否则 注入的可能不是代理对象
private AuthUserService authUserService;
@@ -36,7 +38,6 @@ public class F2CRealm extends AuthorizingRealm {
@Override
protected AuthorizationInfo doGetAuthorizationInfo(PrincipalCollection principals) {
Long userId = JWTUtils.tokenInfoByToken(principals.toString()).getUserId();
//SysUserEntity user = authUserService.getUserById(userId);
SimpleAuthorizationInfo simpleAuthorizationInfo = new SimpleAuthorizationInfo();
Set<String> role = authUserService.roles(userId).stream().collect(Collectors.toSet());
simpleAuthorizationInfo.addRoles(role);

17
backend/src/main/java/io/dataease/auth/config/WhitelistConfig.java Normal file
View File

@@ -0,0 +1,17 @@
package io.dataease.auth.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import java.util.List;
@Configuration
@ConfigurationProperties(prefix = "dataease")
@Data
public class WhitelistConfig {
private List<String> whitelist;
}

35
backend/src/main/java/io/dataease/auth/filter/JWTFilter.java
View File

@@ -6,6 +6,9 @@ import io.dataease.auth.entity.TokenInfo;
import io.dataease.auth.service.AuthUserService;
import io.dataease.auth.util.JWTUtils;
import io.dataease.commons.utils.CommonBeanFactory;
import io.dataease.commons.utils.ServletUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.shiro.authc.AuthenticationException;
import org.apache.shiro.subject.Subject;
import org.apache.shiro.web.filter.authc.BasicHttpAuthenticationFilter;
import org.slf4j.Logger;
@@ -23,6 +26,8 @@ public class JWTFilter extends BasicHttpAuthenticationFilter {
private Logger LOGGER = LoggerFactory.getLogger(this.getClass());
public final static String expireMessage = "login token is expire";
/*@Autowired
private AuthUserService authUserService;*/
@@ -45,7 +50,10 @@ public class JWTFilter extends BasicHttpAuthenticationFilter {
protected boolean executeLogin(ServletRequest request, ServletResponse response) throws Exception {
HttpServletRequest httpServletRequest = (HttpServletRequest) request;
String authorization = httpServletRequest.getHeader("Authorization");
// 当没有出现登录超时 且需要刷新token 则执行刷新token
if (JWTUtils.loginExpire(authorization)){
throw new AuthenticationException(expireMessage);
}
if (JWTUtils.needRefresh(authorization)){
authorization = refreshToken(request, response);
}
@@ -67,7 +75,12 @@ public class JWTFilter extends BasicHttpAuthenticationFilter {
boolean loginSuccess = executeLogin(request, response);
return loginSuccess;
} catch (Exception e) {
response401(request, response);
if (e instanceof AuthenticationException && StringUtils.equals(e.getMessage(), expireMessage)){
responseExpire(request, response);
}else {
response401(request, response);
}
}
}
return false;
@@ -81,7 +94,13 @@ public class JWTFilter extends BasicHttpAuthenticationFilter {
AuthUserService authUserService = CommonBeanFactory.getBean(AuthUserService.class);
SysUserEntity user = authUserService.getUserById(tokenInfo.getUserId());
String password = user.getPassword();
// 删除老token操作时间
JWTUtils.removeTokenExpire(token);
String newToken = JWTUtils.sign(tokenInfo, password);
// 记录新token操作时间
JWTUtils.addTokenExpire(newToken);
JWTToken jwtToken = new JWTToken(newToken);
this.getSubject(request, response).login(jwtToken);
// 设置响应的Header头新Token
@@ -123,4 +142,16 @@ public class JWTFilter extends BasicHttpAuthenticationFilter {
LOGGER.error(e.getMessage());
}
}
private void responseExpire(ServletRequest req, ServletResponse resp) {
try {
HttpServletResponse httpServletResponse = (HttpServletResponse) resp;
httpServletResponse.addHeader("Access-Control-Expose-Headers", "authentication-status");
httpServletResponse.setHeader("authentication-status", "login_expire");
httpServletResponse.setStatus(401);
} catch (Exception e) {
LOGGER.error(e.getMessage());
}
}
}

5
backend/src/main/java/io/dataease/auth/server/AuthServer.java
View File

@@ -49,6 +49,8 @@ public class AuthServer implements AuthApi {
Map<String,Object> result = new HashMap<>();
TokenInfo tokenInfo = TokenInfo.builder().userId(user.getUserId()).username(username).lastLoginTime(System.currentTimeMillis()).build();
String token = JWTUtils.sign(tokenInfo, realPwd);
// 记录token操作时间
JWTUtils.addTokenExpire(token);
result.put("token", token);
ServletUtils.setToken(token);
return result;
@@ -79,6 +81,9 @@ public class AuthServer implements AuthApi {
@Override
public String test() {
SysUserEntity userById = authUserService.getUserById(4L);
String nickName = userById.getNickName();
System.out.println(nickName);
return "apple";
}
}

74
backend/src/main/java/io/dataease/auth/service/impl/ShiroServiceImpl.java
View File

@@ -1,17 +1,22 @@
package io.dataease.auth.service.impl;
import io.dataease.auth.config.WhitelistConfig;
import io.dataease.auth.service.ShiroService;
import org.apache.commons.collections.CollectionUtils;
import org.apache.shiro.spring.web.ShiroFilterFactoryBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.StringJoiner;
@Service
public class ShiroServiceImpl implements ShiroService {
private final static String ANON = "anon";
@Autowired
private WhitelistConfig whitelistConfig;
@Override
public Map<String, String> loadFilterChainDefinitionMap() {
@@ -20,47 +25,38 @@ public class ShiroServiceImpl implements ShiroService {
// 配置过滤:不会被拦截的链接 -> 放行 start ----------------------------------------------------------
// 放行Swagger2页面，需要放行这些
filterChainDefinitionMap.put("/swagger-ui.html","anon");
filterChainDefinitionMap.put("/swagger-ui/**","anon");
filterChainDefinitionMap.put("/swagger/**","anon");
filterChainDefinitionMap.put("/webjars/**", "anon");
filterChainDefinitionMap.put("/swagger-resources/**","anon");
filterChainDefinitionMap.put("/v2/**","anon");
filterChainDefinitionMap.put("/v3/**","anon");
filterChainDefinitionMap.put("/static/**", "anon");
filterChainDefinitionMap.put("/common-files/**", "anon");
// filterChainDefinitionMap.put("/401", "anon");
// filterChainDefinitionMap.put("/404", "anon");
// 登陆
// filterChainDefinitionMap.put("/api/auth/logout", "anon");
filterChainDefinitionMap.put("/api/auth/login", "anon");
// 退出
// 放行未授权接口，重定向使用
filterChainDefinitionMap.put("/unauth", "anon");
filterChainDefinitionMap.put("/display/**", "anon");
// token过期接口
filterChainDefinitionMap.put("/tokenExpired", "anon");
// 被挤下线
filterChainDefinitionMap.put("/downline", "anon");
// 放行 end ----------------------------------------------------------
/*List<ExtPermissionBean> extPermissionBeans = extUserMapper.getPermissions();
extPermissionBeans.forEach(item -> {
StringJoiner f2cPerms = new StringJoiner(",", "f2cPerms[", "]");
f2cPerms.add(item.getPermission());
filterChainDefinitionMap.put(item.getPath(), "jwt," + f2cPerms);
filterChainDefinitionMap.put("/swagger-ui.html",ANON);
filterChainDefinitionMap.put("/swagger-ui/**",ANON);
filterChainDefinitionMap.put("/swagger/**",ANON);
filterChainDefinitionMap.put("/webjars/**", ANON);
filterChainDefinitionMap.put("/swagger-resources/**",ANON);
filterChainDefinitionMap.put("/v2/**",ANON);
filterChainDefinitionMap.put("/v3/**",ANON);
filterChainDefinitionMap.put("/static/**", ANON);
filterChainDefinitionMap.put("/css/**", ANON);
filterChainDefinitionMap.put("/js/**", ANON);
filterChainDefinitionMap.put("/img/**", ANON);
filterChainDefinitionMap.put("/fonts/**", ANON);
filterChainDefinitionMap.put("/favicon.ico", ANON);
filterChainDefinitionMap.put("/", ANON);
filterChainDefinitionMap.put("/index.html", ANON);
filterChainDefinitionMap.put("/api/auth/login", ANON);
filterChainDefinitionMap.put("/unauth", ANON);
filterChainDefinitionMap.put("/display/**", ANON);
filterChainDefinitionMap.put("/tokenExpired", ANON);
filterChainDefinitionMap.put("/downline", ANON);
List<String> whitelist = whitelistConfig.getWhitelist();
if (CollectionUtils.isNotEmpty(whitelist))
whitelist.forEach(path -> {
filterChainDefinitionMap.put(path, ANON);
});
*/
filterChainDefinitionMap.put("/api/auth/logout", "logout");
filterChainDefinitionMap.put("/**", "jwt");
return filterChainDefinitionMap;
}
@Override
public void updatePermission(ShiroFilterFactoryBean shiroFilterFactoryBean, Integer roleId, Boolean isRemoveSession) {

61
backend/src/main/java/io/dataease/auth/util/JWTUtils.java
View File

@@ -6,20 +6,23 @@ import com.auth0.jwt.algorithms.Algorithm;
import com.auth0.jwt.exceptions.JWTDecodeException;
import com.auth0.jwt.interfaces.DecodedJWT;
import io.dataease.auth.entity.TokenInfo;
import io.dataease.commons.utils.ServletUtils;
import io.dataease.auth.filter.JWTFilter;
import io.dataease.commons.utils.CommonBeanFactory;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import javax.servlet.http.HttpServletResponse;
import org.apache.shiro.authc.AuthenticationException;
import org.springframework.cache.Cache;
import org.springframework.cache.CacheManager;
import java.util.Date;
public class JWTUtils {
// token过期时间5min (过期会自动刷新续命 目的是避免一直都是同一个token )
private static final long EXPIRE_TIME = 1*60*1000;
// token过期时间1min (过期会自动刷新续命 目的是避免一直都是同一个token )
private static final long EXPIRE_TIME = 1*60*1000/2;
// 登录间隔时间10min 超过这个时间强制重新登录
private static final long Login_Interval = 10*60*1000;
private static final long Login_Interval = 20*60*1000;
/**
@@ -35,16 +38,12 @@ public class JWTUtils {
.withClaim("username", tokenInfo.getUsername())
.withClaim("userId", tokenInfo.getUserId())
.build();
DecodedJWT jwt = verifier.verify(token);
Long lastLoginTime = jwt.getClaim("lastLoginTime").asLong();
long now = System.currentTimeMillis();
if (now - lastLoginTime > Login_Interval){
verifier.verify(token);
if (loginExpire(token)){
// 登录超时
HttpServletResponse response = ServletUtils.response();
response.addHeader("Access-Control-Expose-Headers", "authentication-status");
response.setHeader("authentication-status", "login_expire");
throw new AuthenticationException(JWTFilter.expireMessage);
// 前端拦截 登录超时状态 直接logout
return false;
//return false;
}
return true;
}
@@ -74,6 +73,17 @@ public class JWTUtils {
return new Date().getTime() >= exp.getTime();
}
/**
* 当前token是否登录超时
* @param token
* @return
*/
public static boolean loginExpire(String token){
Long now = System.currentTimeMillis();
Long lastOperateTime = tokenLastOperateTime(token);
return now - lastOperateTime > Login_Interval;
}
public static Date getExp(String token) {
try {
DecodedJWT jwt = JWT.decode(token);
@@ -106,4 +116,29 @@ public class JWTUtils {
return null;
}
}
/**
* 获取当前token上次操作时间
* @param token
* @return
*/
public static Long tokenLastOperateTime(String token){
CacheManager cacheManager = CommonBeanFactory.getBean(CacheManager.class);
Cache tokens_expire = cacheManager.getCache("tokens_expire");
Long expTime = tokens_expire.get(token, Long.class);
return expTime;
}
public static void removeTokenExpire(String token){
CacheManager cacheManager = CommonBeanFactory.getBean(CacheManager.class);
Cache tokens_expire = cacheManager.getCache("tokens_expire");
tokens_expire.evict(token);
}
public static void addTokenExpire(String token){
CacheManager cacheManager = CommonBeanFactory.getBean(CacheManager.class);
Cache tokens_expire = cacheManager.getCache("tokens_expire");
long now = System.currentTimeMillis();
tokens_expire.put(token, now);
}
}

5
backend/src/main/java/io/dataease/base/domain/DatasetTableField.java
View File

@@ -1,9 +1,12 @@
package io.dataease.base.domain;
import java.io.Serializable;
import lombok.Builder;
import lombok.Data;
@Data
@Builder
public class DatasetTableField implements Serializable {
private String id;
@@ -24,4 +27,4 @@ public class DatasetTableField implements Serializable {
private Integer deType;
private static final long serialVersionUID = 1L;
}
}

12
backend/src/main/java/io/dataease/base/mapper/ext/ExtDeptMapper.java
View File

@@ -1,9 +1,8 @@
package io.dataease.base.mapper.ext;
import org.apache.ibatis.annotations.Delete;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Update;
import io.dataease.base.mapper.ext.query.GridExample;
import io.dataease.controller.sys.request.SimpleTreeNode;
import org.apache.ibatis.annotations.*;
import java.util.List;
@@ -22,4 +21,9 @@ public interface ExtDeptMapper {
" <foreach collection='ids' item='id' open='(' separator=',' close=')'>#{id}</foreach> " +
"</script>")
int batchDelete(@Param("ids") List<Long> ids);
List<SimpleTreeNode> allNodes();
List<SimpleTreeNode> nodesByExample(GridExample example);
}

21
backend/src/main/java/io/dataease/base/mapper/ext/ExtDeptMapper.xml Normal file
View File

@@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="io.dataease.base.mapper.ext.ExtDeptMapper">
<resultMap id="simpleNode" type="io.dataease.controller.sys.request.SimpleTreeNode">
<id property="id" column="id" javaType="java.lang.Long" />
<result property="pid" column="pid" javaType="java.lang.Long"/>
</resultMap>
<select id="allNodes" resultMap="simpleNode">
select dept_id as id, pid from sys_dept
</select>
<select id="nodesByExample" resultType="io.dataease.base.mapper.ext.query.GridExample" resultMap="simpleNode">
select dept_id as id, pid from sys_dept
<include refid="io.dataease.base.mapper.ext.query.GridSql.gridCondition" />
</select>
</mapper>

3
backend/src/main/java/io/dataease/base/mapper/ext/ExtSysRoleMapper.java
View File

@@ -1,6 +1,7 @@
package io.dataease.base.mapper.ext;
import io.dataease.base.domain.SysRole;
import io.dataease.base.mapper.ext.query.GridExample;
import io.dataease.controller.sys.request.RoleGridRequest;
import io.dataease.controller.sys.response.RoleUserItem;
import org.apache.ibatis.annotations.Param;
@@ -12,7 +13,7 @@ import java.util.Map;
public interface ExtSysRoleMapper {
List<SysRole> query(@Param("request")RoleGridRequest request);
List<SysRole> query(GridExample example);
int deleteRoleMenu(@Param("roleId") Long roleId);

18
backend/src/main/java/io/dataease/base/mapper/ext/ExtSysRoleMapper.xml
View File

@@ -9,15 +9,19 @@
<result property="name" column="name"/>
</resultMap>
<select id="query" resultMap="io.dataease.base.mapper.SysRoleMapper.BaseResultMap">
<select id="query" parameterType="io.dataease.base.mapper.ext.query.GridExample" resultMap="io.dataease.base.mapper.SysRoleMapper.BaseResultMap">
select r.*
from sys_role r
<where>
<if test="request.name != null">
AND r.name like CONCAT('%', #{request.name},'%')
</if>
</where>
order by r.update_time desc
<if test="_parameter != null">
<include refid="io.dataease.base.mapper.ext.query.GridSql.gridCondition" />
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
<if test="orderByClause == null">
order by r.update_time desc
</if>
</select>

4
backend/src/main/java/io/dataease/base/mapper/ext/ExtSysUserMapper.java
View File

@@ -1,5 +1,7 @@
package io.dataease.base.mapper.ext;
import io.dataease.base.mapper.ext.query.GridExample;
import io.dataease.controller.sys.base.BaseGridRequest;
import io.dataease.controller.sys.request.UserGridRequest;
import io.dataease.controller.sys.response.SysUserGridResponse;
import org.apache.ibatis.annotations.Param;
@@ -8,5 +10,5 @@ import java.util.List;
public interface ExtSysUserMapper {
List<SysUserGridResponse> query(@Param("request")UserGridRequest request);
List<SysUserGridResponse> query(GridExample example);
}

21
backend/src/main/java/io/dataease/base/mapper/ext/ExtSysUserMapper.xml
View File

@@ -27,17 +27,24 @@
</collection>
</resultMap>
<select id="query" resultMap="BaseResultMap">
<select id="query" parameterType="io.dataease.base.mapper.ext.query.GridExample" resultMap="BaseResultMap">
select u.*,u.user_id as id, r.role_id,r.name as role_name , d.pid, d.name as dept_name
from sys_user u left join sys_users_roles ur on u.user_id = ur.user_id
left join sys_role r on r.role_id = ur.role_id
left join sys_dept d on d.dept_id = u.dept_id
<where>
<if test="request.name != null">
AND u.name like CONCAT('%', #{request.name},'%')
</if>
</where>
order by u.update_time desc
<if test="_parameter != null">
<include refid="io.dataease.base.mapper.ext.query.GridSql.gridCondition" />
</if>
<if test="orderByClause != null">
order by ${orderByClause}
</if>
<if test="orderByClause == null">
order by u.update_time desc
</if>
</select>

259
backend/src/main/java/io/dataease/base/mapper/ext/query/GridExample.java Normal file
View File

@@ -0,0 +1,259 @@
package io.dataease.base.mapper.ext.query;
import io.dataease.controller.sys.base.ConditionEntity;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.List;
public class GridExample {
protected String orderByClause;
protected boolean distinct;
protected List<Criteria> oredCriteria;
public GridExample() {
oredCriteria = new ArrayList<Criteria>();
}
public void setOrderByClause(String orderByClause) {
this.orderByClause = orderByClause;
}
public String getOrderByClause() {
return orderByClause;
}
public void setDistinct(boolean distinct) {
this.distinct = distinct;
}
public boolean isDistinct() {
return distinct;
}
public List<Criteria> getOredCriteria() {
return oredCriteria;
}
public void or(Criteria criteria) {
oredCriteria.add(criteria);
}
public Criteria or() {
Criteria criteria = createCriteriaInternal();
oredCriteria.add(criteria);
return criteria;
}
public Criteria createCriteria() {
Criteria criteria = createCriteriaInternal();
if (oredCriteria.size() == 0) {
oredCriteria.add(criteria);
}
return criteria;
}
protected Criteria createCriteriaInternal() {
Criteria criteria = new Criteria();
return criteria;
}
public void clear() {
oredCriteria.clear();
orderByClause = null;
distinct = false;
}
protected abstract static class GeneratedCriteria {
protected List<Criterion> criteria;
protected GeneratedCriteria() {
super();
criteria = new ArrayList<Criterion>();
}
public boolean isValid() {
return criteria.size() > 0;
}
public List<Criterion> getAllCriteria() {
return criteria;
}
public List<Criterion> getCriteria() {
return criteria;
}
protected void addCriterion(String condition) {
if (condition == null) {
throw new RuntimeException("Value for condition cannot be null");
}
criteria.add(new Criterion(condition));
}
protected void addCriterion(String condition, Object value, String property) {
if (value == null) {
throw new RuntimeException("Value for " + property + " cannot be null");
}
criteria.add(new Criterion(condition, value));
}
protected void addCriterion(String condition, Object value1, Object value2, String property) {
if (value1 == null || value2 == null) {
throw new RuntimeException("Between values for " + property + " cannot be null");
}
criteria.add(new Criterion(condition, value1, value2));
}
public Criteria addCondtion(ConditionEntity conditionEntity){
String field = conditionEntity.getField();
Object value = conditionEntity.getValue();
String operator = conditionEntity.getOperator();
if (StringUtils.isEmpty(operator))
operator = "like";
switch (operator){
case "eq":
addCriterion(field+" = ", value, field);
break;
case "ne":
addCriterion(field+" <> ", value, field);
break;
case "like":
addCriterion(field+" like ", "%"+value+"%", field);
break;
case "not like":
addCriterion(field+" not like ", value, field);
break;
case "in":
List<Object> invalues = (List<Object>)value;
addCriterion(field+" in", invalues, field);
break;
case "not in":
List<Object> notinvalues = (List<Object>)value;
addCriterion(field+" not in", notinvalues, field);
break;
case "between":
List<Object> values = (List<Object>)value;
Object v1 = values.get(0);
Object v2 = values.get(1);
addCriterion(field+" between", v1, v2, field);
break;
case "gt":
addCriterion(field+" > ", value, field);
break;
case "ge":
addCriterion(field+" >= ", value, field);
break;
case "lt":
addCriterion(field+" < ", value, field);
break;
case "le":
addCriterion(field+" <= ", value, field);
break;
}
return (Criteria) this;
}
}
public static class Criteria extends GeneratedCriteria {
protected Criteria() {
super();
}
}
public static class Criterion {
private String condition;
private Object value;
private Object secondValue;
private boolean noValue;
private boolean singleValue;
private boolean betweenValue;
private boolean listValue;
private String typeHandler;
public String getCondition() {
return condition;
}
public Object getValue() {
return value;
}
public Object getSecondValue() {
return secondValue;
}
public boolean isNoValue() {
return noValue;
}
public boolean isSingleValue() {
return singleValue;
}
public boolean isBetweenValue() {
return betweenValue;
}
public boolean isListValue() {
return listValue;
}
public String getTypeHandler() {
return typeHandler;
}
protected Criterion(String condition) {
super();
this.condition = condition;
this.typeHandler = null;
this.noValue = true;
}
protected Criterion(String condition, Object value, String typeHandler) {
super();
this.condition = condition;
this.value = value;
this.typeHandler = typeHandler;
if (value instanceof List<?>) {
this.listValue = true;
} else {
this.singleValue = true;
}
}
protected Criterion(String condition, Object value) {
this(condition, value, null);
}
protected Criterion(String condition, Object value, Object secondValue, String typeHandler) {
super();
this.condition = condition;
this.value = value;
this.secondValue = secondValue;
this.typeHandler = typeHandler;
this.betweenValue = true;
}
protected Criterion(String condition, Object value, Object secondValue) {
this(condition, value, secondValue, null);
}
}
}

4
backend/src/main/java/io/dataease/base/mapper/ext/query/GridSql.java Normal file
View File

@@ -0,0 +1,4 @@
package io.dataease.base.mapper.ext.query;
public interface GridSql {
}

35
backend/src/main/java/io/dataease/base/mapper/ext/query/GridSql.xml Normal file
View File

@@ -0,0 +1,35 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="io.dataease.base.mapper.ext.query.GridSql">
<sql id="gridCondition">
<where>
<foreach collection="oredCriteria" item="criteria" separator="or">
<if test="criteria.valid">
<trim prefix="(" prefixOverrides="and" suffix=")">
<foreach collection="criteria.criteria" item="criterion">
<choose>
<when test="criterion.noValue">
and ${criterion.condition}
</when>
<when test="criterion.singleValue">
and ${criterion.condition} #{criterion.value}
</when>
<when test="criterion.betweenValue">
and ${criterion.condition} #{criterion.value} and #{criterion.secondValue}
</when>
<when test="criterion.listValue">
and ${criterion.condition}
<foreach close=")" collection="criterion.value" item="listItem" open="(" separator=",">
#{listItem}
</foreach>
</when>
</choose>
</foreach>
</trim>
</if>
</foreach>
</where>
</sql>
</mapper>

6
backend/src/main/java/io/dataease/commons/constants/DatasetMode.java Normal file
View File

@@ -0,0 +1,6 @@
package io.dataease.commons.constants;
public class DatasetMode {
public static final String EXTRACT = "1";
public static final String DIRECT = "0";
}

5
backend/src/main/java/io/dataease/commons/constants/JobStatus.java Normal file
View File

@@ -0,0 +1,5 @@
package io.dataease.commons.constants;
public enum JobStatus {
Prepare, Underway, Completed, Error
}

5
backend/src/main/java/io/dataease/commons/constants/TestPlanStatus.java
View File

@@ -1,5 +0,0 @@
package io.dataease.commons.constants;
public enum TestPlanStatus {
Prepare, Underway, Completed
}

30
backend/src/main/java/io/dataease/config/HbaseConfig.java Normal file
View File

@@ -0,0 +1,30 @@
package io.dataease.config;
import com.fit2cloud.autoconfigure.QuartzAutoConfiguration;
import org.springframework.boot.autoconfigure.AutoConfigureBefore;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.DependsOn;
import org.springframework.core.env.Environment;
import javax.annotation.Resource;
@Configuration
@AutoConfigureBefore(QuartzAutoConfiguration.class)
public class HbaseConfig {
@Resource
private Environment env; // 保存了配置文件的信息
@Bean
@ConditionalOnMissingBean
public org.apache.hadoop.conf.Configuration configuration(){
org.apache.hadoop.conf.Configuration configuration = new org.apache.hadoop.conf.Configuration();
configuration.set("hbase.zookeeper.quorum", env.getProperty("hbase.zookeeper.quorum"));
configuration.set("hbase.zookeeper.property.clientPort", env.getProperty("hbase.zookeeper.property.clientPort"));
configuration.set("hbase.client.retries.number", env.getProperty("hbase.client.retries.number", "1"));
return configuration;
}
}

2
backend/src/main/java/io/dataease/controller/dataset/DataSetTableFieldController.java
View File

@@ -19,7 +19,7 @@ public class DataSetTableFieldController {
@PostMapping("list/{tableId}")
public List<DatasetTableField> list(@PathVariable String tableId) {
DatasetTableField datasetTableField = new DatasetTableField();
DatasetTableField datasetTableField = DatasetTableField.builder().build();
datasetTableField.setTableId(tableId);
return dataSetTableFieldsService.list(datasetTableField);
}

14
backend/src/main/java/io/dataease/controller/sys/SysDeptController.java
View File

@@ -3,6 +3,7 @@ package io.dataease.controller.sys;
import io.dataease.base.domain.SysDept;
import io.dataease.commons.utils.BeanUtils;
import io.dataease.controller.ResultHolder;
import io.dataease.controller.sys.base.BaseGridRequest;
import io.dataease.controller.sys.request.DeptCreateRequest;
import io.dataease.controller.sys.request.DeptDeleteRequest;
import io.dataease.controller.sys.request.DeptStatusRequest;
@@ -39,6 +40,19 @@ public class SysDeptController extends ResultHolder {
return nodeResponses;
}
@PostMapping("/search")
public List<DeptNodeResponse> search(@RequestBody BaseGridRequest request){
List<SysDept> nodes = deptService.nodesTreeByCondition(request);
//List<SysDept> nodes = deptService.nodesByPid(pid);
List<DeptNodeResponse> nodeResponses = nodes.stream().map(node -> {
DeptNodeResponse deptNodeResponse = BeanUtils.copyBean(new DeptNodeResponse(), node);
deptNodeResponse.setHasChildren(node.getSubCount() > 0);
deptNodeResponse.setTop(node.getPid() == deptService.DEPT_ROOT_PID);
return deptNodeResponse;
}).collect(Collectors.toList());
return nodeResponses;
}
@ApiOperation("查询部门")
@PostMapping("/root")
public ResultHolder rootData(){

3
backend/src/main/java/io/dataease/controller/sys/SysRoleController.java
View File

@@ -6,6 +6,7 @@ import com.github.pagehelper.PageHelper;
import io.dataease.base.domain.SysRole;
import io.dataease.commons.utils.PageUtils;
import io.dataease.commons.utils.Pager;
import io.dataease.controller.sys.base.BaseGridRequest;
import io.dataease.controller.sys.request.RoleGridRequest;
import io.dataease.controller.sys.request.RoleMenusRequest;
import io.dataease.controller.sys.response.RoleUserItem;
@@ -49,7 +50,7 @@ public class SysRoleController {
@ApiOperation("查询角色")
@PostMapping("/roleGrid/{goPage}/{pageSize}")
public Pager<List<SysRole>> roleGrid(@PathVariable int goPage, @PathVariable int pageSize, @RequestBody RoleGridRequest request) {
public Pager<List<SysRole>> roleGrid(@PathVariable int goPage, @PathVariable int pageSize, @RequestBody BaseGridRequest request) {
Page<Object> page = PageHelper.startPage(goPage, pageSize, true);
Pager<List<SysRole>> listPager = PageUtils.setPageInfo(page, sysRoleService.query(request));
return listPager;

7
backend/src/main/java/io/dataease/controller/sys/SysUserController.java
View File

@@ -5,6 +5,7 @@ import com.github.pagehelper.Page;
import com.github.pagehelper.PageHelper;
import io.dataease.commons.utils.PageUtils;
import io.dataease.commons.utils.Pager;
import io.dataease.controller.sys.base.BaseGridRequest;
import io.dataease.controller.sys.request.SysUserCreateRequest;
import io.dataease.controller.sys.request.SysUserPwdRequest;
import io.dataease.controller.sys.request.SysUserStateRequest;
@@ -27,10 +28,14 @@ public class SysUserController {
@ApiOperation("查询用户")
@PostMapping("/userGrid/{goPage}/{pageSize}")
public Pager<List<SysUserGridResponse>> userGrid(@PathVariable int goPage, @PathVariable int pageSize, @RequestBody UserGridRequest request) {
public Pager<List<SysUserGridResponse>> userGrid(@PathVariable int goPage, @PathVariable int pageSize, @RequestBody BaseGridRequest request) {
Page<Object> page = PageHelper.startPage(goPage, pageSize, true);
return PageUtils.setPageInfo(page, sysUserService.query(request));
}
/*public Pager<List<SysUserGridResponse>> userGrid(@PathVariable int goPage, @PathVariable int pageSize, @RequestBody UserGridRequest request) {
Page<Object> page = PageHelper.startPage(goPage, pageSize, true);
return PageUtils.setPageInfo(page, sysUserService.query(request));
}*/
@ApiOperation("创建用户")
@PostMapping("/create")

24
backend/src/main/java/io/dataease/controller/sys/base/BaseGridRequest.java Normal file
View File

@@ -0,0 +1,24 @@
package io.dataease.controller.sys.base;
import io.dataease.base.mapper.ext.query.GridExample;
import lombok.Data;
import org.apache.commons.collections.CollectionUtils;
import java.io.Serializable;
import java.util.List;
@Data
public class BaseGridRequest implements Serializable {
private List<ConditionEntity> conditions;
public GridExample convertExample(){
GridExample gridExample = new GridExample();
if (CollectionUtils.isEmpty(conditions))return gridExample;
GridExample.Criteria criteria = gridExample.createCriteria();
conditions.forEach(criteria::addCondtion);
return gridExample;
}
}

16
backend/src/main/java/io/dataease/controller/sys/base/ConditionEntity.java Normal file
View File

@@ -0,0 +1,16 @@
package io.dataease.controller.sys.base;
import lombok.Data;
import java.io.Serializable;
@Data
public class ConditionEntity implements Serializable {
private String field;
private String operator;
private Object value;
}

11
backend/src/main/java/io/dataease/controller/sys/request/SimpleTreeNode.java Normal file
View File

@@ -0,0 +1,11 @@
package io.dataease.controller.sys.request;
import lombok.Data;
@Data
public class SimpleTreeNode {
private Long id;
private Long pid;
}

8
backend/src/main/java/io/dataease/controller/sys/request/UserGridRequest.java
View File

@@ -1,10 +1,18 @@
package io.dataease.controller.sys.request;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.io.Serializable;
@Data
public class UserGridRequest implements Serializable {
@ApiModelProperty("快速检索")
private String quick;
@ApiModelProperty("名称")
private String name;
@ApiModelProperty("组织")
private String deptId;
@ApiModelProperty("状态")
private String enabled;
}

4
backend/src/main/java/io/dataease/datasource/provider/DatasourceProvider.java
View File

@@ -23,4 +23,8 @@ public abstract class DatasourceProvider {
getData(datasourceRequest);
}
abstract public Long count(DatasourceRequest datasourceRequest)throws Exception;
abstract public List<String[]> getPageData(DatasourceRequest datasourceRequest) throws Exception;
}

72
backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java
View File

@@ -9,6 +9,7 @@ import io.dataease.datasource.request.DatasourceRequest;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import java.sql.*;
import java.text.MessageFormat;
import java.util.*;
@Service("jdbc")
@@ -23,23 +24,7 @@ public class JdbcProvider extends DatasourceProvider{
Statement stat = connection.createStatement();
ResultSet rs = stat.executeQuery(datasourceRequest.getQuery())
) {
ResultSetMetaData metaData = rs.getMetaData();
int columnCount = metaData.getColumnCount();
while (rs.next()) {
String[] row = new String[columnCount];
for (int j = 0; j < columnCount; j++) {
int columType = metaData.getColumnType(j + 1);
switch (columType) {
case java.sql.Types.DATE:
row[j] = rs.getDate(j + 1).toString();
break;
default:
row[j] = rs.getString(j + 1);
break;
}
}
list.add(row);
}
list = fetchResult(rs);
} catch (SQLException e){
throw new Exception("ERROR:" + e.getMessage(), e);
}catch (Exception e) {
@@ -48,6 +33,46 @@ public class JdbcProvider extends DatasourceProvider{
return list;
}
@Override
public List<String[]> getPageData(DatasourceRequest datasourceRequest) throws Exception {
List<String[]> list = new LinkedList<>();
try (
Connection connection = getConnection(datasourceRequest);
Statement stat = connection.createStatement();
ResultSet rs = stat.executeQuery(datasourceRequest.getQuery() + MessageFormat.format(" LIMIT {0}, {1}", (datasourceRequest.getStartPage() -1)*datasourceRequest.getPageSize(), datasourceRequest.getPageSize()))
) {
list = fetchResult(rs);
} catch (SQLException e){
throw new Exception("ERROR:" + e.getMessage(), e);
}catch (Exception e) {
throw new Exception("ERROR:" + e.getMessage(), e);
}
return list;
}
private List<String[]> fetchResult( ResultSet rs) throws Exception{
List<String[]> list = new LinkedList<>();
ResultSetMetaData metaData = rs.getMetaData();
int columnCount = metaData.getColumnCount();
while (rs.next()) {
String[] row = new String[columnCount];
for (int j = 0; j < columnCount; j++) {
int columType = metaData.getColumnType(j + 1);
switch (columType) {
case java.sql.Types.DATE:
row[j] = rs.getDate(j + 1).toString();
break;
default:
row[j] = rs.getString(j + 1);
break;
}
}
list.add(row);
}
return list;
}
@Override
public List<String> getTables(DatasourceRequest datasourceRequest) throws Exception {
List<String> tables = new ArrayList<>();
@@ -106,6 +131,19 @@ public class JdbcProvider extends DatasourceProvider{
}
}
public Long count(DatasourceRequest datasourceRequest)throws Exception{
try (Connection con = getConnection(datasourceRequest); Statement ps = con.createStatement()) {
ResultSet resultSet = ps.executeQuery(datasourceRequest.getQuery());
while (resultSet.next()) {
return resultSet.getLong(1);
}
} catch (Exception e) {
throw new Exception("ERROR: " + e.getMessage(), e);
}
return 0L;
}
private Connection getConnection(DatasourceRequest datasourceRequest) throws Exception {
String username = null;
String password = null;

3
backend/src/main/java/io/dataease/datasource/request/DatasourceRequest.java
View File

@@ -11,5 +11,8 @@ public class DatasourceRequest {
protected String query;
protected String table;
protected Datasource datasource;
private Long pageSize;
private Long startPage;
}

30
backend/src/main/java/io/dataease/dto/chart/ChartViewFieldDTO.java Normal file
View File

@@ -0,0 +1,30 @@
package io.dataease.dto.chart;
import lombok.Data;
/**
* @Author gin
* @Date 2021/3/11 1:18 下午
*/
@Data
public class ChartViewFieldDTO {
private String id;
private String tableId;
private String originName;
private String name;
private String type;
private Boolean checked;
private Integer columnIndex;
private Long lastSyncTime;
private Integer deType;
private String summary;
}

25
backend/src/main/java/io/dataease/job/sechedule/DeScheduleJob.java
View File

@@ -1,22 +1,25 @@
package io.dataease.job.sechedule;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import io.dataease.commons.utils.LogUtil;
import org.quartz.*;
public abstract class DeScheduleJob implements Job {
protected String datasetTableId;
protected String expression;
protected String taskId;
@Override
public void execute(JobExecutionContext context) throws JobExecutionException {
JobKey jobKey = context.getTrigger().getJobKey();
JobDataMap jobDataMap = context.getJobDetail().getJobDataMap();
this.datasetTableId = jobDataMap.getString("datasetTableId");
this.expression = jobDataMap.getString("expression");
this.taskId = jobDataMap.getString("taskId");
// JobKey jobKey = context.getTrigger().getJobKey();
// JobDataMap jobDataMap = context.getJobDetail().getJobDataMap();
// this.resourceId = jobDataMap.getString("resourceId");
// this.userId = jobDataMap.getString("userId");
// this.expression = jobDataMap.getString("expression");
//
// LogUtil.info(jobKey.getGroup() + " Running: " + resourceId);
// LogUtil.info("CronExpression: " + expression);
LogUtil.info(jobKey.getGroup() + " Running: " + datasetTableId);
LogUtil.info(jobKey.getName() + " Running: " + datasetTableId);
LogUtil.info("CronExpression: " + expression);
businessExecute(context);
}

22
backend/src/main/java/io/dataease/job/sechedule/ExtractDataJob.java Normal file
View File

@@ -0,0 +1,22 @@
package io.dataease.job.sechedule;
import io.dataease.commons.utils.CommonBeanFactory;
import io.dataease.service.dataset.ExtractDataService;
import org.quartz.JobExecutionContext;
import org.springframework.stereotype.Component;
@Component
public class ExtractDataJob extends DeScheduleJob{
private ExtractDataService extractDataService;
public ExtractDataJob() {
extractDataService = (ExtractDataService) CommonBeanFactory.getBean(ExtractDataService.class);
}
@Override
void businessExecute(JobExecutionContext context) {
extractDataService.extractData(datasetTableId, taskId);
}
}

6
backend/src/main/java/io/dataease/job/sechedule/ScheduleManager.java
View File

@@ -369,11 +369,11 @@ public class ScheduleManager {
addOrUpdateCronJob(jobKey, triggerKey, jobClass, cron, startTime, endTime, null);
}
public JobDataMap getDefaultJobDataMap(String resourceId, String expression, String userId) {
public JobDataMap getDefaultJobDataMap(String resourceId, String expression, String taskId) {
JobDataMap jobDataMap = new JobDataMap();
jobDataMap.put("resourceId", resourceId);
jobDataMap.put("datasetTableId", resourceId);
jobDataMap.put("taskId", taskId);
jobDataMap.put("expression", expression);
jobDataMap.put("userId", userId);
return jobDataMap;
}

8
backend/src/main/java/io/dataease/listener/AppStartListener.java
View File

@@ -20,13 +20,7 @@ public class AppStartListener implements ApplicationListener<ApplicationReadyEve
@Override
public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) {
System.out.println("================= 应用启动 =================");
/* cron schedule */
// scheduleManager.addCronJob(new JobKey("abc", "def"), new TriggerKey("abc", "def"), TestJob.class, "*/10 * * * * ?");
/* single schedule*/
// long timestamp = System.currentTimeMillis() + 90 * 1000;
// Date date = new Date(timestamp);
// scheduleManager.addSingleJob(new JobKey("abc", "def"), new TriggerKey("abc", "def"), TestJob.class, date);
System.out.println("================= Application start =================");
// 项目启动，从数据库读取任务加入到Quartz
List<DatasetTableTask> list = dataSetTableTaskService.list(new DatasetTableTask());
for (DatasetTableTask task : list) {

11
backend/src/main/java/io/dataease/service/ScheduleService.java
View File

@@ -1,8 +1,8 @@
package io.dataease.service;
import io.dataease.base.domain.DatasetTableTask;
import io.dataease.job.sechedule.ExtractDataJob;
import io.dataease.job.sechedule.ScheduleManager;
import io.dataease.job.sechedule.TestJob;
import org.apache.commons.lang3.StringUtils;
import org.quartz.JobKey;
import org.quartz.TriggerKey;
@@ -24,8 +24,8 @@ public class ScheduleService {
if (StringUtils.equalsIgnoreCase(datasetTableTask.getRate(), "0")) {
scheduleManager.addOrUpdateSingleJob(new JobKey(datasetTableTask.getId(), datasetTableTask.getTableId()),
new TriggerKey(datasetTableTask.getId(), datasetTableTask.getTableId()),
TestJob.class,//TODO
new Date(datasetTableTask.getStartTime()));
ExtractDataJob.class,
new Date(datasetTableTask.getStartTime()), scheduleManager.getDefaultJobDataMap(datasetTableTask.getTableId(), datasetTableTask.getCron(), datasetTableTask.getId()));
} else if (StringUtils.equalsIgnoreCase(datasetTableTask.getRate(), "1")) {
Date endTime;
if (datasetTableTask.getEndTime() == null || datasetTableTask.getEndTime() == 0) {
@@ -36,8 +36,9 @@ public class ScheduleService {
scheduleManager.addOrUpdateCronJob(new JobKey(datasetTableTask.getId(), datasetTableTask.getTableId()),
new TriggerKey(datasetTableTask.getId(), datasetTableTask.getTableId()),
TestJob.class,// TODO
datasetTableTask.getCron(), new Date(datasetTableTask.getStartTime()), endTime);
ExtractDataJob.class,
datasetTableTask.getCron(), new Date(datasetTableTask.getStartTime()), endTime,
scheduleManager.getDefaultJobDataMap(datasetTableTask.getTableId(), datasetTableTask.getCron(), datasetTableTask.getId()));
}
}

39
backend/src/main/java/io/dataease/service/chart/ChartViewService.java
View File

@@ -12,6 +12,7 @@ import io.dataease.datasource.provider.ProviderFactory;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.datasource.service.DatasourceService;
import io.dataease.dto.chart.ChartViewDTO;
import io.dataease.dto.chart.ChartViewFieldDTO;
import io.dataease.dto.chart.Series;
import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.service.dataset.DataSetTableFieldsService;
@@ -76,9 +77,9 @@ public class ChartViewService {
public ChartViewDTO getData(String id) throws Exception {
ChartViewWithBLOBs view = chartViewMapper.selectByPrimaryKey(id);
List<DatasetTableField> xAxis = new Gson().fromJson(view.getXAxis(), new TypeToken<List<DatasetTableField>>() {
List<ChartViewFieldDTO> xAxis = new Gson().fromJson(view.getXAxis(), new TypeToken<List<ChartViewFieldDTO>>() {
}.getType());
List<DatasetTableField> yAxis = new Gson().fromJson(view.getYAxis(), new TypeToken<List<DatasetTableField>>() {
List<ChartViewFieldDTO> yAxis = new Gson().fromJson(view.getYAxis(), new TypeToken<List<ChartViewFieldDTO>>() {
}.getType());
List<String> x = new ArrayList<>();
@@ -88,25 +89,25 @@ public class ChartViewService {
BeanUtils.copyBean(dto, view);
return dto;
}
List<String> xIds = xAxis.stream().map(DatasetTableField::getId).collect(Collectors.toList());
List<String> yIds = yAxis.stream().map(DatasetTableField::getId).collect(Collectors.toList());
List<DatasetTableField> xList = dataSetTableFieldsService.getListByIds(xIds);
List<DatasetTableField> yList = dataSetTableFieldsService.getListByIds(yIds);
// List<String> xIds = xAxis.stream().map(DatasetTableField::getId).collect(Collectors.toList());
// List<String> yIds = yAxis.stream().map(DatasetTableField::getId).collect(Collectors.toList());
// List<DatasetTableField> xList = dataSetTableFieldsService.getListByIds(xIds);
// List<DatasetTableField> yList = dataSetTableFieldsService.getListByIds(yIds);
// 获取数据源
// 获取数据集
DatasetTable table = dataSetTableService.get(view.getTableId());
// todo 判断连接方式，直连或者定时抽取 table.mode
Datasource ds = datasourceService.get(table.getDataSourceId());
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
DataTableInfoDTO dataTableInfoDTO = new Gson().fromJson(table.getInfo(), DataTableInfoDTO.class);
datasourceRequest.setTable(dataTableInfoDTO.getTable());
datasourceRequest.setQuery(getSQL(ds.getType(), dataTableInfoDTO.getTable(), xList, yList));
datasourceRequest.setQuery(getSQL(ds.getType(), dataTableInfoDTO.getTable(), xAxis, yAxis));
List<String[]> data = datasourceProvider.getData(datasourceRequest);
// todo 处理结果,目前做一个单系列图表，后期图表组件再扩展
for (DatasetTableField y : yList) {
for (ChartViewFieldDTO y : yAxis) {
Series series1 = new Series();
series1.setName(y.getName());
series1.setType(view.getType());
@@ -116,16 +117,16 @@ public class ChartViewService {
for (String[] d : data) {
StringBuilder a = new StringBuilder();
BigDecimal b = new BigDecimal("0");
for (int i = 0; i < xList.size(); i++) {
if (i == xList.size() - 1) {
for (int i = 0; i < xAxis.size(); i++) {
if (i == xAxis.size() - 1) {
a.append(d[i]);
} else {
a.append(d[i]).append("\n");
}
}
x.add(a.toString());
for (int i = xList.size(); i < xList.size() + yList.size(); i++) {
int j = i - xList.size();
for (int i = xAxis.size(); i < xAxis.size() + yAxis.size(); i++) {
int j = i - xAxis.size();
series.get(j).getData().add(new BigDecimal(d[i]));
}
}
@@ -139,7 +140,7 @@ public class ChartViewService {
return dto;
}
public String getSQL(String type, String table, List<DatasetTableField> xAxis, List<DatasetTableField> yAxis) {
public String getSQL(String type, String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis) {
DatasourceTypes datasourceType = DatasourceTypes.valueOf(type);
switch (datasourceType) {
case mysql:
@@ -150,10 +151,10 @@ public class ChartViewService {
}
}
public String transMysqlSQL(String table, List<DatasetTableField> xAxis, List<DatasetTableField> yAxis) {
// TODO 此处sum后期由用户前端传入
String[] field = yAxis.stream().map(y -> "sum(" + y.getOriginName() + ")").toArray(String[]::new);
String[] group = xAxis.stream().map(DatasetTableField::getOriginName).toArray(String[]::new);
public String transMysqlSQL(String table, List<ChartViewFieldDTO> xAxis, List<ChartViewFieldDTO> yAxis) {
// TODO 字段汇总 排序等
String[] field = yAxis.stream().map(y -> "CAST(" + y.getSummary() + "(" + y.getOriginName() + ") AS DECIMAL(20,2))").toArray(String[]::new);
String[] group = xAxis.stream().map(ChartViewFieldDTO::getOriginName).toArray(String[]::new);
return MessageFormat.format("SELECT {0},{1} FROM {2} GROUP BY {3}", StringUtils.join(group, ","), StringUtils.join(field, ","), table, StringUtils.join(group, ","));
}
}

76
backend/src/main/java/io/dataease/service/dataset/DataSetTableService.java
View File

@@ -37,6 +37,8 @@ public class DataSetTableService {
private DatasourceMapper datasourceMapper;
@Resource
private DataSetTableFieldsService dataSetTableFieldsService;
@Resource
private DataSetTableTaskService dataSetTableTaskService;
public void batchInsert(List<DatasetTable> datasetTable) throws Exception {
for (DatasetTable table : datasetTable) {
@@ -67,11 +69,15 @@ public class DataSetTableService {
public void delete(String id) {
datasetTableMapper.deleteByPrimaryKey(id);
dataSetTableFieldsService.deleteByTableId(id);
// 删除同步任务
dataSetTableTaskService.deleteByTableId(id);
}
public List<DatasetTable> list(DataSetTableRequest dataSetTableRequest) {
DatasetTableExample datasetTableExample = new DatasetTableExample();
datasetTableExample.createCriteria().andSceneIdEqualTo(dataSetTableRequest.getSceneId());
if (StringUtils.isNotEmpty(dataSetTableRequest.getSceneId())) {
datasetTableExample.createCriteria().andSceneIdEqualTo(dataSetTableRequest.getSceneId());
}
if (StringUtils.isNotEmpty(dataSetTableRequest.getSort())) {
datasetTableExample.setOrderByClause(dataSetTableRequest.getSort());
}
@@ -92,7 +98,7 @@ public class DataSetTableService {
}
public Map<String, List<DatasetTableField>> getFieldsFromDE(DataSetTableRequest dataSetTableRequest) throws Exception {
DatasetTableField datasetTableField = new DatasetTableField();
DatasetTableField datasetTableField = DatasetTableField.builder().build();
datasetTableField.setTableId(dataSetTableRequest.getId());
datasetTableField.setChecked(Boolean.TRUE);
List<DatasetTableField> fields = dataSetTableFieldsService.list(datasetTableField);
@@ -121,7 +127,7 @@ public class DataSetTableService {
datasourceRequest.setDatasource(ds);
String table = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class).getTable();
DatasetTableField datasetTableField = new DatasetTableField();
DatasetTableField datasetTableField = DatasetTableField.builder().build();
datasetTableField.setTableId(dataSetTableRequest.getId());
datasetTableField.setChecked(Boolean.TRUE);
List<DatasetTableField> fields = dataSetTableFieldsService.list(datasetTableField);
@@ -137,15 +143,13 @@ public class DataSetTableService {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
String table = new Gson().fromJson(dataSetTableRequest.getInfo(), DataTableInfoDTO.class).getTable();
// datasourceRequest.setTable(table);
DatasetTableField datasetTableField = new DatasetTableField();
DatasetTableField datasetTableField = DatasetTableField.builder().build();
datasetTableField.setTableId(dataSetTableRequest.getId());
datasetTableField.setChecked(Boolean.TRUE);
List<DatasetTableField> fields = dataSetTableFieldsService.list(datasetTableField);
String[] fieldArray = fields.stream().map(DatasetTableField::getOriginName).toArray(String[]::new);
// datasourceRequest.setQuery("SELECT " + StringUtils.join(fieldArray, ",") + " FROM " + table + " LIMIT 0,10;");
datasourceRequest.setQuery(createQuerySQL(ds.getType(), table, fieldArray) + " LIMIT 0,10");
List<String[]> data = new ArrayList<>();
@@ -154,17 +158,6 @@ public class DataSetTableService {
} catch (Exception e) {
}
/*JSONArray jsonArray = new JSONArray();
if (CollectionUtils.isNotEmpty(data)) {
data.forEach(ele -> {
JSONObject jsonObject = new JSONObject();
for (int i = 0; i < ele.length; i++) {
jsonObject.put(fieldArray[i], ele[i]);
}
jsonArray.add(jsonObject);
});
}*/
List<Map<String, Object>> jsonArray = new ArrayList<>();
if (CollectionUtils.isNotEmpty(data)) {
jsonArray = data.stream().map(ele -> {
@@ -184,6 +177,53 @@ public class DataSetTableService {
return map;
}
public List<String[]> getDataSetData(String datasourceId, String table, List<DatasetTableField> fields) {
List<String[]> data = new ArrayList<>();
Datasource ds = datasourceMapper.selectByPrimaryKey(datasourceId);
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
String[] fieldArray = fields.stream().map(DatasetTableField::getOriginName).toArray(String[]::new);
datasourceRequest.setQuery(createQuerySQL(ds.getType(), table, fieldArray) + " LIMIT 0, 10");
try {
data.addAll(datasourceProvider.getData(datasourceRequest));
} catch (Exception e) {
}
return data;
}
public Long getDataSetTotalData(String datasourceId, String table) {
List<String[]> data = new ArrayList<>();
Datasource ds = datasourceMapper.selectByPrimaryKey(datasourceId);
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
datasourceRequest.setQuery("select count(*) from " + table);
try {
return datasourceProvider.count(datasourceRequest);
} catch (Exception e) {
}
return 0l;
}
public List<String[]> getDataSetPageData(String datasourceId, String table, List<DatasetTableField> fields, Long startPage, Long pageSize) {
List<String[]> data = new ArrayList<>();
Datasource ds = datasourceMapper.selectByPrimaryKey(datasourceId);
DatasourceProvider datasourceProvider = ProviderFactory.getProvider(ds.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(ds);
String[] fieldArray = fields.stream().map(DatasetTableField::getOriginName).toArray(String[]::new);
datasourceRequest.setPageSize(pageSize);
datasourceRequest.setStartPage(startPage);
datasourceRequest.setQuery(createQuerySQL(ds.getType(), table, fieldArray));
try {
return datasourceProvider.getData(datasourceRequest);
} catch (Exception e) {
}
return data;
}
public void saveTableField(DatasetTable datasetTable) throws Exception {
Datasource ds = datasourceMapper.selectByPrimaryKey(datasetTable.getDataSourceId());
DataSetTableRequest dataSetTableRequest = new DataSetTableRequest();
@@ -193,7 +233,7 @@ public class DataSetTableService {
if (CollectionUtils.isNotEmpty(fields)) {
for (int i = 0; i < fields.size(); i++) {
TableFiled filed = fields.get(i);
DatasetTableField datasetTableField = new DatasetTableField();
DatasetTableField datasetTableField = DatasetTableField.builder().build();
datasetTableField.setTableId(datasetTable.getId());
datasetTableField.setOriginName(filed.getFieldName());
datasetTableField.setName(filed.getRemarks());

7
backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskLogService.java
View File

@@ -48,4 +48,11 @@ public class DataSetTableTaskLogService {
return extDataSetTaskMapper.list(request);
}
public void deleteByTaskId(String taskId){
DatasetTableTaskLogExample datasetTableTaskLogExample = new DatasetTableTaskLogExample();
DatasetTableTaskLogExample.Criteria criteria = datasetTableTaskLogExample.createCriteria();
criteria.andTaskIdEqualTo(taskId);
datasetTableTaskLogMapper.deleteByExample(datasetTableTaskLogExample);
}
}

24
backend/src/main/java/io/dataease/service/dataset/DataSetTableTaskService.java
View File

@@ -7,6 +7,7 @@ import io.dataease.service.ScheduleService;
import org.apache.commons.lang3.StringUtils;
import org.quartz.CronExpression;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.annotation.Resource;
import java.util.List;
@@ -17,10 +18,12 @@ import java.util.UUID;
* @Date 2021/3/4 1:26 下午
*/
@Service
@Transactional
public class DataSetTableTaskService {
@Resource
private DatasetTableTaskMapper datasetTableTaskMapper;
@Resource
private DataSetTableTaskLogService dataSetTableTaskLogService;
@Resource
private ScheduleService scheduleService;
@@ -46,6 +49,25 @@ public class DataSetTableTaskService {
DatasetTableTask datasetTableTask = datasetTableTaskMapper.selectByPrimaryKey(id);
datasetTableTaskMapper.deleteByPrimaryKey(id);
scheduleService.deleteSchedule(datasetTableTask);
dataSetTableTaskLogService.deleteByTaskId(id);
}
public void delete(DatasetTableTask task) {
datasetTableTaskMapper.deleteByPrimaryKey(task.getId());
scheduleService.deleteSchedule(task);
dataSetTableTaskLogService.deleteByTaskId(task.getId());
}
public void deleteByTableId(String id) {
DatasetTableTaskExample datasetTableTaskExample = new DatasetTableTaskExample();
DatasetTableTaskExample.Criteria criteria = datasetTableTaskExample.createCriteria();
criteria.andTableIdEqualTo(id);
List<DatasetTableTask> datasetTableTasks = datasetTableTaskMapper.selectByExample(datasetTableTaskExample);
datasetTableTasks.forEach(this::delete);
}
public DatasetTableTask get(String id) {
return datasetTableTaskMapper.selectByPrimaryKey(id);
}
public List<DatasetTableTask> list(DatasetTableTask datasetTableTask) {

93
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java Normal file
View File

@@ -0,0 +1,93 @@
package io.dataease.service.dataset;
import com.google.gson.Gson;
import io.dataease.base.domain.DatasetTable;
import io.dataease.base.domain.DatasetTableField;
import io.dataease.base.domain.DatasetTableTaskLog;
import io.dataease.commons.constants.JobStatus;
import io.dataease.commons.utils.CommonBeanFactory;
import io.dataease.dto.dataset.DataTableInfoDTO;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@Service
public class ExtractDataService {
@Resource
private DataSetTableService dataSetTableService;
@Resource
private DataSetTableFieldsService dataSetTableFieldsService;
@Resource
private DataSetTableTaskLogService dataSetTableTaskLogService;
private Long pageSize = 10000l;
private static ExecutorService pool = Executors.newScheduledThreadPool(50); //设置连接池
private Connection connection;
public void extractData(String datasetTableId, String taskId) {
DatasetTableTaskLog datasetTableTaskLog = new DatasetTableTaskLog();
try {
datasetTableTaskLog.setTableId(datasetTableId);
datasetTableTaskLog.setTaskId(taskId);
datasetTableTaskLog.setStatus(JobStatus.Underway.name());
datasetTableTaskLog.setStartTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
Admin admin = getConnection().getAdmin();
DatasetTable datasetTable = dataSetTableService.get(datasetTableId);
List<DatasetTableField> datasetTableFields = dataSetTableFieldsService.list(DatasetTableField.builder().tableId(datasetTable.getId()).build());
String table = new Gson().fromJson(datasetTable.getInfo(), DataTableInfoDTO.class).getTable();
TableName tableName = TableName.valueOf(table + "-" + datasetTable.getDataSourceId());
if(!admin.tableExists(tableName)){
TableDescriptorBuilder descBuilder = TableDescriptorBuilder.newBuilder(tableName);
ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.of("cf");
descBuilder.setColumnFamily(hcd);
TableDescriptor desc = descBuilder.build();
admin.createTable(desc);
}
admin.disableTable(tableName);
admin.truncateTable(tableName, true);
Table tab = getConnection().getTable(tableName);
Long total = dataSetTableService.getDataSetTotalData(datasetTable.getDataSourceId(), table);
Long pageCount = total % pageSize == 0 ? total / pageSize : (total / pageSize) + 1;
for (Long pageIndex = 1l; pageIndex <= pageCount; pageIndex++) {
List<String[]> data = dataSetTableService.getDataSetPageData(datasetTable.getDataSourceId(), table, datasetTableFields, pageIndex, pageSize);
for (String[] d : data) {
for(int i=0;i<datasetTableFields.size();i++){
Put put = new Put(UUID.randomUUID().toString().getBytes());
String value = d[i];
if(value == null){
value = "null";
}
put.addColumn("cf".getBytes(), datasetTableFields.get(i).getOriginName().getBytes(), value.getBytes());
tab.put(put);
}
}
}
datasetTableTaskLog.setStatus(JobStatus.Completed.name());
datasetTableTaskLog.setEndTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
}catch (Exception e){
datasetTableTaskLog.setStatus(JobStatus.Error.name());
datasetTableTaskLog.setEndTime(System.currentTimeMillis());
dataSetTableTaskLogService.save(datasetTableTaskLog);
}
}
private synchronized Connection getConnection() throws Exception{
if(connection == null || connection.isClosed()){
Configuration cfg = CommonBeanFactory.getBean(Configuration.class);
connection = ConnectionFactory.createConnection(cfg, pool);
}
return connection;
}
}

67
backend/src/main/java/io/dataease/service/sys/DeptService.java
View File

@@ -4,16 +4,20 @@ import io.dataease.base.domain.SysDept;
import io.dataease.base.domain.SysDeptExample;
import io.dataease.base.mapper.SysDeptMapper;
import io.dataease.base.mapper.ext.ExtDeptMapper;
import io.dataease.base.mapper.ext.query.GridExample;
import io.dataease.commons.utils.BeanUtils;
import io.dataease.commons.utils.CommonBeanFactory;
import io.dataease.controller.sys.base.BaseGridRequest;
import io.dataease.controller.sys.request.DeptCreateRequest;
import io.dataease.controller.sys.request.DeptDeleteRequest;
import io.dataease.controller.sys.request.DeptStatusRequest;
import io.dataease.controller.sys.request.SimpleTreeNode;
import org.apache.commons.lang3.ObjectUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.annotation.Resource;
import java.util.Date;
import java.util.List;
import java.util.*;
import java.util.stream.Collectors;
@Service
@@ -23,11 +27,11 @@ public class DeptService {
private final static Integer DEFAULT_SUBCOUNT = 0;
public final static Long DEPT_ROOT_PID = 0L;
@Resource
@Autowired(required = false)
private SysDeptMapper sysDeptMapper;
@Resource
@Autowired(required = false)
private ExtDeptMapper extDeptMapper;
public List<SysDept> nodesByPid(Long pid){
@@ -120,4 +124,59 @@ public class DeptService {
return sysDeptMapper.updateByPrimaryKeySelective(sysDept);
}
public List<SysDept> nodesTreeByCondition(BaseGridRequest request){
//DeptService proxy = proxy();
List<SimpleTreeNode> allNodes = allNodes();
List<SimpleTreeNode> targetNodes = nodeByCondition(request);
List<Long> ids = upTree(allNodes, targetNodes);
SysDeptExample example = new SysDeptExample();
SysDeptExample.Criteria criteria = example.createCriteria();
criteria.andDeptIdIn(ids);
example.setOrderByClause("dept_sort");
List<SysDept> sysDepts = sysDeptMapper.selectByExample(example);
return sysDepts;
}
private DeptService proxy(){
return CommonBeanFactory.getBean(DeptService.class);
}
private List<SimpleTreeNode> allNodes(){
List<SimpleTreeNode> simpleTreeNodes = extDeptMapper.allNodes();
return simpleTreeNodes;
}
private List<SimpleTreeNode> nodeByCondition(BaseGridRequest request){
GridExample gridExample = request.convertExample();
List<SimpleTreeNode> simpleTreeNodes = extDeptMapper.nodesByExample(gridExample);
return simpleTreeNodes;
}
/**
* 找出目标节点所在路径上的所有节点 向上找
* @param allNodes 所有节点
* @param targetNodes 目标节点
* @return
*/
private List<Long> upTree(List<SimpleTreeNode> allNodes, List<SimpleTreeNode> targetNodes){
final Map<Long, SimpleTreeNode> map = allNodes.stream().collect(Collectors.toMap(SimpleTreeNode::getId, node -> node));
List<Long> results = targetNodes.parallelStream().flatMap(targetNode -> {
//向上逐级找爹
List<Long> ids = new ArrayList<>();
SimpleTreeNode node = targetNode;
while (node != null) {
ids.add(node.getId());
Long pid = node.getPid();
node = map.get(pid);
}
return ids.stream();
}).distinct().collect(Collectors.toList());
return results;
}
}

5
backend/src/main/java/io/dataease/service/sys/SysRoleService.java
View File

@@ -6,6 +6,7 @@ import io.dataease.base.domain.SysUsersRolesExample;
import io.dataease.base.mapper.SysRoleMapper;
import io.dataease.base.mapper.SysUsersRolesMapper;
import io.dataease.base.mapper.ext.ExtSysRoleMapper;
import io.dataease.controller.sys.base.BaseGridRequest;
import io.dataease.controller.sys.request.RoleGridRequest;
import io.dataease.controller.sys.request.RoleMenusRequest;
import io.dataease.controller.sys.response.RoleUserItem;
@@ -56,8 +57,8 @@ public class SysRoleService {
}
public List<SysRole> query(RoleGridRequest request){
List<SysRole> result = extSysRoleMapper.query(request);
public List<SysRole> query(BaseGridRequest request){
List<SysRole> result = extSysRoleMapper.query(request.convertExample());
return result;
}

9
backend/src/main/java/io/dataease/service/sys/SysUserService.java
View File

@@ -7,8 +7,10 @@ import io.dataease.base.domain.SysUsersRolesKey;
import io.dataease.base.mapper.SysUserMapper;
import io.dataease.base.mapper.SysUsersRolesMapper;
import io.dataease.base.mapper.ext.ExtSysUserMapper;
import io.dataease.base.mapper.ext.query.GridExample;
import io.dataease.commons.utils.BeanUtils;
import io.dataease.commons.utils.CodingUtil;
import io.dataease.controller.sys.base.BaseGridRequest;
import io.dataease.controller.sys.request.SysUserCreateRequest;
import io.dataease.controller.sys.request.SysUserPwdRequest;
import io.dataease.controller.sys.request.SysUserStateRequest;
@@ -40,8 +42,10 @@ public class SysUserService {
@Resource
private ExtSysUserMapper extSysUserMapper;
public List<SysUserGridResponse> query(UserGridRequest request){
List<SysUserGridResponse> lists = extSysUserMapper.query(request);
public List<SysUserGridResponse> query(BaseGridRequest request){
GridExample gridExample = request.convertExample();
List<SysUserGridResponse> lists = extSysUserMapper.query(gridExample);
lists.forEach(item -> {
List<SysUserRole> roles = item.getRoles();
List<Long> roleIds = roles.stream().map(SysUserRole::getRoleId).collect(Collectors.toList());
@@ -111,6 +115,7 @@ public class SysUserService {
return sysUserMapper.updateByPrimaryKeySelective(sysUser);
}
@CacheEvict(value = USER_CACHE_NAME, key = "'user' + #request.userId")
public int adminUpdatePwd(SysUserPwdRequest request){
SysUser sysUser = new SysUser();
sysUser.setUserId(request.getUserId());