diff --git a/README.md b/README.md index e8d3d76..7b42790 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # elasticsearch-engine ## 介绍 +![img.png](img.png) elasticsearch-engine是基于 HighLevelRestClient 封装的 ElasticSearch 查询引擎框架. 支持ElasticSearch基于注解的结构化查询; 基于sql语句的方式查询; @@ -32,6 +33,7 @@ github地址: https://github.com/wanghuan9/elasticsearch-engine 4. elasticsearch-engine-jooq 基于aop,jooq执行监听器 实现sql拦截,改写,执行elasticsearch查询 ## 使用说明 + 所有完整示例 请参考 [使用示例](https://gitee.com/my-source-project/elasticsearch-engine-demo) ### 1.注解查询 diff --git "a/doc/\350\256\241\345\210\222" "b/doc/\350\256\241\345\210\222" index 5e15129..500f3d4 100644 --- "a/doc/\350\256\241\345\210\222" +++ "b/doc/\350\256\241\345\210\222" @@ -6,6 +6,7 @@ 5)自定义注解sql查询 对象参数 (参考 mybatis和jpa实现参数解析) 6)工具类整理, http工具类优化 7)sql日志打印支持配置 +8)es 和 mysql字段映射关系支持自定义 其他: 1)@EsCondition 思考 diff --git a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/parse/sql/EsSqlQueryHelper.java b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/parse/sql/EsSqlQueryHelper.java index 13dde20..0d8d76e 100644 --- a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/parse/sql/EsSqlQueryHelper.java +++ b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/parse/sql/EsSqlQueryHelper.java @@ -3,14 +3,17 @@ import com.elasticsearch.engine.base.common.proxy.handler.exannotation.AnnotationQueryCommon; import com.elasticsearch.engine.base.common.queryhandler.sql.EsSqlExecuteHandler; import com.elasticsearch.engine.base.common.utils.ThreadLocalUtil; +import com.elasticsearch.engine.base.config.EsEngineConfig; import com.elasticsearch.engine.base.model.constant.CommonConstant; import com.elasticsearch.engine.base.model.domain.BackDto; import com.elasticsearch.engine.base.model.emenu.SqlParamParse; +import com.elasticsearch.engine.base.model.exception.EsEngineExecuteException; import com.elasticsearch.engine.base.model.exception.EsEngineJpaExecuteException; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.statement.select.Select; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.aspectj.lang.ProceedingJoinPoint; import org.aspectj.lang.reflect.MethodSignature; import org.springframework.stereotype.Component; @@ -30,9 +33,14 @@ @Component public class EsSqlQueryHelper { + private static final String ENABLE_LOG_OUT_PROPERTIES = "es.engine.config.sql-trace-log"; + @Resource private EsSqlExecuteHandler esSqlExecuteHandler; + @Resource + private EsSqlQueryHelper esSqlQueryHelper; + /** * es aop 查询逻辑 * @@ -45,25 +53,40 @@ public Object esSqlQueryAopCommon(ProceedingJoinPoint pjp, BackDto backDto) thro MethodSignature signature = (MethodSignature) pjp.getSignature(); Method method = signature.getMethod(); Object[] args = pjp.getArgs(); + //不走es查询直接返回(全局开关) + if (!EsEngineConfig.isEsquery(method)) { + return pjp.proceed(args); + } + //获取回表查询参数 Object result = null; try { + //设置标记,在sql拦截器中抛出异常->回到后面的异常处理逻辑中实现es查询 ThreadLocalUtil.set(CommonConstant.IS_ES_QUERY, Boolean.TRUE); result = pjp.proceed(args); } catch (EsEngineJpaExecuteException e) { - if (Objects.nonNull(backDto)) { + String esSql = e.getMessage(); + //判断是否需要回表查询 + if (Objects.isNull(backDto)) { + //无需回表直接执行es查询 + //原生es执行 直接使用绑定参数后的sql + result = esSqlQueryHelper.esQuery(method, esSql, args, backDto); + } else { + //需要回表es查询并回表查询 //回表sql执行, sql重新时使用 原生未绑定参数的sql - List esResult = esQueryBack(method, e.getMessage(), args, backDto); - if(CollectionUtils.isEmpty(esResult)){ + String bakSql = ThreadLocalUtil.remove(CommonConstant.JPA_NATIVE_SQL); + if (StringUtils.isEmpty(bakSql)) { + throw new EsEngineExecuteException("回表sql异常"); + } + List esResult = esSqlQueryHelper.esQueryBack(method, esSql, bakSql, args, backDto); + if (CollectionUtils.isEmpty(esResult)) { return result; } result = pjp.proceed(args); - } else { - //原生es执行 直接使用绑定参数后的sql - result = esQuery(method, e.getMessage(), args, backDto); } } finally { ThreadLocalUtil.remove(CommonConstant.IS_ES_QUERY); ThreadLocalUtil.remove(CommonConstant.BACK_QUERY_SQL); + ThreadLocalUtil.remove(CommonConstant.JPA_NATIVE_SQL); } return result; } @@ -92,16 +115,18 @@ public Object esQuery(Method method, String sql, Object[] args, BackDto backDto) * @param backDto * @throws Exception */ - public List esQueryBack(Method method, String sql, Object[] args, BackDto backDto) throws Exception { - String paramSql = fillParamSql(method, sql, args, backDto); + public List esQueryBack(Method method, String esSql, String sql, Object[] args, BackDto backDto) throws Exception { + String paramSql = fillParamSql(method, esSql, args, backDto); //执行ES查询 List esResult = esSqlExecuteHandler.queryBySql(paramSql, backDto.getBackColumnTyp(), Boolean.TRUE); - if (CollectionUtils.isEmpty(esResult)){ + if (CollectionUtils.isEmpty(esResult)) { return null; } //将原sql改写成回表sql String backSql = SqlParserHelper.rewriteBackSql(sql, backDto, esResult); - log.info("回表sql : {}", backSql); + if (EsEngineConfig.getSqlTraceLog()) { + log.info("回表sql : {}", backSql); + } //将回表sql添加到threadLocal ThreadLocalUtil.set(CommonConstant.BACK_QUERY_SQL, backSql); return esResult; @@ -120,7 +145,9 @@ public List esQueryBack(Method method, String sql, Object[] args, BackDto bac private String fillParamSql(Method method, String sql, Object[] args, BackDto backDto) throws JSQLParserException { //jpa判断是否清除as别名 Boolean isCleanAs = Boolean.TRUE; - log.info("原始sql: {}", sql); + if (EsEngineConfig.getSqlTraceLog()) { + log.info("原始sql: {}", sql); + } //jpa原生查询 则不清楚 as别名 // Query query = method.getAnnotation(Query.class); // if (Objects.nonNull(query) && query.nativeQuery()) { @@ -128,13 +155,15 @@ private String fillParamSql(Method method, String sql, Object[] args, BackDto ba // } //改写sql Select select = SqlParserHelper.rewriteSql(method, sql, isCleanAs, backDto); - log.info("改写后sql: {}", select); + if (EsEngineConfig.getSqlTraceLog()) { + log.info("改写后sql: {}", select); + } //参数替换 // 解析sql参数 - //jooq 需要替换"`" - String selectSql = select.toString().replaceAll("`", ""); - String paramSql = SqlParamParseHelper.getMethodArgsSqlJpa(selectSql, method, args, SqlParamParse.JAP_SQL_PARAM); - log.info("替换参数后sql: {}", paramSql); + String paramSql = SqlParamParseHelper.getMethodArgsSqlJpa(select.toString(), method, args, SqlParamParse.JAP_SQL_PARAM); + if (EsEngineConfig.getSqlTraceLog()) { + log.info("替换参数后sql: {}", paramSql); + } return paramSql; } diff --git a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/parse/sql/SqlParamParseHelper.java b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/parse/sql/SqlParamParseHelper.java index 03b7b8e..b224d9e 100644 --- a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/parse/sql/SqlParamParseHelper.java +++ b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/parse/sql/SqlParamParseHelper.java @@ -237,12 +237,11 @@ private static String renderStringJpa(String sql, List param, SqlParamPa */ public static String getListParameterValue(Object val) { List listParam = (List) val; - StringBuffer sb = new StringBuffer(); + StringJoiner sj = new StringJoiner(","); if (!listParam.isEmpty()) { - listParam.forEach(item -> sb.append(getParameterValue(item)).append(",")); + listParam.forEach(item -> sj.add(getParameterValue(item))); } - String param = sb.toString(); - return param.substring(0, param.length() - 1); + return sj.toString(); } /** diff --git a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/parse/sql/SqlParserHelper.java b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/parse/sql/SqlParserHelper.java index 42ed578..521b2c9 100644 --- a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/parse/sql/SqlParserHelper.java +++ b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/parse/sql/SqlParserHelper.java @@ -1,6 +1,8 @@ package com.elasticsearch.engine.base.common.parse.sql; import com.elasticsearch.engine.base.common.utils.CaseFormatUtils; +import com.elasticsearch.engine.base.common.utils.LocalStringUtils; +import com.elasticsearch.engine.base.common.utils.ReflectionUtils; import com.elasticsearch.engine.base.config.EsEngineConfig; import com.elasticsearch.engine.base.model.annotion.EsQueryIndex; import com.elasticsearch.engine.base.model.domain.BackDto; @@ -10,6 +12,7 @@ import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.*; import net.sf.jsqlparser.expression.operators.conditional.AndExpression; +import net.sf.jsqlparser.expression.operators.relational.Between; import net.sf.jsqlparser.expression.operators.relational.InExpression; import net.sf.jsqlparser.parser.CCJSqlParserManager; import net.sf.jsqlparser.parser.CCJSqlParserUtil; @@ -19,6 +22,7 @@ import net.sf.jsqlparser.util.SelectUtils; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; import java.io.StringReader; import java.lang.reflect.Method; @@ -47,24 +51,26 @@ public static Select rewriteSql(Method method, String oldSql, Boolean isCleanAs, CCJSqlParserManager parserManager = new CCJSqlParserManager(); Select select = (Select) parserManager.parse(new StringReader(oldSql)); PlainSelect plain = (PlainSelect) select.getSelectBody(); + Map tableNames = getJoinTableName(plain); + Map tableAlias = ReflectionUtils.getEsAlias(method, tableNames); //替换表名 setTableItem(method, plain); //清除关联 plain.setJoins(Lists.newArrayList()); //from 别名清除 if (Objects.isNull(backDto)) { - setSelectItem(plain, isCleanAs); + setSelectItem(plain, isCleanAs, tableAlias); } else { setSelectItem(select, backDto); } //where 别名清除 - setWhereItem(plain.getWhere()); + setWhereItem(plain.getWhere(), tableAlias); //group by 别名清除 - setGroupItem(plain.getGroupBy()); + setGroupItem(plain.getGroupBy(), tableAlias); //having - setHavingItem(plain.getHaving()); + setHavingItem(plain.getHaving(), tableAlias); //order by 别名清除 - setOrderItem(plain.getOrderByElements()); + setOrderItem(plain.getOrderByElements(), tableAlias); return select; } @@ -116,6 +122,7 @@ private static void setBackWhereItem(PlainSelect plain, BackDto backDto, List /** * 回去回表查询表名 + * * @param plain * @param backDto * @return @@ -165,7 +172,7 @@ private static Map getJoinTableName(PlainSelect plain) { fromItems.forEach(fromItem -> { String fromItemName = ""; if (fromItem instanceof Table) { - fromItemName = ((Table) fromItem).getName().replaceAll("`", ""); + fromItemName = ((Table) fromItem).getName(); } tableNames.put(fromItemName, fromItem.getAlias() == null ? fromItemName : fromItem.getAlias().getName()); }); @@ -194,19 +201,19 @@ private static void setTableItem(Method method, PlainSelect plain) { * @param plainSelect * @return */ - public static void setSelectItem(PlainSelect plainSelect, Boolean isCleanAs) { + public static void setSelectItem(PlainSelect plainSelect, Boolean isCleanAs, Map tableAlias) { for (SelectItem selectItem : plainSelect.getSelectItems()) { selectItem.accept(new SelectItemVisitorAdapter() { @Override public void visit(SelectExpressionItem item) { if (item.getExpression() instanceof Function) { - setFunction((Function) item.getExpression()); + setFunction((Function) item.getExpression(), tableAlias); } else if (item.getExpression() instanceof CaseExpression) { - setCaseExpression((CaseExpression) item.getExpression()); + setCaseExpression((CaseExpression) item.getExpression(), tableAlias); } else if (item.getExpression() instanceof Column) { Column column = (Column) item.getExpression(); //清除t. - reNameColumnName(column); + reNameColumnName(column, tableAlias); column.setTable(new Table()); } //清除as @@ -240,7 +247,7 @@ private static void setSelectItem(Select select, BackDto backDto) { * @param * @return */ - public static void setWhereItem(Expression where) { + public static void setWhereItem(Expression where, Map tableAlias) { if (where == null) { return; } @@ -261,25 +268,30 @@ public static void setWhereItem(Expression where) { rightExpression = inExpression.getRightExpression() instanceof Parenthesis ? ((Parenthesis) inExpression.getRightExpression()).getExpression() : inExpression.getRightExpression(); leftExpression = inExpression.getLeftExpression() instanceof Parenthesis ? ((Parenthesis) inExpression.getLeftExpression()).getExpression() : inExpression.getLeftExpression(); } + + if (where instanceof Between) { + Between between = (Between) where; + leftExpression = between.getLeftExpression() instanceof Parenthesis ? ((Parenthesis) between.getLeftExpression()).getExpression() : between.getLeftExpression(); + } if (rightExpression instanceof Column) { Column rightColumn = (Column) rightExpression; - reNameColumnName(rightColumn); + reNameColumnName(rightColumn, tableAlias); //清除表别名 rightColumn.setTable(new Table()); } else if (rightExpression instanceof Function) { - setFunction((Function) rightExpression); + setFunction((Function) rightExpression, tableAlias); } else { - setWhereItem(rightExpression); + setWhereItem(rightExpression, tableAlias); } if (leftExpression instanceof Column) { Column leftColumn = (Column) leftExpression; - reNameColumnName(leftColumn); + reNameColumnName(leftColumn, tableAlias); //清除表别名 leftColumn.setTable(new Table()); } else if (leftExpression instanceof Function) { - setFunction((Function) leftExpression); + setFunction((Function) leftExpression, tableAlias); } else { - setWhereItem(leftExpression); + setWhereItem(leftExpression, tableAlias); } } @@ -288,7 +300,7 @@ public static void setWhereItem(Expression where) { * * @param groupBy */ - private static void setGroupItem(GroupByElement groupBy) { + private static void setGroupItem(GroupByElement groupBy, Map tableAlias) { if (groupBy == null) { return; } @@ -296,7 +308,7 @@ private static void setGroupItem(GroupByElement groupBy) { groupByExpressions.forEach(item -> { if (item instanceof Column) { Column groupColumn = (Column) item; - reNameColumnName(groupColumn); + reNameColumnName(groupColumn, tableAlias); groupColumn.setTable(new Table()); } }); @@ -307,11 +319,11 @@ private static void setGroupItem(GroupByElement groupBy) { * * @param having */ - private static void setHavingItem(Expression having) { + private static void setHavingItem(Expression having, Map tableAlias) { if (having == null) { return; } - setWhereItem(having); + setWhereItem(having, tableAlias); } /** @@ -319,7 +331,7 @@ private static void setHavingItem(Expression having) { * * @param orderByElements */ - private static void setOrderItem(List orderByElements) { + private static void setOrderItem(List orderByElements, Map tableAlias) { if (CollectionUtils.isEmpty(orderByElements)) { return; } @@ -327,7 +339,7 @@ private static void setOrderItem(List orderByElements) { Expression expression = item.getExpression(); if (expression instanceof Column) { Column groupColumn = (Column) expression; - reNameColumnName(groupColumn); + reNameColumnName(groupColumn, tableAlias); groupColumn.setTable(new Table()); } }); @@ -339,17 +351,17 @@ private static void setOrderItem(List orderByElements) { * * @param function */ - private static void setFunction(Function function) { + private static void setFunction(Function function, Map tableAlias) { if (function.getParameters() == null || function.getParameters().getExpressions() == null) { return; } List list = function.getParameters().getExpressions(); list.forEach(data -> { if (data instanceof Function) { - setFunction((Function) data); + setFunction((Function) data, tableAlias); } else if (data instanceof Column) { Column column = (Column) data; - reNameColumnName(column); + reNameColumnName(column, tableAlias); //清除表别名 column.setTable(new Table()); } @@ -362,14 +374,14 @@ private static void setFunction(Function function) { * @param caseExpression * @return */ - public static void setCaseExpression(CaseExpression caseExpression) { + public static void setCaseExpression(CaseExpression caseExpression, Map tableAlias) { if (caseExpression.getWhenClauses() == null) { return; } List list = caseExpression.getWhenClauses(); list.forEach(data -> { - if (data instanceof WhenClause) { - setWhereItem(((WhenClause) data).getWhenExpression()); + if (data != null) { + setWhereItem(data.getWhenExpression(), tableAlias); } }); } @@ -379,7 +391,20 @@ public static void setCaseExpression(CaseExpression caseExpression) { * * @param column */ - private static void reNameColumnName(Column column) { + private static void reNameColumnName(Column column, Map tableAlias) { + //替换mysql和es的别名映射 + if (!tableAlias.isEmpty()) { + String columnName = column.toString(); + //判断是否包含两个'.' 替换掉jooq的库名(jooq生成的字段格式为 `user`.`person`.`person_no`) + int n = columnName.length() - LocalStringUtils.replaceSpot(columnName).length(); + if (n > NumberUtils.INTEGER_ONE) { + columnName = columnName.substring(columnName.indexOf(".") + 1); + } + if (tableAlias.containsKey(columnName)) { + column.setColumnName(tableAlias.get(columnName)); + } + } + //清除t. if (!EsEngineConfig.isNamingStrategy()) { String columnName = column.getColumnName(); columnName = CaseFormatUtils.underscoreToCamel(columnName); diff --git a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/utils/BeanTools.java b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/parse/sql/SqlResponseParse.java similarity index 58% rename from elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/utils/BeanTools.java rename to elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/parse/sql/SqlResponseParse.java index 93d4fb3..7846db8 100644 --- a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/utils/BeanTools.java +++ b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/parse/sql/SqlResponseParse.java @@ -1,47 +1,75 @@ -package com.elasticsearch.engine.base.common.utils; +package com.elasticsearch.engine.base.common.parse.sql; +import com.elasticsearch.engine.base.common.utils.CaseFormatUtils; +import com.elasticsearch.engine.base.common.utils.DateUtils; +import com.elasticsearch.engine.base.common.utils.ReflectionUtils; +import com.elasticsearch.engine.base.config.EsEngineConfig; +import com.elasticsearch.engine.base.model.domain.SqlResponse; import com.elasticsearch.engine.base.model.emenu.DataType; import com.elasticsearch.engine.base.model.exception.EsEngineExecuteException; -import org.springframework.beans.BeanWrapper; -import org.springframework.beans.BeanWrapperImpl; -import org.springframework.util.Assert; -import org.springframework.util.StringUtils; -import java.beans.PropertyDescriptor; import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.math.BigDecimal; -import java.util.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; /** -* @author wanghuan -* @description BeanTools -* @mail 958721894@qq.com -* @date 2022/6/17 15:30 -*/ -public class BeanTools { - public static Object mapToObject(Map map, Class beanClass) throws Exception { - if (map == null) { - return null; - } - - Object obj = beanClass.newInstance(); + * @author wanghuan + * @description BeanTools + * @mail 958721894@qq.com + * @date 2022/6/17 15:30 + */ +public class SqlResponseParse { - Field[] fields = obj.getClass().getDeclaredFields(); - for (Field field : fields) { - if (map.get(field.getName()) == null || StringUtils.isEmpty(map.get(field.getName()))) { - continue; - } - int mod = field.getModifiers(); - if (Modifier.isStatic(mod) || Modifier.isFinal(mod)) { - continue; + /** + * 将sql查询结果转换为指定类 + * + * @param columns + * @param rows + * @param clazz + * @param + * @return + * @throws Exception + */ + public static T generateObjBySqlReps(List columns, List rows, Class clazz, Boolean isExtendQuery) throws Exception { + if (rows.size() != columns.size()) { + throw new Exception("sql column not match"); + } + //单个结果: count,sum 结果转换 + boolean check = rows.size() == 1 && (ReflectionUtils.isBaseType(clazz) || clazz.equals(BigDecimal.class)); + if (check) { + return (T) SqlResponseParse.fieldTypeCovert(DataType.getDataTypeByStr(columns.get(0).getType()), rows.get(0), clazz); + } + //entity listEntity + Map valueMap = new HashMap(32); + for (int i = 0; i < rows.size(); i++) { + SqlResponseParse.NameTypeValueMap m = new SqlResponseParse.NameTypeValueMap(); + m.setDataType(DataType.getDataTypeByStr(columns.get(i).getType())); + String paramName = columns.get(i).getName(); + //是否下划线转驼峰转 + if (EsEngineConfig.isNamingStrategy()) { + paramName = CaseFormatUtils.underscoreToCamel(paramName); } - field.setAccessible(true); - field.set(obj, map.get(field.getName())); + m.setFieldName(paramName); + m.setValue(rows.get(i)); + valueMap.put(paramName, m); } - return obj; + T t = (T) SqlResponseParse.typeMapToObject(valueMap, clazz); + return t; } + /** + * map to object + * + * @param map + * @param beanClass + * @param + * @return + * @throws Exception + */ public static T typeMapToObject(Map map, Class beanClass) throws Exception { if (map == null) { return null; @@ -65,35 +93,6 @@ public static T typeMapToObject(Map map, Class return t; } - public static String[] getNoValuePropertyNames(Object source) { - Assert.notNull(source, "传递的参数对象不能为空"); - final BeanWrapper beanWrapper = new BeanWrapperImpl(source); - PropertyDescriptor[] pds = beanWrapper.getPropertyDescriptors(); - - Set noValuePropertySet = new HashSet<>(); - Arrays.stream(pds).forEach(pd -> { - Object propertyValue = beanWrapper.getPropertyValue(pd.getName()); - if (StringUtils.isEmpty(propertyValue)) { - noValuePropertySet.add(pd.getName()); - } else { - if (Iterable.class.isAssignableFrom(propertyValue.getClass())) { - Iterable iterable = (Iterable) propertyValue; - Iterator iterator = iterable.iterator(); - if (!iterator.hasNext()) { - noValuePropertySet.add(pd.getName()); - } - } - if (Map.class.isAssignableFrom(propertyValue.getClass())) { - Map map = (Map) propertyValue; - if (map.isEmpty()) { - noValuePropertySet.add(pd.getName()); - } - } - } - }); - String[] result = new String[noValuePropertySet.size()]; - return noValuePropertySet.toArray(result); - } /** * es参数类型转化 diff --git a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/proxy/EsEngineInterfaceScanner.java b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/proxy/EsEngineInterfaceScanner.java index 48ef2c6..ae5a1a9 100644 --- a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/proxy/EsEngineInterfaceScanner.java +++ b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/proxy/EsEngineInterfaceScanner.java @@ -66,7 +66,7 @@ public void setResourceLoader(ResourceLoader resourceLoader) { } /** - * 对应添加了@EsHelperProxy的类生成代理 + * 对继承了BaseEsRepository的类生成代理 * 修改 BeanDefinition 并重新注册到 beanDefinitionMap * * @param registry diff --git a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/proxy/EsEngineProxyBeanFactory.java b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/proxy/EsEngineProxyBeanFactory.java index f9b09e7..4f441c0 100644 --- a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/proxy/EsEngineProxyBeanFactory.java +++ b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/proxy/EsEngineProxyBeanFactory.java @@ -1,6 +1,5 @@ package com.elasticsearch.engine.base.common.proxy; -import com.elasticsearch.engine.base.common.queryhandler.EsProxyExecuteHandler; import org.elasticsearch.client.RestHighLevelClient; import org.springframework.beans.BeansException; import org.springframework.beans.factory.FactoryBean; @@ -43,6 +42,12 @@ public void setApplicationContext(ApplicationContext applicationContext) throws this.applicationContext = applicationContext; } + /** + * 返回FactoryBean创建的bean + * 调用 getBean("EsEngineProxyBeanFactory"),会调用EsEngineProxyBeanFactory.getObject() 返回对象 + * + * @return + */ @Override public T getObject() { return (T) Proxy.newProxyInstance( diff --git a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/queryhandler/EsProxyExecuteHandler.java b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/proxy/EsProxyExecuteHandler.java similarity index 95% rename from elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/queryhandler/EsProxyExecuteHandler.java rename to elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/proxy/EsProxyExecuteHandler.java index e82b821..ff94ac7 100644 --- a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/queryhandler/EsProxyExecuteHandler.java +++ b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/proxy/EsProxyExecuteHandler.java @@ -1,4 +1,4 @@ -package com.elasticsearch.engine.base.common.queryhandler; +package com.elasticsearch.engine.base.common.proxy; import com.elasticsearch.engine.base.common.proxy.enums.EsQueryType; import com.elasticsearch.engine.base.common.proxy.handler.EsQueryProxyExecuteFactory; diff --git a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/proxy/EsQueryProxy.java b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/proxy/EsQueryProxy.java index f535fa0..076b3fb 100644 --- a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/proxy/EsQueryProxy.java +++ b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/proxy/EsQueryProxy.java @@ -1,6 +1,5 @@ package com.elasticsearch.engine.base.common.proxy; -import com.elasticsearch.engine.base.common.queryhandler.EsProxyExecuteHandler; import com.elasticsearch.engine.base.common.utils.ThreadLocalUtil; import com.elasticsearch.engine.base.model.constant.CommonConstant; import org.slf4j.Logger; @@ -24,18 +23,18 @@ public class EsQueryProxy implements InvocationHandler { private EsProxyExecuteHandler esProxyExecuteHandler; - private boolean enableLogOutEsQueryJson = false; + private boolean enableLogOut = false; public EsQueryProxy(Class targetInterface, boolean visitQueryBeanParent) { this.targetInterface = targetInterface; this.visitQueryBeanParent = visitQueryBeanParent; } - public EsQueryProxy(Class targetInterface, boolean visitQueryBeanParent, EsProxyExecuteHandler esProxyExecuteHandler, boolean enableLogOutEsQueryJson) { + public EsQueryProxy(Class targetInterface, boolean visitQueryBeanParent, EsProxyExecuteHandler esProxyExecuteHandler, boolean enableLogOut) { this.targetInterface = targetInterface; this.visitQueryBeanParent = visitQueryBeanParent; this.esProxyExecuteHandler = esProxyExecuteHandler; - this.enableLogOutEsQueryJson = enableLogOutEsQueryJson; + this.enableLogOut = enableLogOut; } @Override diff --git a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/queryhandler/ann/model/EsBaseExecuteHandle.java b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/queryhandler/ann/model/EsBaseExecuteHandle.java index cb56165..6fc1151 100644 --- a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/queryhandler/ann/model/EsBaseExecuteHandle.java +++ b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/queryhandler/ann/model/EsBaseExecuteHandle.java @@ -4,11 +4,13 @@ import com.elasticsearch.engine.base.common.parse.ann.model.EsQueryEngine; import com.elasticsearch.engine.base.common.parse.ann.model.QueryAnnParser; import com.elasticsearch.engine.base.common.utils.JsonParser; +import com.elasticsearch.engine.base.common.utils.ThreadLocalUtil; import com.elasticsearch.engine.base.config.EsEngineConfig; import com.elasticsearch.engine.base.holder.AbstractEsRequestHolder; import com.elasticsearch.engine.base.hook.RequestHook; import com.elasticsearch.engine.base.hook.ResponseHook; import com.elasticsearch.engine.base.mapping.annotation.Aggs; +import com.elasticsearch.engine.base.model.constant.CommonConstant; import com.elasticsearch.engine.base.model.constant.EsConstant; import com.elasticsearch.engine.base.model.domain.BaseResp; import com.elasticsearch.engine.base.model.domain.DefaultAggResp; @@ -54,6 +56,7 @@ public class EsBaseExecuteHandle extends AbstractEsBaseExecuteHandle { */ @Override public void executePostProcessorBefore(Object param, AbstractEsRequestHolder esHolder) { + String methodName = ThreadLocalUtil.get(CommonConstant.INTERFACE_METHOD_NAME); //前置处理扩展 嵌套扩展处理 List hooks = esHolder.getRequestHooks(); if (!hooks.isEmpty()) { @@ -71,16 +74,22 @@ public void executePostProcessorBefore(Object param, AbstractEsRequestHolder esH } //前置处理es索引名动态配置 resetIndexName(esHolder); - log.info("execute-es-query-json is\n{}", esHolder.getSource().toString()); + if (EsEngineConfig.getQueryJsonLog()) { + log.info("{} execute-es-query-json is\n{}", methodName, esHolder.getSource().toString()); + } + } @Override public void executePostProcessorAfter(Object param, SearchResponse resp, BaseResp result) { + String methodName = ThreadLocalUtil.get(CommonConstant.INTERFACE_METHOD_NAME); ResponseHook responseHook; if ((responseHook = checkResponseHook(param)) != null) { result.setResult(responseHook.handleResponse(resp)); } - log.info("execute-es-result-json is\n{}", JsonParser.asJson(result)); + if (EsEngineConfig.getQueryJsonLog()) { + log.info("{} execute-es-result-json is\n{}", methodName, JsonParser.asJson(result)); + } } @@ -97,14 +106,15 @@ public SearchResponse execute(SearchSourceBuilder sourceBuilder, String indexNam sourceBuilder.timeout(new TimeValue(EsEngineConfig.getQueryTimeOut(), TimeUnit.SECONDS)); //ES的查询请求对象 SearchRequest searchRequest = new SearchRequest().indices(indexName).source(sourceBuilder); - log.info("execute-es-query-json is\n{}", searchRequest); + if (EsEngineConfig.getQueryJsonLog()) { + log.info("execute-es-query-json is\n{}", searchRequest); + } try { searchResponse = restClient.search(searchRequest, RequestOptions.DEFAULT); } catch (IOException e) { throw new EsEngineQueryException("Execute Query Error, Method-invokeNative ,cause:", e); } SearchHits hitsResult = searchResponse.getHits(); - log.info("命中总记录数 ={}", hitsResult.getTotalHits()); return searchResponse; } @@ -123,14 +133,15 @@ public BaseResp execute(SearchSourceBuilder sourceBuilder, String indexNa sourceBuilder.timeout(new TimeValue(EsEngineConfig.getQueryTimeOut(), TimeUnit.SECONDS)); //ES的查询请求对象 SearchRequest searchRequest = new SearchRequest().indices(indexName).source(sourceBuilder); - log.info("execute-es-query-json is\n{}", searchRequest); + if (EsEngineConfig.getQueryJsonLog()) { + log.info("execute-es-query-json is\n{}", searchRequest); + } try { searchResponse = restClient.search(searchRequest, RequestOptions.DEFAULT); } catch (IOException e) { throw new EsEngineQueryException("Execute Query Error, Method-invokeNativeBuildRes ,cause:", e); } SearchHits hitsResult = searchResponse.getHits(); - log.info("命中总记录数 ={}", hitsResult.getTotalHits()); return EsResponseParse.returnDefaultResult(searchResponse, responseClazz); } @@ -204,6 +215,7 @@ protected BaseResp baseExecute(Object param, Class responseClazz, Abst * @return */ public BaseResp executeAggs(Method method, Object param) { + String methodName = ThreadLocalUtil.get(CommonConstant.INTERFACE_METHOD_NAME); if (!checkExistsAggAnnotation(param)) { throw new EsEngineQueryException("param field Missing @Aggs annotation"); } @@ -219,11 +231,15 @@ public BaseResp executeAggs(Method method, Object param) { defaultAgg.setCount(bucketOneAgg.getDocCount()); records.add(defaultAgg); } - log.info("execute-es-response-json is\n{}", JsonParser.asJson(searchResponse)); + if (EsEngineConfig.getQueryJsonLog()) { + log.info("{} execute-es-response-json is\n{}", methodName, JsonParser.asJson(searchResponse)); + } BaseResp resp = new BaseResp<>(); resp.setRecords(records); resp.setTotalHit((long) records.size()); - log.info("execute-es-result-json is\n{}", JsonParser.asJson(resp)); + if (EsEngineConfig.getQueryJsonLog()) { + log.info("{} execute-es-result-json is\n{}", methodName, JsonParser.asJson(resp)); + } return resp; } diff --git a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/queryhandler/ann/param/EsParamExecuteHandler.java b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/queryhandler/ann/param/EsParamExecuteHandler.java index 37a18ba..03560e9 100644 --- a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/queryhandler/ann/param/EsParamExecuteHandler.java +++ b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/queryhandler/ann/param/EsParamExecuteHandler.java @@ -39,13 +39,20 @@ public class EsParamExecuteHandler extends AbstractEsBaseExecuteHandle { @Override public void executePostProcessorBefore(Object param, AbstractEsRequestHolder esHolder) { + String methodName = ThreadLocalUtil.get(CommonConstant.INTERFACE_METHOD_NAME); //前置处理es索引名动态配置 resetIndexName(esHolder); + if (EsEngineConfig.getQueryJsonLog()) { + log.info("{} execute-es-query-json is\n{}", methodName, esHolder.getSource().toString()); + } } @Override public void executePostProcessorAfter(Object param, SearchResponse resp, BaseResp result) { - + String methodName = ThreadLocalUtil.get(CommonConstant.INTERFACE_METHOD_NAME); + if (EsEngineConfig.getQueryJsonLog()) { + log.info("{} execute-es-result-json is\n{}", methodName, JsonParser.asJson(result)); + } } @@ -108,7 +115,6 @@ protected BaseResp baseExecute(Class responseClazz, AbstractEsRequestH source.timeout(new TimeValue(EsEngineConfig.getQueryTimeOut(), TimeUnit.SECONDS)); //前置扩展 executePostProcessorBefore(null, esHolder); - log.info("{} execute-es-query-json is\n{}", methodName, esHolder.getSource().toString()); try { resp = restClient.search(esHolder.getRequest(), RequestOptions.DEFAULT); } catch (IOException e) { @@ -117,7 +123,6 @@ protected BaseResp baseExecute(Class responseClazz, AbstractEsRequestH //后置处理扩展 加入自定义结果解析 BaseResp result = EsResponseParse.returnDefaultResult(resp, responseClazz); executePostProcessorAfter(null, resp, result); - log.info("{} execute-es-result-json is\n{}", methodName, JsonParser.asJson(result)); ThreadLocalUtil.remove(); return result; } finally { diff --git a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/queryhandler/sql/EsSqlExecuteHandler.java b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/queryhandler/sql/EsSqlExecuteHandler.java index 3b68ba5..75ba7aa 100644 --- a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/queryhandler/sql/EsSqlExecuteHandler.java +++ b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/queryhandler/sql/EsSqlExecuteHandler.java @@ -1,11 +1,12 @@ package com.elasticsearch.engine.base.common.queryhandler.sql; -import com.elasticsearch.engine.base.common.utils.*; +import com.elasticsearch.engine.base.common.parse.sql.SqlResponseParse; +import com.elasticsearch.engine.base.common.utils.HttpClientTool; +import com.elasticsearch.engine.base.common.utils.JsonParser; import com.elasticsearch.engine.base.config.ElasticSearchProperties; import com.elasticsearch.engine.base.config.EsEngineConfig; import com.elasticsearch.engine.base.model.constant.CommonConstant; import com.elasticsearch.engine.base.model.domain.SqlResponse; -import com.elasticsearch.engine.base.model.emenu.DataType; import com.elasticsearch.engine.base.model.emenu.EsVersionConstant; import com.elasticsearch.engine.base.model.emenu.SqlFormat; import com.elasticsearch.engine.base.model.exception.EsEngineQueryException; @@ -15,8 +16,10 @@ import org.springframework.stereotype.Component; import javax.annotation.Resource; -import java.math.BigDecimal; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Random; /** * @author wanghuan @@ -53,8 +56,10 @@ public String queryBySql(String sql, SqlFormat sqlFormat) { ipport = hosts[randomindex]; } ipport = "http://" + ipport; - log.info(ipport + constant.getSqlQueryPrefix() + sqlFormat.getFormat()); - log.info("{\"query\":\"" + sql + "\"}"); + if (EsEngineConfig.getQueryJsonLog()) { + log.info(ipport + constant.getSqlQueryPrefix() + sqlFormat.getFormat()); + log.info("{\"query\":\"" + sql + "\"}"); + } String username = elasticSearchProperties.getUsername(); String password = elasticSearchProperties.getPassword(); @@ -126,7 +131,7 @@ public List queryBySql(String sql, Class clazz, Boolean isExtendQuery) if (!CollectionUtils.isEmpty(sqlResponse.getRows())) { for (List row : sqlResponse.getRows()) { try { - result.add(generateObjBySqlReps(sqlResponse.getColumns(), row, clazz, isExtendQuery)); + result.add(SqlResponseParse.generateObjBySqlReps(sqlResponse.getColumns(), row, clazz, isExtendQuery)); } catch (Exception e) { throw new RuntimeException(e); } @@ -136,41 +141,4 @@ public List queryBySql(String sql, Class clazz, Boolean isExtendQuery) } - /** - * 将sql查询结果转换为指定类 - * - * @param columns - * @param rows - * @param clazz - * @param - * @return - * @throws Exception - */ - private T generateObjBySqlReps(List columns, List rows, Class clazz, Boolean isExtendQuery) throws Exception { - if (rows.size() != columns.size()) { - throw new Exception("sql column not match"); - } - //单个结果: count,sum 结果转换 - boolean check = rows.size() == 1 && (ReflectionUtils.isBaseType(clazz) || clazz.equals(BigDecimal.class)); - if (check) { - return (T) BeanTools.fieldTypeCovert(DataType.getDataTypeByStr(columns.get(0).getType()), rows.get(0), clazz); - } - //entity listEntity - Map valueMap = new HashMap(32); - for (int i = 0; i < rows.size(); i++) { - BeanTools.NameTypeValueMap m = new BeanTools.NameTypeValueMap(); - m.setDataType(DataType.getDataTypeByStr(columns.get(i).getType())); - String paramName = columns.get(i).getName(); - //是否下划线转驼峰转 - if (EsEngineConfig.isNamingStrategy()) { - paramName = CaseFormatUtils.underscoreToCamel(paramName); - } - m.setFieldName(paramName); - m.setValue(rows.get(i)); - valueMap.put(paramName, m); - } - T t = (T) BeanTools.typeMapToObject(valueMap, clazz); - return t; - } - } diff --git a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/utils/LocalStringUtils.java b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/utils/LocalStringUtils.java new file mode 100644 index 0000000..07e436b --- /dev/null +++ b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/utils/LocalStringUtils.java @@ -0,0 +1,30 @@ +package com.elasticsearch.engine.base.common.utils; + +/** + * @author wanghuan + * @description StringUtils + * @mail 958721894@qq.com + * @date 2022-07-15 22:24 + */ +public class LocalStringUtils { + + /** + * 替换string中的'`' + * + * @param str + * @return + */ + public static String replaceSlightPauseMark(String str) { + return str.replaceAll("`", ""); + } + + /** + * 替换string中的'.' + * + * @param str + * @return + */ + public static String replaceSpot(String str) { + return str.replaceAll("\\.", ""); + } +} diff --git a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/utils/ReflectionUtils.java b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/utils/ReflectionUtils.java index f1efa54..e3f636b 100644 --- a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/utils/ReflectionUtils.java +++ b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/common/utils/ReflectionUtils.java @@ -2,7 +2,12 @@ import com.elasticsearch.engine.base.common.parse.sql.SqlParamParseHelper; +import com.elasticsearch.engine.base.common.proxy.handler.exannotation.AnnotationQueryCommon; +import com.elasticsearch.engine.base.model.annotion.ESColumn; +import com.elasticsearch.engine.base.model.annotion.EsQueryIndex; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.commons.lang3.StringUtils; import java.lang.reflect.*; import java.math.BigDecimal; @@ -190,16 +195,16 @@ public static boolean isExtendsType(Class type) { /** * 获取对象的filed name 和 value,支持嵌套 * file 的name 格式为 为 objectName.filedName的形式 - * + * * @param view * @param paramName * @return * @throws IllegalAccessException */ - public static Map getNestedFieldsMap(String paramName,Object view) { + public static Map getNestedFieldsMap(String paramName, Object view) { Map map = new HashMap<>(26); try { - getFields(view,map, paramName); + getFields(view, map, paramName); } catch (IllegalAccessException e) { throw new RuntimeException(e); } @@ -208,6 +213,7 @@ public static Map getNestedFieldsMap(String paramName,Object vie /** * 递归获取对象的 filed name 和 value + * * @param view * @param map * @param parentName @@ -220,13 +226,61 @@ public static void getFields(Object view, Map map, String parent field.setAccessible(true); String name = parentName + "." + field.getName(); Object val = field.get(view); - if(isBaseTypeAndExtend(field.getType())){ + if (isBaseTypeAndExtend(field.getType())) { String parameterVal = SqlParamParseHelper.getParameterValue(val); map.put(name, parameterVal); - }else{ - getFields(val,map,name); + } else { + getFields(val, map, name); + } + } + } + + /** + * 获取mybatis映射es的字段别名 + * + * @param method + * @return + */ + public static Map getEsAlias(Method method, Map tableNames) { + //方法返回值 + Class queryIndex = null; + Class returnType = method.getReturnType(); + if (Objects.nonNull(returnType.getAnnotation(EsQueryIndex.class))) { + queryIndex = returnType; + } + //方法返回值的泛型 + Class returnGenericType = AnnotationQueryCommon.getReturnGenericType(method); + if (Objects.isNull(queryIndex) && Objects.nonNull(returnGenericType) && Objects.nonNull(returnGenericType.getAnnotation(EsQueryIndex.class))) { + queryIndex = returnGenericType; + } + + Map table = Maps.newHashMap(); + if (Objects.nonNull(queryIndex)) { + Field[] fieldArr = queryIndex.getDeclaredFields(); + for (Field field : fieldArr) { + field.setAccessible(true); + ESColumn esColumn = field.getAnnotation(ESColumn.class); + if (Objects.nonNull(esColumn)) { + table.put(getSqlColumn(esColumn, tableNames), esColumn.esColumn()); + } } } + return table; + } + + /** + * getSqlColumn + * + * @param esColumn + * @param tableNames + * @return + */ + private static String getSqlColumn(ESColumn esColumn, Map tableNames) { + String tableName = tableNames.get(esColumn.table()); + if (StringUtils.isNotEmpty(tableName)) { + return tableName + "." + esColumn.sqlColumn(); + } + return esColumn.sqlColumn(); } } diff --git a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/config/EsEngineConfig.java b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/config/EsEngineConfig.java index cba7a98..e23ba3c 100644 --- a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/config/EsEngineConfig.java +++ b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/config/EsEngineConfig.java @@ -112,6 +112,24 @@ public static boolean getEsQuery() { return globalConfig.esEngineConfigProperties.isEsQuery(); } + /** + * es extend(mybatis,jpa,jooq) sql转换日志开关 + * + * @return + */ + public static boolean getSqlTraceLog() { + return globalConfig.esEngineConfigProperties.isSqlTraceLog(); + } + + /** + * es query json log 日志开关 + * + * @return + */ + public static boolean getQueryJsonLog() { + return globalConfig.esEngineConfigProperties.isQueryJsonLog(); + } + /** * es extend(mybatis,jpa,jooq)查询降级 包含的接口,仅再esQuery=false时才生效 * diff --git a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/config/EsEngineConfigProperties.java b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/config/EsEngineConfigProperties.java index 79a4983..c22feae 100644 --- a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/config/EsEngineConfigProperties.java +++ b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/config/EsEngineConfigProperties.java @@ -18,7 +18,7 @@ public class EsEngineConfigProperties { /** * elasticSearch version */ - private Integer elasticVersion = 7; + private Integer elasticVersion = 6; /** * 对没有添加注解的字段 默然按照trem/trems查询 @@ -26,7 +26,7 @@ public class EsEngineConfigProperties { private boolean isBuildDefault = Boolean.TRUE; /** - * 查询参及响应参数数解析是否是下划线 true解析成下划线 false按照参数名驼峰 + * 查询参及响应参数数解析是否是下划线 true解析成下划线 false按照参数名驼峰 默认值false */ private boolean namingStrategy = Boolean.FALSE; @@ -50,6 +50,16 @@ public class EsEngineConfigProperties { */ private boolean esQuery = Boolean.TRUE; + /** + * es extend(mybatis,jpa,jooq) sql转换日志开关 + */ + private boolean sqlTraceLog = Boolean.FALSE; + + /** + * es query json log 日志开关 + */ + private boolean queryJsonLog = Boolean.FALSE; + /** * es extend(mybatis,jpa,jooq)查询降级 包含的接口,仅再esQuery=false时才生效 * item 为接口名 diff --git a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/model/annotion/ESColumn.java b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/model/annotion/ESColumn.java new file mode 100644 index 0000000..e7de8a0 --- /dev/null +++ b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/model/annotion/ESColumn.java @@ -0,0 +1,20 @@ +package com.elasticsearch.engine.base.model.annotion; + +import java.lang.annotation.*; + +/** + * @author wanghuan + * @description ESColumn + * @date 2022/7/15 17:03 + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD, ElementType.ANNOTATION_TYPE}) +@Documented +public @interface ESColumn { + + String table() default ""; + + String sqlColumn() default ""; + + String esColumn() default ""; +} diff --git a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/model/constant/CommonConstant.java b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/model/constant/CommonConstant.java index f5e0dc7..7129705 100644 --- a/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/model/constant/CommonConstant.java +++ b/elasticsearch-engine-base/src/main/java/com/elasticsearch/engine/base/model/constant/CommonConstant.java @@ -71,8 +71,23 @@ public class CommonConstant { /** * es客户端版本为7, es版本为6的格式为 GMT+8 - * 日期查询默认的时间时区 + * 日期查询默认的时间时区 */ public static final String DEFAULT_TIME_ZONE = "+08:00"; + /** + * jooq BETWEEN 格式 + */ + public static final String JOOQ_SQL_BETWEEN = "between {ts"; + + /** + * jooq BETWEEN 前缀 + */ + public static final String JOOQ_SQL_BETWEEN_PREFIX = "\\{ts"; + + /** + * jooq BETWEEN 后缀 + */ + public static final String JOOQ_SQL_BETWEEN_SUFFIX = "\\}"; + } diff --git a/elasticsearch-engine-jooq/src/main/java/com/elasticsearch/engine/jooq/annotion/JooqEsQuery.java b/elasticsearch-engine-jooq/src/main/java/com/elasticsearch/engine/jooq/annotion/JooqEsQuery.java index 32fd4ae..5820b9c 100644 --- a/elasticsearch-engine-jooq/src/main/java/com/elasticsearch/engine/jooq/annotion/JooqEsQuery.java +++ b/elasticsearch-engine-jooq/src/main/java/com/elasticsearch/engine/jooq/annotion/JooqEsQuery.java @@ -18,7 +18,7 @@ * * @return */ - String tableName() default ""; + String backTable() default ""; /** * 回表字段 diff --git a/elasticsearch-engine-jooq/src/main/java/com/elasticsearch/engine/jooq/aop/JooqEsQueryAop.java b/elasticsearch-engine-jooq/src/main/java/com/elasticsearch/engine/jooq/aop/JooqEsQueryAop.java index 980cf82..1a57c30 100644 --- a/elasticsearch-engine-jooq/src/main/java/com/elasticsearch/engine/jooq/aop/JooqEsQueryAop.java +++ b/elasticsearch-engine-jooq/src/main/java/com/elasticsearch/engine/jooq/aop/JooqEsQueryAop.java @@ -1,16 +1,8 @@ package com.elasticsearch.engine.jooq.aop; import com.elasticsearch.engine.base.common.parse.sql.EsSqlQueryHelper; -import com.elasticsearch.engine.base.common.utils.ThreadLocalUtil; -import com.elasticsearch.engine.base.config.EsEngineConfig; -import com.elasticsearch.engine.base.model.constant.CommonConstant; -import com.elasticsearch.engine.base.model.domain.BackDto; -import com.elasticsearch.engine.base.model.exception.EsEngineExecuteException; -import com.elasticsearch.engine.base.model.exception.EsEngineJpaExecuteException; import com.elasticsearch.engine.jooq.model.JooqBackDto; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.aspectj.lang.ProceedingJoinPoint; import org.aspectj.lang.annotation.Around; import org.aspectj.lang.annotation.Aspect; @@ -19,9 +11,6 @@ import org.springframework.stereotype.Component; import javax.annotation.Resource; -import java.lang.reflect.Method; -import java.util.List; -import java.util.Objects; /** @@ -47,42 +36,7 @@ public void esQueryCut() { @Around(value = "esQueryCut()") public Object retryAdvice(ProceedingJoinPoint pjp) throws Throwable { MethodSignature signature = (MethodSignature) pjp.getSignature(); - Method method = signature.getMethod(); - Object[] args = pjp.getArgs(); - //不走es查询直接返回(全局开关) - if(!EsEngineConfig.isEsquery(method)){ - return pjp.proceed(args); - } - //获取回表查询参数 - BackDto backDto = JooqBackDto.hasJooqBack(method); - Object result = null; - try { - //设置标记,在sql拦截器中抛出异常->回到后面的异常处理逻辑中实现es查询 - ThreadLocalUtil.set(CommonConstant.IS_ES_QUERY, Boolean.TRUE); - result = pjp.proceed(args); - } catch (EsEngineJpaExecuteException e) { - //判断是否需要回表查询 - if (Objects.isNull(backDto)) { - //无需回表直接执行es查询 - result = esSqlQueryHelper.esQuery(method, e.getMessage(), args, backDto); - } else { - //回表sql执行, sql重新时使用 原生未绑定参数的sql - String bakSql = ThreadLocalUtil.remove(CommonConstant.JPA_NATIVE_SQL); - if (StringUtils.isEmpty(bakSql)) { - throw new EsEngineExecuteException("jpa 回表sql异常"); - } - List esResult = esSqlQueryHelper.esQueryBack(method, bakSql, args, backDto); - if (CollectionUtils.isEmpty(esResult)) { - return result; - } - result = pjp.proceed(args); - } - } finally { - ThreadLocalUtil.remove(CommonConstant.IS_ES_QUERY); - ThreadLocalUtil.remove(CommonConstant.BACK_QUERY_SQL); - ThreadLocalUtil.remove(CommonConstant.JPA_NATIVE_SQL); - } - return result; + return esSqlQueryHelper.esSqlQueryAopCommon(pjp, JooqBackDto.hasJooqBack(signature.getMethod())); } } diff --git a/elasticsearch-engine-jooq/src/main/java/com/elasticsearch/engine/jooq/listener/JooqEsQueryExecuteListener.java b/elasticsearch-engine-jooq/src/main/java/com/elasticsearch/engine/jooq/listener/JooqEsQueryExecuteListener.java index 0e7db44..949ae6a 100644 --- a/elasticsearch-engine-jooq/src/main/java/com/elasticsearch/engine/jooq/listener/JooqEsQueryExecuteListener.java +++ b/elasticsearch-engine-jooq/src/main/java/com/elasticsearch/engine/jooq/listener/JooqEsQueryExecuteListener.java @@ -1,5 +1,6 @@ package com.elasticsearch.engine.jooq.listener; +import com.elasticsearch.engine.base.common.utils.LocalStringUtils; import com.elasticsearch.engine.base.common.utils.ThreadLocalUtil; import com.elasticsearch.engine.base.model.constant.CommonConstant; import com.elasticsearch.engine.base.model.exception.EsEngineJpaExecuteException; @@ -59,7 +60,7 @@ public void renderEnd(ExecuteContext ctx) { // ctx.sql("SELECT `user`.`person`.`id`, `user`.`person`.`person_no`, `user`.`person`.`person_name`, `user`.`person`.`phone`, `user`.`person`.`salary`, `user`.`person`.`company`, `user`.`person`.`status`, `user`.`person`.`sex`, `user`.`person`.`address`, `user`.`person`.`create_time`, `user`.`person`.`create_user` FROM `user`.`person` WHERE `user`.`person`.`status` = ? AND person_no IN ('US2022060100001', 'US2022060100023')"); ctx.sql(backSql); } else { - ThreadLocalUtil.set(CommonConstant.JPA_NATIVE_SQL, ctx.sql()); + ThreadLocalUtil.set(CommonConstant.JPA_NATIVE_SQL, sqlTransform(Objects.requireNonNull(ctx.sql()))); } } @@ -70,7 +71,7 @@ public void renderEnd(ExecuteContext ctx) { */ @Override public void executeEnd(ExecuteContext ctx) { - log.info("jooq回表执行sql: " + ctx.sql()); +// log.info("jooq回表执行sql: " + ctx.sql()); } @Override @@ -98,7 +99,7 @@ public void bindEnd(ExecuteContext ctx) { if (Objects.isNull(isEsQuery)) { return; } - String sql = ctx.query().toString(); + String sql = Objects.requireNonNull(ctx.query()).toString(); //非select语句直接返回 if (!sql.trim().startsWith(CommonConstant.SELECT_SQL_PREFIX_LOWER) && !sql.trim().startsWith(CommonConstant.SELECT_SQL_PREFIX_UPPER)) { return; @@ -106,7 +107,7 @@ public void bindEnd(ExecuteContext ctx) { String backSql = ThreadLocalUtil.get(CommonConstant.BACK_QUERY_SQL); if (StringUtils.isEmpty(backSql)) { - throw new EsEngineJpaExecuteException(sql); + throw new EsEngineJpaExecuteException(sqlTransform(sql)); } } @@ -131,4 +132,14 @@ public void bindEnd(ExecuteContext ctx) { // } // } + private String sqlTransform(String sql) { + //jooq 需要替换"`" + sql = LocalStringUtils.replaceSlightPauseMark(sql); + //处理jooq生成的between + if (sql.contains(CommonConstant.JOOQ_SQL_BETWEEN)) { + return sql.replaceAll(CommonConstant.JOOQ_SQL_BETWEEN_PREFIX, "").replaceAll(CommonConstant.JOOQ_SQL_BETWEEN_SUFFIX, ""); + } + return sql; + } + } \ No newline at end of file diff --git a/elasticsearch-engine-jooq/src/main/java/com/elasticsearch/engine/jooq/model/JooqBackDto.java b/elasticsearch-engine-jooq/src/main/java/com/elasticsearch/engine/jooq/model/JooqBackDto.java index 9fc0c1c..a1b0f27 100644 --- a/elasticsearch-engine-jooq/src/main/java/com/elasticsearch/engine/jooq/model/JooqBackDto.java +++ b/elasticsearch-engine-jooq/src/main/java/com/elasticsearch/engine/jooq/model/JooqBackDto.java @@ -18,7 +18,7 @@ public class JooqBackDto extends BackDto { public static BackDto hasJooqBack(Method method) { JooqEsQuery esQuery = method.getAnnotation(JooqEsQuery.class); String backColumn = esQuery.backColumn(); - String tableName = esQuery.tableName(); + String tableName = esQuery.backTable(); Class backColumnTyp = esQuery.backColumnType(); if (StringUtils.isNotEmpty(backColumn) && Objects.nonNull(backColumnTyp) && !backColumnTyp.equals(Objects.class)) { return BackDto.builder().tableName(tableName).backColumn(backColumn).backColumnTyp(backColumnTyp).build(); diff --git a/elasticsearch-engine-jpa/src/main/java/com/elasticsearch/engine/jpa/annotion/JpaEsQuery.java b/elasticsearch-engine-jpa/src/main/java/com/elasticsearch/engine/jpa/annotion/JpaEsQuery.java index 65e6231..5819421 100644 --- a/elasticsearch-engine-jpa/src/main/java/com/elasticsearch/engine/jpa/annotion/JpaEsQuery.java +++ b/elasticsearch-engine-jpa/src/main/java/com/elasticsearch/engine/jpa/annotion/JpaEsQuery.java @@ -18,7 +18,7 @@ * * @return */ - String tableName() default ""; + String backTable() default ""; /** * 回表字段 diff --git a/elasticsearch-engine-jpa/src/main/java/com/elasticsearch/engine/jpa/aop/JpaEsQueryAop.java b/elasticsearch-engine-jpa/src/main/java/com/elasticsearch/engine/jpa/aop/JpaEsQueryAop.java index 2202a20..554b664 100644 --- a/elasticsearch-engine-jpa/src/main/java/com/elasticsearch/engine/jpa/aop/JpaEsQueryAop.java +++ b/elasticsearch-engine-jpa/src/main/java/com/elasticsearch/engine/jpa/aop/JpaEsQueryAop.java @@ -1,16 +1,8 @@ package com.elasticsearch.engine.jpa.aop; import com.elasticsearch.engine.base.common.parse.sql.EsSqlQueryHelper; -import com.elasticsearch.engine.base.common.utils.ThreadLocalUtil; -import com.elasticsearch.engine.base.config.EsEngineConfig; -import com.elasticsearch.engine.base.model.constant.CommonConstant; -import com.elasticsearch.engine.base.model.domain.BackDto; -import com.elasticsearch.engine.base.model.exception.EsEngineExecuteException; -import com.elasticsearch.engine.base.model.exception.EsEngineJpaExecuteException; import com.elasticsearch.engine.jpa.model.JpaBackDto; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.aspectj.lang.ProceedingJoinPoint; import org.aspectj.lang.annotation.Around; import org.aspectj.lang.annotation.Aspect; @@ -19,9 +11,6 @@ import org.springframework.stereotype.Component; import javax.annotation.Resource; -import java.lang.reflect.Method; -import java.util.List; -import java.util.Objects; /** @@ -47,43 +36,6 @@ public void esQueryCut() { @Around(value = "esQueryCut()") public Object retryAdvice(ProceedingJoinPoint pjp) throws Throwable { MethodSignature signature = (MethodSignature) pjp.getSignature(); - Method method = signature.getMethod(); - Object[] args = pjp.getArgs(); - //不走es查询直接返回(全局开关) - if(!EsEngineConfig.isEsquery(method)){ - return pjp.proceed(args); - } - //获取回表查询参数 - BackDto backDto = JpaBackDto.hasJpaBack(method); - Object result = null; - try { - //设置标记,在sql拦截器中抛出异常->回到后面的异常处理逻辑中实现es查询 - ThreadLocalUtil.set(CommonConstant.IS_ES_QUERY, Boolean.TRUE); - result = pjp.proceed(args); - } catch (EsEngineJpaExecuteException e) { - //判断是否需要回表查询 - if (Objects.isNull(backDto)) { - //无需回表直接执行es查询 - //原生es执行 直接使用绑定参数后的sql - result = esSqlQueryHelper.esQuery(method, e.getMessage(), args, backDto); - } else { - //需要回表es查询并回表查询 - //回表sql执行, sql重新时使用 原生未绑定参数的sql - String bakSql = ThreadLocalUtil.remove(CommonConstant.JPA_NATIVE_SQL); - if (StringUtils.isEmpty(bakSql)) { - throw new EsEngineExecuteException("jpa 回表sql异常"); - } - List esResult = esSqlQueryHelper.esQueryBack(method, bakSql, args, backDto); - if (CollectionUtils.isEmpty(esResult)) { - return result; - } - result = pjp.proceed(args); - } - } finally { - ThreadLocalUtil.remove(CommonConstant.IS_ES_QUERY); - ThreadLocalUtil.remove(CommonConstant.BACK_QUERY_SQL); - ThreadLocalUtil.remove(CommonConstant.JPA_NATIVE_SQL); - } - return result; + return esSqlQueryHelper.esSqlQueryAopCommon(pjp, JpaBackDto.hasJpaBack(signature.getMethod())); } } diff --git a/elasticsearch-engine-jpa/src/main/java/com/elasticsearch/engine/jpa/model/JpaBackDto.java b/elasticsearch-engine-jpa/src/main/java/com/elasticsearch/engine/jpa/model/JpaBackDto.java index 59b0786..8de9c55 100644 --- a/elasticsearch-engine-jpa/src/main/java/com/elasticsearch/engine/jpa/model/JpaBackDto.java +++ b/elasticsearch-engine-jpa/src/main/java/com/elasticsearch/engine/jpa/model/JpaBackDto.java @@ -18,7 +18,7 @@ public class JpaBackDto extends BackDto { public static BackDto hasJpaBack(Method method) { JpaEsQuery esQuery = method.getAnnotation(JpaEsQuery.class); String backColumn = esQuery.backColumn(); - String tableName = esQuery.tableName(); + String tableName = esQuery.backTable(); Class backColumnTyp = esQuery.backColumnType(); if (StringUtils.isNotEmpty(backColumn) && Objects.nonNull(backColumnTyp) && !backColumnTyp.equals(Objects.class)) { return BackDto.builder().tableName(tableName).backColumn(backColumn).backColumnTyp(backColumnTyp).build(); diff --git a/elasticsearch-engine-mybatis/src/main/java/com/elasticsearch/engine/mybatis/annotion/MybatisEsQuery.java b/elasticsearch-engine-mybatis/src/main/java/com/elasticsearch/engine/mybatis/annotion/MybatisEsQuery.java index 6d562c5..a659bf8 100644 --- a/elasticsearch-engine-mybatis/src/main/java/com/elasticsearch/engine/mybatis/annotion/MybatisEsQuery.java +++ b/elasticsearch-engine-mybatis/src/main/java/com/elasticsearch/engine/mybatis/annotion/MybatisEsQuery.java @@ -19,7 +19,7 @@ * * @return */ - String tableName() default ""; + String backTable() default ""; /** * 回表字段 diff --git a/elasticsearch-engine-mybatis/src/main/java/com/elasticsearch/engine/mybatis/interceptor/MybatisEsQueryInterceptor.java b/elasticsearch-engine-mybatis/src/main/java/com/elasticsearch/engine/mybatis/interceptor/MybatisEsQueryInterceptor.java index b06fa48..5e89593 100644 --- a/elasticsearch-engine-mybatis/src/main/java/com/elasticsearch/engine/mybatis/interceptor/MybatisEsQueryInterceptor.java +++ b/elasticsearch-engine-mybatis/src/main/java/com/elasticsearch/engine/mybatis/interceptor/MybatisEsQueryInterceptor.java @@ -142,17 +142,23 @@ private List doQueryEs(Method method, BoundSql boundSql, Configuration config Class returnType = method.getReturnType(); //方法返回值的泛型 Class returnGenericType = AnnotationQueryCommon.getReturnGenericType(method); - log.info("原始sql: {}", boundSql.getSql()); + if (EsEngineConfig.getSqlTraceLog()) { + log.info("原始sql: {}", boundSql.getSql()); + } //改写sql Select select = SqlParserHelper.rewriteSql(method, boundSql.getSql(), Boolean.FALSE, null); //通过反射修改sql语句 Field field = boundSql.getClass().getDeclaredField("sql"); field.setAccessible(true); field.set(boundSql, select.toString()); - log.info("改写后sql: {}", boundSql.getSql()); + if (EsEngineConfig.getSqlTraceLog()) { + log.info("改写后sql: {}", boundSql.getSql()); + } //参数替换 String sql = SqlParamParseMybatisHelper.paramParse(configuration, boundSql); - log.info("替换参数后sql: {}", sql); + if (EsEngineConfig.getSqlTraceLog()) { + log.info("替换参数后sql: {}", sql); + } //执行ES查询 if (List.class.isAssignableFrom(returnType) && Objects.nonNull(returnGenericType)) { result = esSqlExecuteHandler.queryBySql(sql, returnGenericType, Boolean.TRUE); @@ -164,28 +170,44 @@ private List doQueryEs(Method method, BoundSql boundSql, Configuration config } + /** + * 执行es回表查询 + * + * @param method + * @param boundSql + * @param ms + * @param backDto + * @return + * @throws Exception + */ private MappedStatement doQueryEsBack(Method method, BoundSql boundSql, MappedStatement ms, BackDto backDto) throws Exception { Configuration configuration = ms.getConfiguration(); String originalSql = boundSql.getSql(); - log.info("原始sql: {}", originalSql); + if (EsEngineConfig.getSqlTraceLog()) { + log.info("原始sql: {}", originalSql); + } //改写sql Select select = SqlParserHelper.rewriteSql(method, boundSql.getSql(), Boolean.FALSE, backDto); //通过反射修改sql语句 Field field = boundSql.getClass().getDeclaredField("sql"); field.setAccessible(true); field.set(boundSql, select.toString()); - - log.info("改写后sql: {}", boundSql.getSql()); + if (EsEngineConfig.getSqlTraceLog()) { + log.info("改写后sql: {}", boundSql.getSql()); + } //参数替换 String sql = SqlParamParseMybatisHelper.paramParse(configuration, boundSql); - log.info("替换参数后sql: {}", sql); + if (EsEngineConfig.getSqlTraceLog()) { + log.info("替换参数后sql: {}", sql); + } //执行ES查询 List esResult = esSqlExecuteHandler.queryBySql(sql, backDto.getBackColumnTyp(), Boolean.TRUE); //将原sql改写成回表sql String backSql = SqlParserHelper.rewriteBackSql(originalSql, backDto, esResult); - log.info("回表sql : {}", backSql); - + if (EsEngineConfig.getSqlTraceLog()) { + log.info("回表sql : {}", backSql); + } //替换mybatis执行的sql MappedStatement qs = newMappedStatement(ms, new BoundSqlSqlSource(boundSql)); MetaObject msObject = SystemMetaObject.forObject(qs); diff --git a/elasticsearch-engine-mybatis/src/main/java/com/elasticsearch/engine/mybatis/model/MybatisBackDto.java b/elasticsearch-engine-mybatis/src/main/java/com/elasticsearch/engine/mybatis/model/MybatisBackDto.java index 7667518..41e6a29 100644 --- a/elasticsearch-engine-mybatis/src/main/java/com/elasticsearch/engine/mybatis/model/MybatisBackDto.java +++ b/elasticsearch-engine-mybatis/src/main/java/com/elasticsearch/engine/mybatis/model/MybatisBackDto.java @@ -18,7 +18,7 @@ public class MybatisBackDto extends BackDto { public static BackDto hasBack(Method method) { MybatisEsQuery esQuery = method.getAnnotation(MybatisEsQuery.class); String backColumn = esQuery.backColumn(); - String tableName = esQuery.tableName(); + String tableName = esQuery.backTable(); Class backColumnTyp = esQuery.backColumnType(); if (StringUtils.isNotEmpty(backColumn) && Objects.nonNull(backColumnTyp) && !backColumnTyp.equals(Objects.class)) { return BackDto.builder().tableName(tableName).backColumn(backColumn).backColumnTyp(backColumnTyp).build(); diff --git a/img.png b/img.png new file mode 100644 index 0000000..55aeaf8 Binary files /dev/null and b/img.png differ