Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj

# Conflicts:
#	projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy
#	projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy
#	projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy
This commit is contained in:
2025-11-19 11:47:42 +08:00
8 changed files with 35 additions and 50 deletions
@@ -1,6 +1,7 @@
package com.fangjia.open.interceptors;
import com.fangjia.core.util.AnalyticsUtil;
import com.fangjia.core.util.JsonUtils;
import com.fangjia.dal.mongo.open.model.OpenResource;
import com.fangjia.dal.mongo.tool.model.OpenUriLog;
import com.fangjia.dal.sql.model.ApiUser;
@@ -12,6 +13,8 @@ import com.fangjia.open.model.BasicModel;
import com.fangjia.open.model.RestfulModelAndView;
import com.fangjia.service.open.OpenResourceService;
import com.fangjia.service.open.OpenUriLogService;
import com.fasterxml.jackson.databind.ObjectMapper;
import net.sf.json.JSONNull;
import net.sf.json.JSONObject;
import net.sf.json.JsonConfig;
import net.sf.json.processors.JsonValueProcessor;
@@ -22,17 +25,13 @@ import org.sothis.web.mvc.ActionContext;
import org.sothis.web.mvc.ActionInvocation;
import org.sothis.web.mvc.ActionInvocationException;
import org.sothis.web.mvc.Interceptor;
import javax.servlet.http.HttpServletRequest;
import java.math.BigDecimal;
import java.text.SimpleDateFormat;
import java.util.Collection;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.*;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
/**
* API审计日志监控器
*
@@ -116,7 +115,6 @@ public class AuditInterceptor implements Interceptor {
}
return params;
}
/**
* @param request
* @param apiUser
@@ -163,7 +161,7 @@ public class AuditInterceptor implements Interceptor {
if(isSaveResult(request)){
JSONObject json = JSONObject.fromObject(result, DEFAULT_JSON_CONFIG);
json.remove("tid");
openUriLog.setResMap(json);
openUriLog.setResMap((Map<String, Object>)JsonUtils.convertJsonToJavaObj(json, Map.class));
}
} catch (Exception e) {
LOGGER.error("", e);
@@ -37,7 +37,7 @@ class DistrictSellStatBatch extends BaseBatch {
@Override
void process(Object args) {
def months = DateUtils.getMonthList(202209)
def months = DateUtils.getMonthList(202510)
months = months[0..(months.size()-1)]
println months
for(def month : months){
@@ -69,9 +69,9 @@ class DistrictSellStatBatch extends BaseBatch {
if(!curDate){
curDate = new Date()
}
def citys = CommonUtils.findAllCity()
citys.remove("上海")
citys.remove("福州")
def citys = ["上海"]//CommonUtils.findAllCity()
// citys.remove("上海")
// citys.remove("福州")
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1)
def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(curDate))
@@ -89,9 +89,7 @@ class DistrictSellStatBatch extends BaseBatch {
for(def city : citys){
index++
if(index < 242){
continue
}
def districts = []
if(city == "福州"){
if(day == 29){
@@ -847,23 +847,6 @@ class OpenApiUtils {
}
// static def addrVerifyShBank(city, address){
// address = URLEncoder.encode(address, "utf-8");
//// def url = "http://192.168.0.100:32080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}"
// def url = "http://localhost:8080/addrSvc/boscEnterpriseVillageIdentify?address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
// if(city){
// url = url + "&city=${city}"
// }
// String content = new HttpRequest().get(url, "utf-8");
// if(content){
// def doc = new JsonSlurper().parseText(content)
// if(doc && doc.code ==200 && doc.result){
// def dtos = doc.result
// return dtos
// }
// }
// return null;
// }
static def addrVerifyBase(city, address){
@@ -6,8 +6,13 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
<<<<<<< .mine
import net.sf.json.JSONNull
import org.bson.types.ObjectId
=======
>>>>>>> .theirs
class DistrictLocationGeo {
@@ -145,7 +150,8 @@ class DistrictLocationGeo {
ExcelUtils.modify("D:\\数据\\昆仑\\263未出居委数据.xlsx",{ d->
def jsonObject = GaoDeApiUtils.geocodeByAddress(d['C'], d['B'])
Thread.sleep(1000)
if(jsonObject){
def count = 0
if (jsonObject) {
def rets = jsonObject.geocodes
if(rets){
def dbo = rets[0]
@@ -154,7 +160,9 @@ class DistrictLocationGeo {
d['E'] = dbo.level
}
}
count = jsonObject.regeocode.pois.size()
}
d['D'] = count
})
}
}
@@ -6,17 +6,15 @@ import com.fangjia.script.Watcher
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.google.common.hash.*
import com.google.common.hash.BloomFilter
import com.google.common.hash.Funnels
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.commons.lang.StringUtils
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
@@ -24,15 +22,12 @@ import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.nio.charset.StandardCharsets
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
/**
* 建行模型估值项目
* 出售房源数据同步
@@ -201,7 +196,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
} finally {
latch.countDown()
}
};
};
})
}
latch.await()
@@ -492,8 +487,8 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
// def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
def date = DateUtils.addMonth(endDate.toString(), -12, "yyyyMMdd")
def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
// def date = DateUtils.addMonth(endDate.toString(), -12, "yyyyMMdd")
def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")//DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")
@@ -10,7 +10,7 @@ class XiamenVillage {
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\厦门\\涉农组织类型判定(to估家,2025-09-30)-结果.xlsx",{ d->
ExcelUtils.multiThreadingModify("D:\\数据\\厦门\\涉农组织类型判定(to估家,2025-10-31)-以此为准.xlsx",{ d->
if(!d['R']){
def dto = OpenApiUtils.addrVerifyXiameng(null, d['B'], 2)
@@ -1,19 +1,17 @@
package com.fangjia.script.yinjihaun.amap
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.thirdPriceCrawl.UserAgents
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.JacksonUtil
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.jsoup.Jsoup
import java.util.zip.GZIPInputStream
class GaoDeApiUtils {
//94c74c60e102972a8d17b7b75d7403be
def static GAO_DE_AK = "947cd51c2f3110a93cbd755f697a526d" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390
//67423021cae3b58e5c2ac595a389b036 :临时买的key
def static GAO_DE_AK = "67423021cae3b58e5c2ac595a389b036" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390
//15021384995:dfc2245bac8a7b2d5b62e060e59d4eca
/**
@@ -42,6 +40,7 @@ class GaoDeApiUtils {
try {
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
} catch (Exception e) {
println location + " 出错:"+content
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
}
jsonObj = JSONObject.fromObject(content)
@@ -339,9 +339,13 @@ public class AddressParserServiceImpl implements AddressParserService {
}
}
}
district = districtService.findById(district.getId(), MATCH_PROPERTY_FIELDS);
if(district == null){
logger.error("未找到匹配的CPDB数据:" + addr + "ID:" + district.getId());
logger.error("未找到匹配的CPDB数据:" + addr + "ID:" + matchedMate.getId());
}else{
district = districtService.findById(district.getId(), MATCH_PROPERTY_FIELDS);
}
if(district == null){
logger.error("未找到匹配的CPDB数据:" + addr + "ID:" + matchedMate.getId());
}
return new Building(district, scoreResult.getScore(matchedMate));
}else{