Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj

This commit is contained in:
2024-11-08 10:05:05 +08:00
4 changed files with 25 additions and 13 deletions
@@ -177,7 +177,7 @@ class ExportPaimai {
def run(){
def city = "合肥"
def city = "福州"
def region = null
def datas = findPaimaiData(city,region)
println "总量:"+datas.size()
@@ -241,7 +241,7 @@ class ExportPaimai {
println d.title +" "+d.url
}
}
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\合肥法拍.xlsx",rows,"数据")
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\福州法拍.xlsx",rows,"数据")
// batchCreateFile(rows, "D:\\数据\\重庆农商行\\法拍\\淘宝拍卖(新).csv",null,"GBK")
println "生成完成........"
@@ -333,21 +333,29 @@ class PaimaiDataSend {
def aliFpRun(){
def index = 0
def q = new BasicDBObject()
q.put("province","福建")
q.put("city","福州")
q.put("sub_type" , "住宅用房")
// q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
// q.put("province","江苏")
// q.put("sub_type","土地")
// q.put("parser_status", new BasicDBObject('$ne',2))
q.put("url","https://sf-item.taobao.com/sf_item/673646300374.htm")
// q.put("url","https://sf-item.taobao.com/sf_item/673646300374.htm")
//https://sf-item.taobao.com/sf_item/657131607486.htm
// q.put("parser_status", new BasicDBObject('$exists',false))
// q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01")))
q.put("create_time", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-05-01")))
// q.put("area", new BasicDBObject('$lt', 30))
def dbcursor = ali_fp_coll.find(q).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def dbcursor = ali_fp_coll.find(q).setOptions(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(100)
println "总记录共:"+dbcursor.count()
def rets = []
for(def d : dbcursor){
index++
if(index==119){
println "测试"
}
println "ali-fp: "+index
JSON json = new JSONObject()
json.put("project", "paimai_taobao")
@@ -373,7 +381,7 @@ class PaimaiDataSend {
json.put("result", result)
rets << json
if(rets.size()==5000){
if(rets.size()==1000){
println "共:"+rets.size()
send(rets)
rets.clear()
@@ -396,7 +404,7 @@ class PaimaiDataSend {
try{
totalCount.addAndGet(1)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
CommonUtils.sendMessage("result2parse", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 0)
@@ -27,8 +27,8 @@ class QiliangTest {
// task.run()
def data = [
'address': '浙江省嘉兴市南湖区昌盛南路36号19幢嘉兴智慧产业创业园智慧大厦A座168室',
'type': '1' //1:对公 0:对私
'address': '厦门市同安区厦门市同安区思明园',
'type': '2' //1:对公 0:对私
]
def s = task.getProxyResult(data);
print(s);
@@ -814,6 +814,9 @@ class DataStandardBatch extends BaseBatch{
if(d.name == copyObj.name){
dbo = d;
break;
}else if(d.estate_type=="别墅" && d.name == copyObj.name+"别墅"){
dbo = d;
break;
}else{
def dbNamePy = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ")
def keywordNamePy = com.fangjia.core.util.DataUtil.hanyuQuanpin(name, " ")
@@ -909,12 +912,12 @@ class DataStandardBatch extends BaseBatch{
if(StringUtils.isNotEmpty(data.name)){
def jsonObject = null
if(DataUtil.isPreciseAddress(data.address)){
jsonObject = OpenApiClient.addressMatch(data.city, data.address, "property", false, data.region, data.name, "fuzzy", false)
jsonObject = OpenApiClient.addressMatch(data.city, data.address, "property", false, data.region, data.name, "strict", false)
}else{
jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false)
jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "strict", false)
}
if(jsonObject && jsonObject.credit>0 ){
if(jsonObject && jsonObject.credit>=0 ){
def dbo = [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address,estate_type:jsonObject.propertyType, credit:jsonObject.credit]
if(dbo){
return getFilerMatchDistrict(dbo, data)
@@ -931,6 +934,7 @@ class DataStandardBatch extends BaseBatch{
DBObject qdata = new BasicDBObject();
qdata.put("s_date", new BasicDBObject('$gte', SHORT_DATE))
// qdata.put("city","上海")
// qdata.put("name","白金瀚宫")
// qdata.put("source","安居客")
// qdata.put("name",Pattern.compile("龚家宅.*"))
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));