Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into script_gj

# Conflicts:
#	projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy
#	projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy
This commit is contained in:
2021-11-20 18:03:24 +08:00
157 changed files with 6232 additions and 574 deletions
@@ -38,9 +38,7 @@ class GaoDeMapCrawl extends BaseBatch{
DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.235", 27018)
def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc"
final def dbUrl = "mongodb://192.168.0.235:27018";
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
final AtomicInteger count = new AtomicInteger()
def updateLists = []
@@ -0,0 +1,32 @@
package com.fangjia.script.wenyan.auto
import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
//def keys = new BasicDBObject()
//keys.city = 1
//keys.region = 1
//keys.name = 1
//keys.estate_type2 = 1
//["ÖØÇì"].each {def city->
// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject seaweed->
// BasicDBObject stat = DbUtils.findDistrictStat(seaweed, DateUtil.getFirstDayOfWeek(new Date()))
// if(!stat){
// CommonUtils.recalculationAverage(seaweed.city, seaweed.region, seaweed.name, seaweed.estate_type2)
// println(seaweed)
// }
// }, keys)
//
//}
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\³É½»¼Û¡ª¡ªÔ¤²â¼Ò.xlsx",{List item->
BasicDBObject query = DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"])
BasicDBObject seaweed = DbUtils.seaweed.findOne(query)
if(seaweed){
BasicDBObject stat = DbUtils.findDistrictStat(seaweed, item["E"])
item["H"] = stat?.credit
}
println(item)
})
@@ -0,0 +1,113 @@
package com.fangjia.script.wenyan.auto
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.apache.lucene.index.Term
import org.apache.lucene.search.BooleanClause
import org.apache.lucene.search.BooleanQuery
import org.apache.lucene.search.TermQuery
/**
* 移除小区名中的区域信息
* @param name
* @param region
* @return
*/
String removeRegion(String name, String region){
if(name.startsWith(region) && name.length() - region.length() > 2){
return name.replaceFirst(region, "")
}
return name;
}
def String getStandardAddress(def city, def addr) {
if (StringUtils.isBlank(addr)) {
return null
}
JSONObject result = OpenApiClient.addressStandardize(city, addr)
if (result.containsKey("standardAddress") && result.getBoolean("standardAddress")) {
def standardAddr = result.getString("street")
if (result.containsKey("streetNumber")) {
standardAddr += result.getString("streetNumber")
}
return standardAddr
}
return null
}
def NOW = new Date()
def getType(String type){
if(type == null){
return "普通住宅"
}
if(type.contains("商住两用")){
return "商住"
}else if(type.contains("别墅")){
return "别墅"
}else{
return "普通住宅"
}
}
ExcelUtils.modify("D:\\nextcloud\\共享文档\\项目文档\\重庆农商行\\新增小区4.xlsx",{List item->
if(item["C"] != "明佳园"){
return
}
BasicDBObject seaweed = new BasicDBObject()
seaweed.city = item["A"]
seaweed.region = item["B"]
seaweed.name = item["C"]
def query = new BasicDBObject("city", seaweed.city).append("region", seaweed.region)
.append('name', seaweed.name)
.append("visible", 0)
.append("cat", "district");
BasicDBObject one = DbUtils.seaweed.findOne(query)
if(!one){
query.removeField("name")
query.append("alias", seaweed.name)
one = DbUtils.seaweed.findOne(query)
}
if (one && one.status == 0) {
item["i"] = "已存在"
return;
}
seaweed.address = getStandardAddress( seaweed.city, item["D"])
String source = item["H"]
Double lng = item["E"]
Double lat = item["F"]
if(lng){
if(source.contains("高德")){
seaweed.lng = (long)(lng * 10000000000l)
seaweed.lat = (long)(lat * 10000000000l)
JSONObject geo = BaiDuApiUtils.googleToBaidu(lat, lng)
seaweed.lng2 = (long)(geo.lng * 10000000000l)
seaweed.lat2 = (long)(geo.lat * 10000000000l)
}else if(source.contains("百度")){
Map<String, Double> point = GeoUtils.geoconvBaiduToGoogle(lng, lat);
long google_lat = (long)(point.get("lat") * 10000000000l);
long google_lng = (long)(point.get("lng") * 10000000000l);
long baiDu_lat = (long)(lat * 10000000000l)
long baiDu_lng = (long)(lng * 10000000000l)
seaweed.lng = google_lng
seaweed.lat = google_lat
seaweed.lat2 = baiDu_lat
seaweed.lng2 = baiDu_lng
}
}
seaweed.cat = "district"
seaweed.status = 0
seaweed.c_date = NOW
seaweed.ip = "MACHINE_SAVE"
seaweed.estate_type2 = getType(item["G"])
Map<String, Object> markMap = new HashMap<String, Object>()
seaweed.mark_map = markMap
markMap.put("data_source", "机器入库")
DbUtils.seaweed.save(seaweed)
println(seaweed)
})
@@ -1,16 +1,42 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.RegionData
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.common.ServerAreaData
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import net.sf.json.JSONObject
def data = [["城市", "区域", "小区名", "别名", "地址", "地址别名"]]
["ID", "北京", "重庆", "哈尔滨"].each {def city->
DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(city)).each {BasicDBObject district->
def item = [district._id.toString(), district.city, district.region, district.name, district.alias?.join(","), district.address, district.addr_alias?.join(",")]
println(item)
data << item
//city_sell_stat = MongoUtils.getDbCollection("fangjia_stat", "city_sell_stat")
//data = [["城市", "202101", "202106"]]
//CommonUtils.findAllCity().each {def city->
// BasicDBObject serverArea = ServerAreaData.getServerAreaByDisplay(city)
// if(!serverArea.ranking || !serverArea.ranking.toString().contains("县级行政单位")){
// BasicDBObject pre = city_sell_stat.findOne(new BasicDBObject("city", city).append("date", 202101))
// BasicDBObject now = city_sell_stat.findOne(new BasicDBObject("city", city).append("date", 202106))
// println([serverArea.display, pre?.avg_price, now?.avg_price])
// data << [serverArea.display, pre?.avg_price, now?.avg_price]
// }
//}
//ExcelUtils.write("D:\\城市均价数据.xlsx", data, "城市均价")
//ExcelUtils.multiThreadingModify("C:\\Users\\wenyan\\Downloads\\经纬度错误数据修正.xlsx",{List item->
// String city = item["B"]
// String region = RegionData.formatRegion(city, item["D"])
// String sql = "select count(1) as count from latest_seawater where cat='sell' and city='${item["B"]}' and region='${region}' and district_name='${item["F"]}'"
// ElasticUtils.executeSqlQuery(sql,{Map result->
// item["P"] = result.get("count")
// })
// println(item)
//})
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\装修案例.xlsx",{List item->
JSONObject json = OpenApiClient.addressMatch("西安", item["H"], "property", false, item["C"])
if(json.name){
item["I"] = json.district
item["J"] = json.name
}
}
ExcelUtils.write("D:\\待抓取小区.xlsx", data, "数据")
})
@@ -25,7 +25,7 @@ while (startDate.before(endDate)) {
OPEN_URI_LOG.find(and(eq("user_id", "1007705"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
if (!hashSet.contains(result)) {
def item = ["5c6535b99c213017e0768ee1", "qlkj", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
def item = ["60a3202c0b42a00001aad385", "qlkjgx", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
data << item
hashSet << result
} else {
@@ -42,7 +42,7 @@ while (startDate.before(endDate)) {
keys.addAll(params.keys())
def key = "${params.address}_${params.size}".toString()
if (!hashSet.contains(key)) {
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date.format("yyy"), doc.data_status, doc.code, doc.msg]
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg]
param_keys.each {
item << params[it]
}
@@ -261,15 +261,16 @@ import org.dom4j.io.SAXReader
// data << [dbo.parameter_map.city, dbo.parameter_map.address, dbo.res_map.result.province, dbo.res_map.result.city, dbo.res_map.result.district, dbo.res_map.result.developmentZone, dbo.res_map.result.township, dbo.res_map.result.street,dbo.res_map.result.streetNumber,dbo.res_map.result.name,dbo.res_map.result.buildingNumber,dbo.res_map.result.unitNumber,dbo.res_map.result.roomNumber,dbo.res_map.result.formattedAddress]
//}
//ExcelUtils.write("D:\\建行测试地址.xlsx", data, "结果")
DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com")
def query = new BasicDBObject()
query.cat = "region"
query.city= "上海"
query.weekend = Date.parse("yyyyMMdd", "20210228")
def doc = []
def prices = []
SQMETRE_RENT_COLL.find(query, new BasicDBObject("region", 1).append("avg_price", 1)).each {
doc.add(new BasicDBObject("name", it.region+"区").append("value", it.avg_price))
prices << it.avg_price
}
println(JSONArray.fromObject(doc).toString())
//DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com")
//def query = new BasicDBObject()
//query.cat = "region"
//query.city= "上海"
//query.weekend = Date.parse("yyyyMMdd", "20210228")
//def doc = []
//def prices = []
//SQMETRE_RENT_COLL.find(query, new BasicDBObject("region", 1).append("avg_price", 1)).each {
// doc.add(new BasicDBObject("name", it.region+"区").append("value", it.avg_price))
// prices << it.avg_price
//}
//println(JSONArray.fromObject(doc).toString())
DbUtils.seaweed.remove(new BasicDBObject("_id", DbUtils.seaweed.findById("6103bf1c64baf60001e7e1d0")._id))
@@ -1,6 +1,11 @@
import com.fangjia.core.util.CNNMFilter
import com.fangjia.core.util.HttpRequest
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import net.sf.json.JSONObject
import org.apache.commons.io.FileUtils
//
@@ -35,7 +40,79 @@ import org.apache.commons.io.FileUtils
// println("copy:" + file.getAbsolutePath() + "\tto:" + ("E:\\成交数据\\DBF汇总\\" + file.getName()))
// }
//}
DSCollection seawater = new DataStore("192.168.9.118", 27777).getCollection("fangjia.seawater")
seawater.find(new BasicDBObject("cat", "sell").append("city", "上海").append("region", "长宁").append("district_name", "新泾三村")).each {
println(it)
}
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\副本地址标准化验证结果分析.xlsx",{List item->
// HttpRequest request = new HttpRequest()
// JSONObject json = request.getJsonContent2("http://localhost:8082/structure", ["address": item["A"]], "UTF-8")
// if(json){
// item["D"] = json.result.province
// item["E"] = json.result.city
// item["F"] = json.result.district
// item["G"] = json.result.dev
// item["H"] = json.result.township
// item["I"] = json.result.village
// item["J"] = json.result.group
// item["K"] = json.result.street
// item["L"] = json.result.streetNumber
// item["M"] = json.result.name
// item["N"] = json.result.buildingNumber
// item["O"] = json.result.unitNumber
// item["P"] = json.result.roomNumber
// item["q"] = json.result.floor
// item["r"] = json.result.formatAddress
// }
// println(item)
//})
//for(int i=1; i < 100; i++){
// println("${i}号楼#@#building_number")
// println("${i}栋#@#building_number")
// println("${i}幢#@#building_number")
// println("${i}单元#@#unit_number")
// if(i< 20){
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}号楼#@#building_number")
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}栋#@#building_number")
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}幢#@#building_number")
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}单元#@#unit_number")
// }
//}
//nums= "abcdefghijklmn"
//for(def num : nums.toCharArray()){
// println("${num}栋#@#building_number")
// println("${num}幢#@#building_number")
// println("${num}单元#@#unit_number")
//}
//nums= "abcdefghijklmn"
//for(def num : nums.toCharArray()){
// println("${num}座#@#building_number")
// for(int i = 1; i <11; i++){
// println("${num}${i}座#@#building_number")
//// println("${num}${i}幢#@#building_number")
//// println("${num}${i}单元#@#unit_number")
// }
//}
//for(int i = 1; i <20; i++){
// println("${i}区#@#part_number")
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}区#@#part_number")
// println("${i}期#@#stage_number")
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}期#@#stage_number")
//}
//
//for(def num : "东南北中西甲乙丙丁".toCharArray()){
// println("${num}区#@#part_number")
//}
//for(int i = -5; i < 100; i++){
// println("${i}楼#@#floor")
// if(i < 10 && i > 0){
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}楼#@#floor")
// }
//}
//for(def num : "abcdefg东南西北".toCharArray()){
// println("${num}户#@#room")
//}
//for(i in 1.. 40){
// for(j in 1..9){
// println("${i}0${j}室#@#room")
// }
//}
ElasticUtils.executeSqlQuery("select * from seawater where cat='sell' and district_name='新泾三村' and city='上海' and p_date > 20210101",{Map doc->
println(doc.avg_price + "\t" + doc.root + "\t" + doc.p_date)
})