Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into script_gj
# Conflicts: # projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy # projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy
This commit is contained in:
@@ -38,9 +38,7 @@ class GaoDeMapCrawl extends BaseBatch{
|
||||
|
||||
DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.235", 27018)
|
||||
def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc"
|
||||
|
||||
final def dbUrl = "mongodb://192.168.0.235:27018";
|
||||
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||
final AtomicInteger count = new AtomicInteger()
|
||||
def updateLists = []
|
||||
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
package com.fangjia.script.wenyan.auto
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
//def keys = new BasicDBObject()
|
||||
//keys.city = 1
|
||||
//keys.region = 1
|
||||
//keys.name = 1
|
||||
//keys.estate_type2 = 1
|
||||
//["ÖØÇì"].each {def city->
|
||||
// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject seaweed->
|
||||
// BasicDBObject stat = DbUtils.findDistrictStat(seaweed, DateUtil.getFirstDayOfWeek(new Date()))
|
||||
// if(!stat){
|
||||
// CommonUtils.recalculationAverage(seaweed.city, seaweed.region, seaweed.name, seaweed.estate_type2)
|
||||
// println(seaweed)
|
||||
// }
|
||||
// }, keys)
|
||||
//
|
||||
//}
|
||||
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\³É½»¼Û¡ª¡ªÔ¤²â¼Ò.xlsx",{List item->
|
||||
BasicDBObject query = DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"])
|
||||
BasicDBObject seaweed = DbUtils.seaweed.findOne(query)
|
||||
if(seaweed){
|
||||
BasicDBObject stat = DbUtils.findDistrictStat(seaweed, item["E"])
|
||||
item["H"] = stat?.credit
|
||||
}
|
||||
println(item)
|
||||
})
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
package com.fangjia.script.wenyan.auto
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.lucene.index.Term
|
||||
import org.apache.lucene.search.BooleanClause
|
||||
import org.apache.lucene.search.BooleanQuery
|
||||
import org.apache.lucene.search.TermQuery
|
||||
|
||||
/**
|
||||
* 移除小区名中的区域信息
|
||||
* @param name
|
||||
* @param region
|
||||
* @return
|
||||
*/
|
||||
String removeRegion(String name, String region){
|
||||
if(name.startsWith(region) && name.length() - region.length() > 2){
|
||||
return name.replaceFirst(region, "")
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
def String getStandardAddress(def city, def addr) {
|
||||
if (StringUtils.isBlank(addr)) {
|
||||
return null
|
||||
}
|
||||
JSONObject result = OpenApiClient.addressStandardize(city, addr)
|
||||
if (result.containsKey("standardAddress") && result.getBoolean("standardAddress")) {
|
||||
def standardAddr = result.getString("street")
|
||||
if (result.containsKey("streetNumber")) {
|
||||
standardAddr += result.getString("streetNumber")
|
||||
}
|
||||
return standardAddr
|
||||
}
|
||||
return null
|
||||
}
|
||||
def NOW = new Date()
|
||||
def getType(String type){
|
||||
if(type == null){
|
||||
return "普通住宅"
|
||||
}
|
||||
if(type.contains("商住两用")){
|
||||
return "商住"
|
||||
}else if(type.contains("别墅")){
|
||||
return "别墅"
|
||||
}else{
|
||||
return "普通住宅"
|
||||
}
|
||||
}
|
||||
|
||||
ExcelUtils.modify("D:\\nextcloud\\共享文档\\项目文档\\重庆农商行\\新增小区4.xlsx",{List item->
|
||||
if(item["C"] != "明佳园"){
|
||||
return
|
||||
}
|
||||
BasicDBObject seaweed = new BasicDBObject()
|
||||
seaweed.city = item["A"]
|
||||
seaweed.region = item["B"]
|
||||
seaweed.name = item["C"]
|
||||
def query = new BasicDBObject("city", seaweed.city).append("region", seaweed.region)
|
||||
.append('name', seaweed.name)
|
||||
.append("visible", 0)
|
||||
.append("cat", "district");
|
||||
BasicDBObject one = DbUtils.seaweed.findOne(query)
|
||||
if(!one){
|
||||
query.removeField("name")
|
||||
query.append("alias", seaweed.name)
|
||||
one = DbUtils.seaweed.findOne(query)
|
||||
}
|
||||
if (one && one.status == 0) {
|
||||
item["i"] = "已存在"
|
||||
return;
|
||||
}
|
||||
seaweed.address = getStandardAddress( seaweed.city, item["D"])
|
||||
String source = item["H"]
|
||||
Double lng = item["E"]
|
||||
Double lat = item["F"]
|
||||
if(lng){
|
||||
if(source.contains("高德")){
|
||||
seaweed.lng = (long)(lng * 10000000000l)
|
||||
seaweed.lat = (long)(lat * 10000000000l)
|
||||
JSONObject geo = BaiDuApiUtils.googleToBaidu(lat, lng)
|
||||
seaweed.lng2 = (long)(geo.lng * 10000000000l)
|
||||
seaweed.lat2 = (long)(geo.lat * 10000000000l)
|
||||
}else if(source.contains("百度")){
|
||||
Map<String, Double> point = GeoUtils.geoconvBaiduToGoogle(lng, lat);
|
||||
long google_lat = (long)(point.get("lat") * 10000000000l);
|
||||
long google_lng = (long)(point.get("lng") * 10000000000l);
|
||||
long baiDu_lat = (long)(lat * 10000000000l)
|
||||
long baiDu_lng = (long)(lng * 10000000000l)
|
||||
seaweed.lng = google_lng
|
||||
seaweed.lat = google_lat
|
||||
seaweed.lat2 = baiDu_lat
|
||||
seaweed.lng2 = baiDu_lng
|
||||
}
|
||||
}
|
||||
seaweed.cat = "district"
|
||||
seaweed.status = 0
|
||||
seaweed.c_date = NOW
|
||||
seaweed.ip = "MACHINE_SAVE"
|
||||
seaweed.estate_type2 = getType(item["G"])
|
||||
Map<String, Object> markMap = new HashMap<String, Object>()
|
||||
seaweed.mark_map = markMap
|
||||
markMap.put("data_source", "机器入库")
|
||||
DbUtils.seaweed.save(seaweed)
|
||||
println(seaweed)
|
||||
})
|
||||
|
||||
+35
-9
@@ -1,16 +1,42 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.common.ServerAreaData
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
def data = [["城市", "区域", "小区名", "别名", "地址", "地址别名"]]
|
||||
["ID", "北京", "重庆", "哈尔滨"].each {def city->
|
||||
DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(city)).each {BasicDBObject district->
|
||||
def item = [district._id.toString(), district.city, district.region, district.name, district.alias?.join(","), district.address, district.addr_alias?.join(",")]
|
||||
println(item)
|
||||
data << item
|
||||
//city_sell_stat = MongoUtils.getDbCollection("fangjia_stat", "city_sell_stat")
|
||||
//data = [["城市", "202101", "202106"]]
|
||||
//CommonUtils.findAllCity().each {def city->
|
||||
// BasicDBObject serverArea = ServerAreaData.getServerAreaByDisplay(city)
|
||||
// if(!serverArea.ranking || !serverArea.ranking.toString().contains("县级行政单位")){
|
||||
// BasicDBObject pre = city_sell_stat.findOne(new BasicDBObject("city", city).append("date", 202101))
|
||||
// BasicDBObject now = city_sell_stat.findOne(new BasicDBObject("city", city).append("date", 202106))
|
||||
// println([serverArea.display, pre?.avg_price, now?.avg_price])
|
||||
// data << [serverArea.display, pre?.avg_price, now?.avg_price]
|
||||
// }
|
||||
//}
|
||||
//ExcelUtils.write("D:\\城市均价数据.xlsx", data, "城市均价")
|
||||
|
||||
//ExcelUtils.multiThreadingModify("C:\\Users\\wenyan\\Downloads\\经纬度错误数据修正.xlsx",{List item->
|
||||
// String city = item["B"]
|
||||
// String region = RegionData.formatRegion(city, item["D"])
|
||||
// String sql = "select count(1) as count from latest_seawater where cat='sell' and city='${item["B"]}' and region='${region}' and district_name='${item["F"]}'"
|
||||
// ElasticUtils.executeSqlQuery(sql,{Map result->
|
||||
// item["P"] = result.get("count")
|
||||
// })
|
||||
// println(item)
|
||||
//})
|
||||
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\装修案例.xlsx",{List item->
|
||||
JSONObject json = OpenApiClient.addressMatch("西安", item["H"], "property", false, item["C"])
|
||||
if(json.name){
|
||||
item["I"] = json.district
|
||||
item["J"] = json.name
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\待抓取小区.xlsx", data, "数据")
|
||||
})
|
||||
+1
-1
@@ -25,7 +25,7 @@ while (startDate.before(endDate)) {
|
||||
OPEN_URI_LOG.find(and(eq("user_id", "1007705"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
|
||||
def result = JSONObject.fromObject(doc.res_map.result)
|
||||
if (!hashSet.contains(result)) {
|
||||
def item = ["5c6535b99c213017e0768ee1", "qlkj", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
|
||||
def item = ["60a3202c0b42a00001aad385", "qlkjgx", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
|
||||
data << item
|
||||
hashSet << result
|
||||
} else {
|
||||
|
||||
+1
-1
@@ -42,7 +42,7 @@ while (startDate.before(endDate)) {
|
||||
keys.addAll(params.keys())
|
||||
def key = "${params.address}_${params.size}".toString()
|
||||
if (!hashSet.contains(key)) {
|
||||
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date.format("yyy"), doc.data_status, doc.code, doc.msg]
|
||||
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg]
|
||||
param_keys.each {
|
||||
item << params[it]
|
||||
}
|
||||
|
||||
+13
-12
@@ -261,15 +261,16 @@ import org.dom4j.io.SAXReader
|
||||
// data << [dbo.parameter_map.city, dbo.parameter_map.address, dbo.res_map.result.province, dbo.res_map.result.city, dbo.res_map.result.district, dbo.res_map.result.developmentZone, dbo.res_map.result.township, dbo.res_map.result.street,dbo.res_map.result.streetNumber,dbo.res_map.result.name,dbo.res_map.result.buildingNumber,dbo.res_map.result.unitNumber,dbo.res_map.result.roomNumber,dbo.res_map.result.formattedAddress]
|
||||
//}
|
||||
//ExcelUtils.write("D:\\建行测试地址.xlsx", data, "结果")
|
||||
DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com")
|
||||
def query = new BasicDBObject()
|
||||
query.cat = "region"
|
||||
query.city= "上海"
|
||||
query.weekend = Date.parse("yyyyMMdd", "20210228")
|
||||
def doc = []
|
||||
def prices = []
|
||||
SQMETRE_RENT_COLL.find(query, new BasicDBObject("region", 1).append("avg_price", 1)).each {
|
||||
doc.add(new BasicDBObject("name", it.region+"区").append("value", it.avg_price))
|
||||
prices << it.avg_price
|
||||
}
|
||||
println(JSONArray.fromObject(doc).toString())
|
||||
//DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com")
|
||||
//def query = new BasicDBObject()
|
||||
//query.cat = "region"
|
||||
//query.city= "上海"
|
||||
//query.weekend = Date.parse("yyyyMMdd", "20210228")
|
||||
//def doc = []
|
||||
//def prices = []
|
||||
//SQMETRE_RENT_COLL.find(query, new BasicDBObject("region", 1).append("avg_price", 1)).each {
|
||||
// doc.add(new BasicDBObject("name", it.region+"区").append("value", it.avg_price))
|
||||
// prices << it.avg_price
|
||||
//}
|
||||
//println(JSONArray.fromObject(doc).toString())
|
||||
DbUtils.seaweed.remove(new BasicDBObject("_id", DbUtils.seaweed.findById("6103bf1c64baf60001e7e1d0")._id))
|
||||
+81
-4
@@ -1,6 +1,11 @@
|
||||
import com.fangjia.core.util.CNNMFilter
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
|
||||
//
|
||||
@@ -35,7 +40,79 @@ import org.apache.commons.io.FileUtils
|
||||
// println("copy:" + file.getAbsolutePath() + "\tto:" + ("E:\\成交数据\\DBF汇总\\" + file.getName()))
|
||||
// }
|
||||
//}
|
||||
DSCollection seawater = new DataStore("192.168.9.118", 27777).getCollection("fangjia.seawater")
|
||||
seawater.find(new BasicDBObject("cat", "sell").append("city", "上海").append("region", "长宁").append("district_name", "新泾三村")).each {
|
||||
println(it)
|
||||
}
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\副本地址标准化验证结果分析.xlsx",{List item->
|
||||
// HttpRequest request = new HttpRequest()
|
||||
// JSONObject json = request.getJsonContent2("http://localhost:8082/structure", ["address": item["A"]], "UTF-8")
|
||||
// if(json){
|
||||
// item["D"] = json.result.province
|
||||
// item["E"] = json.result.city
|
||||
// item["F"] = json.result.district
|
||||
// item["G"] = json.result.dev
|
||||
// item["H"] = json.result.township
|
||||
// item["I"] = json.result.village
|
||||
// item["J"] = json.result.group
|
||||
// item["K"] = json.result.street
|
||||
// item["L"] = json.result.streetNumber
|
||||
// item["M"] = json.result.name
|
||||
// item["N"] = json.result.buildingNumber
|
||||
// item["O"] = json.result.unitNumber
|
||||
// item["P"] = json.result.roomNumber
|
||||
// item["q"] = json.result.floor
|
||||
// item["r"] = json.result.formatAddress
|
||||
// }
|
||||
// println(item)
|
||||
//})
|
||||
//for(int i=1; i < 100; i++){
|
||||
// println("${i}号楼#@#building_number")
|
||||
// println("${i}栋#@#building_number")
|
||||
// println("${i}幢#@#building_number")
|
||||
// println("${i}单元#@#unit_number")
|
||||
// if(i< 20){
|
||||
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}号楼#@#building_number")
|
||||
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}栋#@#building_number")
|
||||
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}幢#@#building_number")
|
||||
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}单元#@#unit_number")
|
||||
// }
|
||||
//}
|
||||
//nums= "abcdefghijklmn"
|
||||
//for(def num : nums.toCharArray()){
|
||||
// println("${num}栋#@#building_number")
|
||||
// println("${num}幢#@#building_number")
|
||||
// println("${num}单元#@#unit_number")
|
||||
//}
|
||||
//nums= "abcdefghijklmn"
|
||||
//for(def num : nums.toCharArray()){
|
||||
// println("${num}座#@#building_number")
|
||||
// for(int i = 1; i <11; i++){
|
||||
// println("${num}${i}座#@#building_number")
|
||||
//// println("${num}${i}幢#@#building_number")
|
||||
//// println("${num}${i}单元#@#unit_number")
|
||||
// }
|
||||
//}
|
||||
//for(int i = 1; i <20; i++){
|
||||
// println("${i}区#@#part_number")
|
||||
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}区#@#part_number")
|
||||
// println("${i}期#@#stage_number")
|
||||
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}期#@#stage_number")
|
||||
//}
|
||||
//
|
||||
//for(def num : "东南北中西甲乙丙丁".toCharArray()){
|
||||
// println("${num}区#@#part_number")
|
||||
//}
|
||||
//for(int i = -5; i < 100; i++){
|
||||
// println("${i}楼#@#floor")
|
||||
// if(i < 10 && i > 0){
|
||||
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}楼#@#floor")
|
||||
// }
|
||||
//}
|
||||
//for(def num : "abcdefg东南西北".toCharArray()){
|
||||
// println("${num}户#@#room")
|
||||
//}
|
||||
//for(i in 1.. 40){
|
||||
// for(j in 1..9){
|
||||
// println("${i}0${j}室#@#room")
|
||||
// }
|
||||
//}
|
||||
ElasticUtils.executeSqlQuery("select * from seawater where cat='sell' and district_name='新泾三村' and city='上海' and p_date > 20210101",{Map doc->
|
||||
println(doc.avg_price + "\t" + doc.root + "\t" + doc.p_date)
|
||||
})
|
||||
Reference in New Issue
Block a user