This commit is contained in:
2021-07-17 21:42:28 +08:00
parent 61d7d1a5a8
commit 6ff9f665b0
6 changed files with 233 additions and 13 deletions
@@ -38,9 +38,7 @@ class GaoDeMapCrawl extends BaseBatch{
DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.235", 27018)
def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc"
final def dbUrl = "mongodb://192.168.0.235:27018";
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
final AtomicInteger count = new AtomicInteger()
def updateLists = []
@@ -0,0 +1,32 @@
package com.fangjia.script.wenyan.auto
import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
//def keys = new BasicDBObject()
//keys.city = 1
//keys.region = 1
//keys.name = 1
//keys.estate_type2 = 1
//["ÖØÇì"].each {def city->
// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject seaweed->
// BasicDBObject stat = DbUtils.findDistrictStat(seaweed, DateUtil.getFirstDayOfWeek(new Date()))
// if(!stat){
// CommonUtils.recalculationAverage(seaweed.city, seaweed.region, seaweed.name, seaweed.estate_type2)
// println(seaweed)
// }
// }, keys)
//
//}
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\³É½»¼Û¡ª¡ªÔ¤²â¼Ò.xlsx",{List item->
BasicDBObject query = DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"])
BasicDBObject seaweed = DbUtils.seaweed.findOne(query)
if(seaweed){
BasicDBObject stat = DbUtils.findDistrictStat(seaweed, item["E"])
item["H"] = stat?.credit
}
println(item)
})
@@ -0,0 +1,113 @@
package com.fangjia.script.wenyan.auto
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.apache.lucene.index.Term
import org.apache.lucene.search.BooleanClause
import org.apache.lucene.search.BooleanQuery
import org.apache.lucene.search.TermQuery
/**
* 移除小区名中的区域信息
* @param name
* @param region
* @return
*/
String removeRegion(String name, String region){
if(name.startsWith(region) && name.length() - region.length() > 2){
return name.replaceFirst(region, "")
}
return name;
}
def String getStandardAddress(def city, def addr) {
if (StringUtils.isBlank(addr)) {
return null
}
JSONObject result = OpenApiClient.addressStandardize(city, addr)
if (result.containsKey("standardAddress") && result.getBoolean("standardAddress")) {
def standardAddr = result.getString("street")
if (result.containsKey("streetNumber")) {
standardAddr += result.getString("streetNumber")
}
return standardAddr
}
return null
}
def NOW = new Date()
def getType(String type){
if(type == null){
return "普通住宅"
}
if(type.contains("商住两用")){
return "商住"
}else if(type.contains("别墅")){
return "别墅"
}else{
return "普通住宅"
}
}
ExcelUtils.modify("D:\\nextcloud\\共享文档\\项目文档\\重庆农商行\\新增小区4.xlsx",{List item->
if(item["C"] != "明佳园"){
return
}
BasicDBObject seaweed = new BasicDBObject()
seaweed.city = item["A"]
seaweed.region = item["B"]
seaweed.name = item["C"]
def query = new BasicDBObject("city", seaweed.city).append("region", seaweed.region)
.append('name', seaweed.name)
.append("visible", 0)
.append("cat", "district");
BasicDBObject one = DbUtils.seaweed.findOne(query)
if(!one){
query.removeField("name")
query.append("alias", seaweed.name)
one = DbUtils.seaweed.findOne(query)
}
if (one && one.status == 0) {
item["i"] = "已存在"
return;
}
seaweed.address = getStandardAddress( seaweed.city, item["D"])
String source = item["H"]
Double lng = item["E"]
Double lat = item["F"]
if(lng){
if(source.contains("高德")){
seaweed.lng = (long)(lng * 10000000000l)
seaweed.lat = (long)(lat * 10000000000l)
JSONObject geo = BaiDuApiUtils.googleToBaidu(lat, lng)
seaweed.lng2 = (long)(geo.lng * 10000000000l)
seaweed.lat2 = (long)(geo.lat * 10000000000l)
}else if(source.contains("百度")){
Map<String, Double> point = GeoUtils.geoconvBaiduToGoogle(lng, lat);
long google_lat = (long)(point.get("lat") * 10000000000l);
long google_lng = (long)(point.get("lng") * 10000000000l);
long baiDu_lat = (long)(lat * 10000000000l)
long baiDu_lng = (long)(lng * 10000000000l)
seaweed.lng = google_lng
seaweed.lat = google_lat
seaweed.lat2 = baiDu_lat
seaweed.lng2 = baiDu_lng
}
}
seaweed.cat = "district"
seaweed.status = 0
seaweed.c_date = NOW
seaweed.ip = "MACHINE_SAVE"
seaweed.estate_type2 = getType(item["G"])
Map<String, Object> markMap = new HashMap<String, Object>()
seaweed.mark_map = markMap
markMap.put("data_source", "机器入库")
DbUtils.seaweed.save(seaweed)
println(seaweed)
})
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20210401", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210501", "yyyyMMdd")
def startDate = DateUtil.parseDate("20210501", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210601", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -22,10 +22,10 @@ while (startDate.before(endDate)) {
calendar.add(Calendar.DATE, 1)
def nextDate = calendar.getTime()
count++
OPEN_URI_LOG.find(and(eq("user_id", "1007705"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
OPEN_URI_LOG.find(and(eq("user_id", "1007757"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
if (!hashSet.contains(result)) {
def item = ["5c6535b99c213017e0768ee1", "qlkj", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
def item = ["60a3202c0b42a00001aad385", "qlkjgx", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
data << item
hashSet << result
} else {
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20210401", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210501", "yyyyMMdd")
def startDate = DateUtil.parseDate("20210601", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210701", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -42,7 +42,7 @@ while (startDate.before(endDate)) {
keys.addAll(params.keys())
def key = "${params.address}_${params.size}".toString()
if (!hashSet.contains(key)) {
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date.format("yyy"), doc.data_status, doc.code, doc.msg]
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg]
param_keys.each {
item << params[it]
}
@@ -1,6 +1,11 @@
import com.fangjia.core.util.CNNMFilter
import com.fangjia.core.util.HttpRequest
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import net.sf.json.JSONObject
import org.apache.commons.io.FileUtils
//
@@ -35,7 +40,79 @@ import org.apache.commons.io.FileUtils
// println("copy:" + file.getAbsolutePath() + "\tto:" + ("E:\\成交数据\\DBF汇总\\" + file.getName()))
// }
//}
DSCollection seawater = new DataStore("192.168.9.118", 27777).getCollection("fangjia.seawater")
seawater.find(new BasicDBObject("cat", "sell").append("city", "上海").append("region", "长宁").append("district_name", "新泾三村")).each {
println(it)
}
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\副本地址标准化验证结果分析.xlsx",{List item->
// HttpRequest request = new HttpRequest()
// JSONObject json = request.getJsonContent2("http://localhost:8082/structure", ["address": item["A"]], "UTF-8")
// if(json){
// item["D"] = json.result.province
// item["E"] = json.result.city
// item["F"] = json.result.district
// item["G"] = json.result.dev
// item["H"] = json.result.township
// item["I"] = json.result.village
// item["J"] = json.result.group
// item["K"] = json.result.street
// item["L"] = json.result.streetNumber
// item["M"] = json.result.name
// item["N"] = json.result.buildingNumber
// item["O"] = json.result.unitNumber
// item["P"] = json.result.roomNumber
// item["q"] = json.result.floor
// item["r"] = json.result.formatAddress
// }
// println(item)
//})
//for(int i=1; i < 100; i++){
// println("${i}号楼#@#building_number")
// println("${i}栋#@#building_number")
// println("${i}幢#@#building_number")
// println("${i}单元#@#unit_number")
// if(i< 20){
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}号楼#@#building_number")
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}栋#@#building_number")
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}幢#@#building_number")
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}单元#@#unit_number")
// }
//}
//nums= "abcdefghijklmn"
//for(def num : nums.toCharArray()){
// println("${num}栋#@#building_number")
// println("${num}幢#@#building_number")
// println("${num}单元#@#unit_number")
//}
//nums= "abcdefghijklmn"
//for(def num : nums.toCharArray()){
// println("${num}座#@#building_number")
// for(int i = 1; i <11; i++){
// println("${num}${i}座#@#building_number")
//// println("${num}${i}幢#@#building_number")
//// println("${num}${i}单元#@#unit_number")
// }
//}
//for(int i = 1; i <20; i++){
// println("${i}区#@#part_number")
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}区#@#part_number")
// println("${i}期#@#stage_number")
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}期#@#stage_number")
//}
//
//for(def num : "东南北中西甲乙丙丁".toCharArray()){
// println("${num}区#@#part_number")
//}
//for(int i = -5; i < 100; i++){
// println("${i}楼#@#floor")
// if(i < 10 && i > 0){
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}楼#@#floor")
// }
//}
//for(def num : "abcdefg东南西北".toCharArray()){
// println("${num}户#@#room")
//}
//for(i in 1.. 40){
// for(j in 1..9){
// println("${i}0${j}室#@#room")
// }
//}
ElasticUtils.executeSqlQuery("select * from seawater where cat='sell' and district_name='新泾三村' and city='上海' and p_date > 20210101",{Map doc->
println(doc.avg_price + "\t" + doc.root + "\t" + doc.p_date)
})