geng xin
This commit is contained in:
@@ -38,9 +38,7 @@ class GaoDeMapCrawl extends BaseBatch{
|
||||
|
||||
DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.235", 27018)
|
||||
def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc"
|
||||
|
||||
final def dbUrl = "mongodb://192.168.0.235:27018";
|
||||
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||
final AtomicInteger count = new AtomicInteger()
|
||||
def updateLists = []
|
||||
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
package com.fangjia.script.wenyan.auto
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
//def keys = new BasicDBObject()
|
||||
//keys.city = 1
|
||||
//keys.region = 1
|
||||
//keys.name = 1
|
||||
//keys.estate_type2 = 1
|
||||
//["ÖØÇì"].each {def city->
|
||||
// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject seaweed->
|
||||
// BasicDBObject stat = DbUtils.findDistrictStat(seaweed, DateUtil.getFirstDayOfWeek(new Date()))
|
||||
// if(!stat){
|
||||
// CommonUtils.recalculationAverage(seaweed.city, seaweed.region, seaweed.name, seaweed.estate_type2)
|
||||
// println(seaweed)
|
||||
// }
|
||||
// }, keys)
|
||||
//
|
||||
//}
|
||||
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\³É½»¼Û¡ª¡ªÔ¤²â¼Ò.xlsx",{List item->
|
||||
BasicDBObject query = DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"])
|
||||
BasicDBObject seaweed = DbUtils.seaweed.findOne(query)
|
||||
if(seaweed){
|
||||
BasicDBObject stat = DbUtils.findDistrictStat(seaweed, item["E"])
|
||||
item["H"] = stat?.credit
|
||||
}
|
||||
println(item)
|
||||
})
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
package com.fangjia.script.wenyan.auto
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.lucene.index.Term
|
||||
import org.apache.lucene.search.BooleanClause
|
||||
import org.apache.lucene.search.BooleanQuery
|
||||
import org.apache.lucene.search.TermQuery
|
||||
|
||||
/**
|
||||
* 移除小区名中的区域信息
|
||||
* @param name
|
||||
* @param region
|
||||
* @return
|
||||
*/
|
||||
String removeRegion(String name, String region){
|
||||
if(name.startsWith(region) && name.length() - region.length() > 2){
|
||||
return name.replaceFirst(region, "")
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
def String getStandardAddress(def city, def addr) {
|
||||
if (StringUtils.isBlank(addr)) {
|
||||
return null
|
||||
}
|
||||
JSONObject result = OpenApiClient.addressStandardize(city, addr)
|
||||
if (result.containsKey("standardAddress") && result.getBoolean("standardAddress")) {
|
||||
def standardAddr = result.getString("street")
|
||||
if (result.containsKey("streetNumber")) {
|
||||
standardAddr += result.getString("streetNumber")
|
||||
}
|
||||
return standardAddr
|
||||
}
|
||||
return null
|
||||
}
|
||||
def NOW = new Date()
|
||||
def getType(String type){
|
||||
if(type == null){
|
||||
return "普通住宅"
|
||||
}
|
||||
if(type.contains("商住两用")){
|
||||
return "商住"
|
||||
}else if(type.contains("别墅")){
|
||||
return "别墅"
|
||||
}else{
|
||||
return "普通住宅"
|
||||
}
|
||||
}
|
||||
|
||||
ExcelUtils.modify("D:\\nextcloud\\共享文档\\项目文档\\重庆农商行\\新增小区4.xlsx",{List item->
|
||||
if(item["C"] != "明佳园"){
|
||||
return
|
||||
}
|
||||
BasicDBObject seaweed = new BasicDBObject()
|
||||
seaweed.city = item["A"]
|
||||
seaweed.region = item["B"]
|
||||
seaweed.name = item["C"]
|
||||
def query = new BasicDBObject("city", seaweed.city).append("region", seaweed.region)
|
||||
.append('name', seaweed.name)
|
||||
.append("visible", 0)
|
||||
.append("cat", "district");
|
||||
BasicDBObject one = DbUtils.seaweed.findOne(query)
|
||||
if(!one){
|
||||
query.removeField("name")
|
||||
query.append("alias", seaweed.name)
|
||||
one = DbUtils.seaweed.findOne(query)
|
||||
}
|
||||
if (one && one.status == 0) {
|
||||
item["i"] = "已存在"
|
||||
return;
|
||||
}
|
||||
seaweed.address = getStandardAddress( seaweed.city, item["D"])
|
||||
String source = item["H"]
|
||||
Double lng = item["E"]
|
||||
Double lat = item["F"]
|
||||
if(lng){
|
||||
if(source.contains("高德")){
|
||||
seaweed.lng = (long)(lng * 10000000000l)
|
||||
seaweed.lat = (long)(lat * 10000000000l)
|
||||
JSONObject geo = BaiDuApiUtils.googleToBaidu(lat, lng)
|
||||
seaweed.lng2 = (long)(geo.lng * 10000000000l)
|
||||
seaweed.lat2 = (long)(geo.lat * 10000000000l)
|
||||
}else if(source.contains("百度")){
|
||||
Map<String, Double> point = GeoUtils.geoconvBaiduToGoogle(lng, lat);
|
||||
long google_lat = (long)(point.get("lat") * 10000000000l);
|
||||
long google_lng = (long)(point.get("lng") * 10000000000l);
|
||||
long baiDu_lat = (long)(lat * 10000000000l)
|
||||
long baiDu_lng = (long)(lng * 10000000000l)
|
||||
seaweed.lng = google_lng
|
||||
seaweed.lat = google_lat
|
||||
seaweed.lat2 = baiDu_lat
|
||||
seaweed.lng2 = baiDu_lng
|
||||
}
|
||||
}
|
||||
seaweed.cat = "district"
|
||||
seaweed.status = 0
|
||||
seaweed.c_date = NOW
|
||||
seaweed.ip = "MACHINE_SAVE"
|
||||
seaweed.estate_type2 = getType(item["G"])
|
||||
Map<String, Object> markMap = new HashMap<String, Object>()
|
||||
seaweed.mark_map = markMap
|
||||
markMap.put("data_source", "机器入库")
|
||||
DbUtils.seaweed.save(seaweed)
|
||||
println(seaweed)
|
||||
})
|
||||
|
||||
+4
-4
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20210401", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20210501", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20210501", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20210601", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
@@ -22,10 +22,10 @@ while (startDate.before(endDate)) {
|
||||
calendar.add(Calendar.DATE, 1)
|
||||
def nextDate = calendar.getTime()
|
||||
count++
|
||||
OPEN_URI_LOG.find(and(eq("user_id", "1007705"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
|
||||
OPEN_URI_LOG.find(and(eq("user_id", "1007757"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
|
||||
def result = JSONObject.fromObject(doc.res_map.result)
|
||||
if (!hashSet.contains(result)) {
|
||||
def item = ["5c6535b99c213017e0768ee1", "qlkj", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
|
||||
def item = ["60a3202c0b42a00001aad385", "qlkjgx", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
|
||||
data << item
|
||||
hashSet << result
|
||||
} else {
|
||||
|
||||
+3
-3
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20210401", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20210501", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20210601", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20210701", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
@@ -42,7 +42,7 @@ while (startDate.before(endDate)) {
|
||||
keys.addAll(params.keys())
|
||||
def key = "${params.address}_${params.size}".toString()
|
||||
if (!hashSet.contains(key)) {
|
||||
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date.format("yyy"), doc.data_status, doc.code, doc.msg]
|
||||
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg]
|
||||
param_keys.each {
|
||||
item << params[it]
|
||||
}
|
||||
|
||||
+81
-4
@@ -1,6 +1,11 @@
|
||||
import com.fangjia.core.util.CNNMFilter
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
|
||||
//
|
||||
@@ -35,7 +40,79 @@ import org.apache.commons.io.FileUtils
|
||||
// println("copy:" + file.getAbsolutePath() + "\tto:" + ("E:\\成交数据\\DBF汇总\\" + file.getName()))
|
||||
// }
|
||||
//}
|
||||
DSCollection seawater = new DataStore("192.168.9.118", 27777).getCollection("fangjia.seawater")
|
||||
seawater.find(new BasicDBObject("cat", "sell").append("city", "上海").append("region", "长宁").append("district_name", "新泾三村")).each {
|
||||
println(it)
|
||||
}
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\副本地址标准化验证结果分析.xlsx",{List item->
|
||||
// HttpRequest request = new HttpRequest()
|
||||
// JSONObject json = request.getJsonContent2("http://localhost:8082/structure", ["address": item["A"]], "UTF-8")
|
||||
// if(json){
|
||||
// item["D"] = json.result.province
|
||||
// item["E"] = json.result.city
|
||||
// item["F"] = json.result.district
|
||||
// item["G"] = json.result.dev
|
||||
// item["H"] = json.result.township
|
||||
// item["I"] = json.result.village
|
||||
// item["J"] = json.result.group
|
||||
// item["K"] = json.result.street
|
||||
// item["L"] = json.result.streetNumber
|
||||
// item["M"] = json.result.name
|
||||
// item["N"] = json.result.buildingNumber
|
||||
// item["O"] = json.result.unitNumber
|
||||
// item["P"] = json.result.roomNumber
|
||||
// item["q"] = json.result.floor
|
||||
// item["r"] = json.result.formatAddress
|
||||
// }
|
||||
// println(item)
|
||||
//})
|
||||
//for(int i=1; i < 100; i++){
|
||||
// println("${i}号楼#@#building_number")
|
||||
// println("${i}栋#@#building_number")
|
||||
// println("${i}幢#@#building_number")
|
||||
// println("${i}单元#@#unit_number")
|
||||
// if(i< 20){
|
||||
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}号楼#@#building_number")
|
||||
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}栋#@#building_number")
|
||||
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}幢#@#building_number")
|
||||
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}单元#@#unit_number")
|
||||
// }
|
||||
//}
|
||||
//nums= "abcdefghijklmn"
|
||||
//for(def num : nums.toCharArray()){
|
||||
// println("${num}栋#@#building_number")
|
||||
// println("${num}幢#@#building_number")
|
||||
// println("${num}单元#@#unit_number")
|
||||
//}
|
||||
//nums= "abcdefghijklmn"
|
||||
//for(def num : nums.toCharArray()){
|
||||
// println("${num}座#@#building_number")
|
||||
// for(int i = 1; i <11; i++){
|
||||
// println("${num}${i}座#@#building_number")
|
||||
//// println("${num}${i}幢#@#building_number")
|
||||
//// println("${num}${i}单元#@#unit_number")
|
||||
// }
|
||||
//}
|
||||
//for(int i = 1; i <20; i++){
|
||||
// println("${i}区#@#part_number")
|
||||
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}区#@#part_number")
|
||||
// println("${i}期#@#stage_number")
|
||||
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}期#@#stage_number")
|
||||
//}
|
||||
//
|
||||
//for(def num : "东南北中西甲乙丙丁".toCharArray()){
|
||||
// println("${num}区#@#part_number")
|
||||
//}
|
||||
//for(int i = -5; i < 100; i++){
|
||||
// println("${i}楼#@#floor")
|
||||
// if(i < 10 && i > 0){
|
||||
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}楼#@#floor")
|
||||
// }
|
||||
//}
|
||||
//for(def num : "abcdefg东南西北".toCharArray()){
|
||||
// println("${num}户#@#room")
|
||||
//}
|
||||
//for(i in 1.. 40){
|
||||
// for(j in 1..9){
|
||||
// println("${i}0${j}室#@#room")
|
||||
// }
|
||||
//}
|
||||
ElasticUtils.executeSqlQuery("select * from seawater where cat='sell' and district_name='新泾三村' and city='上海' and p_date > 20210101",{Map doc->
|
||||
println(doc.avg_price + "\t" + doc.root + "\t" + doc.p_date)
|
||||
})
|
||||
Reference in New Issue
Block a user