Merge branch 'dev' into 'master'

增加杭银消金四级标准化接口

See merge request !810
This commit is contained in:
2024-02-18 16:24:39 +08:00
6 changed files with 280 additions and 73 deletions
@@ -0,0 +1,175 @@
package com.fangjia.script.majintao
import com.fangjia.core.RegionData
import com.fangjia.core.ProvinceData
import com.fangjia.core.util.MongoUtils
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
def map = [
["学校", "高等院校", "中学", "小学", "幼儿园", "成人教育", "职业技术学校"]: "学校",
["综合医院", "专科医院"] : "医院",
["银行"] : "银行",
["超级市场", "家电电子卖场", "便民商店/便利店"] : "超市",
["商场"] : "商场",
["美容美发店"] : "美容院",
["健身中心"] : "健身中心",
["公园广场", "公园"] : "公园/绿地",
["公交车站"] : "公交",
["飞机场"] : "机场",
["火车站"] : "火车站",
["快餐厅", "外国餐厅", "中餐厅"] : "餐馆",
["宾馆酒店", "旅馆招待所"] : "酒店"
] as HashMap<List<String>, String>
def type3In = ["学校", "高等院校", "中学", "小学", "幼儿园", "成人教育", "职业技术学校", "健身中心", "公园广场", "公园", "地铁站", "飞机场", "火车站"]
// 输出 Map 的内容
map.each { key, value ->
println("$key: $value")
}
def provinceList = []
def provinceMap = ProvinceData.data
provinceMap.each { pro ->
provinceList.add(pro.getValue()[0])
}
def cityMap = ProvinceData.GOVERNMENT_CITY_LIST
cityMap.each { city ->
provinceList.add(city.getValue()[0])
}
def specialMap = ProvinceData.SPECIAL_ADMIN_REGION
specialMap.each { spe ->
provinceList.add(spe.getValue()[1])
}
def pcmap = [:]
provinceList.remove("广西省")
provinceList.add("广西壮族自治区")
DSCollection seabird = new DataStore("192.168.0.43", 28017).getCollection("fangjia_base.baidu_place_poi2")
DBCollection m43 = MongoUtils.getCollectionByName("fangjia_base", "baidu_place_poi2", "192.168.0.43", 28017)
//def type3Not = ["学校内部设施", "进港", "出港", "候船室", "住宿服务相关"]
//
//def type2Not = ["学校", "专科医院", "综合医院", "诊所", "银行", "自动提款机", "便民商店/便利店", "家电电子卖场", "超级市场", "商场", "美容美发店", "地铁站", "港口码头", "住宿服务"]
//def type2In = ["运动场馆", "公园广场", "公交车站", "机场相关", "火车站", "餐饮服务"]
// 运动场馆 且为 健身中心
// 公园广场 公园广场
//公园广场 公园
// 公交车站 公交车站相关
// 机场相关 飞机场
// 火车站 火车站
// 餐饮服务 快餐厅 外国餐厅 中餐厅
println(provinceList)
//provinceList = provinceList.subList(0,1)
def c = 0
def citys = []
def citys2 = []
for (p in provinceList) {
println(p)
def result = []
ElasticUtils.executeSqlQuery("SELECT location,name,adname,cityname,m_date,id,address,type FROM amap_poi WHERE pname = '" + p + "'", { Map data ->
// println(data["type"] + "-----")
c = c+1
if (c%10000==0){
println("已处理$c 条")
}
def locationString = String.valueOf(data.get("location"))
def typeString = String.valueOf(data.get("type"))
def cityString = String.valueOf(data.get("cityname"))
// citys2.add(cityS)
// print(cityString)
String city
if ("台湾省".equals(cityString) | "香港特别行政区".equals(cityString) | "澳门特别行政区".equals(cityString)) {
city = cityString
} else {
city = RegionData.formatCity(cityString)
}
def m_date = data["m_date"]
def region = data["adname"]
region = RegionData.formatRegion(city,region)
def address = String.valueOf(data.get("address"))
def uid = String.valueOf(data.get("id"))
def name = data.get("name")
if (typeString.contains("|")) {
typeString = typeString.split("'|'")[0]
}
// println(typeString)
def secondType = typeString.split(";")[1]
def smallType = typeString.split(";")[2]
// println(smallType.getClass().getName())
// 使用正则表达式匹配经度和纬度
def matcher = locationString =~ /(-?\d+\.\d+)/
def keyword = ""
def flag = 0
// 类型映射
// 用三级分类判断
if (smallType in type3In) {
// 遍历Map的键集合,检查目标字符串是否存在于键集合中
map.keySet().each { key ->
if (key.contains(smallType)) {
def value = map[key]
// println("$smallType 存在于Map的键中,对应的值为: $value")
keyword = value
}
}
} else {
//用二级分类判断
map.keySet().each { key ->
if (key.contains(secondType)) {
def value = map[key]
// println("$secondType 存在于Map的键中,对应的值为: $value")
keyword = value
}
}
}
// 获取匹配到的经度和纬度值
def lng = matcher[0][0]
lng = lng.toDouble()
def lat = matcher[1][0]
lat = lat.toDouble()
// println "纬度: $lng, 经度: $lat,"
if (!"".equals(keyword)) {
def dbo = new BasicDBObject()
dbo["_id"] = new ObjectId()
dbo["keyword"] = keyword
dbo["lat"] = lat
dbo["name"] = name
dbo["city"] = city
dbo["m_date"] = m_date
dbo["lng"] = lng
dbo["c_date"] = m_date
dbo["address"] = address
dbo["uid"] = uid
dbo["status"] = 0
dbo["source"] = "gaode"
dbo["region"] = region
result.add(dbo)
}
// print(result)
// println("888")
})
if (null != result && result.size() > 0) {
m43.insert(result)
}
println("已处理完$p 的数据")
}
@@ -0,0 +1,25 @@
package com.fangjia.script.majintao
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject
def count = 0
//ElasticUtils.executeSqlQuery("SELECT * FROM seawater WHERE city = '重庆' and region = '渝北' and district_name = '棕榈泉国际花园四期'and p_date > 20230801 ",{Map doc->
ElasticUtils.executeSqlQuery("SELECT * FROM seawater WHERE p_date > 20230801 order by p_date desc limit 100",{Map doc->
def keys = ["c_date","_id","cat","city","region","name","address","block","district_name","total_price","area","height","floor","fitment","estate_type","hall","toilet","publish_date","url","direction","build_type","pay_type","line","rent_type","status","room"]
// def dbo = new BasicDBObject()
// for( String key in keys){
// dbo[key] = doc[key]
// }
// println(doc['m_date'])
// print(doc['name'])
if (count==0)print(doc)
count+=1
// DbUtils.seabird.save(dbo)
})
//DbUtils.seabird.find(new BasicDBObject("district_name","棕榈泉国际花园四期")).each {
// println(it)
//}
print(count)
@@ -7,26 +7,26 @@ import jxl.Sheet
import jxl.Workbook
import org.apache.poi.ss.usermodel.WorkbookFactory
String excelFilePath = "F:\\估家\\文件\\安徽省联社\\全国省级行政区房屋覆盖情况表 - 副本.xlsx"
ExcelUtils.removeEmptyRow(excelFilePath)
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
province1 = items['A']
if (!province1.equals('黑龙江省') || !province1.equals('内蒙古自治区')) {
province = province1.substring(0, 2)
} else {
province = province1.substring(0, 3)
}
println('province: ' + province)
def citys = ProvinceData.getAllCity(province)
println(citys)
if (citys.size() > 0) {
ExcelUtils.writeToExcel(excelFilePath, citys, province)
}
})
//String excelFilePath = "F:\\估家\\文件\\安徽省联社\\全国省级行政区房屋覆盖情况表 - 副本.xlsx"
//ExcelUtils.removeEmptyRow(excelFilePath)
//ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
//
// province1 = items['A']
// if (!province1.equals('黑龙江省') || !province1.equals('内蒙古自治区')) {
// province = province1.substring(0, 2)
// } else {
// province = province1.substring(0, 3)
// }
//
// println('province: ' + province)
//
// def citys = ProvinceData.getAllCity(province)
// println(citys)
//// if (citys.size() > 0) {
//// ExcelUtils.writeToExcel(excelFilePath, citys, province)
//// }
//
//})
//
@@ -1,54 +1,4 @@
package com.fangjia.script.majintao
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils
import net.sf.json.JSONObject
String excelFilePath = "F:\\估家\\文件\\北交所\\暂时不能估值数据 - 副本.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath)
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
String comprehensiveUrl = "http://open.fangjia.com/comprehensiveQuery/houseValueInformation";
String communityUrl = "http://open.fangjia.com/addrSvc/avgPrice";
String valueUrl = "http://open.fangjia.com/property/addressEvaluate";
String caseUrl = "http://open.fangjia.com/house/case";
String poiUrl = "http://open.fangjia.com/district/addrFacilitys";
HttpRequest httpRequest = new HttpRequest();
map = new HashMap<String, String>()
String city = items['A']
String address = items['B']
double size = items['C']
System.out.println(city + "---" + address + "----" + size);
map.put("city", city)
map.put("address", address)
map.put("size", size.toString())
map.put("strctureArea", size.toString())
map.put("token", "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB")
if (!address.contains("%")) {
// 发送HTTP请求
long start
long end
def addressValue = httpRequest.getJsonContent(communityUrl, map, "UTF-8")
items['D'] = addressValue.get('result').toString()
def value = httpRequest.getJsonContent(valueUrl, map, "UTF-8")
items['E'] = value.get('result').toString()
def houseCase = httpRequest.getJsonContent(caseUrl, map, "UTF-8")
items['F'] = houseCase.get('result').toString()
def poi = httpRequest.getJsonContent(poiUrl, map, "UTF-8")
items['G'] = poi.get('result').toString()
}
})
import org.bson.types.ObjectId
a = new ObjectId()
print(a)