Merge branch 'script_gj' into 'master'

update

See merge request !260
This commit is contained in:
2022-02-22 11:53:13 +08:00
27 changed files with 424 additions and 261 deletions
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-1", "2022-02-1")
def prices = CcbCommonUtils.findPersonPrice(city, "2022-02-1", "2022-03-1")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202112//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202110
def endDate = 202112
def sDate= 202201//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202111
def endDate = 202201
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -39,8 +39,8 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
// task.run()//统计最近3个月抓取的成交数据量
task.exportTrade()//导出未入库成交
task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
@@ -73,13 +73,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys;
def rets = []
rets << ["城市","10月总量","11月总量","12月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","11月总量","12月总量","1月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2021-10-01"
def date = "2021-11-01"
def endDate = "2021-12-01"
def newDate = "2022-01-01"
def startDate = "2021-11-01"
def date = "2021-12-01"
def endDate = "2022-01-01"
def newDate = "2022-02-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -129,7 +129,7 @@ class TradeStatLog {
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\12月成交统计.xlsx", rets, "数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-01成交统计.xlsx", rets, "数据")
println "表格生成完成"
}
@@ -12,9 +12,90 @@ import com.fangjia.core.util.BaiDuApiUtils
class DistrictMarge {
static main(args) {
def task = new DistrictMarge()
task.run()
//task.run()
task.repeatAddress()
}
def repeatAddress(){
def citys = CommonDatautils.findAllCity()
for(def c : citys){
def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name", "lng","address","name_cat"])
findRepead(c, districts)
}
}
def findRepead(city, districts){
def map = [:]
for(def d : districts){
if(d.name_cat == 0 && d.lng){
if(d.address && d.address.contains("号")){
def list = map.get(d.address)
if(!list){
list = []
map.put(d.address, list)
}
list << d
}
}
}
def newMap = [:]
map.each {k,vals->
if(vals.size()>=2){
vals.sort{o1,o2->
return o1.name.size() <=> o2.name.size()
}
newMap.put(k,vals)
}
}
newMap.each {k,vals->
def names = []
for(def d : vals){
def newKey = d.name
if(d.name.contains("小区")){
newKey = d.name.toString().replace("小区", "")
}
if(!names.contains(newKey)){
names << newKey
}else{
d.flag = "删除"
def udata = new BasicDBObject()
udata.put("name_cat", 11)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId( d._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
}
def values = newMap.values()
def allRets = []
for(def list : values){
allRets.addAll(list)
}
def results = []
def delcount = 0
for(def r : allRets){
def rows = []
rows << r._id.toString()
rows << r.city
rows << r.name
rows << r.address
rows << r.name_cat
if(r.flag){
rows << r.flag
delcount++
}
results << rows
}
// ExcelUtils.write("d:\\数据\\小区\\删除重复地址(${city}).xlsx", results, "sheet")
println "${city} 共:"+map.size()+" 2个以上:"+newMap.size()+" delcount:"+delcount
}
def run(){
def rets = getDatas()
def deleteIndex = 0
@@ -433,23 +433,43 @@ def city = "
// }
//})
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\漏映射别名.xlsx", { d ->
if (d['B']) {
def dist = CommonDatautils.findDistrictFieldById(d['B'],["city","region","name","mapping_alias"])
if(dist){
def alias = dist.mapping_alias
if(!alias){
alias = []
}
if(alias && !alias.contains(d['C'])){
index++
println index
alias << d['C']
def udata = new BasicDBObject()
udata.put("mapping_alias",alias)
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
}
//def index = 0
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\漏映射别名.xlsx", { d ->
// if (d['B']) {
// def dist = CommonDatautils.findDistrictFieldById(d['B'],["city","region","name","mapping_alias"])
// if(dist){
// def alias = dist.mapping_alias
// if(!alias){
// alias = []
// }
// if(alias && !alias.contains(d['C'])){
// index++
// println index
// alias << d['C']
// def udata = new BasicDBObject()
// udata.put("mapping_alias",alias)
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
// }
// }
// }
//})
ExcelUtils.modify("D:\\数据\\小区\\评级小区.xls", { d ->
if(d['A']){
try{
def dist = CommonDatautils.findDistrictFieldById(d['A'],["city","region","name"])
if(dist){
d['B'] = dist.city
d['C'] = dist.region
d['D'] = dist.name
def price = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name)
d['E'] = price
}
}catch(Exception e){
println d['A']
}
}
})
@@ -218,7 +218,7 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name","金辉伯爵山")
// q.put("name","玉泉路甲46号院")
DBObject keys = new BasicDBObject();
if(fields){
@@ -23,8 +23,8 @@ class FzDataResultStat {
// task.regionPriceTrendRun()
}
def priceMarkupRun(){
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
def map1 = [:]
@@ -53,7 +53,7 @@ class FzDataResultStat {
}
rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString(), markup]
}
//ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
@@ -94,7 +94,7 @@ class FzDataResultStat {
}
def prices = exportRegion(city, months)
def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")
ExcelUtils.write("D:\\数据\\福州\\区域均价半年走势_${curMonth}.xlsx", prices, "数据")
ExcelUtils.write("D:\\数据\\福州\\福州6区均价半年走势_${curMonth}.xlsx", prices, "数据")
}
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202112.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202201.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -143,8 +143,8 @@ class ThirdPriceExportBatch extends BaseBatch{
if(!retMap.zhugezhaofang_esf_price && d.zhugezhaofang_esf_price){
retMap.zhugezhaofang_esf_price = d.zhugezhaofang_esf_price
}
if(!retMap.anjuke_esf_price && d.anjuke_esf_price){
retMap.anjuke_esf_price = d.anjuke_esf_price
if(!retMap.anjuke_esf_price && d.get("58tc_esf_price")){
retMap.anjuke_esf_price = d.get("58tc_esf_price")
}
if(!retMap.soufang_esf_price && d.soufang_esf_price){
retMap.soufang_esf_price = d.soufang_esf_price
@@ -152,6 +152,9 @@ class ThirdPriceExportBatch extends BaseBatch{
if(!retMap.lianjia_esf_price && d.lianjia_esf_price){
retMap.lianjia_esf_price = d.lianjia_esf_price
}
if(!retMap.lianjia_esf_price && d.beike_esf_price){
retMap.lianjia_esf_price = d.beike_esf_price
}
if(!retMap.cityhouse_esf_price && d.cityhouse_esf_price){
retMap.cityhouse_esf_price = d.cityhouse_esf_price
@@ -45,7 +45,7 @@ class EsHouseStat {
public static void main(String[] args){
def task = new EsHouseStat();
//task.run();
task.run();
// task.testRun("深圳")
// task.exportOpenLog();
// task.exportMongoLog()
@@ -59,7 +59,7 @@ class EsHouseStat {
// task.szRentHouseStat();
// task.addPrice();
task.statNoHouseCCBData();
// task.statNoHouseCCBData();
}
def addPrice(){
@@ -337,21 +337,6 @@ class EsHouseStat {
return total;
}
def findHouse(def city, def startDate, def endDate){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1")
sqlBuffer.append(" and city = '"+city+"'")
sqlBuffer.append(" and cat = 'sell'")
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def total = 0;
while (rs.next()){
total = rs.getInt("count")
}
return total;
}
def findHouseSource(def city, def startDate, def endDate){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT * FROM \"seawater-20*\" WHERE 1=1")
@@ -373,6 +358,21 @@ class EsHouseStat {
return statMap;
}
def findHouse(def city, def startDate, def endDate){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1")
sqlBuffer.append(" and city = '"+city+"'")
sqlBuffer.append(" and cat = 'sell'")
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def total = 0;
while (rs.next()){
total = rs.getInt("count")
}
return total;
}
def testRun(def city){
@@ -149,6 +149,7 @@ class OpenApiUtils {
ret.lng2 = gmap.get("lng")
ret.lat2 = gmap.get("lat")
ret.estateType = ret.type
ret.source = "高德"
}catch (Exception e){
}
@@ -250,14 +251,14 @@ class OpenApiUtils {
}
if(type && type.contains("出入口")){
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】-${type}"
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】-${type}"
}else{
println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
poiList.add(jo);
}
}else{
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
}
}
}else{
@@ -302,7 +303,7 @@ class OpenApiUtils {
def distname = formatName(ret.get("name"))
address = ret.get("address")
if((distname && distname.contains(name)) || (distname && name.contains(distname)) || (address && address.contains(name)) || ( address && name.contains(address))){
if(ret.get("area") && (region.contains(ret.get("area")) || ret.get("area").contains(region))){
// if(ret.get("area") && (region.contains(ret.get("area")) || ret.get("area").contains(region))){
lng2 = location.lng
lat2 = location.lat
def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2)
@@ -311,7 +312,7 @@ class OpenApiUtils {
def mapRegion = ret.get("area")
distRegion = mapRegion
districtName = distname
if(address && address.contains(mapRegion)){
if(address && mapRegion && address.contains(mapRegion)){
address = address.substring(address.indexOf(mapRegion)+mapRegion.size());
}
if(ret.get("detail_info")){
@@ -330,6 +331,7 @@ class OpenApiUtils {
map.name = districtName
map.region = distRegion
map.estateType = estateType
map.source = "百度"
rets << map
lat = null
@@ -340,12 +342,18 @@ class OpenApiUtils {
println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
}
}else{
println "X百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+ret.get("address")+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
}
// }else{
// println "X百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+ret.get("address")+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
// }
}else{
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(distname, name);
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(address, name);
def score1 = 0
if(distname && name){
score1 = CosineSimilarAlgorithm.getSimilarityRatio(distname, name);
}
def score2 = 0
if(address && name){
score2 = CosineSimilarAlgorithm.getSimilarityRatio(address, name);
}
if(score1 || score2){
rets << map
println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】(score:"+score1+"#"+score2+")"
@@ -31,7 +31,7 @@ class RegionPointCrawl {
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
//def provinces = ["河北省","山西省","辽宁省","吉林省","黑龙江省","江苏省","浙江省","安徽省","福建省","江西省","山东省","河南省","湖北省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
def provinces = ["内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"]
def provinces = ["福建省"]//"内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"
for(def p : provinces){
task.crawlCityAndDistrictCode(p);
task.run(p);
@@ -7,13 +7,17 @@ import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.regex.Matcher
import java.util.regex.Pattern
import com.fangjia.script.util.DbUtils
/**
* 模糊地址经纬度更新
@@ -44,13 +48,16 @@ class VagueAddressLocationBatch extends BaseBatch {
void process(Object args) {
//"北京","常州",
// def citys =["无锡","重庆","东莞","中山","合肥","苏州","广州","深圳","佛山","长沙","韶关","惠州","清远","珠海","福州","厦门","龙岩","漳州","成都","天津","南京","青岛"] //CommonDatautils.findAllCity();
def citys =["上海"] //CommonDatautils.findAllCity();
def citys =["北京"] //CommonDatautils.findAllCity();
//,"济南","青岛","烟台","威海","潍坊","淄博","临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄"
//市谊大厦 华三新村 阳光家园
for (def city : citys) {
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat2", "lat", "lng2", "lng"]);
def rets = findVagueAddress(city, districts);
findAndUpdateLocation(rets);
writeCSV(results, "d:\\数据\\结果\\" + city + "地址更新.csv")
// writeCSV(results, "d:\\数据\\结果\\" + city + "地址更新.csv")
writeCSV(results, "d:\\数据\\结果\\" + city + "地址模糊.csv")
// ExcelUtils.write("d:\\数据\\结果\\" + city + "地址模糊.xlsx", rets, "数据")
}
@@ -58,6 +65,7 @@ class VagueAddressLocationBatch extends BaseBatch {
def findVagueAddress(def city, def districts) {
def rets = []
def rows = []
for (def district : districts) {
def address = district.address;
if (address) {
@@ -66,10 +74,13 @@ class VagueAddressLocationBatch extends BaseBatch {
//没地址
if (!address) {
rets << district;
// rows << [district.city, district.region, district.name, district.address]
println district.city + " " + district.name + " ******[" + district.address + "]******** 模糊地址"
} else {
if (!findPattern(address)) {
rets << district;
// rows << [district.city, district.region, district.name, district.address]
println district.city + " " + district.name + " ******[" + district.address + "]******** 模糊地址"
} else {
println district.city + " " + district.address + " 精确地址"
@@ -97,7 +108,7 @@ class VagueAddressLocationBatch extends BaseBatch {
} else if (address.indexOf("路") == -1 && address.indexOf("街") == -1 && address.indexOf("大道") == -1 && address.indexOf("巷") == -1 && address.indexOf("北里") == -1 && address.indexOf("南里") == -1 && address.indexOf("东里") == -1 && address.indexOf("西里") == -1) {
return null;
} else {
pattern = Pattern.compile("[路|街|大道|巷]\\d+");
pattern = Pattern.compile("[路甲|路|街|大道|巷|弄|里]\\d+");
matcher = pattern.matcher(address)
if(matcher.find()){
return s;
@@ -123,16 +134,14 @@ class VagueAddressLocationBatch extends BaseBatch {
for (def ret : rets) {
index++;
if(ret.name!="药师新村"){
continue
}
def locationMap = findLatAndLng(ret);
if (locationMap) {
updateLocation(locationMap, ret);
} else {
println "[没找到] " + ret.city + " " + ret.region + " " + ret.name + " " + ret.address
}
println "【${index}/${rets.size()}】************************************************************************************"
println "【${index}/${rets.size()}】**************【${ret.region}---${ret.name}----${ret.address}】**********************************************************************"
}
}
@@ -148,14 +157,14 @@ class VagueAddressLocationBatch extends BaseBatch {
msgMap.db_lng2 = ret.lng2
BasicDBObject udata = new BasicDBObject()
if (ret.lat2 != locationMap.lat2) {
udata.put('lat2', locationMap.lat2)
udata.put('lng2', locationMap.lng2)
if (ret.lat2 && ret.lat2 != locationMap.lat2) {
msgMap.m_lat2 = locationMap.lat2
msgMap.m_lng2 = locationMap.lng2
def distance = GeoHelper.GetDistance(ret.lat2, ret.lng2, locationMap.lat2, locationMap.lng2)
msgMap.updateLngFlag = true
msgMap.distance = distance
udata.put('lat2', locationMap.lat2)
udata.put('lng2', locationMap.lng2)
}
if (ret.lat != locationMap.lat) {
@@ -171,31 +180,39 @@ class VagueAddressLocationBatch extends BaseBatch {
if (udata) {
println ret.city + " " + ret.region + " " + ret.name + " [ 修改其经纬度!]"
udata.put("m_date", new Date())
// DBUtils.seaweed.update(new BasicDBObject('_id',locationMap._id),new BasicDBObject('$set',udata))
DbUtils.seaweed.update(new BasicDBObject('_id',locationMap.id),new BasicDBObject('$set',udata))
}
results << msgMap
}
def findStandardObject(def city, def name, def rets) {
def findStandardObject(def city, def region, def name, def rets) {
def map;
if (rets) {
if (rets.size() > 1) {
//相似里找有精确地址
for (def mapObj : rets) {
mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
if (mapObj.address && findPattern(mapObj.address)) {
map = mapObj;
break;
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
if(mapObj.name.contains(name) || name.contains(mapObj.name)){
mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
if (mapObj.address && findPattern(mapObj.address)) {
map = mapObj;
break;
}
}
}
}
if(!map){
//交带有交叉口地址
for (def mapObj : rets) {
if (mapObj.address && (mapObj.address.contains("交汇")||mapObj.address.contains("交叉"))) {
if(!mapObj.address.contains("与;") && !mapObj.address.startsWith("路") && !mapObj.address.startsWith("路;")){
map = mapObj;
break;
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
if(mapObj.name.contains(name) || name.contains(mapObj.name)) {
if (mapObj.address && (mapObj.address.contains("交汇") || mapObj.address.contains("交叉"))) {
if (!mapObj.address.contains("与;") && !mapObj.address.startsWith("路") && !mapObj.address.startsWith("路;")) {
map = mapObj;
break;
}
}
}
}
}
@@ -211,18 +228,18 @@ class VagueAddressLocationBatch extends BaseBatch {
def findLatAndLng(ret) {
//https://restapi.amap.com/v3/place/text?keywords=
def rets = OpenApiUtils.getGaoDeSearchLocation(ret.city, ret.region, ret.name)
def map = findStandardObject(ret.city, ret.name, rets);
def map = findStandardObject(ret.city, ret.region, ret.name, rets);
def locationMap = [:]
if (!map || !map.lat) {
//http://api.map.baidu.com/place/v2/search
rets = OpenApiUtils.getBiDuLocation(ret.city, ret.region, ret.name);
map = findStandardObject(ret.city, ret.name, rets);
map = findStandardObject(ret.city, ret.region, ret.name, rets);
}
if (!map || !map.lat) {
//https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap
def code = codeMap.get(ret.city);
rets = OpenApiUtils.searchBaiDuMap(ret.name, ret.region, code.toInteger());
map = findStandardObject(ret.city, ret.name, rets);
map = findStandardObject(ret.city, ret.region, ret.name, rets);
}
// if((!map || !map.lat)){
@@ -237,7 +254,7 @@ class VagueAddressLocationBatch extends BaseBatch {
locationMap.lng2 = (map.lng2.toDouble() * 10000000000).longValue()
locationMap.id = ret._id
locationMap.city = ret.city
if (ret.address.toString().endsWith("路")) {
if (map.address) {
locationMap.address = map.address.replace("(","").replace(")","");
}
@@ -48,7 +48,7 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-12-01"
static def statDate = "2022-01-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2022年1月\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2022年2月\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -12,7 +12,19 @@ class TempMatchDistrict {
static main(args) {
def task = new TempMatchDistrict()
// task.run()
task.regionConver()
// task.regionConver()
task.stat()
}
def stat(){
def citys = ["广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"]
def total = 0
for(def city : citys){
def count = CommonDatautils.findAllDistrictCount("district", city)
total = total + count
println city +" "+count
}
println "共:"+total
}
def regionConver(){
@@ -29,7 +41,7 @@ class TempMatchDistrict {
}
def run(){
def path = "D:\\数据\\小区\\收集数据-匹配\\收集数据"
def files = ["合肥.xlsx","湖北.xlsx","陕西.xlsx","上海.xlsx","深圳.xlsx","苏州.xlsx","天津.xlsx","雄安.xlsx","浙江.xlsx","重庆.xlsx"]
def files = ["杭州.xlsx"]
for(def f : files){
def filePath = path+"\\"+f;
match(filePath);
@@ -10,7 +10,7 @@ class FangjiaClientUtils {
public static def getMapData(def city, def region, def name, def address, def types, def mapCode){
JSONObject jsonObj = null
try {
def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"&region="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
def url = "http://192.168.0.216:38080/FangJiaDataService/match?city="+city+"&region="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
//def url = "http://192.168.10.166:8080/FangJiaDataService/match?city="+city+"&region="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
if(address){
url = url + "&address="+address
@@ -40,9 +40,9 @@ class LocationTest {
// System.out.println(str);
def task = new LocationTest()
// task.run()
task.run()
// task.run2()
task.run3()
// task.run3()
// task.run4()
// task.run5()
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.paimai
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
@@ -33,8 +34,8 @@ public class LianjiaCrawlByChrome {
static main(args) {
def task = new LianjiaCrawlByChrome();
task.run()
// task.run1()
// task.run()
// task.run1()
task.addDealRegion();
}
@@ -94,12 +95,12 @@ public class LianjiaCrawlByChrome {
}
def run(){
// System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
// System.setProperty("http.nonProxyHosts", "localhost");
// WebDriver driver = new ChromeDriver()
// def cityMap = getAllCityUrls(driver)
// getLogin(driver)
// crawlData(driver, cityMap)
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def cityMap = getAllCityUrls(driver)
getLogin(driver)
crawlData(driver, cityMap)
}
@@ -109,7 +110,7 @@ public class LianjiaCrawlByChrome {
lastMonth = lastMonth +"-01"
def q = new BasicDBObject()
q.put("region", null)
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2021-12-01","yyyy-MM-dd")))
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
// q.put("source", new BasicDBObject('$in', ["Á´¼ÒÔÚÏß","±´¿ÇÕÒ·¿"]))
def dbcursor = DEAL_PRICE_COLL.find(q)
def keys = []
@@ -193,7 +194,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2021-12-31", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
@@ -219,6 +220,9 @@ public class LianjiaCrawlByChrome {
detail.city = city
detail.district_name = arrs[0]
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
if(detail.region){
detail.region = RegionData.formatRegion(city, detail.region)
}
def roomStr = arrs[1]
if (roomStr) {
@@ -391,9 +395,9 @@ public class LianjiaCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
if(!href.contains("fang.lianjia")){
// if(!href.contains("fang.lianjia") && e.text().contains("¸£ÖÝ")){
cityMap.put(e.text(), href)
}
// }
}
}
return cityMap
@@ -60,12 +60,12 @@ class CrawlDistrictIdSelect {
def findCrawlData(){
def q = new BasicDBObject()
q.put("city","福州")
q.put("source", "城市房产")
q.put("s_date",202012)
q.put("city","±±¾©")
q.put("source", "ËÑ·¿")
q.put("s_date",202112)
def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:]
def prefix = ".cityhouse.cn/ha/"
def prefix = "esf.fang.com/loupan/"
def keyList =new HashSet()
for(def db : dbcursor){
def url = null
@@ -77,11 +77,14 @@ class CrawlDistrictIdSelect {
url = db.url
}
if(url && url.contains(prefix) ){
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".html"))
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".htm"))
if(district_id=="0"){
continue
}
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(prefix))
if(!shortCity){
shortCity = "bj"
}
def key = shortCity +"#"+district_id
def ids = retMap.get(key)
@@ -92,7 +95,7 @@ class CrawlDistrictIdSelect {
if(!keyList.contains(key)){
def map = new BasicDBObject()
map.root = "cityhouse.com"
map.root = "fang.com"
map.city_short= shortCity
map.city = db.city
map.region = db.region
@@ -68,7 +68,7 @@ class FZloationUpdate {
Long lng0 = lng*10000000000L
udata.put("lat", lat0)
udata.put("lng", lng0 )
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
println "[ÐÞ¸Ä-${total}] "+ dist.region+" "+dist.name+" "+dist.address+" "+upldateFlag
}
@@ -153,7 +153,7 @@ class FZloationUpdate {
if(addr){
def rMap = [:]
def reps = GaoDeApiUtils.geo(dist.city, addr);
if(reps && reps.count.toInteger()>=1){
if(reps && reps.count && reps.count.toInteger()>=1){
def map = reps.geocodes.get(0)
def location = map.location
def level = map.level
@@ -40,11 +40,11 @@ class DataStat {
def run(){
def source = "58ͬ³Ç"//
def citys = findCrawlCity(source, 202111);
def source = "城市房产"//
def citys = findCrawlCity(source, 202112);
for(def city : citys){
def count1 = findCount(source, city, 202111);
def count2 = findCount(source, city, 202112);
def count1 = findCount(source, city, 202112);
def count2 = findCount(source, city, 202201);
println city +" "+count1+" "+count2
// println count1
// println "--- ----"
File diff suppressed because one or more lines are too long
@@ -315,11 +315,11 @@ class AddMetadataBatchQfang{
private static void saveThirdPartyPrice(def saveDate){
def q = new BasicDBObject()
q.put("url", saveDate.get("xqUrl"))
q.put("s_date", SHORT_DATE);
q.put("city", saveDate.get("city"));
q.put("region", saveDate.get("region"));
q.put("name", saveDate.get("xqName"));
q.put("source", saveDate.get("source"));
def udata = THIRD_PARTY_PRICE_COLL.findOne(q)
if(!udata){
@@ -336,6 +336,14 @@ class AddMetadataBatchQfang{
obj.put("source", saveDate.get("source"));
obj.put("c_date", new Date());
THIRD_PARTY_PRICE_COLL.save(obj);
println "[save] "+saveDate.toString()
}else{
def updata = new BasicDBObject()
updata.put("avg_price", saveDate.get("price"))
updata.put("url", saveDate.get("xqUrl"));
updata.put("m_date", new Date());
THIRD_PARTY_PRICE_COLL.update(q,new BasicDBObject('$set',updata))
println "[update] "+saveDate.toString()
}
}
@@ -87,7 +87,7 @@ class AddSeedBatchQfang extends BaseBatch{
href = "https:"+href;
}
if(href&&!CITYS.contains(city)){
ret.put(city, href);
ret.put(city, href);
}
}
}catch(Exception e){
@@ -18,7 +18,7 @@ class SynchDistrictPrice {
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
//"fang.com":"搜房",
def sourceMap = ["fang.com":"搜房"]
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()
@@ -94,11 +94,16 @@ class SynchDistrictPrice {
for(def r : rets){
def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
if(obj){
updateIndex++
thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
addIndex++
thirdPartyColl.save(r)
}else{
thirdPartyColl.save(r)
try{
addIndex++
thirdPartyColl.save(r)
}catch(Exception e){
e.printStackTrace()
}
}
// if(!obj){
// addIndex++
@@ -112,7 +117,7 @@ class SynchDistrictPrice {
// }
// }
}
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count +" 修改:"+updateIndex
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count +" 已存在:"+updateIndex
}
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
def count1 = findTotalCount(source, 202111);
def count2 = findTotalCount(source, 202112);
def count1 = findTotalCount(source, 202112);
def count2 = findTotalCount(source, 202201);
println source +" "+count1 +" "+count2
}
}
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20211201", "yyyyMMdd")
def endDate = DateUtil.parseDate("20220101", "yyyyMMdd")
def startDate = DateUtil.parseDate("20220101", "yyyyMMdd")
def endDate = DateUtil.parseDate("20220201", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0