diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LandDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LandDetailCrawl.groovy index 1f7d635e9..6a44c0463 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LandDetailCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LandDetailCrawl.groovy @@ -31,7 +31,7 @@ class LandDetailCrawl { def task = new LandDetailCrawl(); // task.run() task.updateData(); - task.undateOldData(); +// task.undateOldData(); } def run(){ @@ -65,7 +65,7 @@ class LandDetailCrawl { } def updateData(){ - def datas = ExcelUtils.read("D:\\数据\\友达\\土地--12月.xlsx",0, 1) + def datas = ExcelUtils.read("D:\\数据\\土地\\土地成交3月(2).xlsx",0, 1) if(datas && datas.size()>0){ for(def data : datas){ def id = data['A'] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy index 3c4ca8b31..9d191a6d4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy @@ -60,7 +60,8 @@ class HouseExport { static main(args) { def task = new HouseExport(); - task.run() +// task.run() + task.marge() } def run(){ @@ -777,4 +778,23 @@ def GenNonDuplicateID(randomLength){ + def marge(){ + File file = new File("D:\\数据\\建行\\新建文本文档.txt"); + def ids = new HashSet(); + def index = 0 + + file.eachLine{it-> + index++ + if(index>1){ + def arrs = it.split("\\|@\\|") + def id = arrs[1] + if(!ids.contains(id)){ + ids << id + } + } + } + println index + println ids.size() + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy new file mode 100644 index 000000000..6131ce01e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy @@ -0,0 +1,40 @@ +package com.fangjia.script.gj.data + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils + +class FieldStat { + + static main(args) { + def task = new FieldStat(); + task.run() + } + + def run(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区融合结果概况2.xlsx", { row -> + def city = row['A'] + if(city){ + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]); + def total = districts.size() + + def aliasIndex = 0 + def aliasTotalCount = 0 + for(def d : districts){ + def alias = d.alias + if(alias){ + aliasIndex++ + aliasTotalCount = aliasTotalCount + alias.size() + } + } + row['J'] = aliasTotalCount + row['K'] = aliasIndex + def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total) + row['L'] = p+"%" + row['M'] = total + println city +" "+p + } + + }); + println "finish" + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZBlankDataExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZBlankDataExport.groovy index 9d46a2d08..20d3b1dff 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZBlankDataExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZBlankDataExport.groovy @@ -630,128 +630,133 @@ class CZBlankDataExport { sqlBuffer.append(" and m_date >='"+startDate+"'").append(" and m_date <'"+endDate+"'"); // sqlBuffer.append(" and m_date >='"+pDate+"'").append(" and m_date <='"+pDate+"'"); sqlBuffer.append(" ORDER BY p_date desc limit 200") - ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + + ResultSet rs = null def index = 0; def houses = []; - while (rs.next()) { + try{ + rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) - def area = rs.getObject("area") + while (rs.next()) { - - - if(area && (area>1000||area<30)){ - continue; - } - - def floor = rs.getObject("floor") - if(floor && floor>100){ - continue; - } - - def height = rs.getObject("height") - if(height && (height>100||height==0)){ - continue; - } - - def total_price = rs.getObject("total_price") - def house_area = area; - def avg_price = total_price/house_area; - - def distPrice = distPriceMaps.get(dbo.city +"##"+dbo.region+"##"+dbo.name); - if(distPrice){ - def maxValue = distPrice + distPrice*0.15; - def minValue = distPrice - distPrice*0.15; - if(avg_price>maxValue.toInteger() || avg_price1000||area<30)){ continue; } - } - def district_name = rs.getObject("district_name") - if(district_name.endsWith("别墅") && height && height>=10){ - continue; - } + def floor = rs.getObject("floor") + if(floor && floor>100){ + continue; + } - def m = [:]; - //小区名称 - m.name = dbo.name; - if(dbo.alias){ - def newAlias = []; - for(def tname : dbo.alias){ - if(tname && tname.length()>=2){ - newAlias << tname; + def height = rs.getObject("height") + if(height && (height>100||height==0)){ + continue; + } + + def total_price = rs.getObject("total_price") + def house_area = area; + def avg_price = total_price/house_area; + + def distPrice = distPriceMaps.get(dbo.city +"##"+dbo.region+"##"+dbo.name); + if(distPrice){ + def maxValue = distPrice + distPrice*0.15; + def minValue = distPrice - distPrice*0.15; + if(avg_price>maxValue.toInteger() || avg_price=10){ + continue; + } + + def m = [:]; + //小区名称 + m.name = dbo.name; + if(dbo.alias){ + def newAlias = []; + for(def tname : dbo.alias){ + if(tname && tname.length()>=2){ + newAlias << tname; + } + } + if(newAlias){ + m.all_name = dbo.name + "("+newAlias.join(",")+")" + }else{ + m.all_name = dbo.name; + } }else{ m.all_name = dbo.name; } - }else{ - m.all_name = dbo.name; + m.city = rs.getObject("city"); + m.region = rs.getObject("region"); + m.district_id= dbo.num_id; + + m.house_id = createHouseId(rs.getObject("url"));//rs.getObject("_id").toString(); + //建筑物总楼层 + m.height = height>0?height:null; + //所在楼层 + m.floor = getHouseFloor(floor, m.height); + //建筑面积 + m.house_area = area; + //挂牌价 + m.total_price = total_price; + + def keys = m.region +"-"+ m.name+"-"+m.house_area+"-"+m.height+"-"+m.floor+"-"+m.total_price; + if(!houses.contains(keys)){ + houses << keys; + }else{ + println "此房源已存在....." + continue; + } + + if(index > 30){ + break; + } + m.room = rs.getObject("room"); + m.hall = rs.getObject("hall"); + m.estate_type = rs.getObject("estate_type"); + //建成年代 + m.build_date = getBuildDate(rs.getObject("build_date"), dbo); + //总使用年限 + if(districtIds.contains(m.district_id)){ + m.property_life = 70; + }else{ + m.property_life = getPropertyLife(dbo.property_life); + } + //朝向 + m.direction = getDirectionCode(rs.getObject("direction"), dbo.estate_type2); + //建筑结构 + m.build_structure_code = getBuildStructureCode(dbo.build_date, dbo.build_structure); + //建筑类别 + m.build_type_code = getBuildTypeCode(dbo.name, dbo.build_type, height); + if(m.build_type_code=='3' && m.height>6){ + m.height = null + } + + //内部装修与成新度 + m.fitment_code = getFitmentCode(rs.getObject("fitment")); + //是否有电梯 + m.elevator_code = getElevatorCode(height, m.build_type_code); + //房价网评估价 + //m.evaluation_price = getEvaluationPrice(m); + //println "[${index++}] "+new Date(); + //评估时间 + m.evaluation_date = DateUtils.formatDateByPattern(new Date(),"yyyyMMdd"); + //挂牌时间 + m.publish_date = rs.getObject("p_date").toString()//DateUtils.formatDateByPattern(obj.publish_date,"yyyyMMdd"); + + m.status=1; + + index++; + saveData(m); } - m.city = rs.getObject("city"); - m.region = rs.getObject("region"); - m.district_id= dbo.num_id; + }catch(Exception e){ - m.house_id = createHouseId(rs.getObject("url"));//rs.getObject("_id").toString(); - //建筑物总楼层 - m.height = height>0?height:null; - //所在楼层 - m.floor = getHouseFloor(floor, m.height); - //建筑面积 - m.house_area = area; - //挂牌价 - m.total_price = total_price; - - def keys = m.region +"-"+ m.name+"-"+m.house_area+"-"+m.height+"-"+m.floor+"-"+m.total_price; - if(!houses.contains(keys)){ - houses << keys; - }else{ - println "此房源已存在....." - continue; - } - - if(index > 30){ - break; - } - m.room = rs.getObject("room"); - m.hall = rs.getObject("hall"); - m.estate_type = rs.getObject("estate_type"); - //建成年代 - m.build_date = getBuildDate(rs.getObject("build_date"), dbo); - //总使用年限 - if(districtIds.contains(m.district_id)){ - m.property_life = 70; - }else{ - m.property_life = getPropertyLife(dbo.property_life); - } - //朝向 - m.direction = getDirectionCode(rs.getObject("direction"), dbo.estate_type2); - //建筑结构 - m.build_structure_code = getBuildStructureCode(dbo.build_date, dbo.build_structure); - //建筑类别 - m.build_type_code = getBuildTypeCode(dbo.name, dbo.build_type, height); - if(m.build_type_code=='3' && m.height>6){ - m.height = null - } - - //内部装修与成新度 - m.fitment_code = getFitmentCode(rs.getObject("fitment")); - //是否有电梯 - m.elevator_code = getElevatorCode(height, m.build_type_code); - //房价网评估价 - //m.evaluation_price = getEvaluationPrice(m); - //println "[${index++}] "+new Date(); - //评估时间 - m.evaluation_date = DateUtils.formatDateByPattern(new Date(),"yyyyMMdd"); - //挂牌时间 - m.publish_date = rs.getObject("p_date").toString()//DateUtils.formatDateByPattern(obj.publish_date,"yyyyMMdd"); - - m.status=1; - - index++; - saveData(m); } + return index; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/BasicInfoDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/BasicInfoDataSynch.groovy index cbe84d9f7..69a998c48 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/BasicInfoDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/BasicInfoDataSynch.groovy @@ -4,11 +4,14 @@ import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject - +/** + * 娴欏晢鏈湴鍖栭儴缃插悓姝ヨ剼鏈� + */ @BatchOwner(Watcher.GaoJuan) class BasicInfoDataSynch extends BaseBatch{ // final static DBCollection SERVER_AREA_CONN = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo) @@ -29,10 +32,10 @@ class BasicInfoDataSynch extends BaseBatch{ @Override public void process(Object args) { -// synchCitys(); -// synchRegion() -// synchTown() -// synchDict() + synchCitys(); + synchRegion() + synchTown() + synchDict() synchDistrict("district") synchDistrict("office") } @@ -47,46 +50,49 @@ class BasicInfoDataSynch extends BaseBatch{ zsCitysMap.put(zsCitys._id.toString(), zsCitys) zsDatas << zsCitys } - //数据存在则修改,不存在则保存 + //鏁版嵁瀛樺湪鍒欎慨鏀癸紝涓嶅瓨鍦ㄥ垯淇濆瓨 if(fangjiaCitys){ for(def fjCity : fangjiaCitys){ - if(fjCity.display!="全国"){ + if(fjCity.display!="鍏ㄥ浗"){ fjCitysMap.put(fjCity._id.toString(), fjCity) } def zsCity = zsCitysMap.get(fjCity._id.toString()) - if(zsCity){//修改 + if(zsCity){//淇敼 if(zsCity.province != fjCity.province || zsCity.alias!=fjCity.alias){ def udata = new BasicDBObject() udata.put("province",fjCity.province) udata.put("alias",fjCity.alias) udata.put("status",0) + udata.put("update_date",new Date().getTime()) ZS_CITYS_CONN.update(new BasicDBObject("_id",zsCity._id), new BasicDBObject('$set',udata)) } - }else{//保存 - def addUata = new BasicDBObject() + }else{//淇濆瓨 + def addUata = new BasicDBObject() addUata.put("_id",fjCity._id) addUata.put("display",fjCity.display) addUata.put("province",fjCity.province) addUata.put("alias",fjCity.alias) addUata.put("status",0) + addUata.put("update_date",new Date().getTime()) ZS_CITYS_CONN.save(addUata); } } } - //老数据不在新数据中,被删除时,状态修改为1 + //鑰佹暟鎹笉鍦ㄦ柊鏁版嵁涓紝琚垹闄ゆ椂锛岀姸鎬佷慨鏀逛负1 if(zsDatas){ for(def zsCity : zsDatas){ def fjCity = fjCitysMap.get(zsCity._id.toString()) if(!fjCity){ def udata = new BasicDBObject() udata.put("status",1) + udata.put("update_date",new Date().getTime()) ZS_CITYS_CONN.update(new BasicDBObject("_id",zsCity._id), new BasicDBObject('$set',udata)) } } } - println "[${fangjiaCitys.size()}]个城市同步完成" + println "[${fangjiaCitys.size()}]涓煄甯傚悓姝ュ畬鎴�" } def synchRegion(){ @@ -110,6 +116,7 @@ class BasicInfoDataSynch extends BaseBatch{ udata.put("pattern",fjRegion.pattern) udata.put("org_name",fjRegion.org_name) udata.put("status",0) + udata.put("update_date",new Date().getTime()) if(zsRegionObj){ ZS_REGION_CONN.update(new BasicDBObject("_id",zsRegionObj._id), new BasicDBObject('$set',udata)) }else{ @@ -125,11 +132,12 @@ class BasicInfoDataSynch extends BaseBatch{ if(!fjRegion){ def udata = new BasicDBObject() udata.put("status",1) + udata.put("update_date",new Date().getTime()) ZS_REGION_CONN.update(new BasicDBObject("_id",zsRegion._id), new BasicDBObject('$set',udata)) } } } - println "[${fjRegions.size()}]个行政区域同步完成" + println "[${fjRegions.size()}]涓鏀垮尯鍩熷悓姝ュ畬鎴�" } def synchTown(){ @@ -152,6 +160,7 @@ class BasicInfoDataSynch extends BaseBatch{ udata.put("short_name",fjTown.short_name) udata.put("name",fjTown.name) udata.put("status",0) + udata.put("update_date",new Date().getTime()) if(zsTownObj){ if(fjTown.city!=zsTownObj.city||fjTown.region!=zsTownObj.region||fjTown.short_name!=zsTownObj.short_name||fjTown.name!=zsTownObj.name||zsTownObj.status!=0){ ZS_TOWN_CONN.update(new BasicDBObject("_id",zsTownObj._id), new BasicDBObject('$set',udata)) @@ -169,11 +178,12 @@ class BasicInfoDataSynch extends BaseBatch{ if(!fjTown){ def udata = new BasicDBObject() udata.put("status",1) + udata.put("update_date",new Date().getTime()) ZS_TOWN_CONN.update(new BasicDBObject("_id",zsTown._id), new BasicDBObject('$set',udata)) } } } - println "[${fjTowns.size()}]个乡镇同步完成" + println "[${fjTowns.size()}]涓埂闀囧悓姝ュ畬鎴�" } def synchDict(){ def fjDicts = CommonDatautils.getAllDictionary() @@ -194,6 +204,7 @@ class BasicInfoDataSynch extends BaseBatch{ udata.put("cat",fjDict.cat) udata.put("name",fjDict.name) udata.put("status",0) + udata.put("update_date",new Date().getTime()) if(zsDictObj){ if(fjDict.cat!=zsDictObj.cat||fjDict.name!=zsDictObj.name||zsDictObj.status!=0){ ZS_DICT_CONN.update(new BasicDBObject("_id",zsDictObj._id), new BasicDBObject('$set',udata)) @@ -220,11 +231,12 @@ class BasicInfoDataSynch extends BaseBatch{ if(!fjDict){ def udata = new BasicDBObject() udata.put("status",1) + udata.put("update_date",new Date().getTime()) ZS_DICT_CONN.update(new BasicDBObject("_id",zsDict._id), new BasicDBObject('$set',udata)) } } } - println "[${fjDicts.size()}]个字典数据同步完成" + println "[${fjDicts.size()}]涓瓧鍏告暟鎹悓姝ュ畬鎴�" } def synchDistrict(cat){ def fangjiaCitys = CommonDatautils.findAllCity() @@ -259,18 +271,25 @@ class BasicInfoDataSynch extends BaseBatch{ udata.put("township",fjDistrict.township) udata.put("alias",fjDistrict.alias) udata.put("cat",cat) + udata.put("update_date",new Date().getTime()) if(fjDistrict.addr_alias){ udata.put("addr_alias",fjDistrict.addr_alias) } udata.put("status",0) - def map = new BasicDBObject() - map.put("lat",fjDistrict.lat/10000000000.0d) - map.put("lon", fjDistrict.lng/10000000000.0d) + def map = null; + if(fjDistrict && fjDistrict.lat){ + map = new BasicDBObject() + map.put("lat",fjDistrict.lat/10000000000.0d) + map.put("lon", fjDistrict.lng/10000000000.0d) + } + if(zsDistrictObj){ if(fjDistrict.city!=zsDistrictObj.city||fjDistrict.region!=zsDistrictObj.region||fjDistrict.block!=zsDistrictObj.block||fjDistrict.name!=zsDistrictObj.name||fjDistrict.address!=zsDistrictObj.address||fjDistrict.estate_type2!=zsDistrictObj.estate_type||fjDistrict.township!=zsDistrictObj.township||fjDistrict.alias!=zsDistrictObj.alias||fjDistrict.addr_alias!=zsDistrictObj.addr_alias||map!=zsDistrictObj.location||zsDistrictObj.status!=0){ - udata.put("location", map) - ZS_SEAWEED_CONN.update(new BasicDBObject("_id",zsDistrictObj._id), new BasicDBObject('$set',udata)) + if(map){ + udata.put("location", map) + ZS_SEAWEED_CONN.update(new BasicDBObject("_id",zsDistrictObj._id), new BasicDBObject('$set',udata)) + } } }else{ udata.put("location", map) @@ -295,10 +314,11 @@ class BasicInfoDataSynch extends BaseBatch{ if(!fjDistrict){ def udata = new BasicDBObject() udata.put("status",1) + udata.put("update_date",new Date().getTime()) ZS_SEAWEED_CONN.update(new BasicDBObject("_id",fjDistrict._id), new BasicDBObject('$set',udata)) } } } - println "[${fjCity}-${cat}][${districts.size()}]个小区数据同步完成" + println "[${fjCity}-${cat}][${districts.size()}]涓皬鍖烘暟鎹悓姝ュ畬鎴�" } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/DistrictLevelBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/DistrictLevelBatch.groovy index e6576a569..6c1344eb9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/DistrictLevelBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/DistrictLevelBatch.groovy @@ -9,6 +9,9 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +/** + * 娴欏晢鏈湴鍖栭儴缃插悓姝ヨ剼鏈� + */ @BatchOwner(Watcher.GaoJuan) class DistrictLevelBatch extends BaseBatch{ @@ -28,7 +31,7 @@ class DistrictLevelBatch extends BaseBatch{ def datas = addPriceData(dists, priceMap); datas = calcuteLevel(datas); saveOrUpdateDB(city, datas); - println city +" 楼盘评级同步完成" + println city +" 妤肩洏璇勭骇鍚屾瀹屾垚" } } @@ -109,6 +112,7 @@ class DistrictLevelBatch extends BaseBatch{ udata.put("level", d.level) udata.put("avg_price", d.avg_price) udata.put("status",0) + udata.put("update_date",new Date().getTime()) if(zsDistLevel){ if(zsDistLevel.city!=d.city||zsDistLevel.region!=d.region||zsDistLevel.name!=d.name||zsDistLevel.level!=d.level||zsDistLevel.avg_price!=d.avg_price||zsDistLevel.status!=0){ ZS_LEVEL_CONN.update(new BasicDBObject("_id",zsDistLevel._id), new BasicDBObject('$set',udata)) @@ -127,10 +131,11 @@ class DistrictLevelBatch extends BaseBatch{ } if(zsDistLevels.size()>0){ for(def zs : zsDistLevels){ - def newData = newDataMap.get(zs._id.toString()); + def newData = newDataMap.get(zs._id.toString()); if(!newData){ def udata = new BasicDBObject() udata.put("status",1) + udata.put("update_date",new Date().getTime()) ZS_LEVEL_CONN.update(new BasicDBObject("_id",zs._id), new BasicDBObject('$set',udata)) } }