浙商修改时间

This commit is contained in:
2021-03-16 17:37:50 +08:00
parent 138ccf2f18
commit ac85c4159d
6 changed files with 222 additions and 132 deletions
@@ -31,7 +31,7 @@ class LandDetailCrawl {
def task = new LandDetailCrawl();
// task.run()
task.updateData();
task.undateOldData();
// task.undateOldData();
}
def run(){
@@ -65,7 +65,7 @@ class LandDetailCrawl {
}
def updateData(){
def datas = ExcelUtils.read("D:\\数据\\友达\\土地--12月.xlsx",0, 1)
def datas = ExcelUtils.read("D:\\数据\\土地\\土地成交3月(2).xlsx",0, 1)
if(datas && datas.size()>0){
for(def data : datas){
def id = data['A']
@@ -60,7 +60,8 @@ class HouseExport {
static main(args) {
def task = new HouseExport();
task.run()
// task.run()
task.marge()
}
def run(){
@@ -777,4 +778,23 @@ def GenNonDuplicateID(randomLength){
def marge(){
File file = new File("D:\\数据\\建行\\新建文本文档.txt");
def ids = new HashSet();
def index = 0
file.eachLine{it->
index++
if(index>1){
def arrs = it.split("\\|@\\|")
def id = arrs[1]
if(!ids.contains(id)){
ids << id
}
}
}
println index
println ids.size()
}
}
@@ -0,0 +1,40 @@
package com.fangjia.script.gj.data
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
class FieldStat {
static main(args) {
def task = new FieldStat();
task.run()
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区融合结果概况2.xlsx", { row ->
def city = row['A']
if(city){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]);
def total = districts.size()
def aliasIndex = 0
def aliasTotalCount = 0
for(def d : districts){
def alias = d.alias
if(alias){
aliasIndex++
aliasTotalCount = aliasTotalCount + alias.size()
}
}
row['J'] = aliasTotalCount
row['K'] = aliasIndex
def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
row['L'] = p+"%"
row['M'] = total
println city +" "+p
}
});
println "finish"
}
}
@@ -630,128 +630,133 @@ class CZBlankDataExport {
sqlBuffer.append(" and m_date >='"+startDate+"'").append(" and m_date <'"+endDate+"'");
// sqlBuffer.append(" and m_date >='"+pDate+"'").append(" and m_date <='"+pDate+"'");
sqlBuffer.append(" ORDER BY p_date desc limit 200")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
ResultSet rs = null
def index = 0;
def houses = [];
while (rs.next()) {
try{
rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def area = rs.getObject("area")
while (rs.next()) {
if(area && (area>1000||area<30)){
continue;
}
def floor = rs.getObject("floor")
if(floor && floor>100){
continue;
}
def height = rs.getObject("height")
if(height && (height>100||height==0)){
continue;
}
def total_price = rs.getObject("total_price")
def house_area = area;
def avg_price = total_price/house_area;
def distPrice = distPriceMaps.get(dbo.city +"##"+dbo.region+"##"+dbo.name);
if(distPrice){
def maxValue = distPrice + distPrice*0.15;
def minValue = distPrice - distPrice*0.15;
if(avg_price>maxValue.toInteger() || avg_price<minValue.toInteger()){
def area = rs.getObject("area")
if(area && (area>1000||area<30)){
continue;
}
}
def district_name = rs.getObject("district_name")
if(district_name.endsWith("别墅") && height && height>=10){
continue;
}
def floor = rs.getObject("floor")
if(floor && floor>100){
continue;
}
def m = [:];
//小区名称
m.name = dbo.name;
if(dbo.alias){
def newAlias = [];
for(def tname : dbo.alias){
if(tname && tname.length()>=2){
newAlias << tname;
def height = rs.getObject("height")
if(height && (height>100||height==0)){
continue;
}
def total_price = rs.getObject("total_price")
def house_area = area;
def avg_price = total_price/house_area;
def distPrice = distPriceMaps.get(dbo.city +"##"+dbo.region+"##"+dbo.name);
if(distPrice){
def maxValue = distPrice + distPrice*0.15;
def minValue = distPrice - distPrice*0.15;
if(avg_price>maxValue.toInteger() || avg_price<minValue.toInteger()){
continue;
}
}
if(newAlias){
m.all_name = dbo.name + "("+newAlias.join(",")+")"
def district_name = rs.getObject("district_name")
if(district_name.endsWith("别墅") && height && height>=10){
continue;
}
def m = [:];
//小区名称
m.name = dbo.name;
if(dbo.alias){
def newAlias = [];
for(def tname : dbo.alias){
if(tname && tname.length()>=2){
newAlias << tname;
}
}
if(newAlias){
m.all_name = dbo.name + "("+newAlias.join(",")+")"
}else{
m.all_name = dbo.name;
}
}else{
m.all_name = dbo.name;
}
}else{
m.all_name = dbo.name;
m.city = rs.getObject("city");
m.region = rs.getObject("region");
m.district_id= dbo.num_id;
m.house_id = createHouseId(rs.getObject("url"));//rs.getObject("_id").toString();
//建筑物总楼层
m.height = height>0?height:null;
//所在楼层
m.floor = getHouseFloor(floor, m.height);
//建筑面积
m.house_area = area;
//挂牌价
m.total_price = total_price;
def keys = m.region +"-"+ m.name+"-"+m.house_area+"-"+m.height+"-"+m.floor+"-"+m.total_price;
if(!houses.contains(keys)){
houses << keys;
}else{
println "此房源已存在....."
continue;
}
if(index > 30){
break;
}
m.room = rs.getObject("room");
m.hall = rs.getObject("hall");
m.estate_type = rs.getObject("estate_type");
//建成年代
m.build_date = getBuildDate(rs.getObject("build_date"), dbo);
//总使用年限
if(districtIds.contains(m.district_id)){
m.property_life = 70;
}else{
m.property_life = getPropertyLife(dbo.property_life);
}
//朝向
m.direction = getDirectionCode(rs.getObject("direction"), dbo.estate_type2);
//建筑结构
m.build_structure_code = getBuildStructureCode(dbo.build_date, dbo.build_structure);
//建筑类别
m.build_type_code = getBuildTypeCode(dbo.name, dbo.build_type, height);
if(m.build_type_code=='3' && m.height>6){
m.height = null
}
//内部装修与成新度
m.fitment_code = getFitmentCode(rs.getObject("fitment"));
//是否有电梯
m.elevator_code = getElevatorCode(height, m.build_type_code);
//房价网评估价
//m.evaluation_price = getEvaluationPrice(m);
//println "[${index++}] "+new Date();
//评估时间
m.evaluation_date = DateUtils.formatDateByPattern(new Date(),"yyyyMMdd");
//挂牌时间
m.publish_date = rs.getObject("p_date").toString()//DateUtils.formatDateByPattern(obj.publish_date,"yyyyMMdd");
m.status=1;
index++;
saveData(m);
}
m.city = rs.getObject("city");
m.region = rs.getObject("region");
m.district_id= dbo.num_id;
}catch(Exception e){
m.house_id = createHouseId(rs.getObject("url"));//rs.getObject("_id").toString();
//建筑物总楼层
m.height = height>0?height:null;
//所在楼层
m.floor = getHouseFloor(floor, m.height);
//建筑面积
m.house_area = area;
//挂牌价
m.total_price = total_price;
def keys = m.region +"-"+ m.name+"-"+m.house_area+"-"+m.height+"-"+m.floor+"-"+m.total_price;
if(!houses.contains(keys)){
houses << keys;
}else{
println "此房源已存在....."
continue;
}
if(index > 30){
break;
}
m.room = rs.getObject("room");
m.hall = rs.getObject("hall");
m.estate_type = rs.getObject("estate_type");
//建成年代
m.build_date = getBuildDate(rs.getObject("build_date"), dbo);
//总使用年限
if(districtIds.contains(m.district_id)){
m.property_life = 70;
}else{
m.property_life = getPropertyLife(dbo.property_life);
}
//朝向
m.direction = getDirectionCode(rs.getObject("direction"), dbo.estate_type2);
//建筑结构
m.build_structure_code = getBuildStructureCode(dbo.build_date, dbo.build_structure);
//建筑类别
m.build_type_code = getBuildTypeCode(dbo.name, dbo.build_type, height);
if(m.build_type_code=='3' && m.height>6){
m.height = null
}
//内部装修与成新度
m.fitment_code = getFitmentCode(rs.getObject("fitment"));
//是否有电梯
m.elevator_code = getElevatorCode(height, m.build_type_code);
//房价网评估价
//m.evaluation_price = getEvaluationPrice(m);
//println "[${index++}] "+new Date();
//评估时间
m.evaluation_date = DateUtils.formatDateByPattern(new Date(),"yyyyMMdd");
//挂牌时间
m.publish_date = rs.getObject("p_date").toString()//DateUtils.formatDateByPattern(obj.publish_date,"yyyyMMdd");
m.status=1;
index++;
saveData(m);
}
return index;
}
@@ -4,11 +4,14 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
/**
* 浙商本地化部署同步脚本
*/
@BatchOwner(Watcher.GaoJuan)
class BasicInfoDataSynch extends BaseBatch{
// final static DBCollection SERVER_AREA_CONN = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo)
@@ -29,10 +32,10 @@ class BasicInfoDataSynch extends BaseBatch{
@Override
public void process(Object args) {
// synchCitys();
// synchRegion()
// synchTown()
// synchDict()
synchCitys();
synchRegion()
synchTown()
synchDict()
synchDistrict("district")
synchDistrict("office")
}
@@ -47,46 +50,49 @@ class BasicInfoDataSynch extends BaseBatch{
zsCitysMap.put(zsCitys._id.toString(), zsCitys)
zsDatas << zsCitys
}
//数据存在则修改,不存在则保存
//数据存在则修改,不存在则保存
if(fangjiaCitys){
for(def fjCity : fangjiaCitys){
if(fjCity.display!="全国"){
if(fjCity.display!="全国"){
fjCitysMap.put(fjCity._id.toString(), fjCity)
}
def zsCity = zsCitysMap.get(fjCity._id.toString())
if(zsCity){//修改
if(zsCity){//修改
if(zsCity.province != fjCity.province || zsCity.alias!=fjCity.alias){
def udata = new BasicDBObject()
udata.put("province",fjCity.province)
udata.put("alias",fjCity.alias)
udata.put("status",0)
udata.put("update_date",new Date().getTime())
ZS_CITYS_CONN.update(new BasicDBObject("_id",zsCity._id), new BasicDBObject('$set',udata))
}
}else{//保存
def addUata = new BasicDBObject()
}else{//保存
def addUata = new BasicDBObject()
addUata.put("_id",fjCity._id)
addUata.put("display",fjCity.display)
addUata.put("province",fjCity.province)
addUata.put("alias",fjCity.alias)
addUata.put("status",0)
addUata.put("update_date",new Date().getTime())
ZS_CITYS_CONN.save(addUata);
}
}
}
//老数据不在新数据中,被删除时,状态修改为1
//老数据不在新数据中,被删除时,状态修改为1
if(zsDatas){
for(def zsCity : zsDatas){
def fjCity = fjCitysMap.get(zsCity._id.toString())
if(!fjCity){
def udata = new BasicDBObject()
udata.put("status",1)
udata.put("update_date",new Date().getTime())
ZS_CITYS_CONN.update(new BasicDBObject("_id",zsCity._id), new BasicDBObject('$set',udata))
}
}
}
println "[${fangjiaCitys.size()}]个城市同步完成"
println "[${fangjiaCitys.size()}]个城市同步完成"
}
def synchRegion(){
@@ -110,6 +116,7 @@ class BasicInfoDataSynch extends BaseBatch{
udata.put("pattern",fjRegion.pattern)
udata.put("org_name",fjRegion.org_name)
udata.put("status",0)
udata.put("update_date",new Date().getTime())
if(zsRegionObj){
ZS_REGION_CONN.update(new BasicDBObject("_id",zsRegionObj._id), new BasicDBObject('$set',udata))
}else{
@@ -125,11 +132,12 @@ class BasicInfoDataSynch extends BaseBatch{
if(!fjRegion){
def udata = new BasicDBObject()
udata.put("status",1)
udata.put("update_date",new Date().getTime())
ZS_REGION_CONN.update(new BasicDBObject("_id",zsRegion._id), new BasicDBObject('$set',udata))
}
}
}
println "[${fjRegions.size()}]个行政区域同步完成"
println "[${fjRegions.size()}]个行政区域同步完成"
}
def synchTown(){
@@ -152,6 +160,7 @@ class BasicInfoDataSynch extends BaseBatch{
udata.put("short_name",fjTown.short_name)
udata.put("name",fjTown.name)
udata.put("status",0)
udata.put("update_date",new Date().getTime())
if(zsTownObj){
if(fjTown.city!=zsTownObj.city||fjTown.region!=zsTownObj.region||fjTown.short_name!=zsTownObj.short_name||fjTown.name!=zsTownObj.name||zsTownObj.status!=0){
ZS_TOWN_CONN.update(new BasicDBObject("_id",zsTownObj._id), new BasicDBObject('$set',udata))
@@ -169,11 +178,12 @@ class BasicInfoDataSynch extends BaseBatch{
if(!fjTown){
def udata = new BasicDBObject()
udata.put("status",1)
udata.put("update_date",new Date().getTime())
ZS_TOWN_CONN.update(new BasicDBObject("_id",zsTown._id), new BasicDBObject('$set',udata))
}
}
}
println "[${fjTowns.size()}]个乡镇同步完成"
println "[${fjTowns.size()}]个乡镇同步完成"
}
def synchDict(){
def fjDicts = CommonDatautils.getAllDictionary()
@@ -194,6 +204,7 @@ class BasicInfoDataSynch extends BaseBatch{
udata.put("cat",fjDict.cat)
udata.put("name",fjDict.name)
udata.put("status",0)
udata.put("update_date",new Date().getTime())
if(zsDictObj){
if(fjDict.cat!=zsDictObj.cat||fjDict.name!=zsDictObj.name||zsDictObj.status!=0){
ZS_DICT_CONN.update(new BasicDBObject("_id",zsDictObj._id), new BasicDBObject('$set',udata))
@@ -220,11 +231,12 @@ class BasicInfoDataSynch extends BaseBatch{
if(!fjDict){
def udata = new BasicDBObject()
udata.put("status",1)
udata.put("update_date",new Date().getTime())
ZS_DICT_CONN.update(new BasicDBObject("_id",zsDict._id), new BasicDBObject('$set',udata))
}
}
}
println "[${fjDicts.size()}]个字典数据同步完成"
println "[${fjDicts.size()}]个字典数据同步完成"
}
def synchDistrict(cat){
def fangjiaCitys = CommonDatautils.findAllCity()
@@ -259,18 +271,25 @@ class BasicInfoDataSynch extends BaseBatch{
udata.put("township",fjDistrict.township)
udata.put("alias",fjDistrict.alias)
udata.put("cat",cat)
udata.put("update_date",new Date().getTime())
if(fjDistrict.addr_alias){
udata.put("addr_alias",fjDistrict.addr_alias)
}
udata.put("status",0)
def map = new BasicDBObject()
map.put("lat",fjDistrict.lat/10000000000.0d)
map.put("lon", fjDistrict.lng/10000000000.0d)
def map = null;
if(fjDistrict && fjDistrict.lat){
map = new BasicDBObject()
map.put("lat",fjDistrict.lat/10000000000.0d)
map.put("lon", fjDistrict.lng/10000000000.0d)
}
if(zsDistrictObj){
if(fjDistrict.city!=zsDistrictObj.city||fjDistrict.region!=zsDistrictObj.region||fjDistrict.block!=zsDistrictObj.block||fjDistrict.name!=zsDistrictObj.name||fjDistrict.address!=zsDistrictObj.address||fjDistrict.estate_type2!=zsDistrictObj.estate_type||fjDistrict.township!=zsDistrictObj.township||fjDistrict.alias!=zsDistrictObj.alias||fjDistrict.addr_alias!=zsDistrictObj.addr_alias||map!=zsDistrictObj.location||zsDistrictObj.status!=0){
udata.put("location", map)
ZS_SEAWEED_CONN.update(new BasicDBObject("_id",zsDistrictObj._id), new BasicDBObject('$set',udata))
if(map){
udata.put("location", map)
ZS_SEAWEED_CONN.update(new BasicDBObject("_id",zsDistrictObj._id), new BasicDBObject('$set',udata))
}
}
}else{
udata.put("location", map)
@@ -295,10 +314,11 @@ class BasicInfoDataSynch extends BaseBatch{
if(!fjDistrict){
def udata = new BasicDBObject()
udata.put("status",1)
udata.put("update_date",new Date().getTime())
ZS_SEAWEED_CONN.update(new BasicDBObject("_id",fjDistrict._id), new BasicDBObject('$set',udata))
}
}
}
println "[${fjCity}-${cat}][${districts.size()}]个小区数据同步完成"
println "[${fjCity}-${cat}][${districts.size()}]个小区数据同步完成"
}
}
@@ -9,6 +9,9 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 浙商本地化部署同步脚本
*/
@BatchOwner(Watcher.GaoJuan)
class DistrictLevelBatch extends BaseBatch{
@@ -28,7 +31,7 @@ class DistrictLevelBatch extends BaseBatch{
def datas = addPriceData(dists, priceMap);
datas = calcuteLevel(datas);
saveOrUpdateDB(city, datas);
println city +" Â¥ÅÌÆÀ¼¶Í¬²½Íê³É"
println city +" 楼盘评级同步完成"
}
}
@@ -109,6 +112,7 @@ class DistrictLevelBatch extends BaseBatch{
udata.put("level", d.level)
udata.put("avg_price", d.avg_price)
udata.put("status",0)
udata.put("update_date",new Date().getTime())
if(zsDistLevel){
if(zsDistLevel.city!=d.city||zsDistLevel.region!=d.region||zsDistLevel.name!=d.name||zsDistLevel.level!=d.level||zsDistLevel.avg_price!=d.avg_price||zsDistLevel.status!=0){
ZS_LEVEL_CONN.update(new BasicDBObject("_id",zsDistLevel._id), new BasicDBObject('$set',udata))
@@ -127,10 +131,11 @@ class DistrictLevelBatch extends BaseBatch{
}
if(zsDistLevels.size()>0){
for(def zs : zsDistLevels){
def newData = newDataMap.get(zs._id.toString());
def newData = newDataMap.get(zs._id.toString());
if(!newData){
def udata = new BasicDBObject()
udata.put("status",1)
udata.put("update_date",new Date().getTime())
ZS_LEVEL_CONN.update(new BasicDBObject("_id",zs._id), new BasicDBObject('$set',udata))
}
}