Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into open_gj

# Conflicts:
#	projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java
This commit is contained in:
2021-11-15 19:08:41 +08:00
60 changed files with 4473 additions and 328 deletions
@@ -1265,6 +1265,7 @@ public class DistrictController {
district.setAddrAlias(new ArrayList<String>(addrAlias));
}
district.setStatus(0);
district.setNameCat(0);
district.setCategory("district");
district.setPublisherType(11);
district.setCreationDate(new Date());
@@ -176,7 +176,7 @@ table{text-align: left;}
}
})
if(role==12){
if(role==12||role==17){
if(userStr.length>0){
console.log(userStr);
$("input[name=roleUsers]").val(userStr);
@@ -306,7 +306,7 @@ $("input[name=email]").blur(function(){
$("#role").change(function() {
var role = $(this).val();
//只对农行生效
if (role == 12) {
if (role == 12 || role == 17) {
$("#show_chk_all").removeAttr("style");
jQuery.ajax({
type: 'POST',
@@ -340,12 +340,12 @@ $("input[name=email]").blur(function(){
}
return false;
}
<#if user?? && user.role == 12>
<#if user?? && (user.role == 12 || user.role == 17)>
$("#show_chk_all").removeAttr("style");
jQuery.ajax({
type: 'POST',
url: '../fjt/getUsersByRole',
data: {role:12},
data: {role:${user.role}},
dataType: 'json',
timeout: 100000,
success: function(model) {
@@ -1339,7 +1339,7 @@
<td class = "t1">室号</td>
<td>
<#if evaluation.buildType!=3 && evaluation.buildType!=8 && evaluation.buildType!=0>
<#if evaluation.floorDesc!="">
<#if evaluation.floorDesc?? && evaluation.floorDesc!="">
<span class="labelModel2">${evaluation.floorDesc!}</span>
<span class="editModel2 hide"><input type="text" name="floorDesc" value="${evaluation.floorDesc!}" /> </span>
</#if>
@@ -54,7 +54,6 @@
<div class="main_content">
<p>房价网成立于2006年,隶属于上海估家网络科技有限公司,致力于建立中国领先的房产大数据库CPDB及推动基于地理位置的人工智能的发展和应用。公司以“发现地址价值”为使命,始终秉承客观严谨的态度为政府、银行、金融机构等客户提供大数据、建模、咨询等服务。目前,所积累的房产数据已覆盖全国367个城市及地区,技术力量强大,客户查询量行业领先。</p>
<p>公司在贷前估值、内评系统、贷后批量复估、信贷授信、地址反欺诈、数据治理、精准营销等领域都有着丰富的经验,拥有多项专利,被评为高新技术企业,赢得了国家统计局、建设银行、农业银行、平安集团、微众银行等众多客户的信赖。</p>
<p>2020年,南京三悦网络科技有限公司与我司达成全面战略合作,并接受我司委托,全权负责我司产品手机版“房价网”公众号和电脑版的“房价通”的市场推广、客户开发与售后服务;有权直接与客户签订上述产品的销售合同;我司对其所有用户全面提供产品升级、数据更新及线上售后等服务。</p>
</div>
<div class="main_title"><small>数据为大众服务</small></div>
<div class="main_content">
@@ -234,6 +234,8 @@ public class District implements SeaweedEntity {
private String mainTypeArea;
private String createYear;
private Integer nameCat;
@Column(name = "source_web")
public String getSourceWeb() {
@@ -1879,9 +1881,13 @@ public class District implements SeaweedEntity {
public void setTownship(String township) {
this.township = township;
}
@Column(name = "name_cat")
public Integer getNameCat() {
return nameCat;
}
public void setNameCat(Integer nameCat) {
this.nameCat = nameCat;
}
}
@@ -13,8 +13,9 @@ import com.mongodb.DBObject
*/
class CityLatlngBox {
public static final DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.105", 27018)
public static final DBCollection LATLNG_COLL = MongoUtils.getCollectionByName("addrepo", "city_latlng_range", "192.168.0.219", 27017)
//public static final DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.105", 27018)
DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.235", 27018)
public static final DBCollection LATLNG_COLL = MongoUtils.getCollectionByName("addrepo", "city_latlng_range", "192.168.0.206", 27020)
public static void main(String[] args) {
CityLatlngBox sd = new CityLatlngBox();
@@ -22,7 +23,7 @@ class CityLatlngBox {
}
void process() {
def cities = CommonUtils.findAllCity()
def cities = ["白沙","阿坝州","甘孜州","阿勒泰","临沧","那曲","三沙","喀什","昌吉","迪庆州","普洱","德宏州","湘西","塔城","凉山州","屯昌","山南","海西州","阿拉善盟","乐东","定安","怒江州","黄山","图木舒克","楚雄","临高","乌兰察布","延边","文山州","锡林郭勒盟","陵水","天门","淮北","黔东南","万宁","湘潭","海北州","琼中","吐鲁番","惠州","巴音郭楞","恩施","铜仁","黔南","西双版纳州","潜江","林芝","文昌","仙桃","神农架","和田","黔西南","果洛州","儋州","哈密","大理","东方","日喀则","兴安盟","昌江","澄迈","保亭","济源","五指山","阿拉尔","伊犁","临夏","海南州","琼海","石河子","昌都","五家渠","玉树州","博尔塔拉","东营","铜陵","黄南州","廊坊","嘉峪关","马鞍山","宁波","广安","中山","克孜勒苏","厦门","莱芜","鄂州","东莞","海口","鹤壁","六盘水","漯河","三亚","珠海","深圳","乌海","淮南","北海","双鸭山","襄阳","新余","汕头","克拉玛依","鹰潭","盘锦","镇江","石嘴山","嘉兴","铜川","焦作","芜湖","潮州","萍乡","红河","黄石","辽源","辽阳","枣庄","毕节","许昌","阳泉","自贡","遂宁","景德镇","营口","日照","莆田"]//CommonUtils.findAllCity()
for (String city : cities) {
println city
BOUNDS_COLL.remove(new BasicDBObject('city', city))
@@ -70,13 +71,13 @@ class CityLatlngBox {
int row = 0
int inx = 0
def buffer = []
while(seedLatBD < cityBoundBD.getMaxLat()) {
while(seedLatGD < cityBoundGD.getMaxLat()) {
int col = 0
Bounds boundsBD = null;
Bounds boundsQQ = null;
Bounds boundsGD = null;
while (seedLngBD < cityBoundBD.getMaxLng()) {
while (seedLngGD < cityBoundGD.getMaxLng()) {
double[] espBD = GeoUtils.getLatLon(seedLngBD, seedLatBD, offset, 90) // 右侧OFFSET处的经纬度
double[] wnpBD = GeoUtils.getLatLon(seedLngBD, seedLatBD, offset, 360) // 上方OFFSET处的经纬度
@@ -39,7 +39,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
println city +" "+rows.size()
}
def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_commbuildtype.txt",headers, "utf-8", false);
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMBUILDTYPE_GJ.dat",headers, "utf-8", false);
println "txt 生成"
}
@@ -12,6 +12,7 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
@@ -46,16 +47,19 @@ class BigdataCommunityBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"])
println city +" 小区量:"+dists.size()
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city))
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets)
println city +" "+rets.size()
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
}
def fileName = "/home/goojia/ccb/bigdata_community.dat"
def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.dat"
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat"
}
def allDatas = []
@@ -66,8 +70,27 @@ class BigdataCommunityBatch extends BaseBatch{
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
}
def addDeleteDist(dists, ccbMaps, rets){
def fangjiaDists = []
for(def d : dists){
fangjiaDists << d._id.toString()
}
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
def ccbDists = ccbMaps.keySet()
for(def ccbId : ccbDists){
if(!fangjiaDists.contains(ccbId)){
def datas = ccbMaps.get(ccbId)
datas[datas.size()-1] = 1
rets << datas
}
}
println "添加删除状态小区:"+rets.size()
return rets
}
def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){
def rets = Collections.synchronizedList(new ArrayList());
//def rets = Collections.synchronizedList(new ArrayList());
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
@@ -182,6 +205,7 @@ class BigdataCommunityBatch extends BaseBatch{
ret << ""
ret << ""
ret << ""
ret << 0
}catch(Exception e){
println dist.city + " "+dist.name
e.printStackTrace()
@@ -13,6 +13,7 @@ import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
@@ -28,7 +29,7 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataDealBatch extends BaseBatch{
def curDate = 20210401//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
def curDate = 20210901//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
@@ -37,32 +38,45 @@ class BigdataDealBatch extends BaseBatch{
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
runBatch(new BigdataDealBatch(), args);
}
@Override
public void process(Object args) {
def weekend = DateUtils.getFirstDayOfWeek(new Date())
def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
def citys = CcbCommonUtils.citys;
println citys.unique().size()
def rows = []
def headers = ["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
// if(["贵阳","佛山","杭州","沈阳","天津","中山","徐州","绍兴","南通","哈尔滨","惠州","长沙","青岛","长春","烟台","温州","太原","台州","石家庄","宁波","常州","大连","嘉兴","金华","昆明"].contains(city)){
// ppeDists = getPpeGTE15Datas(city)
// }
def statMap = getDistrictStatByCity(city, weekend)
def trades = getCityData(city, statMap)
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println city +" 成交共:"+rets.size()
def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){
def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
}
def trades = getCityData(city, statMap, ppeDists)
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println city +" 成交共:"+rets.size()
// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false);
def cityDatas = []
cityDatas << headers
@@ -70,9 +84,9 @@ class BigdataDealBatch extends BaseBatch{
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
}
def fileName = "/home/goojia/ccb/bigdata_deal.dat"
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.dat"
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
}
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
@@ -143,16 +157,17 @@ class BigdataDealBatch extends BaseBatch{
def street = ""
def streetNumber = ""
if(trade.address){
if(trade.address.toString().endsWith("弄")){
if(trade.address.toString().contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
}
}else{
}
if(!street){
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
if(jsonObj && jsonObj.searchAddress){
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.endsWith("弄")){
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
@@ -169,7 +184,7 @@ class BigdataDealBatch extends BaseBatch{
if(!street){
if(obj.address){
if(obj.address.toString().endsWith("弄")){
if(obj.address.toString().contains("弄")){
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
if(number){
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
@@ -281,7 +296,7 @@ class BigdataDealBatch extends BaseBatch{
}
if(trade.address){
if(trade.address.toString().endsWith("弄")){
if(trade.address.toString().contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
ret << trade.address.toString().substring(0, trade.address.toString().indexOf(number))
@@ -389,7 +404,7 @@ class BigdataDealBatch extends BaseBatch{
return s;
}
def getCityData(def city, def statMap){
def getCityData(def city, def statMap, def ppeDistsMap){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"])
def tradesList = Collections.synchronizedList(new ArrayList());
@@ -400,32 +415,49 @@ class BigdataDealBatch extends BaseBatch{
def startDate = null;
// if(citys.contains(city)){
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210401", "yyyyMMdd"))
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// }else{
// startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// }
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210801", "yyyyMMdd"))
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def price = statMap.get(dist.region+"@"+dist.name)
// def price = statMap.get(dist.region+"@"+dist.name)
def startPrice = null
def endPrice = null
if(price){
if(dist.city=="上海"){
startPrice = (int)(price - price*0.1)
endPrice = (int)(price*0.5 + price)
}else{
startPrice = 1//(int)(price - price*0.5)
endPrice = 500000//(int)(price*0.5 + price)
}
}
// if(price){
// if(dist.city=="上海"){
// startPrice = (int)(price - price*0.1)
// endPrice = (int)(price*0.5 + price)
// }else if(dist.city=="杭州"){
// startPrice = (int)(price - price*0.15)
// endPrice = (int)(price*0.15 + price)
// }else{
// startPrice = (int)(price - price*0.5)
// endPrice = (int)(price*0.5 + price)
// }
// }
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice)
if(!trades && dist.address){
def flag = dist.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(flag){
trades = CcbCommonUtils.findTradesDetailByAddress(dist, startDate, endDate, startPrice, endPrice)
}
}
if(trades){
// def unitPrice= ppeDistsMap.get(dist._id.toString())
// if(unitPrice){
// for(def t : trades){
// t.avg_price = (int)(t.avg_price*0.03+t.avg_price)
// }
// }
tradesList.addAll(trades)
}
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name+" "+trades.size()+" "+new Date().toLocaleString()
@@ -443,31 +475,38 @@ class BigdataDealBatch extends BaseBatch{
def getYouDaData(statMap){
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210801", "yyyyMMdd"))
def q = new BasicDBObject()
q.put("contractTime", new BasicDBObject('$gte', startDate));
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate));
q.put("area", new BasicDBObject('$gte', 20))
// q.put("premisesName" , "呼玛二村")
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
def rets = []
def index = 0
for(def obj : dbcursor){
if(obj.fjName && ["公寓","联列住宅","花园住宅","别墅","里弄房","新工房"].contains(obj.houseType)){
def price = statMap.get(obj.fjRegion+"@"+obj.fjName)
if(price){
def startPrice = 0
def endPrice= 0
if(obj.fjCity=="上海"){
startPrice = (int)(price - price*0.1)
endPrice = (int)(price*0.5 + price)
}else{
startPrice = 1//(int)(price - price*0.5)
endPrice = 500000//(int)(price*0.5 + price)
}
if(obj.price>=startPrice && obj.price<=endPrice){
rets << obj;
}
}
if(obj.fjName && ["公寓","联列住宅","花园住宅","别墅","里弄房","新工房","其它"].contains(obj.houseType)){
rets << obj;
// def price = statMap.get(obj.fjRegion+"@"+obj.fjName)
// if(price){
// def startPrice = 0
// def endPrice= 0
// if(obj.fjCity=="上海"){
// startPrice = (int)(price - price*0.1)
// endPrice = (int)(price*0.5 + price)
// }else{
// startPrice = (int)(price - price*0.5)
// endPrice = (int)(price*0.5 + price)
// }
// if(obj.price>=startPrice && obj.price<=endPrice){
// rets << obj;
// }else{
// index++
// println index + " "+obj
// }
// }
}
}
return rets;
@@ -486,4 +525,19 @@ class BigdataDealBatch extends BaseBatch{
return map
}
def getPpeGTE15Datas(def city){
def sql = "select unit_price,commid from avm_evaluation_item_gj where data_type='成交' and pe>=15 and pe<=20 " +
"and cityname='"+city+"'";
def map = [:]
try{
sqlInstance.eachRow(sql,{
map.put(it.commid, it.unit_price)
});
}catch(Exception e){
getPpeGTE15Datas(city)
}
return map
}
}
@@ -40,8 +40,8 @@ class BigdataDistrictBatch extends BaseBatch{
def rets = []
rets << headers
rets.addAll(allDatas)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_district.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_district.txt",headers, "utf-8",false);
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\BIGDATA_DISTRICT_GJ.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.dat",headers, "utf-8",false);
}
def createDatas(rows){
@@ -55,7 +55,7 @@ class BigdataForsaleBatch extends BaseBatch{
}
public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"];
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -66,15 +66,16 @@ class BigdataForsaleBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def index = 0
def citys = CcbCommonUtils.citys;
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
index++
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210831);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20211018);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
}
}
@@ -95,9 +96,9 @@ class BigdataForsaleBatch extends BaseBatch{
def commId = rMap.get("_id").toString()
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
if(ret && ret.size()<100){
ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
}
// if(ret && ret.size()<100){
// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
// }
if(ret){
def areaData = distCodeMap.get(rMap.region)
if(areaData){
@@ -144,9 +145,9 @@ class BigdataForsaleBatch extends BaseBatch{
}
latch.await()
def fileName = "/home/goojia/ccb/bigdata_forsale.dat"
def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.dat"
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_FORSALE_GJ.dat"
}
println city + " 房源总量:"+retMap.size()
@@ -158,8 +159,6 @@ class BigdataForsaleBatch extends BaseBatch{
}
FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true);
println "txt文件生成 "
}
@@ -284,6 +283,7 @@ class BigdataForsaleBatch extends BaseBatch{
def rets = []
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
@@ -413,6 +413,8 @@ class BigdataForsaleBatch extends BaseBatch{
rets << rowMap
}
}
println name +" "+houses.size()+" last:"+rets.size()
return rets;
}
@@ -65,9 +65,9 @@ class BigdataPoiBatch extends BaseBatch{
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"]
def fileName = "/home/goojia/ccb/bigdata_poi.txt"
def fileName = "/home/goojia/ccb/BIGDATA_POI_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_poi.txt"
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_POI_GJ.dat"
}
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false);
println "txt 生成"
@@ -9,11 +9,12 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.sql.Sql
class CcbCommonUtils {
//
public static def citys = ["扬州", "上海","北京","重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"]
public static def citys = ["扬州", "上海","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
// public static def citys = ["武汉"]
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
@@ -27,10 +28,15 @@ class CcbCommonUtils {
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
public static DSCollection dataStoreConnection = null;
public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
public static def findCityCodeValue(){
def cityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"]
return cityCodeMap;
@@ -75,11 +81,23 @@ class CcbCommonUtils {
return rets
}
public static def findGuidePrice(city){
def q = new BasicDBObject()
q.put("city", city)
def dbcursor = DISTRICT_GUIDE_PRICE_COLL.find(q)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets
}
public static def findTradesDetail(dist, startDate, endDate, startPrice, endPrice){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
q.put("area", new BasicDBObject('$gte', 20))
q.put("avg_price", new BasicDBObject('$gt', 0))
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
@@ -90,9 +108,48 @@ class CcbCommonUtils {
alias << dist.name
}
if(dist.name.endsWith("弄小区") || dist.name.endsWith("小区") ){
def name = dist.name.replace("弄小区","弄").replace("小区","")
if(!alias.contains(name)){
alias << name
}
}
if(alias){
q.put("district_name", new BasicDBObject('$in', alias))
}
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = DEAL_PRICE_COLL.find(q)
def total = dbcursor.count()
def rets = new HashSet()
def lists = []
println dist.name +" "+total
for(def d : dbcursor){
// if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
d.dist_id = dist._id.toString()
d.address = dist.address
d.master_name = dist.name
d.build_type = dist.build_type
d.build_structure = dist.build_structure
d.house_roperty = dist.house_roperty
d.property_life = dist.property_life
d.estate_type2 = dist.estate_type2
lists << d
}
// }
}
return lists
}
public static def findTradesDetailByAddress(dist, startDate, endDate, startPrice, endPrice){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
q.put("area", new BasicDBObject('$gte', 20))
q.put("district_name", dist.address)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def dbcursor = DEAL_PRICE_COLL.find(q)
def total = dbcursor.count()
@@ -168,13 +225,63 @@ class CcbCommonUtils {
return dataStoreConnection
}
public static def findAllDistrictIdsByOracle(def cityid){
def sql = "select commid from bigdata_community_gj where cityId=${cityid} and status=0";
def set = new HashSet()
sqlInstance.eachRow(sql,{r->
set << r.COMMID
});
return set
}
public static def findAllDistrictsByOracle(def cityid){
def sql = "select CITYID,COMMID,COMMNAME,COMMALIAS,DISTID,STREET,STREETNO,ADDRESS,LANDAREA,BUILDAREA,COMMTYPE,IFNEW,GREENINGRATE,PLOTRATIO,PARKINGCNT,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,WORKDATE,ZTS,LONGITUDE,LATITUDE from bigdata_community_gj where cityId='${cityid}' and status=0 ";
def rows = []
try{
sqlInstance.eachRow(sql,{r->
def map = [:]
map.cityid = r.CITYID
map.commid = r.COMMID
map.commname = r.COMMNAME
map.distid = r.DISTID
map.street = r.STREET
map.streetno = r.STREETNO
map.address = r.ADDRESS
map.landarea = r.LANDAREA
map.buildarea = r.BUILDAREA
map.commtype = r.COMMTYPE
map.ifnew = r.IFNEW
map.greeningrate = r.GREENINGRATE
map.plotratio = r.PLOTRATIO
map.parkingcnt = r.PARKINGCNT
map.introduction = ""
map.remark = r.REMARK
map.createtime = r.CREATETIME
map.updatetime = r.UPDATETIME
map.sourcecode = r.SOURCECODE
map.workdate = r.WORKDATE
map.zts = r.ZTS
map.lng = r.LONGITUDE
map.lat = r.LATITUDE
rows << map
});
}catch(Exception e){
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
findAllDistrictsByOracle(cityid)
println "错误:"+e.getMessage()
}
return rows
}
public static def findAllDistrictByCity(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name", "美辰香醒")
// q.put("name","江南人家")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -221,4 +328,48 @@ class CcbCommonUtils {
return null;
}
public static def findCcbDistrict(cityid){
def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" +
"\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj where cityId=${cityid}";
def map = [:]
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DISTID
data << r.COMMID
data << r.DISTNAME
data << r.COMMNAME
data << r.ADDRESS
data << r.LATITUDE
data << r.LONGITUDE
data << r.LANDAREA
data << r.BUILDAREA
data << r.GREENINGRATE
data << r.PLOTRATIO
data << r.PARKINGCNT
data << r.BUILDYEAR
data << r.HOUSEUSE
data << r.ZTS
data << r.WORKDATE
data << r.UPDATEFLAG
data << r.UPDATETIME
data << r.COMMTYPE
data << r.COMMALIAS
data << r.STREET
data << r.STREETNO
data << r.IFNEW
data << ""
data << r.REMARK
data << r.CREATETIME
data << r.SOURCECODE
data << r.XZJDB
data << r.LJXZQHDM
data << r.LJXZQH
data << r.STATUS
map.put(r.COMMID, data)
});
return map
}
}
@@ -0,0 +1,58 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.jiangnanblank.SFTP
import com.jcraft.jsch.ChannelSftp
import org.apache.commons.codec.digest.DigestUtils
class CcbFtpUtils {
static def ftpServer = "192.168.0.45"
static def ftpPort = 17495
static def ftpUser = "ccb"
static def ftpPassword = "ccb!@#"
static def ftpPath = "/home/ccb/output/"
static SFTP sftp = new SFTP();
static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
static def ftpUpload(outputPath_dist, month){
//上传到ftp
sftpChannel = reConnect();
sftp.upload(ftpPath+ month+"/", outputPath_dist, sftpChannel);
}
static def reConnect(){
if(sftpChannel == null){
println "sftpChannel 为空,需重连....."
if(sftp == null){
sftp = new SFTP();
}
sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
}else{
println "sftpChannel 正常....."
}
return sftpChannel;
}
static def uploadSha512Data(def sb, def dataCodeFileName, def charset){
String sha512HexStr= DigestUtils.sha512Hex(sb.getBytes())//生成加密码码
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
}
static def uploadSha512Data2(def dataFileName, def dataCodeFileName, def charset){
def data = new FileInputStream(new File(dataFileName))
String sha512HexStr= DigestUtils.sha512Hex(data)//生成加密码码
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
}
static main(args) {
}
}
@@ -43,7 +43,9 @@ class DistrictDelete {
static main(args) {
def task = new DistrictDelete()
// task.run()
task.gzRun();
// task.gzRun();
// task.deleteDistrict()
task.deleteCommPrice()
}
def run(){
@@ -211,9 +213,9 @@ class DistrictDelete {
def gzRun(){
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\经纬度有问题的小区.xlsx",{d->
if(d['U'] && d['U'].toString().toDouble().toInteger()==1){
def id = d['E']
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\要删除小区 (5).xlsx",{d->
if(d['G']){
def id = d['G']
BasicDBObject q=new BasicDBObject('name_cat',1)
q.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q))
@@ -223,4 +225,38 @@ class DistrictDelete {
},0)
}
def deleteDistrict(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d->
if(d['G']){
def id = d['G']
BasicDBObject q=new BasicDBObject('name_cat',1)
q.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q))
deleteAllRefDistrict(id)
}
},0)
}
def deleteAllRefDistrict(id){
def tables = ["bigdata_forsale_gj","avm_evaluation_item_gj","community_base_price_gj","community_poi_gj","community_sale_gj","community_sale_tmp","community_test_price_tmp"]
for(def tableName : tables){
deleteByCommid(tableName, id)
}
}
def deleteCommPrice(){
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\均价删除.xlsx",{d->
if(d['B']){
def id = d['B']
println id
deleteByCommid("community_base_price_gj",id)
}
},0)
}
def deleteByCommid(tableName, commid){
def sql = "delete from "+tableName+" where commid='${commid}'";
sqlInstance.execute(sql)
println "删除:"+tableName+" "+commid
}
}
@@ -0,0 +1,85 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
@BatchOwner(Watcher.GaoJuan)
class DistrictGuidePriceBatch extends BaseBatch{
static main(args) {
runBatch(new DistrictGuidePriceBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rows = []
for (def city : citys) {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findGuidePrice(city)
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
rows.addAll(rets)
println city + " " + rets.size()
}
}
def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_GUIDE_PRICE_GJ.dat", headers, "utf-8", false);
}
def getDistrictMap(def city){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
def distMap = [:]
for(def d : districts){
distMap.put(d.region+"##"+d.name, d)
}
return distMap;
}
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
def rets = []
if(prices){
for(def ps : prices){
def dist = distMap.get(ps.match_region+"##"+ps.match_name)
if(dist){
def ret = []
def areaData = distCodeMap.get(dist.region)
if(areaData){
ret << ps._id.toString()
ret << dist._id.toString()
ret << dist.name
ret << cityCodeDatas.get(dist.city)
ret << dist.city
ret << areaData.district_code
ret << areaData.district
ret << ps.avg_price
ret << ps.source_type
ret << ""
ret << DateUtils.formatDateByPattern(ps.c_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(ps.c_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}else{
println ps.match_region+"##"+ps.match_name
}
}
}
return rets;
}
}
@@ -29,8 +29,8 @@ class EvaluateRateBatch extends BaseBatch{
def rets = []
rets << headers
rets.addAll(allDatas)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\evaluate_rate.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluate_rate.txt",headers, "utf-8", false);
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\EVALUATE_RATE_GJ.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\EVALUATE_RATE_GJ.dat",headers, "utf-8", false);
}
def createDatas(def datas){
@@ -0,0 +1,808 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.ccbevaluation.CcbFtpUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import groovy.sql.Sql
import java.sql.ResultSet
class ExportOracleData {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new ExportOracleData()
task.run()
}
def run() {
// exportEvaluationRateData()
// exportBigdataPoiData()
// exportCommunityStaging()
// exportCommunityBuileType()
// exportBigdataDistrictData()
// exportBigdataCommunity()
// exportGuidedPrice()
exportBigdataForsale()
// exportThirdPartyPrice()
// exportThirdPartyPrice2()
// exportBigdataDeal()
// exportCommunityGuidedPrice()
// exportAvmEvaluationGJNB()
// exportAVMEvaluationItemGJNB()
// exportCommunityBasePrice()
// exportCommunityBasePriceGj()
// exportCityPrice()
// exportRegionPrice()
// exportLastFile()
}
def exportEvaluationRateData() {
def sql = "select ID,RATE_TYPE,RATE,RANK,CITY,REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from evaluate_rate_gj";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.RATE_TYPE
data << r.RATE
data << r.RANK
data << r.CITY
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/output/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat"
}
def headers = []//["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, curDate)
println "上传Ftp完成......"
}
def exportBigdataPoiData() {
def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.CITYID
data << r.DISTID
data << r.ID
data << r.POI_NAME
data << r.TYPE
data << r.TYPE_CODE
data << r.LATITUDE
data << r.LONGITUDE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat"
}
def headers = []//["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "BIGDATA_POI_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, curDate)
println "BIGDATA_POI_GJ 上传Ftp完成......"
}
def exportCommunityStaging() {
def sql = "select ID,COMMID,MASTER_COMMID,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_staging_gj";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMID
data << r.MASTER_COMMID
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat"
}
def headers = []//["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "COMMUNITY_STAGING_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, curDate)
println "COMMUNITY_STAGING_GJ 上传Ftp完成......"
}
def exportCommunityBuileType() {
def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.CITYID
data << r.COMMID
data << r.BUILDTYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.WORKDATE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat"
}
def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "BIGDATA_COMMBUILDTYPE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, curDate)
println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......"
}
def exportBigdataDistrictData() {
def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.DISTID
data << r.DISTNAME
data << r.DISTSPELL
data << r.CITYID
data << r.DISTCODE
data << r.SEQ
data << r.DISTTYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.LJXZQH
data << r.LJXZQHDM
data << r.XZQHDM
data << r.WORKDATE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat"
}
def headers = []
//["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "BIGDATA_DISTRICT_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, curDate)
println "BIGDATA_DISTRICT_GJ 上传Ftp完成......"
}
def exportGuidedPrice() {
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj ";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMID
data << r.COMMNAME
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.DEALDATE
data << r.AVG_PRICE
data << r.GUIDING_PERSON
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "GUIDED_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, curDate)
println "GUIDED_PRICE_GJ 上传Ftp完成......"
}
def exportCommunityGuidedPrice() {
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMID
data << r.COMMNAME
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.AVGPRICE
data << r.SOURCETYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "COMMUNITY_GUIDE_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, curDate)
println "COMMUNITY_GUIDE_PRICE_GJ 上传Ftp完成......"
}
def exportBigdataDeal() {
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" +
"HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" +
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
"from bigdata_deal_gj";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.CITYID
data << r.DEALID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.STREET
data << r.STREETNO
data << r.COMMID
data << r.COMMNAME
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.UNITID
data << r.UNITNO
data << r.UNITNAME
data << r.HOUSEID
data << r.ADDRESS
data << r.BUILDYEAR
data << r.PROPTYPE
data << r.PROPYEARS
data << r.PERIODNUM
data << r.FLOORNUM
data << r.FLOORTYPE
data << r.FLOORCNT
data << r.BUILDTYPE
data << r.BUILDSTRU
data << r.BUILDAREA
data << r.LIVINGAREA
data << r.HOUSESTYLE
data << r.ROOMCNT
data << r.LIVINGROOMCNT
data << r.KITCHENCNT
data << r.BATHROOMCNT
data << r.HOUSEUSE
data << r.HOUSESTRU
data << r.HOUSEASPECT
data << r.FITMENT
data << r.EVALUATENO
data << r.DEALUNITPRICE
data << r.DEALTOTALPRICE
data << r.DECLAREPRICE
data << r.EVALUATEPRICE
data << r.CONTRACTNO
data << r.DEALDATE
data << r.DEALTYPE
data << r.PAYTYPE
data << r.DEALPERIOD
data << r.IFELEVATOR
data << r.COMPAREAGARDEN
data << r.COMPAREATERRACE
data << r.COMPAREABALCONY
data << r.COMPAREABAYWINDOW
data << r.COMPAREAATTIC
data << r.COMPAREAGARAGE
data << r.COMPAREABASEMENT
data << r.COMPAREAOTHER
data << r.DATASOURCE
data << r.AGENCYID
data << r.AGENCYNAME
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.CJJE
data << r.WORKDATE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "BIGDATA_DEAL_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, curDate)
println "BIGDATA_DEAL_GJ 上传Ftp完成......"
}
def exportThirdPartyPrice() {
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, \n" +
"TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE\n" +
"from third_party_price_gj";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMID
data << r.COMMNAME
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.YEAR_MONTH
data << r.ZHUGE_AVG_PRICE
data << r.ANJUKE_AVG_PRICE
data << r.FANG_AVG_PRICE
data << r.LIANJIA_AVG_PRICE
data << r.CITYHOUSE_AVG_PRICE
data << r.MAITIAN_AVG_PRICE
data << r.KE_AVG_PRICE
data << r.TAOFANG_AVG_PRICE
data << r.ZHONGYUAN_AVG_PRICE
data << r.TBS_AVG_PRICE
data << r.QFANG_AVG_PRICE
data << r.FANGTAN_AVG_PRICE
data << r.FANGDUODUO_AVG_PRICE
data << r.WIWJ_AVG_PRICE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE.replaceAll(~'\\r', '')
allRows << data
})
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "THIRD_PARTY_PRICE_GJ 数据文件输出完成......"
//CcbFtpUtils.ftpUpload(fileName, curDate)
}
def exportThirdPartyPrice2() {
def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from third_party_price";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.CITYID
data << r.DISTID
data << r.COMMID
data << r.COMMNAME
data << r.DATA_SOURCE_ID
data << r.DATA_SOURCE
data << r.PRICE_DATE
data << r.COMMUNITY_PRICE
data << r.COMMUNITY_MAX_PRICE
data << r.COMMUNITY_MIN_PRICE
data << r.UPDATETIME
allRows << data
})
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "THIRD_PARTY_PRICE_GJ 数据文件输出完成......"
//CcbFtpUtils.ftpUpload(fileName, curDate)
}
def exportBigdataCommunity(){
def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" +
"\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj ";//where cityId in (350200,150100)
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DISTID
data << r.COMMID
data << r.DISTNAME
data << r.COMMNAME
data << r.ADDRESS
data << r.LATITUDE
data << r.LONGITUDE
data << r.LANDAREA
data << r.BUILDAREA
data << r.GREENINGRATE
data << r.PLOTRATIO
data << r.PARKINGCNT
data << r.BUILDYEAR
data << r.HOUSEUSE
data << r.ZTS
data << r.WORKDATE
data << r.UPDATEFLAG
data << r.UPDATETIME
data << r.COMMTYPE
data << r.COMMALIAS
data << r.STREET
data << r.STREETNO
data << r.IFNEW
data << ""
data << r.REMARK
data << r.CREATETIME
data << r.SOURCECODE
data << r.XZJDB
data << r.LJXZQHDM
data << r.LJXZQH
data << r.STATUS
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat"
}
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
//"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
//"乡镇/街道办","逻辑行政区划代码","逻辑行政区划","状态"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "BIGDATA_COMMUNITY_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, curDate)
println "BIGDATA_COMMUNITY_GJ 上传Ftp完成......"
}
def exportBigdataForsale(){
//["北京","上海"]
def citys = ["重庆","天津","成都","沈阳","大连","深圳","杭州","合肥","无锡","武汉","广州"]//CcbCommonUtils.citys;
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0002.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0002.dat"
}
def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
for(def city : citys){
def cityCode = cityCodeDataMap.get(city)
def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" +
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}'";
def allRows = []
def index = 0
sqlInstance.eachRow(sql,{r->
index = index+1
println city +" "+index
def data = []
data << r.CITYID
data << r.FORSALEID
data << r.PROPCODE
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.UNITID
data << r.UNITNO
data << r.UNITNAME
data << r.HOUSEID
data << r.ADDRESS
data << r.BUILDYEAR
data << r.PROPTYPE
data << r.PROPYEARS
data << r.UNITPRICE
data << r.TOTALPRICE
data << r.PAYTYPE
data << r.FLOORNUM
data << r.FLOORCNT
data << r.BUILDTYPE
data << r.HOUSESTRU
data << r.BUILDSTRU
data << r.BUILDAREA
data << r.LIVINGAREA
data << r.HOUSESTYLE
data << r.ROOMCNT
data << r.LIVINGROOMCNT
data << r.KITCHENCNT
data << r.BATHROOMCNT
data << r.HOUSEASPECT
data << r.FITMENT
data << r.IFELEVATOR
data << r.ELEVATORHOUSE
data << r.PUBLISHTIME
data << r.EXPIRETIME
data << r.IFFIRST
data << r.DATASOURCE
data << r.AGENCYID
data << r.AGENCYNAME
data << r.PUBLISHER
data << r.PHONE
data << ""
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.ATTACH
allRows << data
if(allRows.size()%200000==0){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
allRows.clear()
}
});
if(allRows.size()>0){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
}
}
println "BIGDATA_FORSALE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, curDate)
println "BIGDATA_FORSALE_GJ 上传Ftp完成......"
}
def exportAvmEvaluationGJNB(){
def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" +
"ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" +
"DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" +
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH >= 202106";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.COMMUNITY_NUM
data << r.HAS_PRICE_COMMUNITY_NUM
data << r.PRICE_COVERAGE
data << r.ARTIFICIAL_CALC_NUM
data << r.ARTIFICIAL_CALC_COVERAGE
data << r.SAMPLE_CALC_NUM
data << r.SAMPLE_CALC_COVERAGE
data << r.ANN_CALC_NUM
data << r.ANN_CALC_COVERAGE
data << r.TEST_NUM
data << r.TEST_COMM_NUM
data << r.TEST_COMM_COVERAGE
data << r.PPE15_COVERAGE
data << r.RATE_MEDIAN
data << r.COEFFICIENT
data << r.DEAL_TEST_CASE_NUM
data << r.DEAL_TEST_COMM_NUM
data << r.DEAL_TEST_COMM_COVERAGE
data << r.DEAL_PPE15_COVERAGE
data << r.DEAL_COEFFICIENT
data << r.DEAL_RATE_MEDIAN
data << r.THIRD_TEST_CASE_NUM
data << r.THIRD_TEST_COMM_NUM
data << r.THIRD_TEST_COMM_COVERAGE
data << r.THIRD_PPE15_COVERAGE
data << r.THIRD_COEFFICIENT
data << r.THIRD_RATE_MEDIAN
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE",
//"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE",
//"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "AVM_EVALUATION_GJNB 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, curDate)
println "AVM_EVALUATION_GJNB 上传Ftp完成......"
}
def exportAVMEvaluationItemGJNB(){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" +
"to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" +
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH >= 202106";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.AVG_PRICE
data << r.CALC_TYPE
data << r.CONFIDENCE
data << r.TARGET_AVG_PRICE
data << r.DEALDATE
data << r.UNIT_PRICE
data << r.BUILDAREA
data << r.FITMENT
data << r.HOUSEASPECT
data << r.FLOORNUM
data << r.FLOORCNT
data << r.DATA_TYPE
data << r.RATE
data << r.PE
data << r.CREATETIME
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","CONFIDENCE","TARGET_AVG_PRICE","DEALDATE","UNIT_PRICE","BUILDAREA","FITMENT","HOUSEASPECT","FLOORNUM","FLOORCNT","DATA_TYPE","RATE","PE","CREATETIME"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "AVM_EVALUATION_ITEM_GJNB 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, curDate)
println "AVM_EVALUATION_ITEM_GJNB 上传Ftp完成......"
}
def exportCommunityBasePrice(){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.AVG_PRICE
data << r.CALC_TYPE
data << r.FAKE_RATE
data << r.DIFF_RATE
data << r.CONFI_RATE
data << r.CORR_RATE
data << r.CONFIDENCE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, curDate)
println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......"
}
def exportCommunityBasePriceGj(){
}
def exportCityPrice(){
def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.PRICE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "CITY_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, curDate)
println "CITY_PRICE_GJ 上传Ftp完成......"
}
def exportRegionPrice(){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.PRICE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "DISTRICT_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, curDate)
println "DISTRICT_PRICE_GJ 上传Ftp完成......"
}
def exportLastFile(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def dataCodeFileName = "/ccb/${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok"
}
CcbFtpUtils.ftpUpload(dataCodeFileName, curDate)
}
}
@@ -4,13 +4,20 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import org.bson.types.ObjectId
/**
* 建行模型估值项目
* 人工指导价同步
@@ -20,6 +27,8 @@ import java.util.concurrent.Executors
class GuiDingPriceBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol();
static main(args) {
runBatch(new GuiDingPriceBatch(), args);
@@ -34,6 +43,7 @@ class GuiDingPriceBatch extends BaseBatch{
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, DateUtils.addMonth(new Date(), -6))
if(prices){
def distMap = getDistrictMap(city)
@@ -42,18 +52,88 @@ class GuiDingPriceBatch extends BaseBatch{
println city + " " + rets.size()
}
//添加搜房的参考价
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
// def datas = findFangDatas(districts)
// def fangRets = createDatas(cityCodeDatas, distCodeMap, datas)
// rows.addAll(fangRets)
//添加保亭文昌的指导价
def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx")
def cityDataMap = [:]
for(def d : cityDatas){
cityDataMap.put(d['A'], d['E'])
}
def personDatas = findPersonDatas(districts, cityDataMap);
def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas)
rows.addAll(personRets)
}
def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.txt", headers, "utf-8", false);
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
def rets = []
rets << headers
rets.addAll(rows)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\guiding_price.xlsx", rets, "数据")
// def rets = []
// rets << headers
// rets.addAll(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\guiding_price.xlsx", rets, "数据")
}
def findFangDatas(def districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println " 小区量:"+districts.size()
def result = Collections.synchronizedList(new ArrayList());
for(def i=0;i<districts.size();i++){
final def dist = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def obj = findMatchThirdPrice(dist)
if(obj && obj.avg_price){
result << ["dist_id":dist._id.toString(),"city":dist.city, "region":dist.region, "name":dist.name, "avg_price":obj.avg_price,"row_id":obj._id.toString(), "m_date":obj.m_date]
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name +" "+obj.avg_price
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return result
}
def findPersonDatas(def districts, def dataMap){
def result = []
for(def i=0;i<districts.size();i++) {
def dist = districts[i];
def price = dataMap.get(dist._id.toString())
if (price) {
result << ["dist_id": dist._id.toString(), "city": dist.city, "region": dist.region, "name": dist.name, "avg_price": price.toInteger(), "row_id": new ObjectId(), "m_date": new Date()]
println dist.city + " " + dist.region + " " + dist.name + " " + price
}
}
return result;
}
def findMatchThirdPrice(dist){
def q = new BasicDBObject()
q.put("match_city", dist.city)
q.put("match_region", dist.region)
q.put("match_name", dist.name)
q.put("s_date", sDate)
q.put("source","搜房")
def obj = dataProcessColl.findOne(q)
return obj
}
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
def rets = []
if(prices){
@@ -66,13 +146,13 @@ class GuiDingPriceBatch extends BaseBatch{
ret << ps._id.toString()
ret << dist._id.toString()
ret << dist.name
ret << cityCodeDatas.city_code
ret << cityCodeDatas.fangjia_city
ret << cityCodeDatas.get(dist.city)
ret << dist.city
ret << areaData.district_code
ret << areaData.district
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << ps.guiding_value
ret << ps.guiding_person
ret << "房价网"
ret << ""
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
@@ -85,8 +165,41 @@ class GuiDingPriceBatch extends BaseBatch{
return rets;
}
def createDatas(cityCodeDatas, distCodeMap, datas){
def rets = []
if(datas) {
for (def ps : datas) {
def ret = []
def areaData = distCodeMap.get(ps.region)
if (areaData) {
ret << ps.row_id
ret << ps.dist_id
ret << ps.name
ret << cityCodeDatas.get(ps.city)
ret << ps.city
ret << areaData.district_code
ret << areaData.district
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
ret << ps.avg_price
ret << "房价网"
ret << ""
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
}
return rets;
}
def createPersonDatas(cityCodeDatas, distCodeMap, dataMap){
}
def getDistrictMap(def city){
def districts = CcbCommonUtils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
def distMap = [:]
for(def d : districts){
distMap.put(d.region+"##"+d.name, d)
@@ -0,0 +1,98 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class LoadGuide {
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
static main(args) {
def task = new LoadGuide()
task.run()
}
def run(){
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞二手房政府指导价.xls")
// saveChengduoPrice("D:\\数据\\建行\\房产估值\\指导价\\成都部分住宅小区二手房成交参考价格201-286-258.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\绍兴指导价小区_0909.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\宁波交易参考价格.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安成交参考价格.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx")
savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
}
def savePrice(filePath){
def datas = ExcelUtils.read(filePath,0, 1)
def rets = []
for(def d : datas){
if(d['E']){
def o = new BasicDBObject()
o.city = d['E']
o.region = d['A']
o.name = d['C']
o.avg_price = d['D'].toString().toDouble().toInteger()
o.match_city = d['E']
o.match_region = d['F']
o.match_name = d['G']
o.source_type = 1 //1:成交参考价 2:参考价
o.c_date = new Date()
rets << o
}
}
if(rets){
println rets.size()
DISTRICT_GUIDE_PRICE_COLL.insert(rets)
}
}
def saveDongGuangPrice(def filePath){
def datas = ExcelUtils.read(filePath,0, 0)
def rets = []
for(def d : datas){
if(d['E']){
def o = new BasicDBObject()
o.city = d['E']
o.region = d['A']
o.name = d['B']
o.avg_price = d['D'].toString().toDouble().toInteger()
o.match_city = d['E']
o.match_region = d['F']
o.match_name = d['G']
o.house_count = d['C'].toString().toDouble().toInteger()
o.source_type = 1 //1:成交参考价 2:参考价
o.c_date = new Date()
rets << o
}
}
if(rets){
DISTRICT_GUIDE_PRICE_COLL.insert(rets)
}
}
def saveChengduoPrice(def filePath){
def datas = ExcelUtils.read(filePath,2, 0)
def rets = []
for(def d : datas){
if(d['E']){
def o = new BasicDBObject()
o.city = d['E']
o.region = d['B']
o.name = d['C']
o.avg_price = d['D'].toString().toDouble().toInteger()
o.match_city = d['E']
o.match_region = d['F']
o.match_name = d['G']
o.source_type = 1 //1:成交参考价 2:参考价
o.c_date = new Date()
rets << o
}
}
if(rets){
DISTRICT_GUIDE_PRICE_COLL.insert(rets)
}
}
}
@@ -0,0 +1,78 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
class LocationUpdate {
static main(args) {
def task = new LocationUpdate();
task.run()
}
def run(){
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\经纬度\\经纬度人工核实结果_0923.xlsx",{d->
index++
if(d['C']){
println index +" "+d['A']+" "+d['B']+" "+d['C']
def region = RegionData.formatRegion(d['A'], d['B'])
if(d['H']){
def status = d['H'].toString().toDouble().toInteger()
if(status==1){
def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
if(obj){
def gmap = GeocodeUtils.getGaodeToBaidu(d['F'].toString().toDouble(), d['G'].toString().toDouble())
def lng2 = null
def lat2 = null
if(gmap){
lng2 = (gmap.get("lng")*10000000000).longValue()
lat2 = (gmap.get("lat")*10000000000).longValue()
}
BasicDBObject q=new BasicDBObject()
q.put("m_date", new Date())
q.put("lat2", lat2)
q.put("lng2", lng2)
q.put("lng", (d['F'].toString().toDouble()*10000000000).longValue())
q.put("lat", (d['G'].toString().toDouble()*10000000000).longValue())
DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q))
}else{
println d['A']+" "+d['B']+" "+d['C']+" 没找到小区"
}
}else{
def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
if(obj){
BasicDBObject q=new BasicDBObject('name_cat',1)
q.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q))
}else{
println d['A']+" "+d['B']+" "+d['C']+" 没找到小区"
}
}
}else{
def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
if(obj){
BasicDBObject q=new BasicDBObject('name_cat',1)
q.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q))
}else{
println d['A']+" "+d['B']+" "+d['C']+" 没找到小区"
}
}
}
})
}
}
@@ -40,8 +40,8 @@ class StagingDistrictBatch extends BaseBatch{
results << headers
results.addAll(rows)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\staging_district.xlsx", results, "数据")
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\staging_district.txt",headers, "utf-8", false);
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "数据")
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_STAGING_GJ.dat",headers, "utf-8", false);
}
@@ -3,6 +3,8 @@ package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.Point
import com.fangjia.script.util.DbUtils
@@ -14,6 +16,11 @@ import com.mongodb.DBCollection
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.*;
@@ -42,7 +49,7 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
// println city +" "+count
//}
//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx"
//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0927.xlsx"
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\7月成交.xlsx",0,1)
//def dealMap = [:]
//for(def d : datas){
@@ -59,7 +66,7 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807 - 副本.xlsx"
//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0903.xlsx"
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1)
//def dealMap = [:]
@@ -83,58 +90,71 @@ def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
// if (city) {
// def cityid = cityCodeDatas.get(city)
// def total = findCount(cityid, sqlInstance)
// d['K'] = total
// d['O'] = total
// }
//},0)
//小区字段覆盖率
ExcelUtils.multiThreadingModify(fileName,{d->
def city = d['A']
if(city){
def cityid = cityCodeDatas.get(city)
def total = findCount(cityid, sqlInstance)
d['AA'] = total
////小区字段覆盖率
//ExcelUtils.multiThreadingModify(fileName,{d->
// def city = d['A']
// if(city){
// def cityid = cityCodeDatas.get(city)
// def total = findCount(cityid, sqlInstance)
// d['AA'] = total
//
// def a = findEstateType(cityid, sqlInstance)
// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total)
//
// def b = findFiledCount(cityid, 'buildyear', sqlInstance)
// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total)
//
// def c = findFiledCount(cityid, 'address', sqlInstance)
// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total)
//
// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance)
// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total)
//
// def e = findFiledCount(cityid, 'landarea', sqlInstance)
// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total)
//
// def f = findFiledCount(cityid, 'buildarea', sqlInstance)
// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total)
//
// def g = findFiledCount(cityid, 'plotratio', sqlInstance)
// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total)
//
// def h = findFiledCount(cityid, 'greeningrate', sqlInstance)
// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total)
//
// def i = findFiledCount(cityid, 'latitude', sqlInstance)
// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total)
//
// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance)
// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total)
// println d
// }
//},1)
def a = findEstateType(cityid, sqlInstance)
d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total)
def b = findFiledCount(cityid, 'buildyear', sqlInstance)
d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total)
def c = findFiledCount(cityid, 'address', sqlInstance)
d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total)
def d1 = findFiledCount(cityid, 'ZTS', sqlInstance)
d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total)
def e = findFiledCount(cityid, 'landarea', sqlInstance)
d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total)
def f = findFiledCount(cityid, 'buildarea', sqlInstance)
d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total)
def g = findFiledCount(cityid, 'plotratio', sqlInstance)
d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total)
def h = findFiledCount(cityid, 'greeningrate', sqlInstance)
d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total)
def i = findFiledCount(cityid, 'latitude', sqlInstance)
d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total)
def j = findFiledCount(cityid, 'parkingcnt', sqlInstance)
d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total)
println d
}
},1)
//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807-1.xlsx"
//def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx"
//def dealMap = findDealCount(sqlInstance)
//ExcelUtils.multiThreadingModify(fileName,{d->
// if(d['C']){
// d['Q'] = dealMap.get(d['C'])
// if(d['A']){
// d['U'] = dealMap.get(d['A'])
// }
//},2)
//},0)
def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交未入库人工部分城市统计.xlsx"
def datas = ExcelUtils.read(fileName, 0, 1)
def dealMap = [:]
for(def d : datas){
dealMap.put(d['A'], d['B'])
}
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx",{d->
if(d['A']){
d['W'] = dealMap.get(d['A'])
}
},0)
//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx"
@@ -170,7 +190,7 @@ ExcelUtils.multiThreadingModify(fileName,{d->
def findDealCount(sqlInstance){
def sql = "select cityname,count(*) as count from bigdata_deal where dealdate>=to_date('2021-06-01','yyyy-mm-dd') and dealdate<to_date('2021-07-01','yyyy-mm-dd') group by cityname";
def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('2021-08-01','yyyy-mm-dd') and dealdate<to_date('2021-09-01','yyyy-mm-dd') group by cityname";
def count = 0
def map = [:]
@@ -184,7 +204,7 @@ def findDealCount(sqlInstance){
def findEstateType(cityid, sqlInstance){
def sql = "select count(*) as count from bigdata_commbuildtype where cityid='${cityid}' and buildtype!='0199'";
def sql = "select count(*) as count from bigdata_commbuildtype_gj where cityid='${cityid}' and buildtype!='0199'";
def count = 0
sqlInstance.eachRow(sql,{r->
@@ -194,7 +214,7 @@ def findEstateType(cityid, sqlInstance){
}
def findCount(cityid, sqlInstance){
def sql = "select count(*) as count from bigdata_community where cityid='${cityid}'";
def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0";
def count = 0
sqlInstance.eachRow(sql,{r->
@@ -204,7 +224,7 @@ def findCount(cityid, sqlInstance){
}
def findFiledCount(cityid, field, sqlInstance){
def sql = "select count(*) as count from bigdata_community where cityid='${cityid}' and "+field+" is not null";
def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0 and "+field+" is not null";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
@@ -353,3 +373,5 @@ def getStreetInfo(def city, def glocation){
@@ -0,0 +1,81 @@
package com.fangjia.script.ccbevaluation
import groovy.sql.Sql
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class TestRepeat {
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new TestRepeat()
task.run()
}
def run(){
}
def checkUpdateCount(){
def newHouses = readTxt("C:\\Users\\gaojuan\\Desktop\\BIGDATA_FORSALE_GJ.dat")
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(newHouses.size())
println " ·¿Ô´Á¿£º"+newHouses.size()
def retMap = new ConcurrentHashMap()
for(def i=0;i<newHouses.size();i++){
final def map = newHouses[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def obj = findHouse(map.houseId, map.commId)
if(obj){
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
}
def findHouse(def houseId, def commId){
def sql = "select * from bigdata_forsale_gj where commid='${commId}' and forsaleid='${houseId}'";
def obj = null
sqlInstance.eachRow(sql,{r->
obj = r
});
return obj
}
def readTxt(def file){
def lines = org.apache.commons.io.FileUtils.readLines(new File(file))
def rets = [];
for(line in lines) {
def arrs = line.split("|@|")
def map = [:]
map.houseId = arrs[1]
map.commId = arrs[6]
map.pdate = arrs[10]
rets << map
}
return rets;
}
}
@@ -11,6 +11,8 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.ConcurrentHashMap
/**
* 建行模型估值项目
* 第三方均价数据同步
@@ -18,11 +20,13 @@ import com.mongodb.DBCollection
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def sDate= 202109//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202107
def endDate = 202109
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static main(args) {
runBatch(new ThirdPartyPriceBatch(), args);
@@ -33,13 +37,15 @@ class ThirdPartyPriceBatch extends BaseBatch {
def citys = CcbCommonUtils.citys;
def rows = []
def index = 0
def weekend = DateUtils.getFirstDayOfWeek(new Date())
for(def city : citys){
index++
def statMap = getDistrictStatByCity(city, weekend)
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
def thirdPriceMap = findThirdPartyPrice(city)
def thirdPriceMap = findThirdPartyPrice(city, statMap)
def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)
rows.addAll(rets)
println index+" "+city+" "+rets.size()
@@ -47,9 +53,9 @@ class ThirdPartyPriceBatch extends BaseBatch {
def fileName = "/home/goojia/ccb/third_party_price.txt"
def fileName = "/home/goojia/ccb/THIRD_PARTY_PRICE_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\third_party_price.txt"
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\THIRD_PARTY_PRICE_GJ.dat"
}
def header = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
@@ -61,6 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){
def rets = []
for(def d : dists){
def third = thirdPriceMap.get(d.region+"##"+d.name)
if(third){
def data = distCodeMap.get(d.region)
@@ -166,10 +173,10 @@ class ThirdPartyPriceBatch extends BaseBatch {
}
def findThirdPartyPrice(def city){
def findThirdPartyPrice(def city, def statMap){
def q = new BasicDBObject()
q.put("city", city)
q.put("s_date", new BasicDBObject('$gte', sDate))
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
@@ -242,13 +249,46 @@ class ThirdPartyPriceBatch extends BaseBatch {
def resultMap = [:]
map.each {key, value->
if(value && value.size()>1){
resultMap.put(key, value)
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){
def price = statMap.get(key)
if(price){
def priceMap = [:]
def startPrice = (int)(price - price*0.3)
def endPrice = (int)(price*0.3 + price)
value.each{pk,pv->
if(pk.contains("price")){
if(pv>=startPrice && pv<=endPrice){
priceMap.put(pk,pv)
}
}else{
priceMap.put(pk,pv)
}
}
if(priceMap.size()>1){
resultMap.put(key, priceMap)
}
}
}else{
resultMap.put(key, value)
}
}
}
return resultMap
}
def getDistrictStatByCity(city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DIST_SELL_STAT.find(q)
def map = new ConcurrentHashMap()
for(def d : dbcursor){
map.put(d.region+"##"+d.district_name, d.avg_price)
}
return map
}
}
@@ -1,12 +1,22 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/***
* 建行成交抓取统计
@@ -21,23 +31,34 @@ class TradeStatLog {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new TradeStatLog()
task.run()
// task.run()
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
task.exportTrade()
// task.deleteDistrict()
}
def run(){
def citys = CcbCommonUtils.citys;
def rets = []
rets << ["城市","5月总量","6月总量","7月总量","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","6月总量","7月总量","8月总量","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
def startDate = "2021-05-01"
def date = "2021-06-01"
def endDate = "2021-07-01"
def newDate = "2021-08-01"
//def startDate = "2021-05-01"
def startDate = "2021-06-01"
def date = "2021-07-01"
def endDate = "2021-08-01"
def newDate = "2021-09-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -64,7 +85,7 @@ class TradeStatLog {
ret << lastMonthCount
ret << curMonthCount
ret << nearMonthCount
if(lastMonthCount){
if(curMonthCount){
def a = (nearMonthCount-curMonthCount)*100/curMonthCount
def pre = DataHandleUtils.getDoubleRound(2, a)
ret << pre
@@ -87,7 +108,7 @@ class TradeStatLog {
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\成交统计.xlsx", rets, "数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx", rets, "数据")
println "表格生成完成"
}
@@ -103,7 +124,7 @@ class TradeStatLog {
rets << [city, districts.size(), m.total, m.count1, m.count2, m.count3, m.count4, m.count5, m.count6, m.count7, m.count8, m.count9, m.count10, m.count11, m.count12, m.count13, m.count14]
println rets.get(rets.size()-1)
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\第三方均价统计.xlsx", rets, "数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\第三方均价统计0927.xlsx", rets, "数据")
println "表格生成完成"
}
@@ -129,7 +150,7 @@ class TradeStatLog {
def getThirdPartyPricePrecent(def city){
def sql = "SELECT * FROM third_party_price WHERE cityname = '"+city+"'";
def sql = "SELECT * FROM third_party_price_gj WHERE cityname = '"+city+"' and year_month=202108";
def count0 = 0
def count1 = 0
@@ -299,5 +320,333 @@ class TradeStatLog {
}
def averageStat(){
def citys = ["北京","上海","广州","深圳","重庆","杭州","苏州","厦门","大连","呼和浩特"]
def map = [:]
for(def city : citys){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]);
def rets = findThirdPriceByDist(districts)
map.put(city, rets.size())
map.put(city+"-count", districts.size())
println districts.size()+" "+ rets.size()
}
println map
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\估值测试数据覆盖率占比.xlsx",{d->
def city = d['A']
def count = map.get(city)
def distCount = map.get(city+"-count")
d['H'] = count
d['J'] = distCount
def pre = DataHandleUtils.getDoubleRound(2, count*100.0/distCount)
d['I'] = pre
},0)
}
def findThirdPriceByDist(districts){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
for(def i=0;i<districts.size();i++){
final def dist = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def count = findAverageStat(dist)
if(count){
rets << dist
println totalCount.get() +"/${districts.size()} "+dist.region+" "+dist.name
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def findAverageStat(dist){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
q.put("name", dist.name)
q.put("category","district")
q.put("s_date", new BasicDBObject('$gte', 202105).append('$lt',202108))
def dbcursor = AVERAGE_STATISTICS.find(q)
def flag = 0
for(def d : dbcursor){
if(!flag){
if(d.zhugezhaofang_esf_price){
flag = 1
continue
}
if(d.anjuke_esf_price){
flag = 1
continue
}
if(d.get("58tc_esf_price")){
flag = 1
continue
}
if(d.soufang_esf_price){
flag = 1
}
if(d.lianjia_esf_price){
flag = 1
continue
}
if(d.cityhouse_esf_price){
continue
flag = 1
continue
}
if(d.maitian_esf_price){
flag = 1
continue
}
if(d.beike_esf_price){
flag = 1
continue
}
if(d.get("365house_esf_price")){
flag = 1
continue
}
if(d.zhongyuan_esf_price){
flag = 1
continue
}
if(d.tuboshi_esf_price){
flag = 1
continue
}
if(d.qfang_esf_price){
flag = 1
continue
}
if(d.fangtan_esf_price){
flag = 1
continue
}
if(d.fangduoduo_esf_price){
flag = 1
continue
}
if(d.get("5i5j_esf_price")){
flag = 1
continue
}
}
}
return flag
}
def exportTrade(){
def citys = CcbCommonUtils.citys;
// def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
def rets = []
rets << headers
for(def city : citys){
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern("2021-07-01","yyyy-MM-dd"), DateUtils.parseFromTightByPattern("2021-08-01","yyyy-MM-dd"))
// def trades = getYouDaData()
//def statMap = getDistrictStatByCity(city, weekend)
def nameMap = getAllDistrict(city)
def rMap = findDealIds(city);
def ids = rMap.ids
def details = rMap.detail
for(def t : trades){
if(!ids.contains(t._id.toString())){
def distId = nameMap.get(t.city +"##"+t.region+"##"+t.district_name)
if(!distId && (t.district_name.toString().endsWith("弄") || t.district_name.toString().endsWith("号"))){
distId = nameMap.get(t.city +"##"+t.region+"##"+t.district_name+"小区")
}
if(!distId){
def flag = t.district_name.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(flag){
def datas = CommonDatautils.findDistrictByAddress2(t.city, t.region, t.district_name, "district")
if(datas && datas.size()==1){
distId = datas.get(0)._id.toString()
}
}
}
def reason = ""
if(distId){
def dist = CommonDatautils.findDistrictById(distId)
// def price = statMap.get(dist.region+"@"+dist.name)
// def startPrice = null
// def endPrice = null
// if(price){
// if(dist.city=="上海"){
// startPrice = (int)(price - price*0.1)
// endPrice = (int)(price*0.5 + price)
// }else if(dist.city=="杭州"){
// startPrice = (int)(price - price*0.15)
// endPrice = (int)(price*0.15 + price)
// }else{
// startPrice = (int)(price - price*0.5)
// endPrice = (int)(price*0.5 + price)
// }
// }
// if(t.avg_price<startPrice || t.avg_price>endPrice){
// reason = "价格过滤"
// }
if(dist.name_cat != 0 || !dist.lng || !["普通住宅","商住"].contains(dist.estate_type2) || dist.name.contains("洋房") || dist.name.contains("公房")){
reason = "小区过滤"
}
if(!reason){
def key=t.city+"##"+dist.region+"##"+dist.name+"##"+t.area+"##"+t.avg_price+"##"+DateUtils.formatDateByPattern(t.trade_date,"yyyy-MM-dd")
if(details.contains(key)){
reason = "数据重复"
}
}
}
def tradeDate = DateUtils.formatDateByPattern(t.trade_date,"yyyy-MM-dd")
//rets << [t._id.toString(), t.city, t.region, t.district_name, t.source, t.avg_price, t.area,t.address, tradeDate, reason, t.estate_type]
rets << [t._id.toString(), t.city, t.region, t.district_name, t.source, t.avg_price, t.area,t.address, tradeDate, reason]
}
}
println city +" "+rets.size()
}
ExcelUtils.write("d:\\数据\\成交未入库记录(7月全).xlsx", rets,"数据")
}
def deleteDistrict(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d->
if(d['G']){
def id = d['G']
BasicDBObject q=new BasicDBObject('name_cat',1)
q.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q))
deleteAllRefDistrict(id)
}
},0)
}
def deleteAllRefDistrict(id){
def tables = ["bigdata_community_gj"]
for(def tableName : tables){
deleteByCommid(tableName, id)
}
}
def deleteByCommid(tableName, commid){
def sql = "delete from "+tableName+" where commid='${commid}'";
sqlInstance.execute(sql)
println "删除:"+tableName+" "+commid
}
def getYouDaData() {
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("2021-07-01", "yyyy-MM-dd"))
def endDate = DateUtils.parseFromTightByPattern("2021-08-01", "yyyy-MM-dd")
def q = new BasicDBObject()
// q.put("premisesName","共富一村")
q.put("contractTime", new BasicDBObject('$gte', startDate));//.append('$lt', endDate)
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
def trades = []
for(def db : dbcursor){
def map = [:]
map._id=db._id.toString()
map.city = "上海"
map.region = db.region
if(db.districtName && db.districtName!="null"){
map.district_name=db.districtName
}else if(db.address){
map.district_name=db.address
}
map.area = db.area
map.trade_date = db.contractTime
map.avg_price= db.price
map.estate_type = db.houseType
trades << map
}
return trades
}
def findDealIds(city){
def sql = "select * from bigdata_deal_gj where dealdate>=to_date('2021-07-01','yyyy-mm-dd') and dealdate<to_date('2021-08-01','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd')
def ids = []
def detailInfos = []
sqlInstance.eachRow(sql,{r->
if(!ids.contains(r.dealid)){
ids << r.dealid
def region = RegionData.formatRegion(r.cityname, r.distname)
def key=r.cityname+"##"+region+"##"+r.commname+"##"+r.buildarea.toDouble()+"##"+r.dealunitprice+"##"+DateUtils.formatDateByPattern(r.dealdate,"yyyy-MM-dd")
if(!detailInfos.contains(key)){
detailInfos << key
}
}
});
def rMap = [:]
rMap.ids = ids
rMap.detail = detailInfos
return rMap
}
public static def findTradesDetail(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
// q.put("district_name","百合花园")
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def dbcursor = DEAL_PRICE_COLL.find(q)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets
}
def getDistrictStatByCity(city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DIST_SELL_STAT.find(q)
def map = new ConcurrentHashMap()
for(def d : dbcursor){
map.put(d.region+"@"+d.district_name, d.avg_price)
}
return map
}
def getAllDistrict(city){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name", "alias"])
def map = [:]
for(def d : dists){
map.put(d.city+"##"+d.region+"##"+d.name, d._id.toString())
def alias = d.alias
if(alias){
for(def name : alias){
map.put(d.city+"##"+d.region+"##"+name, d._id.toString())
}
}
}
return map
}
}
@@ -0,0 +1,392 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.ExcelUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
class UpdateHouseRegion {
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
static main(args) {
def task = new UpdateHouseRegion()
// task.updateSeawaterRegion()
// task.updateDealRegion()
// task.updateThirdPartyRegion()
task.updateNoMatchDealRegion()
// task.addAlias()
// task.addFenQi()
}
def addFenQi(){
def citys = CommonDatautils.findAllCity()
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district",["city","region","name","alias"])
addFenQiAlias(dists)
println city +" 处理完成"
}
}
def addFenQiAlias(dists){
for(def dist : dists){
def s1 = DataUtil.match(dist.name, Pattern.compile("[一|二|三|四|五|六|七|八|九|十|十一|十二|十三|十四|十五|十六|十七|十八|\\w+](期|区)\$"))
if(s1){
def newName = dist.name.replace("Ⅰ","1").replace("Ⅱ","2").replace("A区","a区").replace("B区","b区").replace("C区","c区").replace("D区","d区").replace("E区","e区").replace("F区","f区").replace("一期","1期").replace("二期","2期").replace("三期","3期").replace("四期","4期").replace("五期","5期").replace("六期","6期").replace("七期","7期").replace("八期","8期").replace("九期","9期").replace("十期","10期").replace("十一期","11期").replace("十二期","12期")
if(newName!=dist.name){
def alias = dist.alias
if(!alias){
alias = []
}
if(!alias.contains(newName)){
alias << newName
def udata = new BasicDBObject()
udata.put("alias", alias)
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
println dist.city +" "+dist.name+" "+dist.alias+" 新:"+alias
}
}
}
}
}
def addAlias(){
def citys = CommonDatautils.findAllCity()
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district",["city","region","name","alias"])
addDistricts(dists)
println city +" 处理完成"
}
}
def addDistricts(dists){
for(def dist : dists){
def alias = dist.alias
if(alias){
def oldCount = alias.size()
def newAlias = []
for(def aliasName : alias){
if(aliasName.contains("·") || aliasName.contains(".")){
def newName = aliasName.replace("·","").replace(".","")
newAlias << aliasName
if(!newAlias.contains(newName)){
if(newName!=dist.name){
newAlias << newName
}
}
}else{
if(!newAlias.contains(aliasName)) {
newAlias << aliasName
}
}
}
def newCount = newAlias.size()
if(oldCount && alias!=newAlias){
def udata = new BasicDBObject()
udata.put("alias", newAlias)
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
println dist.city +" "+dist.name+" "+dist.alias+" 新:"+newAlias
}
}
}
}
def updateThirdPartyRegion(){
def city = "东莞"
def rets = findThirdPartyPrice(city)
batchThirdPartyUpdate(city, rets);
}
def findThirdPartyPrice(city){
def q = new BasicDBObject()
q.put("city",city)
q.put("s_date",new BasicDBObject('$gte', 202101))
def dbcursor = thirdPartyColl.find(q)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
def findDealData(def city){
def q = new BasicDBObject()
q.put("city", city)
q.put('trade_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-01-01")));
def cursor = DEAL_PRICE_COLL.find(q).batchSize(2000)
def datas = []
def index = 0
for(def obj : cursor){
index++
datas << obj
println index
}
return datas
}
def updateDealRegion(){
def city = "东莞"
def datas = findDealData(city)
batchDealRegion(city, datas)
}
def batchDealRegion(def city, def datas){
def updateCount = new AtomicLong()
def curCount = new AtomicLong()
def totalCount = datas.size();
final CountDownLatch latch = new CountDownLatch(totalCount)
println city +" 成交总量:"+totalCount
for(def i=0;i<totalCount;i++) {
final def data = datas[i];
fixedThreadPool.execute(new Runnable() {
void run() {
curCount.addAndGet(1)
try {
def region = data.region
def name = data.district_name
def rets = CommonDatautils.findDistrictsByName(city, name, "district")
if (!rets) {
rets = CommonDatautils.findDistrictByAlias(city, null, name, "district")
}
if (rets.size() == 1) {
def dist = rets[0]
if (region != dist.region) {
updateCount.addAndGet(1)
def udata = new BasicDBObject()
udata.put("region", dist.region)
DEAL_PRICE_COLL.update(new BasicDBObject('_id', data.get("_id")), new BasicDBObject('$set', udata))
println updateCount.get() + "/${curCount.get()}/" + totalCount + " " + region + " " + dist.region + " " + dist.name
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
})
}
latch.await()
}
def updateNoMatchDealRegion(){
ExcelUtils.multiThreadingModify("D:\\数据\\成交未入库记录(7月全).xlsx",{d->
if(!d['J']){
def city = d['B']
def region = d['C']
def name = d['D']
def rets = CommonDatautils.findDistrictsByName(city, name, "district")
if (!rets) {
rets = CommonDatautils.findDistrictByAlias(city, null, name, "district")
}
if (rets.size() == 1) {
def dist = rets[0]
if (region != dist.region) {
def udata = new BasicDBObject()
udata.put("region", dist.region)
DEAL_PRICE_COLL.update(new BasicDBObject('_id', new ObjectId(d['A'])), new BasicDBObject('$set', udata))
d['J'] = "原数据区域修改("+dist.region+")"
println d['A']+" "+region +" "+dist.region
}
}
}
},0)
}
def updateSeawaterRegion(){
def city = "东莞"
def rets = findHouseSource(city, 20190701, 20211001)
batchUpdate(city, rets);
}
def batchThirdPartyUpdate(def city, def thirdDatas){
def updateCount = new AtomicLong()
def curCount = new AtomicLong()
def totalCount = thirdDatas.size();
final CountDownLatch latch = new CountDownLatch(totalCount)
println city +" 第三方总量:"+totalCount
for(def i=0;i<totalCount;i++) {
final def data = thirdDatas[i];
fixedThreadPool.execute(new Runnable() {
void run() {
curCount.addAndGet(1)
try {
def region = data.region
def name = data.name
def rets = CommonDatautils.findDistrictsByName(city, name, "district")
if (!rets) {
rets = CommonDatautils.findDistrictByAlias(city, null, name, "district")
}
if (rets.size() == 1) {
def dist = rets[0]
if (region != dist.region) {
updateCount.addAndGet(1)
def udata = new BasicDBObject()
udata.put("region", dist.region)
thirdPartyColl.update(new BasicDBObject('_id', data.get("_id")), new BasicDBObject('$set', udata))
println updateCount.get() + "/${curCount.get()}/" + totalCount + " " + region + " " + dist.region + " " + dist.name
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
})
}
latch.await()
}
def findCount(def city, def startDate, def endDate){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-2020\" WHERE 1=1")
sqlBuffer.append(" and city = '"+city+"'")
sqlBuffer.append(" and cat = 'sell'")
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <="+endDate)
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
while (rs.next()) {
def count = rs.getInt("count")
println count
}
}
def batchUpdate(def city, def houses){
def updateCount = new AtomicLong()
def curCount = new AtomicLong()
def totalCount = houses.size();
final CountDownLatch latch = new CountDownLatch(totalCount)
println city +" 房源总量:"+totalCount
for(def i=0;i<totalCount;i++) {
final def house = houses[i];
fixedThreadPool.execute(new Runnable() {
void run() {
curCount.addAndGet(1)
try {
def region = house.region
def name = house.name
def rets = CommonDatautils.findDistrictsByName(city, name, "district")
if (!rets) {
rets = CommonDatautils.findDistrictByAlias(city, null, name, "district")
}
if (rets.size() == 1) {
def dist = rets[0]
if (region != dist.region) {
updateCount.addAndGet(1)
updateHouseRegion2(house.url, dist.region);
println updateCount.get() + "/${curCount.get()}/" + totalCount + " " + region + " " + dist.region + " " + dist.name
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
})
}
latch.await()
}
def findHouseSource(def city, def startDate, def endDate){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT * FROM \"seawater-2019\" WHERE 1=1")
sqlBuffer.append(" and city = '"+city+"'")
sqlBuffer.append(" and cat = 'sell'")
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def houses = []
while (rs.next()){
houses << ["region":rs.getString("region"), "name":rs.getString("district_name"),"url":rs.getString("url")]
}
return houses;
}
def updateHouseRegion2(def url, def region){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("developer:5e8edd851d2fdfbd7415232c67367cc3".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.206:9200/seawater-2019/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.region='${region}'",
"lang": "painless"
},
"query": {
"term": {
"url": "${url}"
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
// def response = JSONObject.fromObject(res)
// println(response)
}
def updateHouseRegion(def url, def region){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("update seawater-2021")
sqlBuffer.append(" set region = '"+region+"'")
sqlBuffer.append(" where url='"+url+"'")
def result = dataSource.connection.createStatement().executeUpdate(sqlBuffer.toString())
println region +" "+result
}
def getAllDistricts(def city){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias"])
def map = [:]
for(def d : dists){
map.put()
}
}
}
@@ -0,0 +1,375 @@
package com.fangjia.script.ccbevaluation.ronghe
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.jusfoun.JusfounCommon
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.OpenApiClient
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
/**
* 建行融合——小区表
*/
@BatchOwner(Watcher.GaoJuan)
class BigdataCommunityGJRHBatch extends BaseBatch {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
public static def batchsize = 1000
static main(args) {
runBatch(new BigdataCommunityGJRHBatch(), args);
}
@Override
public void process(Object args) {
//"厦门","北京","上海",
def citys = ["广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉"]//CcbCommonUtils.citys;
def rows = []
// addBigdataDistrictFJW()
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for (def city : citys) {
def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"])
for(def dist : districts){
def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"])
if(obj){
if(obj.alias && obj.alias.size()>0){
dist.commalias = obj.alias.get(0)
}else{
dist.commalias = ""
}
dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure)
dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2)
dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date);
dist.houseuse = "11"
dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type)
dist.compid = new ObjectId() //物业公司id
dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2)
}
}
println city +" 小区加载完成 "+districts.size()
addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city))
println city +" 1.小区表完成"
addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city))
println city +" 2.小区建筑结构表"
addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city))
println city +" 3.小区房屋类型表"
addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city))
println city +" 4.小区建筑年代表"
addBigdataCommfeeGj(districts, cityCodeDataMap.get(city))
println city +" 6.小区费用表"
addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city))
println city +" 7.小区房屋用途表"
addBigdataPositionGJ(districts, cityCodeDataMap.get(city))
println city + " 8.地理位置表"
}
}
def addBigdataCommunityGJRH(def districts, def cityid){
sqlInstance.executeUpdate("delete from BIGDATA_COMMUNITY_FJW where CITYID='${cityid}'")
sqlInstance.withBatch(batchsize) { stmt ->
for(def d : districts){
def sql = "INSERT INTO BIGDATA_COMMUNITY_FJW (CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,CREATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.distid}','${d.commid}','${d.commname}',${converValue(d.address)},${d.landarea},${d.buildarea},${d.greeningrate},${d.plotratio},${d.parkingcnt},"
if(d.updatetime){
sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.commtype}','${d.commalias}',"
}else{
sql = sql + "null,'${d.commtype}','${d.commalias}',"
}
if(d.street && d.street!='null'){
sql = sql +"'${d.street}',"
}else{
sql = sql +"null,"
}
if(d.streetno && d.streetno!='null'){
sql = sql + "'${d.streetno}',"
}else{
sql = sql +"null,"
}
sql = sql + "'${d.ifnew}',null,${d.remark},"
if(d.createtime){
sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
}else{
sql = sql + "null,'${d.sourcecode}')"
}
try{
stmt.addBatch sql
}catch(Exception e){
println d
println sql
}
}
}
}
def converValue(def val){
if(val && val!='null'){
return "'"+val+"'";
}else{
return null
}
}
def addBigdataCommbuildstruGJ(def districts, def cityid){
sqlInstance.executeUpdate("delete from BIGDATA_COMMBUILDSTRU_FJW where CITYID='${cityid}'")
sqlInstance.withBatch(batchsize) { stmt ->
for(def d : districts){
def sql = "INSERT INTO BIGDATA_COMMBUILDSTRU_FJW (CITYID,COMMID,BUILDSTRU,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.buildstru}',${d.remark},"
if(d.createtime){
sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
}else{
sql = sql + "null,"
}
if(d.updatetime){
sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
}else{
sql = sql + "null,'${d.sourcecode}')"
}
try{
stmt.addBatch sql
}catch(Exception e){
println d
println sql
}
}
}
}
def addBigdataCommhousetypeGJ(def districts, def cityid){
sqlInstance.executeUpdate("delete from BIGDATA_COMMHOUSETYPE_FJW where CITYID='${cityid}'")
sqlInstance.withBatch(batchsize) { stmt ->
for(def d : districts){
def sql = "INSERT INTO BIGDATA_COMMHOUSETYPE_FJW (CITYID,COMMID,HOUSETYPE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.housetype}',${d.remark},"
if(d.createtime){
sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
}else{
sql = sql + "null,"
}
if(d.updatetime){
sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
}else{
sql = sql + "null,'${d.sourcecode}')"
}
try{
stmt.addBatch sql
}catch(Exception e){
println d
println sql
}
}
}
}
def addBigdataCommbuildyearGj(districts, def cityid){
sqlInstance.executeUpdate("delete from BIGDATA_COMMBUILDYEAR_FJW where CITYID='${cityid}'")
sqlInstance.withBatch(batchsize) { stmt ->
for(def d : districts){
if(d.buildyear){
def sql = "INSERT INTO BIGDATA_COMMBUILDYEAR_FJW (CITYID,COMMID,BUILDYEAR,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}',${d.buildyear},${d.remark},"
if(d.createtime){
sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
}else{
sql = sql + "null,"
}
if(d.updatetime){
sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
}else{
sql = sql + "null,'${d.sourcecode}')"
}
try{
stmt.addBatch sql
}catch(Exception e){
println d
println sql
}
}
}
}
}
def addBigdataCommfeeGj(districts, def cityid){
sqlInstance.executeUpdate("delete from BIGDATA_COMMFEE_FJW where CITYID='${cityid}'")
def sqls = []
for(def d : districts){
if(d.estatecharge){
def sql = "INSERT INTO BIGDATA_COMMFEE_FJW (CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.compid}',${d.estatecharge},'${d.houseuse}','${d.buildtype}','1',${d.note},${d.remark},"
if(d.createtime){
sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
}else{
sql = sql + "null,"
}
if(d.updatetime){
sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
}else{
sql = sql + "null,'${d.sourcecode}')"
}
sqls << sql
if (sqls.size() >= batchsize) {
try {
sqlInstance.withBatch(batchsize, { stmt ->
sqls.each { def s ->
stmt.addBatch s
}
})
} catch (Exception e) {
println sqls[0]
e.printStackTrace()
}
sqls.clear()
}
}
}
if (sqls.size() > 0) {
sqlInstance.withBatch(100, { stmt ->
sqls.each { def s ->
stmt.addBatch s
}
})
}
}
def addBigdataCommhouseuseGJ(districts, def cityid){
sqlInstance.executeUpdate("delete from BIGDATA_COMMHOUSEUSE_FJW where CITYID='${cityid}'")
def sqls = []
for(def d : districts){
def sql = "INSERT INTO BIGDATA_COMMHOUSEUSE_FJW (CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.houseuse}',${d.zts},${d.remark},"
if(d.createtime){
sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
}else{
sql = sql + "null,"
}
if(d.updatetime){
sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
}else{
sql = sql + "null,'${d.sourcecode}')"
}
sqls << sql
if (sqls.size() >= batchsize) {
try {
sqlInstance.withBatch(batchsize, { stmt ->
sqls.each { def s ->
stmt.addBatch s
}
})
} catch (Exception e) {
e.printStackTrace()
}
sqls.clear()
}
}
if (sqls.size() > 0) {
sqlInstance.withBatch(100, { stmt ->
sqls.each { def s ->
stmt.addBatch s
}
})
}
}
def addBigdataPositionGJ(districts, def cityid){
sqlInstance.executeUpdate("delete from BIGDATA_POSITION_FJW where CITYID='${cityid}'")
def sqls = []
def index = 1
for(def d : districts){
def sql = "INSERT INTO BIGDATA_POSITION_FJW (CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}','5', '${d.commid}','1',${index},${d.lng},${d.lat},null,${d.remark},"
index++
if(d.createtime){
sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
}else{
sql = sql + "null,"
}
if(d.updatetime){
sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
}else{
sql = sql + "null,'${d.sourcecode}')"
}
sqls << sql
if (sqls.size() >= batchsize) {
try {
sqlInstance.withBatch(batchsize, { stmt ->
sqls.each { def s ->
stmt.addBatch s
}
})
} catch (Exception e) {
e.printStackTrace()
}
sqls.clear()
}
}
if (sqls.size() > 0) {
sqlInstance.withBatch(100, { stmt ->
sqls.each { def s ->
stmt.addBatch s
}
})
}
}
def addBigdataDistrictFJW(){
def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from bigdata_district_gj";
def rows = []
try{
sqlInstance.eachRow(sql,{r->
def insertSql = "INSERT INTO bigdata_district_fjw(DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${r.distid}','${r.distname}','${r.distspell}','${r.cityid}','${r.distcode}','${r.seq}',${r.disttype},null,to_date('${r.createtime}','yyyy-mm-dd hh24:mi:ss'),to_date('${r.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${r.sourcecode}')"
sqlInstance.withBatch(batchsize, { stmt ->
stmt.addBatch insertSql
})
});
}catch(Exception e){
}
}
}
@@ -0,0 +1,239 @@
package com.fangjia.script.ccbevaluation.ronghe
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import groovy.sql.Sql
import java.sql.ResultSet
import java.sql.ResultSetMetaData
@BatchOwner(Watcher.GaoJuan)
class ExportDistrictRongHe extends BaseBatch{
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
runBatch(new ExportDistrictRongHe(), args);
}
@Override
void process(Object args) {
exportBigdataCommunityGJRH()
exportBigdataCommbuildstruGJ()
exportBigdataCommhousetypeGJ()
exportBigdataCommbuildyearGj()
exportBigdataCommfeeGj()
exportBigdataCommhouseuseGJ()
exportBigdataPositionGJ()
exportBigdataDistrictFJW()
}
def exportBigdataCommunityGJRH(){
def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DISTID
data << r.COMMID
data << r.COMMNAME
data << r.ADDRESS
data << r.LANDAREA
data << r.BUILDAREA
data << r.GREENINGRATE
data << r.PLOTRATIO
data << r.PARKINGCNT
data << r.UPDATETIME
data << r.COMMTYPE
data << r.COMMALIAS
data << r.STREET
data << r.STREETNO
data << r.IFNEW
data << r.INTRODUCTION
data << r.REMARK
data << r.CREATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18"
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_FJW_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_COMMUNITY_GJRH_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
}
def exportBigdataCommbuildstruGJ(){
def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.BUILDSTRU
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18"
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_COMMBUILDSTRU_GJ_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
}
def exportBigdataCommhousetypeGJ(){
def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.HOUSETYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18"
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_COMMHOUSETYPE_GJ_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
}
def exportBigdataCommbuildyearGj(){
def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.BUILDYEAR
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18"
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_COMMBUILDYEAR_GJ_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
}
def exportBigdataCommfeeGj(){
def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.COMPID
data << r.FEE
data << r.HOUSEUSE
data << r.BUILDTYPE
data << r.FEETYPE
data << r.NOTE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18"
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMFEE_FJW_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_COMMFEE_GJ_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
}
def exportBigdataCommhouseuseGJ(){
def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.HOUSEUSE
data << r.HOUSECNT
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18"
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_COMMHOUSEUSE_GJ_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
}
def exportBigdataPositionGJ(){
def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.OBJECTTYPE
data << r.OBJECTID
data << r.POSITIONTYPE
data << r.SEQNO
data << r.LONGITUDE
data << r.LATITUDE
data << r.ALTITUDE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18"
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POSITION_FJW_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_POSITION_GJ_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
}
def exportBigdataDistrictFJW(){
def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.DISTID
data << r.DISTNAME
data << r.DISTSPELL
data << r.CITYID
data << r.DISTCODE
data << r.SEQ
data << r.DISTTYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18"
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_FJW_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_DISTRICT_FJW_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
}
}
@@ -536,8 +536,8 @@ class CommonDatautils {
if(name){
alias << name;
}
DBObject q = new BasicDBObject();
q.put("city", city);
q.put("region", region);
@@ -677,6 +677,28 @@ class CommonDatautils {
}
public static def findAllDistricMaptByCity(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def retMap = [:]
for(def d : dbsour){
retMap.put(d._id.toString(), d)
}
return retMap;
}
public static def findAllDistrictByCity(def city, def cat, def fields){
DBObject q = new BasicDBObject();
@@ -760,6 +782,29 @@ class CommonDatautils {
}
return ret;
}
public static def findDistrictFieldById(String id, def fields){
if(!id){
return null;
}
DBObject q = new BasicDBObject();
q.put("_id", new ObjectId(id.toString().replace("[","").replace("]","")));
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
def cursor = getDataStoreConnection().find(q, keys).batchSize(2000)
if(cursor){
for(def obj : cursor){
return obj;
}
}
return null;
}
public static def findDistrictById(String id){
@@ -782,6 +827,7 @@ class CommonDatautils {
keys.put("out_pics", 1);
keys.put("lat", 1);
keys.put("lng", 1);
keys.put("name_cat", 1);
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){
@@ -889,6 +935,25 @@ class CommonDatautils {
}
return null;
}
public static def findDistrictsByName(String city, String name, String cat){
DBObject q = new BasicDBObject();
q.put("city", city);
q.put("cat", cat);
q.put("name", name);
q.put("visible", 0);
def rets = []
DSCursor cursor = getDataStoreConnection().find(q);
if(cursor){
for(def obj : cursor){
rets << obj;
}
}
return rets;
}
public static def getSimpleDistrictMap(def cat, def city){
DBObject q = new BasicDBObject();
q.put("visible", 0);
@@ -1211,6 +1276,28 @@ class CommonDatautils {
return ret;
}
public static def findDistrictByAddress2(def city, def region, def address, def cat){
DBObject q = new BasicDBObject();
if(region){
q.put('region', region)
}
q.put('city',city)
q.put('cat', cat)
q.put('visible',0)
q.put("address", address);
def dsCursor = getDataStoreConnection().find(q);
def ret = [];
for(def obj : dsCursor){
ret << obj;
}
return ret;
}
public static def getCityRegions(city){
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202107.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202109.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -135,7 +135,7 @@ rets << ["9
def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM")
ExcelUtils.write("d:\\数据\\福州\\多方均价对比统计数据"+curMonth+".xlsx",rets,"统计")
def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )"
println s
def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60"
DingDingMessageUtil.sendTextMessage(access_token, s)
@@ -38,9 +38,7 @@ class GaoDeMapCrawl extends BaseBatch{
DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.235", 27018)
def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc"
final def dbUrl = "mongodb://192.168.0.235:27018";
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
final AtomicInteger count = new AtomicInteger()
def updateLists = []
@@ -50,8 +48,8 @@ class GaoDeMapCrawl extends BaseBatch{
// def categorys = ["141202", "141203"]//中学,小学
def prefix = "district_"
def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
// def prefix = "district_"
// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
// def prefix = "road_"
// def categorys = ["190301"]//道路相关
@@ -59,6 +57,10 @@ class GaoDeMapCrawl extends BaseBatch{
// def prefix = "village_"
// def categorys = ["190108","190109"]//村庄
def prefix = "food_"
def categorys = ["050000"]
// def prefix = "company_"
// def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"]
def totalCount = new AtomicLong()
@@ -119,20 +121,24 @@ class GaoDeMapCrawl extends BaseBatch{
def run(){//,"西安","重庆"
//def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def citys =["南昌"]
// def citys = CommonUtils.findAllCity()
def citys = ["南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
def allcitys = ["上海"]//CommonUtils.findAllCity()
def totalCount = 0;
for(def city : citys){
def stop = false
while(stop == false){
def cityLations = findAllLation(city);
if(!cityLations){
stop = true;
}else{
println "开始处理["+city +"]:"+cityLations.size()
gaoDeCitySearch(cityLations);
for(def city : allcitys){
if(!citys.contains(city)){
def stop = false
while(stop == false){
def cityLations = findAllLation(city);
if(!cityLations){
stop = true;
}else{
println "开始处理["+city +"]:"+cityLations.size()
gaoDeCitySearch(cityLations);
}
}
}
}
}
@@ -144,7 +150,11 @@ class GaoDeMapCrawl extends BaseBatch{
q.put(prefix+"status", new BasicDBObject(QueryOperators.EXISTS, false))
// q.put("status", "失败")
// q.put("traffic_status", new BasicDBObject(QueryOperators.EXISTS, false))
def results = BOUNDS_COLL.find(q).limit(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def dbcursor = BOUNDS_COLL.find(q).batchSize(3000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def results = []
for(def d : dbcursor){
results << d
}
return results;
}
@@ -185,14 +195,15 @@ class GaoDeMapCrawl extends BaseBatch{
}
def searching(i, location){
// def bound_gd = location.bound_gd
// def bound_gd = location.bound_gd
// def lats = [bound_gd[1], bound_gd[3]]
// def lngs = [bound_gd[0], bound_gd[2]]
// lats.sort()
// lngs.sort()
// println index
// if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){
search(i, location)
println i+" "+ location
// 120.019556,30.313701
//if(lngs[0]<=120.019556 && lngs[1]>=120.019556 && lats[0]<=30.313701 && lats[1]>=30.313701){
search(i, location)
// }
if(updateLists.size() > 0 && updateLists.size()>1000){
@@ -45,18 +45,20 @@ class OpenUserBatch {
}
def addShenZhenFjtUser(){
def datas = ExcelUtils.read("D:\\数据\\农行\\肇庆农行支行房价通账号.xlsx",0,1)
def datas = ExcelUtils.read("D:\\数据\\渤海\\开通线上评估客户经理名单.xlsx",0,1)
def i = 0;
for(def data : datas){
i++;
def id = 252580 + i;
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+data[2]+"', '"+data[3]+"', '16', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2021-12-10 00:00:00', '0', '', NULL)";
def id = 252608 + i;
BigDecimal bd = new BigDecimal(data[3].toString());
String mobile = bd.toPlainString();
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '17', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2023-01-01 00:00:00', '0', '', NULL)";
onLineSqlInstance.execute(sql);
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[1]+"', '"+data[0]+"', NULL, NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '王辰岑')";
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '姚静梦')";
onLineSqlInstance.execute(sql2)
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'肇庆','肇庆:')"
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'上海','上海:')"
onLineSqlInstance.execute(sql3)
}
}
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-07-01"
static def statDate = "2021-09-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser();//友达二手房存入mongo
task.esfParser();//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2021年8月\\二手房\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2021年10月\\二手房\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -1,7 +1,9 @@
package com.fangjia.script.gj.ccb
import com.fangjia.core.util.DataUtil
import org.apache.commons.lang.StringUtils
import java.util.regex.Matcher
import java.util.regex.Pattern
class CcbCodeUtils {
@@ -173,19 +175,36 @@ class CcbCodeUtils {
}
//钢混结构:1 砖混结构:2 钢结构:3 框架结构:4 框架剪力墙结构:5 剪力墙结构:6 筒体结构:7 框架筒体结构:8 筒中筒:9 砖木结构:10 其他:11
//½¨Öþ½á¹¹
// 钢、钢筋混凝土结构 002
// 钢筋混凝土结构 003
// 砖木结构 004
// 混合结构 005
// 其它结构 006
static def getBuildStructureCode(def buildStructure){
if(buildStructure){
if(buildStructure.contains("钢结构")){
return 3;
return "001";
}else if(buildStructure.contains("钢、钢筋混凝土结构")){
return "002";
}else if(buildStructure.contains("钢筋混凝土结构")){
return 1;
return "003";
}else if(buildStructure.contains("砖木结构")){
return 10;
return "004";
}else if(buildStructure.contains("混合结构")){
return "005";
}
return 11;
}
return null;
return "006";
}
static def getDistHouseTypeCode(def estateType){
if(estateType && estateType=="别墅"){
return "08"
}else if(estateType && (estateType=="普通住宅" || estateType=="商住")){
return "01"
}
return "99"
}
//室内结构
//平层:11 错层:12 复式:13 阁楼:14 团结户:15 老式房:16 地下室:17 带阁楼:21 带车库:22 其他:99 跃层:18 带地下室:24 带储藏室:25 带车位:23
@@ -221,4 +240,67 @@ class CcbCodeUtils {
}
return 99
}
public static Integer getBuildYear(String buildDate){
if(StringUtils.isNotBlank(buildDate)){
Pattern p = Pattern.compile("\\d{4}")
Matcher m = p.matcher(buildDate);
while (m.find()) {
return m.group(0).toInteger();
}
}
return null;
}
static def processEstateCharge(def city, def estateCharege, def estateType2){
if(estateCharege){
try{
estateCharege = trimChar(estateCharege);
estateCharege = estateCharege.replace("元/平米/月","").replace("元/平方米/月","").replace("元/?O?月","")
estateCharege = estateCharege.toDouble()
if(estateType2=="普通住宅"){
if(city=="上海" || city=="北京" || city=="广州" || city=="杭州"){
if(estateCharege>=0.6 && estateCharege<=3.5){
return estateCharege
}else{
return null;
}
}else{
if(estateCharege>=0.2 && estateCharege<=3.5){
return estateCharege
}else{
return null;
}
}
}else if(estateType2=="别墅"){
if(estateCharege>=3){
return estateCharege;
}else{
return null;
}
}else if(estateType2=="商住"){
if(estateCharege>=4){
return estateCharege;
}else{
return null;
}
}
}catch (Exception e){
println "estateCharege 转不了 "+estateCharege
return null;
}
}
return null;
}
static def trimChar(String c) {
try {
if(c){
c = new String(c.getBytes("GBK"), "GBK").replace("?", "/");
}
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
return c;
}
}
@@ -0,0 +1,18 @@
package com.fangjia.script.gj.ccb
import com.fangjia.script.commData.CommonDatautils
class CcbXykData {
static main(args) {
def task = new CcbXykData()
task.run()
}
def run(){
def districts = CommonDatautils.findAllDistrictByCity("ÉϺ£","district",["city","region","name","estate_charge"])
for(def d : districts){
}
}
}
@@ -9,12 +9,20 @@ class FileUtils {
// file.delete();
// }
if (!file.getParentFile().exists()) {
boolean result = file.getParentFile().mkdirs();
if (!result) {
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
}
}
OutputStream out = new FileOutputStream(fileName, isAppend);
def sb = new StringBuffer();
if(headers){
sb.append(headers.join("|@|"))
sb.append(System.getProperty("line.separator"));
sb.append("\n");
}
out.write(sb.toString().getBytes(charset));
@@ -25,7 +33,7 @@ class FileUtils {
if(row){
index++
sb.append(StringUtils.join(row.toArray(), "|@|"))
sb.append(System.getProperty("line.separator"));
sb.append("\n");
if(index%10000==0){
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
@@ -14,12 +14,45 @@ class FieldStat {
static main(args) {
def task = new FieldStat();
// task.run()
task.run1()
// task.run1()
// task.run2()
task.run3()
}
def run3(){
def citys = CommonDatautils.findAllCity()//["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def dist_total = 0
def addr_alias_total = 0
def rets = []
rets << ["城市","有地址别名总量","小区总量","有地址别名小区占比"]
for(def city : citys){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "addr_alias"]);
def alias_index = 0
def alias_total = 0
for(def d : districts){
def alias = d.addr_alias
if(alias){
alias_index++
}
}
def precent = 0
if(districts.size()){
precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size())
}
rets << [city, alias_index, districts.size(), precent+"%"]
}
ExcelUtils.write("d:\\数据\\全国城市地址别名统计.xlsx", rets, "统计")
// println dist_total
// println alias_total
// println addr_alias_total
}
def run1(){
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def citys = CommonDatautils.findAllCity()//["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def dist_total = 0
def addr_alias_total = 0
@@ -37,10 +37,11 @@ class SeaweedDataMatchProcess {
rMap = OpenApiUtils.matchDistrict(city, name, region, false)
}
if(!rMap && address){
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
if(rMap && rMap.region && !region.contains(rMap.region)){
rMap = null
def flag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(flag){
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
}
}
@@ -31,18 +31,24 @@ class GuangfaBlankSelectDistrict {
static main(args) {
def task = new GuangfaBlankSelectDistrict();
task.run()
// task.run()
task.run2()
}
def run(){
def file = "D:\\数据\\广发\\获取楼盘信息需求(发送版).xlsx"
def file = "D:\\数据\\广发\\获取楼盘信息需求(东莞和杭州).xlsx"
def cityMap = standardRegion(file)
def priceMap = getZhiDaoPrice(file)
selectDistricts(cityMap, priceMap)
println "写入完成"
}
def run2(){
selectDistricts2()
println "写入完成"
}
def selectDistricts(cityMap, priceMap){
def rets = []
rets << ["城市","区域","指导价","楼盘名称","楼盘地址","开发商名称","容积率","物业名称","*套数","*楼龄","户型","*建筑面积区间","2021年1-6月成交量","*均价","*成交量","*均价","*成交量","*均价"]
@@ -107,6 +113,70 @@ class GuangfaBlankSelectDistrict {
ExcelUtils.write("d:\\数据\\广发\\楼盘(西安东莞杭州).xlsx", rets, "数据")
}
def selectDistricts2(){
def rets = []
rets << ["城市","区域","指导价","楼盘名称","楼盘地址","开发商名称","容积率","物业名称","*套数","*楼龄","户型","*建筑面积区间","2021年1-6月成交量","*均价","*成交量","*均价","*成交量","*均价"]
def citys = ["东莞","杭州"]
citys.each{city->
def thirdPriceMap = findNearThirdPartyPrice(city)
def retMaps = findCreditHightDistrict3(city, "2021-08-22", thirdPriceMap)
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias","estate_company", "developer","cubage_percent","household_count", "finish_building_date"])
//基本情况合格的小区
def distMap = [:]
for(def d : districts){
// && d.household_count && d.household_count>=50
def distKeys = retMaps.get(d.region)
if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){
if(d.finish_building_date){
def year = CcbCodeUtils.parseBuildYear(d.finish_building_date)
if(year){
def mod = 2021-year
if(mod <=15){
d.house_year = mod
}else{
d.house_year = null
}
}else{
d.house_year = null
}
}else{
d.house_year = null
}
distMap.put(d.city+"##"+d.region+"##"+d.name, d)
}
// if(d.finish_building_date){
// def year = CcbCodeUtils.parseBuildYear(d.finish_building_date)
// if(year){
// def mod = 2021-year
// if(mod <=15){
// def distKeys = retMaps.get(d.region)
// if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){
// d.house_year = mod
// distMap.put(d.city+"##"+d.region+"##"+d.name, d)
// }
// }
// }
// }
}
println city +" 基本情况合格的小区:"+distMap.size()
for(def r : regions){
def names = retMaps.get(r)
if(names){
def results = getRoomTypeAndTradeInfo(names, distMap, priceMap)
rets.addAll(results)
}
}
println "结束"
}
println "开始写入Excel....."+rets.size()
ExcelUtils.write("d:\\数据\\广发\\楼盘(西安东莞杭州).xlsx", rets, "数据")
}
def getRoomTypeAndTradeInfo(def names, def distMap, def priceMap){
def rets = Collections.synchronizedList(new ArrayList());
@@ -332,4 +402,52 @@ class GuangfaBlankSelectDistrict {
}
return thirdPriceMap
}
def findNearThirdPartyPrice(def city){
def q = new BasicDBObject()
q.put("city", city)
q.put("s_date", new BasicDBObject('$gte', 202105))
q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true))
def dbcursor = AVERAGE_STATISTICS.find(q)
def thirdPriceMap = [:]
for(def d : dbcursor){
def key = d.city+"#"+d.region+"#"+d.name
if(!thirdPriceMap.containsKey(key)){
thirdPriceMap.put(d.city+"#"+d.region+"#"+d.name, d.lianjia_esf_price)
}
}
return thirdPriceMap
}
def findCreditHightDistrict3(def city, def regions, def weekend, def thirdPriceMap){
def q = new BasicDBObject()
q.put("city", city)
q.put("value_type", 2)
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd"))
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
def retMaps = [:]
for(def d : dbcursor){
def thirdPrice = thirdPriceMap.get(d.city +"#"+d.region+"#"+d.district_name)
if(thirdPrice){
def precent = (thirdPrice-d.avg_price)*100/d.avg_price
if(Math.abs(precent)>=0 && Math.abs(precent)<=10){
def rets = retMaps.get(d.region)
if(!rets){
rets = []
retMaps.put(d.region, rets)
}
rets << d.city+"##"+d.region+"##"+d.district_name
}
}else{
def rets = retMaps.get(d.region)
if(!rets){
rets = []
retMaps.put(d.region, rets)
}
rets << d.city+"##"+d.region+"##"+d.district_name
}
}
return retMaps;
}
}
@@ -0,0 +1,64 @@
package com.fangjia.script.gj.location
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.NameProcess
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
class DistrictLocationUpdate {
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
static main(args) {
def task = new DistrictLocationUpdate()
task.run()
}
def run(){
def citys = CommonDatautils.findAllCity()
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias", "lng", "lat"])
updateLocation(city, dists);
}
}
def queryPoiMap(def city){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1")
sqlBuffer.append(" and cityname = '" + city + "'")
sqlBuffer.append(" and source = '¸ßµÂ'")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def retMap = [:]
while (rs.next()) {
def map = [:]
map.city = rs.getString("cityname")
map.region = rs.getString("adname")
map.name = rs.getString("name")
map.address = rs.getString("address")
map.type = rs.getString("type")
def location = rs.getObject("location")
if(location){
map.lng = location.x
map.lat = location.y
}
def name = NameProcess.nameStandard(map.city, map.region, map.name)
if(name){
retMap.put(map.region+"##"+name, map)
}
}
return retMap;
}
}
@@ -521,6 +521,10 @@ class GaoDeUtils {
return KeyList.get(count);
}
static void main(def a){
println search("如东县大豫镇豫东村委会","南通")
}
}
@@ -0,0 +1,45 @@
package com.fangjia.script.gj.test
import com.fangjia.core.RegionData
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
class MatchDistrict {
static main(args) {
def task = new MatchDistrict()
task.run()
}
def run(){
def filePath = "D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx"
def city = "深圳"
ExcelUtils.multiThreadingModify(filePath, { data ->
if(data['A']){
def region = RegionData.formatRegion(city, data['A'])
if(region){
def rMap = SeaweedDataMatchProcess.doMatch(city, data['A'], data['C'], null);
if(rMap){
data['E'] = rMap.city
data['F'] = rMap.region
data['G'] = rMap.name
}
}
}
}, 0);
}
}
//{
// "_id" : ObjectId("607963dc5bb87e2a18f4dde6"),
// "guiding_type" : NumberInt(0),
// "estate_type" : "普通住宅",
// "house_type" : NumberInt(0),
// "c_date" : ISODate("2021-04-16T10:15:56.651+0000"),
// "guiding_person" : "深圳房管局",
// "guiding_date" : ISODate("2021-04-10T16:00:00.000+0000"),
// "category" : "district",
// "guiding_value" : NumberInt(48300),
// "name" : "中粮云景国际",
// "city" : "深圳",
// "region" : "光明"
//}
@@ -1,33 +1,55 @@
package com.fangjia.script.gj.test
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.Mongo
import groovy.sql.Sql
import java.text.SimpleDateFormat
//SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
//Date date = new Date(1617211269868);
//print dateFormat.format(date);
DBCollection CITY_INFO_CONN = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_data_ana').getCollection('city_info');
def path = "D:\\数据\\广发\\广东省小区及均价覆盖率统计.xlsx"
ExcelUtils.modify(path,{d->
def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["name"])
d['G'] = districts.size()
//DBCollection CITY_INFO_CONN = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_data_ana').getCollection('city_info');
//def path = "D:\\数据\\广发\\广东省小区及均价覆盖率统计.xlsx"
//ExcelUtils.modify(path,{d->
// def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["name"])
// d['G'] = districts.size()
//
// def q = new BasicDBObject()
// q.put("city", d['B'])
// q.put("weekend", DateUtils.parseFromTightYYYYMMDD("2021-08-08"))
// def obj = CITY_INFO_CONN.findOne(q);
// if(obj){
// d['H'] = obj.price_coverage*100
// }
//})
def q = new BasicDBObject()
q.put("city", d['B'])
q.put("weekend", DateUtils.parseFromTightYYYYMMDD("2021-08-08"))
def obj = CITY_INFO_CONN.findOne(q);
if(obj){
d['H'] = obj.price_coverage*100
def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'";
def countMap = [:]
onLineSqlInstance.eachRow(sql,{r->
def addTime = r.visit_time
def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM")
def ips = countMap.get(month)
if(!ips){
ips = new HashSet();
countMap.put(month, ips)
}
ips << r.ip
});
countMap.each {k,v->
println k+" "+v.size()
}
})
@@ -2,6 +2,8 @@ package com.fangjia.script.jiangnanblank
import com.jcraft.jsch.*
import javax.transaction.SystemException
/**
* Created by gaojuan on 2017/12/6.
*/
@@ -47,12 +49,23 @@ class SFTP {
* @param sftp
*/
public void upload(String directory, String uploadFile, ChannelSftp sftp) {
try {
FileInputStream stream = null
try{
sftp = createDir(directory, sftp)
sftp.cd(directory);
File file=new File(uploadFile);
sftp.put(new FileInputStream(file), file.getName());
stream = new FileInputStream(file)
sftp.put(stream, file.getName());
} catch (Exception e) {
e.printStackTrace();
} finally {
if(stream!=null){
stream.close();
}
// if (sftp != null) {
// sftp.disconnect();
// sftp.exit();
// }
}
}
@@ -100,4 +113,51 @@ class SFTP {
}
/**
* 创建一个文件目录
*/
public ChannelSftp createDir(String createpath, ChannelSftp sftp) {
try {
if (isDirExist(createpath, sftp)) {
sftp.cd(createpath);
}
def pathArry = createpath.split("/");
StringBuffer filePath = new StringBuffer("/");
for (String path : pathArry) {
if (path.equals("")) {
continue;
}
filePath.append(path + "/");
if (isDirExist(filePath.toString(), sftp)) {
sftp.cd(filePath.toString());
} else {
// 建立目录
sftp.mkdir(filePath.toString());
// 进入并设置为当前目录
sftp.cd(filePath.toString());
}
}
sftp.cd(createpath);
} catch (SftpException e) {
throw new SystemException("创建路径错误:" + createpath);
}
return sftp
}
/**
* 判断目录是否存在
*/
public boolean isDirExist(String directory, ChannelSftp sftp) {
boolean isDirExistFlag = false;
try {
SftpATTRS sftpATTRS = sftp.lstat(directory);
isDirExistFlag = true;
return sftpATTRS.isDir();
} catch (Exception e) {
if (e.getMessage().toLowerCase().equals("no such file")) {
isDirExistFlag = false;
}
}
return isDirExistFlag;
}
}
@@ -0,0 +1,58 @@
package com.fangjia.script.test
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class RegionalismCodeStat {
DBCollection REGIONALISM_CODE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "regionalism_code", "192.168.0.235", 27018)
static main(args) {
def task = new RegionalismCodeStat()
task.run()
}
def run(){
def provinces = ["上海市","北京市","天津市","重庆市","福建省","贵州省","湖南省","广东省","湖北省","安徽省","四川省","江苏省","吉林省","河北省","河南省","海南省","江西省","云南省","甘肃省","山东省","陕西省","浙江省","青海省","辽宁省","黑龙江省","山西省","西藏自治区","新疆维吾尔自治区","宁夏回族自治区","内蒙古自治区","广西壮族自治区"]
// def provinces = ["宁夏回族自治区"]
provinces.each {v->
def sets = findCount(v)
println v +" "+ sets.size()
}
}
def findCount(v){
def q = new BasicDBObject()
q.put("province", v)
def dbcursor = REGIONALISM_CODE_COLL.find(q).batchSize(5000)
def set = new HashSet()
for(def obj : dbcursor){
// if(obj.city=="省直辖县级行政区划" || obj.city=="自治区直辖县级行政区划"){
// //set << obj.region
// println "【"+obj.region+"】 过滤"
// continue
// }else{
// if(obj.region && !set.contains(obj.region)){
// set << obj.region
// println "【"+obj.region+"】 add"
// }
// }
if(obj.juweihui){
def k = obj.street+"##"+obj.juweihui
if(!set.contains(k)){
set << k
// println "【"+k+"】 add"
}
}
}
// set.each {k->
// println k
// }
return set;
}
}
@@ -4,6 +4,8 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.linxsh.util.CommonUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
@@ -59,7 +61,7 @@ class DataStandardBatch extends BaseBatch{
private static def limit = 5000;
def static final SHORT_DATE = Utils.getLastMonth()
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6]
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
@@ -81,7 +83,6 @@ class DataStandardBatch extends BaseBatch{
@Override
void process(Object args) {
def flag = true;
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
@@ -120,26 +121,35 @@ class DataStandardBatch extends BaseBatch{
void run() {
try{
println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】"
// if(dbo.avg_price){
if(dbo.name){
//标准化城市(包含region)
def data = standardCity(dbo);
if(data){
//匹配区域
data = standardRegion(data);
if(data){
//标准化小区名
def data3 = standardDistrict(data, dbo.avg_price);
if(data.city=="三亚" && data.region=="陵水"){
data.city = "陵水"
data.region = ""
def data3 = standardDistrictSimple(data, dbo.avg_price);
if(data3){
successCount++;
}
}else if(data){
data = standardRegion(data);
if(data){
//标准化小区名
def data3 = standardDistrict(data, dbo.avg_price);
if(data3){
successCount++;
}
}
}
}
// }else{
// saveLog(dbo, "无价格");
// updateStatus(dbo,statusMap.get("无价格"))
// }
}else{
saveLog(dbo, "无名称");
updateStatus(dbo,statusMap.get("无名称"))
}
count++;
}catch (Exception e){
@@ -253,6 +263,104 @@ class DataStandardBatch extends BaseBatch{
return data;
}
def standardDistrict2(def data, def avg_price){
def newName = NameProcess.nameStandard(data.city, data.region, data.name)
def district = null
if(newName){
district = SeaweedDataMatchProcess.doMatch(data.city, data.region, data.name, data.address);
if(district){
}
}
if(district && avg_price){
removeErrorLog(data);//删除错误日志表中区域不匹配的,正式运行时删除
saveSuccessLog(data, district, type);
updateStatus(data,statusMap.get("小区匹配成功"))
println district.city+" "+district.name+" success"
}else{
if(district){
// data.type = type
saveLog(data, "无价格");
updateStatus(data,statusMap.get("无价格"))
println data.city+" "+data.name+" fail(无价格)"
}else{
println "[匹配失败]"
saveLog(data, "小区匹配失败");
updateStatus(data,statusMap.get("小区匹配失败"))
println data.city+" "+data.name+" fail"
}
}
return district;
}
def standardDistrictSimple(def data, def avg_price){
def newName1 = null
def newName2 = null
if(data.name.contains("别名")){
newName1 = data.name.toString().substring(0, data.name.toString().indexOf("别名")-1)
newName2 = data.name.toString().substring(data.name.toString().indexOf("别名")+3)
}else{
newName1 = NameProcess.nameStandard(data.city, data.region, data.name)
}
def district = null
def type = null
if(newName1){
district = SeaweedDataMatchProcess.doMatch(data.city, data.region, newName1, data.address);
if(district && (district.name == newName1 || (district.alias && district.alias.contains(newName1)))){
district._id = data._id
type = 1
}else{
if(newName2){
district = SeaweedDataMatchProcess.doMatch(data.city, data.region, newName2, data.address);
if(district && (district.name == newName2 || (district.alias && district.alias.contains(newName2)))){
district._id = data._id
type = 1
}else{
def score = CosineSimilarAlgorithm.getSimilarity(district.name, newName2)
if(score>0.8){
district._id = data._id
type = 11
}else{
district = null
}
}
}else{
if(district){
def score = CosineSimilarAlgorithm.getSimilarity(district.name, newName1)
if(score>0.8){
district._id = data._id
type = 11
}else{
district = null
}
}
}
}
}
if(district && avg_price){
removeErrorLog(data);//删除错误日志表中区域不匹配的,正式运行时删除
saveSuccessLog(data, district, type);
updateStatus(data,statusMap.get("小区匹配成功"))
println district.city+" "+district.name+" success"
}else{
if(district){
data.type = type
saveLog(data, "无价格");
updateStatus(data,statusMap.get("无价格"))
println data.city+" "+data.name+" fail(无价格)"
}else{
println "[匹配失败]"
saveLog(data, "小区匹配失败");
updateStatus(data,statusMap.get("小区匹配失败"))
println data.city+" "+data.name+" fail"
}
}
return district;
}
def standardDistrict(def data, def avg_price){
//名称匹配,名称别名匹配,地址别名匹配
//def district = getMatchName(data.city, data.region, data.name, data.address,"district")
@@ -368,30 +476,10 @@ class DataStandardBatch extends BaseBatch{
}
if(district){
if(data.region != district.region){
//type = 12;//区域不同
type = 12;//区域不同
district = null;
}else{
def score1 = 0;
if(data.address && district.address){
score1 = CosineSimilarAlgorithm.getSimilarityRatio(data.address, district.address);
}
if(score1 && score1 >=0.8){
type = 13;//地址相似度在0.8以上
}else{
if(addr && (district.address && addr.indexOf(district.address)>-1||district.address.toString().indexOf(addr)>-1) && (data.name.indexOf(district.name)>-1 || district.name.toString().indexOf(data.name)>-1)){
type = 13;
}else{
if(!addr){
def openDistName = district.name.replace("小区","")
def name = data.name.replace("小区","")
if(openDistName.contains(name) || name.contains(openDistName)){
type = 15;//匹配接口通过名称配上,名称包含
}
}
}
//type = 11;
district = null;
}
type=15
}
}
}
@@ -495,10 +583,16 @@ class DataStandardBatch extends BaseBatch{
}*/
def getMatchNameFromMap(def city, def region, def disName, def address){
def getMatchNameFromMap(def city, def region, def disName, String address){
//查询房价网是否存在该小区
def dbo = findSameNameFromMap(city, region, disName);
dbo = addressCheck(dbo, region, address);
def addrFlag = null
if(address){
addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(addrFlag){
dbo = addressCheck(dbo, region, address);
}
}
if(!dbo){
//别名
def distrs = findAliasNameFromMap(city, region, disName);
@@ -506,7 +600,7 @@ class DataStandardBatch extends BaseBatch{
dbo = distrs[0]
}else if(distrs && distrs.size()>1 && address){
for(def d : distrs){
if(d.address.contains(address)||d.address==address || address.contains(d.address)){
if(addrFlag && (d.address.contains(address)||d.address==address || address.contains(d.address))){
dbo = d;
break;
}
@@ -570,10 +664,16 @@ class DataStandardBatch extends BaseBatch{
return dbo;
}*/
def getMatchAddressFromMap(def city, def region, def address, def name){
def dbo = findSameNameFromMap(city, region, address);
def dbo = null
if(address && isStandardRoad(address)){
dbo = findSameNameFromMap(city, region, address);
}
if(!dbo){
def result = findAddressFromMap(city, region, address);
if(result.size()==1){
def result = null
if(address && isStandardRoad(address)){
result = findAddressFromMap(city, region, address);
}
if(result && result.size()==1){
dbo = result[0];
}else if(result && result.size()>1 && name){
def score = null;
@@ -621,6 +721,15 @@ class DataStandardBatch extends BaseBatch{
return dbo;
}
def isStandardRoad(def s){
def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)");
def matcher = pattern.matcher(s)
if(matcher.find()){
return true;
}
return false
}
def addressCheck(def dbo, def region, def address){
if(dbo && !region && address && address.length()>3){
if(dbo.address.contains(address)||address.contains(dbo.address)||dbo.address==address){
@@ -673,7 +782,6 @@ class DataStandardBatch extends BaseBatch{
def findAllNoMatchData(){
DBObject qdata = new BasicDBObject();
qdata.put("s_date", SHORT_DATE)
// qdata.put("name", "银河湾花园(别墅)")
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
def ret = []
try{
@@ -38,6 +38,152 @@ class Esf58Crawl extends BaseBatch{
def static useDistKeys = []
static def totalCount = new AtomicLong()
def cityList = [
"合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319",
"安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","淮北":"huaibei|9357","滁州":"chuzhou|10266",
"马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323",
"池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296",
"宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232",
"福州":"fz|304","厦门":"xm|606","泉州":"qz|291","莆田":"pt|2429","漳州":"zhangzhou|710",
"宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752","武夷山":"wuyishan|10761",
"石狮":"shishi|296","晋江":"jinjiangshi|297","南安":"nananshi|293","龙海":"longhai|713","上杭":"shanghangxian|6757",
"福安":"fuanshi|7969","福鼎":"fudingshi|7970","安溪":"anxixian|7100","永春":"yongchunxian|7101","永安":"yongan|2133",
"漳浦":"zhangpu|717",
"深圳":"sz|4","广州":"gz|3","东莞":"dg|413","佛山":"fs|222","中山":"zs|771",
"珠海":"zh|910","惠州":"huizhou|722","江门":"jm|629","汕头":"st|783","湛江":"zhanjiang|791",
"肇庆":"zq|901","茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389","清远":"qingyuan|7303",
"阳江":"yj|2284","韶关":"sg|2192","河源":"heyuan|10467","云浮":"yf|10485","汕尾":"sw|9449",
"潮州":"chaozhou|10461","台山":"taishan|11263","阳春":"yangchun|8566","顺德":"sd|8716","惠东":"huidong|725",
"博罗":"boluo|726","海丰":"haifengxian|9444","开平":"kaipingshi|634","陆丰":"lufengshi|9456",
"南宁":"nn|845","柳州":"liuzhou|7133","桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046",
"北海":"bh|10536","贵港":"gg|6770","钦州":"qinzhou|2335","百色":"baise|10513","河池":"hc|2340",
"来宾":"lb|10552","贺州":"hezhou|10549","防城港":"fcg|10539","崇左":"chongzuo|10524","桂平":"guipingqu|6774",
"北流":"beiliushi|9168","博白":"bobaixian|9173","岑溪":"cenxi|2119",
"贵阳":"gy|2015","遵义":"zunyi|7620","黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506",
"毕节":"bijie|10564","铜仁":"tr|10417","安顺":"anshun|7468","黔西南":"qxn|10434","仁怀":"renhuaishi|7628",
"清镇":"qingzhen|12703",
"兰州":"lz|952","天水":"tianshui|8601","白银":"by|10304","庆阳":"qingyang|10475","平凉":"pl|7154",
"酒泉":"jq|10387","张掖":"zhangye|10454","武威":"wuwei|10448","定西":"dx|10322","金昌":"jinchang|7428",
"陇南":"ln|10415","临夏":"linxia|7112","嘉峪关":"jyg|10362","甘南":"gn|10343","敦煌":"dunhuang|10390",
"海口":"haikou|2053","三亚":"sanya|2422","五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136",
"文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184",
"东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","保亭":"baoting|10367","白沙":"baish|10380",
"儋州":"danzhou|10394",
"郑州":"zz|342","洛阳":"luoyang|556","新乡":"xx|1016","南阳":"ny|592","许昌":"xc|977",
"平顶山":"pds|1005","安阳":"ay|1096","焦作":"jiaozuo|3266","商丘":"sq|1029","开封":"kaifeng|2342",
"濮阳":"puyang|2346","周口":"zk|933","信阳":"xy|8694","驻马店":"zmd|1067","漯河":"luohe|2347",
"三门峡":"smx|9317","鹤壁":"hb|2344","济源":"jiyuan|9918","明港":"mg|8541","鄢陵":"yanling|9123",
"禹州":"yuzhou|979","长葛":"changge|9344","灵宝":"lingbaoshi|9307","杞县":"qixianqu|7389","汝州":"ruzhou|1010",
"项城":"xiangchengshi|935","偃师":"yanshiqu|7121","长垣":"changyuan|5936","滑县":"huaxian|5405","林州":"linzhou|1101",
"沁阳":"qinyang|3268","孟州":"mengzhou|3267","温县":"wenxian|7312","尉氏":"weishixian|7391","兰考":"lankaoxian|7393",
"通许":"tongxuxian|7390","新安":"lyxinan|11217","伊川":"yichuan|11220","孟津":"mengjinqu|7122","宜阳":"lyyiyang|11219",
"舞钢":"wugang|1011","永城":"yongcheng|1032","睢县":"suixian|1038","鹿邑":"luyi|939","渑池":"yingchixian|9322",
"沈丘":"shenqiu|942","太康":"taikang|938","商水":"shangshui|936","淇县":"qixianq|9186","浚县":"junxian|9185",
"范县":"fanxian|7285","固始":"gushixian|8698","淮滨":"huaibinxian|8702","邓州":"dengzhou|595","新野":"xinye|603",
"哈尔滨":"hrb|202","大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718",
"佳木斯":"jms|6776","鸡西":"jixi|7289","双鸭山":"sys|9837","鹤岗":"hegang|9061","黑河":"heihe|9862",
"伊春":"yich|9773","七台河":"qth|9848","大兴安岭":"dxal|9878","安达":"shanda|6720","肇东":"shzhaodong|6721",
"肇州":"zhaozhou|382",
"武汉":"wh|158","宜昌":"yc|858","襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032",
"黄石":"hshi|1734","孝感":"xiaogan|3434","黄冈":"hg|2299","恩施":"es|2302","荆门":"jingmen|2296",
"咸宁":"xianning|9617","鄂州":"ez|9709","随州":"suizhou|9656","潜江":"qianjiang|9669","天门":"tm|9517",
"仙桃":"xiantao|9736","神农架":"snj|9605","宜都":"yidou|864","汉川":"hanchuan|3439","枣阳":"zaoyang|896",
"武穴":"wuxueshi|7362","钟祥":"zhongxiangshi|9119","京山":"jingshanxian|9117","沙洋":"shayangxian|9118","松滋":"songzi|3484",
"广水":"guangshuishi|9657","赤壁":"chibishi|9623","老河口":"laohekou|895","谷城":"gucheng|899","宜城":"yichengshi|897",
"南漳":"nanzhang|898","云梦":"yunmeng|3438","安陆":"anlu|3442","大悟":"dawu|3437","孝昌":"xiaochang|3436",
"当阳":"dangyang|865","枝江":"zhijiang|866","嘉鱼":"jiayuxian|9624","随县":"suixia|9660",
"长沙":"cs|414","株洲":"zhuzhou|1086","益阳":"yiyang|10198","常德":"changde|872","衡阳":"hy|914",
"湘潭":"xiangtan|2047","岳阳":"yy|821","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756",
"永州":"yongzhou|2307","娄底":"ld|9481","湘西":"xiangxi|10219","张家界":"zjj|6788","醴陵":"liling|1091",
"澧县":"lixian|876","桂阳":"czguiyang|5699","资兴":"zixing|5698","永兴":"yongxing|5701","常宁":"changningshi|921",
"祁东":"qidongxian|5690","衡东":"hengdong|5693","冷水江":"lengshuijiangshi|9470","涟源":"lianyuanshi|9471","双峰":"shuangfengxian|9473",
"邵阳县":"shaoyangxian|6955","邵东":"shaodongxian|6954","沅江":"yuanjiangs|10201","南县":"nanxian|10202","祁阳":"qiyang|8532",
"湘阴":"xiangyin|828","华容":"huarong|830","慈利":"cilixian|6791","攸县":"zzyouxian|1095",
"石家庄":"sjz|241","保定":"bd|424","唐山":"ts|276","廊坊":"lf|772","邯郸":"hd|572",
"秦皇岛":"qhd|1078","沧州":"cangzhou|652","邢台":"xt|751","衡水":"hs|993","张家口":"zjk|3328",
"承德":"chengde|6760","定州":"dingzhou|8398","馆陶":"gt|8706","张北":"zhangbei|11201","赵县":"zx|9048",
"正定":"zd|3198","迁安市":"qianan|284","任丘":"renqiu|656","三河":"sanhe|776","武安":"wuan|577",
"雄安新区":"xionganxinqu|111234","燕郊":"lfyanjiao|12804","涿州":"zhuozhou|428","河间":"hejian|658","黄骅":"huanghua|657",
"沧县":"cangxian|659","磁县":"cixian|591","涉县":"shexian|14059","霸州":"bazhou|775","香河":"xianghe|5395",
"固安":"lfguan|12803","遵化市":"zunhua|283","迁西":"qianxixian|7061","玉田":"yutianxian|7060","滦南":"luannanxian|7066",
"沙河":"shaheshi|755",
"苏州":"su|5","南京":"nj|172","无锡":"wx|93","常州":"cz|463","徐州":"xz|471",
"南通":"nt|394","扬州":"yz|637","盐城":"yancheng|613","淮安":"ha|968","连云港":"lyg|2049",
"泰州":"taizhou|693","宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279",
"如皋":"rugao|397","启东":"qidong|400","溧阳":"liyang|469","海门":"haimen|399","东海":"donghai|2147",
"扬中":"yangzhong|649","兴化":"xinghuashi|699","新沂":"xinyishi|478","泰兴":"taixing|696","如东":"rudong|402",
"邳州":"pizhou|477","沛县":"xzpeixian|11349","靖江":"jingjiang|698","建湖":"jianhu|618","海安":"haian|401",
"东台":"dongtai|615","丹阳":"danyang|648","宝应县":"baoyingx|14451","灌南":"guannan|2150","灌云":"guanyun|2148",
"姜堰":"jiangyan|697","金坛":"jintan|468","昆山":"szkunshan|16","泗洪":"sihong|5958","泗阳":"siyang|5959",
"句容":"jurong|650","射阳":"sheyang|621","阜宁":"funingxian|620","响水":"xiangshui|619","盱眙":"xuyi|976",
"金湖":"jinhu|975","江阴":"jiangyins|34984",
"南昌":"nc|669","赣州":"ganzhou|2363","九江":"jj|2247","宜春":"yichun|5709","吉安":"ja|2364",
"上饶":"sr|10120","萍乡":"px|2248","抚州":"fuzhou|10134","景德镇":"jdz|2360","新余":"xinyu|10115",
"鹰潭":"yingtan|3209","永新":"yxx|11077","乐平":"lepingshi|9048","进贤":"jinxian|677","分宜":"fenyi|10118",
"丰城":"fengchengshi|5711","樟树":"zhangshu|5713","高安":"gaoan|5712","余江":"yujiang|3210","南城":"nanchengx|10137",
"浮梁":"fuliangxian|9071",
"长春":"cc|319","吉林":"jl|700","四平":"sp|10171","延边":"yanbian|3184","松原":"songyuan|2315",
"白城":"bc|5918","通化":"th|10159","白山":"baishan|10179","辽源":"liaoyuan|2501","公主岭":"gongzhuling|10171",
"梅河口":"meihekou|10162","扶余":"fuyuxian|9085","长岭":"changlingxian|9084","桦甸":"huadian|706","磐石":"panshi|708",
"梨树县":"lishu|10176",
"沈阳":"sy|188","大连":"dl|147","鞍山":"as|523","锦州":"jinzhou|2354","抚顺":"fushun|5722",
"营口":"yk|5898","盘锦":"pj|2041","朝阳":"cy|10106","丹东":"dandong|3445","辽阳":"liaoyang|2038",
"本溪":"benxi|5845","葫芦岛":"hld|10088","铁岭":"tl|6729","阜新":"fx|10097","庄河":"pld|3306",
"瓦房店":"wfd|3279","灯塔":"dengta|2071","凤城":"fengcheng|3450","北票":"beipiao|10109","开原":"kaiyuan|6733",
"银川":"yinchuan|2054","吴忠":"wuzhong|9962","石嘴山":"szs|9971","中卫":"zw|9951","固原":"guyuan|2421",
"呼和浩特":"hu|811","包头":"bt|801","赤峰":"chifeng|6700","鄂尔多斯":"erds|2037","通辽":"tongliao|10015",
"呼伦贝尔":"hlbe|10039","巴彦淖尔市":"bycem|10070","乌兰察布":"wlcb|9993","锡林郭勒":"xl|2408","兴安盟":"xam|9976",
"乌海":"wuhai|2404","阿拉善盟":"alsm|10083","海拉尔":"hlr|2043",
"西宁":"xn|2052","海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909",
"黄南":"huangnan|9896","玉树":"ys|9888","海南":"hainan|10574","格尔木":"geermushi|9904",
"青岛":"qd|122","济南":"jn|265","烟台":"yt|228","潍坊":"wf|362","临沂":"linyi|505",
"淄博":"zb|385","济宁":"jining|450","泰安":"ta|686","聊城":"lc|882","威海":"weihai|518",
"枣庄":"zaozhuang|961","德州":"dz|728","日照":"rizhao|3177","东营":"dy|623","菏泽":"heze|5632",
"滨州":"bz|944","莱芜":"lw|2292","章丘":"zhangqiu|8680","垦利":"kl|11313","诸城":"zc|9146",
"寿光":"shouguang|369","龙口":"longkou|233","曹县":"caoxian|5638","单县":"shanxian|5636","肥城":"feicheng|690",
"高密":"gaomi|371","广饶":"guangrao|627","桓台":"huantaixian|7335","莒县":"juxian|3180","莱州":"laizhou|235",
"蓬莱":"penglai|237","青州":"qingzhou|367","荣成":"rongcheng|522","乳山":"rushan|520","滕州":"tengzhou|967",
"新泰":"xintai|689","招远":"zhaoyuan|3325","邹城":"zoucheng|455","邹平":"zouping|946","临清":"linqing|884",
"茌平":"chiping|887","郓城":"hzyc|5637","博兴":"boxing|949","东明":"dongming|5641","巨野":"juye|5640",
"无棣":"wudi|951","齐河":"qihe|734","微山":"weishan|459","禹城":"yuchengshi|731","临邑":"linyixianq|739",
"乐陵":"leling|730","莱阳":"laiyang|234","宁津":"ningjin|733","高唐":"gaotang|885","莘县":"shenxian|888",
"阳谷":"yanggu|886","冠县":"guanxian|890","平邑":"pingyi|514","郯城":"tancheng|510","沂源":"yiyuanxian|7334",
"汶上":"wenshang|460","梁山":"liangshanx|462","利津":"lijin|628","沂南":"yinanxian|7301","栖霞":"qixia|238",
"宁阳":"ningyang|691","东平":"dongping|692","昌邑":"changyishi|372","安丘":"anqiu|370","昌乐":"changle|373",
"临朐":"linqu|374","鄄城":"juancheng|5635",
"太原":"ty|740","临汾":"linfen|5669","大同":"dt|6964","运城":"yuncheng|5653","晋中":"jz|8854",
"长治":"changzhi|6921","晋城":"jincheng|3350","阳泉":"yq|8760","吕梁":"lvliang|3222","忻州":"xinzhou|3453",
"朔州":"shuozhou|9871","临猗":"linyixian|9193","清徐":"qingxu|10908","柳林":"liulin|3225","高平":"gaoping|3354",
"泽州":"zezhou|3353","襄垣":"xiangyuanxian|6928","孝义":"xiaoyi|3227",
"西安":"xa|483","咸阳":"xianyang|7453","宝鸡":"baoji|2044","渭南":"wn|5733","汉中":"hanzhong|3163",
"榆林":"yl|5942","延安":"yanan|8973","安康":"ankang|3157","商洛":"sl|9854","铜川":"tc|9832",
"神木":"shenmu|5944","韩城":"hancheng|5735","府谷":"fugu|5945","靖边":"jingbian|5947","定边":"dingbian|5948",
"成都":"cd|102","绵阳":"mianyang|1057","德阳":"deyang|2373","南充":"nanchong|2378","宜宾":"yb|2380",
"自贡":"zg|6745","乐山":"ls|3237","泸州":"luzhou|2372","达州":"dazhou|9799","内江":"scnj|5928",
"遂宁":"suining|9688","攀枝花":"panzhihua|2371","眉山":"ms|9704","广安":"ga|2381","资阳":"zy|6803",
"凉山":"liangshan|9717","广元":"guangyuan|9749","雅安":"ya|9687","巴中":"bazhong|9811","阿坝":"ab|9817",
"甘孜":"ganzi|9764","安岳":"anyuexian|6806","广汉":"guanghanshi|8719","简阳":"jianyangshi|6805","仁寿":"renshouxian|9706",
"射洪":"shehongxian|9694","大竹":"dazu|9806","宣汉":"xuanhan|9804","渠县":"qux|9807","长宁":"changningx|7148",
"乌鲁木齐":"xj|984","昌吉":"changji|8582","巴音郭楞":"bygl|9530","伊犁":"yili|9472","阿克苏":"aks|9499",
"喀什":"ks|9326","哈密":"hami|7452","克拉玛依":"klmy|2042","博尔塔拉":"betl|9529","吐鲁番":"tlf|9475",
"和田":"ht|9489","石河子":"shz|9551","克孜勒苏":"kzls|9519","阿拉尔":"ale|9539","五家渠":"wjq|9562",
"图木舒克":"tmsk|9559","库尔勒":"kel|7168","阿勒泰":"alt|18837","塔城":"tac|18845",
"拉萨":"lasa|2055","日喀则":"rkz|9615","山南":"sn|9576","林芝":"linzhi|9646","昌都":"changdu|9648",
"那曲":"nq|9618","阿里":"al|9678","日土":"rituxian|9682","改则":"gaizexian|9684",
"昆明":"km|541","曲靖":"qj|2389","大理":"dali|2398","红河":"honghe|2394","玉溪":"yx|2040",
"丽江":"lj|2392","文山":"ws|2395","楚雄":"cx|2393","西双版纳":"bn|2397","昭通":"zt|9409",
"德宏":"dh|9437","普洱":"pe|9444","保山":"bs|2390","临沧":"lincang|9422","迪庆":"diqing|9432",
"怒江":"nujiang|9462","弥勒":"milexian|8892","安宁":"anningshi|547","宣威":"xuanwushi|7533",
"杭州":"hz|79","宁波":"nb|135","温州":"wz|330","金华":"jh|531","嘉兴":"jx|497",
"台州":"tz|403","绍兴":"sx|355","湖州":"huzhou|831","丽水":"lishui|7921","衢州":"quzhou|6793",
"舟山":"zhoushan|8481","乐清":"yueqingcity|13950","瑞安":"ruiancity|13951","义乌":"yiwu|12291","余姚":"yuyao|5333",
"诸暨":"zhuji|357","象山":"xiangshanxian|6738","温岭":"wenling|408","桐乡":"tongxiang|502","慈溪":"cixi|5334",
"长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536",
"安吉":"anji|836","苍南":"cangnanxian|7579","临海":"linhai|407","永康":"yongkang|537","玉环":"yuhuan|409",
"平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361",
"江山":"jiangshanshi|6796","平阳":"pingyangxian|7575"
]
static main(args) {
runBatch(new Esf58Crawl(), args)
}
@@ -45,28 +191,41 @@ class Esf58Crawl extends BaseBatch{
void process(Object args) {
//addContent("¸£ÖÝ","https://fz.58.com/xiaoqu/", 70);
for(def i=0;i<5;i++){
def city = "¸£ÖÝ"
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
distMap.put(city, districts)
def citys = ["成都","广州","深圳","杭州","南京","郑州","苏州","武汉","青岛","石家庄","太原","济南","长沙","沈阳","宁波","佛山","珠海","昆明","温州","无锡","哈尔滨","南宁","东莞","大连","海口","潍坊","绵阳","乌鲁木齐","惠州"]
cityList.each{city, str->
for(def i=0;i<1;i++){
// def city = "福州"
if(citys.contains(city)){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
if(districts){
def arrs = str.split("\\|")
def url = "https://"+arrs[0]+".58.com/xiaoqu/"
distMap.put(city, districts)
addCrawlKeys(city);
addDetailContent(city, "https://fz.58.com/xiaoqu/");
addCrawlKeys(city);
addDetailContent(city, url);
Thread.sleep(1000*60*10)
// Thread.sleep(1000*60*10)
}
}
}
}
}
def addCrawlKeys(city){
def districts = distMap.get(city);
for(def i=0;i<districts.size();i++){
def districtName = districts[i].name
println "${city}¡¾${i}/${districts.size()}¡¿ "+districtName
for(def j=1;j<=districtName.length()-1;j++) {
def keyword = districtName.toString().substring(0, j);
def keyStr = city+"##"+keyword;
if(!useDistKeys.contains(keyStr)){
useDistKeys << keyStr
if(j<=2){
def keyword = districtName.toString().substring(0, j);
def keyStr = city+"##"+keyword;
if(!useDistKeys.contains(keyStr)){
useDistKeys << keyStr
}
}
}
}
@@ -17,7 +17,7 @@ class SynchDistrictPrice {
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产"]
def sourceMap = ["lianjia.com":"链家"]
def sourceMap = ["cityhouse.cn":"城市房产"]//["zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()
@@ -28,23 +28,23 @@ class SynchDistrictPrice {
def citys = CommonUtils.findAllCity()
sourceMap.each { k, v ->
for(def city : citys){
if(!["上海","北京","海口"].contains(city)){
def rets = findCrawlDistrictDetails(city, k, v)
println city +" "+rets.size()
synchData(city, rets, v)
}
def rets = findCrawlDistrictDetails(city, k, v)
println city +" "+rets.size()
synchData(city, rets, v)
}
}
}
def findCrawlDistrictDetails(city, root, source){
def curMonthFirstDay = DateUtils.getFirstDayOfMonth(new Date())
def q = new BasicDBObject()
q.put("city", city)
q.put("root", root)
q.put('m_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-07-01")));
// q.put("name","兰州万达城")
q.put('m_date', new BasicDBObject('$gte', curMonthFirstDay));
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
@@ -53,6 +53,7 @@ class SynchDistrictPrice {
keys.put("address", 1)
keys.put("avg_price", 1)
keys.put("root", 1)
keys.put("c_date", 1)
keys.put("url", 1)
def rets = []
def dbcursor = districtDetailColl.find(q, keys)
@@ -95,6 +96,8 @@ class SynchDistrictPrice {
thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
addIndex++
thirdPartyColl.save(r)
}else{
thirdPartyColl.save(r)
}
// if(!obj){
// addIndex++
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
for(def source :sources){
def count1 = findTotalCount(source, 202106);
def count2 = findTotalCount(source, 202107);
def count1 = findTotalCount(source, 202108);
def count2 = findTotalCount(source, 202109);
println source +" "+count1 +" "+count2
}
}
@@ -41,8 +41,10 @@ class ThirdDataToGatherBatch extends BaseBatch{
@Override
void process(Object args) {
println SHORT_DATE
def citys = CommonDatautils.findAllCity();
def weekend = DateUtils.getFirstDayOfWeek(new Date());
//def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD("2021-07-15"));
def cityIndex = 0;
final CountDownLatch latch = new CountDownLatch(citys.size())
citys.each { city ->
@@ -80,7 +80,7 @@ class ElasticUtils {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 180);
return requestConfigBuilder;
}
})
@@ -0,0 +1,32 @@
package com.fangjia.script.wenyan.auto
import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
//def keys = new BasicDBObject()
//keys.city = 1
//keys.region = 1
//keys.name = 1
//keys.estate_type2 = 1
//["ÖØÇì"].each {def city->
// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject seaweed->
// BasicDBObject stat = DbUtils.findDistrictStat(seaweed, DateUtil.getFirstDayOfWeek(new Date()))
// if(!stat){
// CommonUtils.recalculationAverage(seaweed.city, seaweed.region, seaweed.name, seaweed.estate_type2)
// println(seaweed)
// }
// }, keys)
//
//}
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\³É½»¼Û¡ª¡ªÔ¤²â¼Ò.xlsx",{List item->
BasicDBObject query = DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"])
BasicDBObject seaweed = DbUtils.seaweed.findOne(query)
if(seaweed){
BasicDBObject stat = DbUtils.findDistrictStat(seaweed, item["E"])
item["H"] = stat?.credit
}
println(item)
})
@@ -0,0 +1,113 @@
package com.fangjia.script.wenyan.auto
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.apache.lucene.index.Term
import org.apache.lucene.search.BooleanClause
import org.apache.lucene.search.BooleanQuery
import org.apache.lucene.search.TermQuery
/**
* 移除小区名中的区域信息
* @param name
* @param region
* @return
*/
String removeRegion(String name, String region){
if(name.startsWith(region) && name.length() - region.length() > 2){
return name.replaceFirst(region, "")
}
return name;
}
def String getStandardAddress(def city, def addr) {
if (StringUtils.isBlank(addr)) {
return null
}
JSONObject result = OpenApiClient.addressStandardize(city, addr)
if (result.containsKey("standardAddress") && result.getBoolean("standardAddress")) {
def standardAddr = result.getString("street")
if (result.containsKey("streetNumber")) {
standardAddr += result.getString("streetNumber")
}
return standardAddr
}
return null
}
def NOW = new Date()
def getType(String type){
if(type == null){
return "普通住宅"
}
if(type.contains("商住两用")){
return "商住"
}else if(type.contains("别墅")){
return "别墅"
}else{
return "普通住宅"
}
}
ExcelUtils.modify("D:\\nextcloud\\共享文档\\项目文档\\重庆农商行\\新增小区4.xlsx",{List item->
if(item["C"] != "明佳园"){
return
}
BasicDBObject seaweed = new BasicDBObject()
seaweed.city = item["A"]
seaweed.region = item["B"]
seaweed.name = item["C"]
def query = new BasicDBObject("city", seaweed.city).append("region", seaweed.region)
.append('name', seaweed.name)
.append("visible", 0)
.append("cat", "district");
BasicDBObject one = DbUtils.seaweed.findOne(query)
if(!one){
query.removeField("name")
query.append("alias", seaweed.name)
one = DbUtils.seaweed.findOne(query)
}
if (one && one.status == 0) {
item["i"] = "已存在"
return;
}
seaweed.address = getStandardAddress( seaweed.city, item["D"])
String source = item["H"]
Double lng = item["E"]
Double lat = item["F"]
if(lng){
if(source.contains("高德")){
seaweed.lng = (long)(lng * 10000000000l)
seaweed.lat = (long)(lat * 10000000000l)
JSONObject geo = BaiDuApiUtils.googleToBaidu(lat, lng)
seaweed.lng2 = (long)(geo.lng * 10000000000l)
seaweed.lat2 = (long)(geo.lat * 10000000000l)
}else if(source.contains("百度")){
Map<String, Double> point = GeoUtils.geoconvBaiduToGoogle(lng, lat);
long google_lat = (long)(point.get("lat") * 10000000000l);
long google_lng = (long)(point.get("lng") * 10000000000l);
long baiDu_lat = (long)(lat * 10000000000l)
long baiDu_lng = (long)(lng * 10000000000l)
seaweed.lng = google_lng
seaweed.lat = google_lat
seaweed.lat2 = baiDu_lat
seaweed.lng2 = baiDu_lng
}
}
seaweed.cat = "district"
seaweed.status = 0
seaweed.c_date = NOW
seaweed.ip = "MACHINE_SAVE"
seaweed.estate_type2 = getType(item["G"])
Map<String, Object> markMap = new HashMap<String, Object>()
seaweed.mark_map = markMap
markMap.put("data_source", "机器入库")
DbUtils.seaweed.save(seaweed)
println(seaweed)
})
@@ -1,16 +1,42 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.RegionData
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.common.ServerAreaData
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import net.sf.json.JSONObject
def data = [["城市", "区域", "小区名", "别名", "地址", "地址别名"]]
["ID", "北京", "重庆", "哈尔滨"].each {def city->
DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(city)).each {BasicDBObject district->
def item = [district._id.toString(), district.city, district.region, district.name, district.alias?.join(","), district.address, district.addr_alias?.join(",")]
println(item)
data << item
//city_sell_stat = MongoUtils.getDbCollection("fangjia_stat", "city_sell_stat")
//data = [["城市", "202101", "202106"]]
//CommonUtils.findAllCity().each {def city->
// BasicDBObject serverArea = ServerAreaData.getServerAreaByDisplay(city)
// if(!serverArea.ranking || !serverArea.ranking.toString().contains("县级行政单位")){
// BasicDBObject pre = city_sell_stat.findOne(new BasicDBObject("city", city).append("date", 202101))
// BasicDBObject now = city_sell_stat.findOne(new BasicDBObject("city", city).append("date", 202106))
// println([serverArea.display, pre?.avg_price, now?.avg_price])
// data << [serverArea.display, pre?.avg_price, now?.avg_price]
// }
//}
//ExcelUtils.write("D:\\城市均价数据.xlsx", data, "城市均价")
//ExcelUtils.multiThreadingModify("C:\\Users\\wenyan\\Downloads\\经纬度错误数据修正.xlsx",{List item->
// String city = item["B"]
// String region = RegionData.formatRegion(city, item["D"])
// String sql = "select count(1) as count from latest_seawater where cat='sell' and city='${item["B"]}' and region='${region}' and district_name='${item["F"]}'"
// ElasticUtils.executeSqlQuery(sql,{Map result->
// item["P"] = result.get("count")
// })
// println(item)
//})
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\装修案例.xlsx",{List item->
JSONObject json = OpenApiClient.addressMatch("西安", item["H"], "property", false, item["C"])
if(json.name){
item["I"] = json.district
item["J"] = json.name
}
}
ExcelUtils.write("D:\\待抓取小区.xlsx", data, "数据")
})
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20210401", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210501", "yyyyMMdd")
def startDate = DateUtil.parseDate("20210901", "yyyyMMdd")
def endDate = DateUtil.parseDate("20211001", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -25,7 +25,7 @@ while (startDate.before(endDate)) {
OPEN_URI_LOG.find(and(eq("user_id", "1007705"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
if (!hashSet.contains(result)) {
def item = ["5c6535b99c213017e0768ee1", "qlkj", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
def item = ["60a3202c0b42a00001aad385", "qlkjgx", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
data << item
hashSet << result
} else {
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20210401", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210501", "yyyyMMdd")
def startDate = DateUtil.parseDate("20210901", "yyyyMMdd")
def endDate = DateUtil.parseDate("20211001", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -42,7 +42,7 @@ while (startDate.before(endDate)) {
keys.addAll(params.keys())
def key = "${params.address}_${params.size}".toString()
if (!hashSet.contains(key)) {
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date.format("yyy"), doc.data_status, doc.code, doc.msg]
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg]
param_keys.each {
item << params[it]
}
@@ -261,15 +261,16 @@ import org.dom4j.io.SAXReader
// data << [dbo.parameter_map.city, dbo.parameter_map.address, dbo.res_map.result.province, dbo.res_map.result.city, dbo.res_map.result.district, dbo.res_map.result.developmentZone, dbo.res_map.result.township, dbo.res_map.result.street,dbo.res_map.result.streetNumber,dbo.res_map.result.name,dbo.res_map.result.buildingNumber,dbo.res_map.result.unitNumber,dbo.res_map.result.roomNumber,dbo.res_map.result.formattedAddress]
//}
//ExcelUtils.write("D:\\建行测试地址.xlsx", data, "结果")
DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com")
def query = new BasicDBObject()
query.cat = "region"
query.city= "ÉϺ£"
query.weekend = Date.parse("yyyyMMdd", "20210228")
def doc = []
def prices = []
SQMETRE_RENT_COLL.find(query, new BasicDBObject("region", 1).append("avg_price", 1)).each {
doc.add(new BasicDBObject("name", it.region+"Çø").append("value", it.avg_price))
prices << it.avg_price
}
println(JSONArray.fromObject(doc).toString())
//DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com")
//def query = new BasicDBObject()
//query.cat = "region"
//query.city= "ÉϺ£"
//query.weekend = Date.parse("yyyyMMdd", "20210228")
//def doc = []
//def prices = []
//SQMETRE_RENT_COLL.find(query, new BasicDBObject("region", 1).append("avg_price", 1)).each {
// doc.add(new BasicDBObject("name", it.region+"Çø").append("value", it.avg_price))
// prices << it.avg_price
//}
//println(JSONArray.fromObject(doc).toString())
DbUtils.seaweed.remove(new BasicDBObject("_id", DbUtils.seaweed.findById("6103bf1c64baf60001e7e1d0")._id))
@@ -1,6 +1,11 @@
import com.fangjia.core.util.CNNMFilter
import com.fangjia.core.util.HttpRequest
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import net.sf.json.JSONObject
import org.apache.commons.io.FileUtils
//
@@ -35,7 +40,79 @@ import org.apache.commons.io.FileUtils
// println("copy:" + file.getAbsolutePath() + "\tto:" + ("E:\\成交数据\\DBF汇总\\" + file.getName()))
// }
//}
DSCollection seawater = new DataStore("192.168.9.118", 27777).getCollection("fangjia.seawater")
seawater.find(new BasicDBObject("cat", "sell").append("city", "上海").append("region", "长宁").append("district_name", "新泾三村")).each {
println(it)
}
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\副本地址标准化验证结果分析.xlsx",{List item->
// HttpRequest request = new HttpRequest()
// JSONObject json = request.getJsonContent2("http://localhost:8082/structure", ["address": item["A"]], "UTF-8")
// if(json){
// item["D"] = json.result.province
// item["E"] = json.result.city
// item["F"] = json.result.district
// item["G"] = json.result.dev
// item["H"] = json.result.township
// item["I"] = json.result.village
// item["J"] = json.result.group
// item["K"] = json.result.street
// item["L"] = json.result.streetNumber
// item["M"] = json.result.name
// item["N"] = json.result.buildingNumber
// item["O"] = json.result.unitNumber
// item["P"] = json.result.roomNumber
// item["q"] = json.result.floor
// item["r"] = json.result.formatAddress
// }
// println(item)
//})
//for(int i=1; i < 100; i++){
// println("${i}号楼#@#building_number")
// println("${i}栋#@#building_number")
// println("${i}幢#@#building_number")
// println("${i}单元#@#unit_number")
// if(i< 20){
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}号楼#@#building_number")
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}栋#@#building_number")
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}幢#@#building_number")
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}单元#@#unit_number")
// }
//}
//nums= "abcdefghijklmn"
//for(def num : nums.toCharArray()){
// println("${num}栋#@#building_number")
// println("${num}幢#@#building_number")
// println("${num}单元#@#unit_number")
//}
//nums= "abcdefghijklmn"
//for(def num : nums.toCharArray()){
// println("${num}座#@#building_number")
// for(int i = 1; i <11; i++){
// println("${num}${i}座#@#building_number")
//// println("${num}${i}幢#@#building_number")
//// println("${num}${i}单元#@#unit_number")
// }
//}
//for(int i = 1; i <20; i++){
// println("${i}区#@#part_number")
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}区#@#part_number")
// println("${i}期#@#stage_number")
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}期#@#stage_number")
//}
//
//for(def num : "东南北中西甲乙丙丁".toCharArray()){
// println("${num}区#@#part_number")
//}
//for(int i = -5; i < 100; i++){
// println("${i}楼#@#floor")
// if(i < 10 && i > 0){
// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}楼#@#floor")
// }
//}
//for(def num : "abcdefg东南西北".toCharArray()){
// println("${num}户#@#room")
//}
//for(i in 1.. 40){
// for(j in 1..9){
// println("${i}0${j}室#@#room")
// }
//}
ElasticUtils.executeSqlQuery("select * from seawater where cat='sell' and district_name='新泾三村' and city='上海' and p_date > 20210101",{Map doc->
println(doc.avg_price + "\t" + doc.root + "\t" + doc.p_date)
})
@@ -912,7 +912,6 @@ public class EvaluationRateServiceImpl implements EvaluationRateService {
evaluatePrice = (long) (evaluatePrice * houseFloorRate);
}
}else{
int mid = evaluation.getHeight()/2;
double num = getFloorRate(mid, flag);
evaluatePrice = (long) (evaluatePrice * num * houseFloorRate);