This commit is contained in:
2022-10-31 16:52:13 +08:00
76 changed files with 4562 additions and 2289 deletions
@@ -693,12 +693,6 @@ public class District2Controller {
if (null != district.getLat2() && district.getLat2() > 0) {
data.put("lat2", district.getLat2());
}
if (null != district.getLng3() && district.getLng3() > 0) {
data.put("lng3", district.getLng3());
}
if (null != district.getLat3() && district.getLat3() > 0) {
data.put("lat3", district.getLat3());
}
if(district.getLat2()>0 && district.getLng2() >0){
Map<String, Double> map = GeocodeUtils.geoconvBaiduToGoogle(district.getLng2()/10000000000d, district.getLat2()/10000000000d);
@@ -2034,14 +2034,6 @@ public class DistrictController {
if (null != district.getLat2() && district.getLat2() > 0) {
data.put("lat2", district.getLat2());
}
// 新加腾讯地图
if (null != district.getLng3() && district.getLng3() > 0) {
data.put("lng3", district.getLng3());
}
if (null != district.getLat3() && district.getLat3() > 0) {
data.put("lat3", district.getLat3());
}
if (district.getLat2() > 0 && district.getLng2() > 0) {
Map<String, Double> map = GeocodeUtils.geoconvBaiduToGoogle(district.getLng2() / 10000000000d,
@@ -2054,12 +2046,8 @@ public class DistrictController {
district.setLat(lat);
district.setLng(lng);
}
district.setLat3(lat);
district.setLng3(lng);
data.put("lat", district.getLat());
data.put("lng", district.getLng());
data.put("lat3", district.getLat3());
data.put("lng3", district.getLng3());
}
data.put("name", district.getName());
@@ -3305,8 +3293,6 @@ public class DistrictController {
put("lng2", "百度经度");
put("lat", "高德纬度");
put("lng", "高德经度");
put("lat3", "腾讯纬度");
put("lng3", "腾讯经度");
put("loopLineInfo", "环线位置");
put("intelligenceLevel", "智能水平");
}
@@ -192,31 +192,6 @@
<input name="district.lat" type="text" id="dLat2" value="${district.lat!0}" /> 注:填入整数,如393422150000
</td>
</tr>
<tr>
<td class = "t1">
<#if markMap?? && markMap["lng3"]?? && markMap["lng3"]?is_number && markMap["lng3"] == 1>
<i class="lock" name="lng3"></i>
<#else>
<i class="unlock" name="lng3"></i>
</#if>
经度(腾讯)<font color="red">*</font></td>
<td>
<input name="district.lng3" type="text" value="${district.lng3!0}" />
</td>
</tr>
<tr>
<td class = "t1">
<#if markMap?? && markMap["la3t"]?? && markMap["lat3"]?is_number && markMap["lat3"] == 1>
<i class="lock" name="lat3"></i>
<#else>
<i class="unlock" name="lat3"></i>
</#if>
纬度(腾讯)<font color="red">*</font></td>
<td>
<input name="district.lat3" type="text" value="${district.lat3!0}" /> 注:填入整数,如393422150000
</td>
</tr>
<tr>
<td class = "t1">
<#if markMap?? && markMap["address"]?? && markMap["address"]?is_number && markMap["address"] == 1>
@@ -202,7 +202,6 @@
<label class="checkbox"><input type="checkbox" id="col31a" temp="31" rank="1" name="chkCol" f_name="calcType" <#if columnIds?index_of('col31a')!=-1>checked</#if>>计算方式</label>
<label class="checkbox"><input type="checkbox" id="col32a" temp="32" rank="1" name="chkCol" f_name="lng2_lat2" <#if columnIds?index_of('col32a')!=-1>checked</#if>>百度</label>
<label class="checkbox"><input type="checkbox" id="col33a" temp="33" rank="1" name="chkCol" f_name="lng_lat" <#if columnIds?index_of('col33a')!=-1>checked</#if>>高德</label>
<label class="checkbox"><input type="checkbox" id="col34a" temp="34" rank="1" name="chkCol" f_name="lng3_lat3" <#if columnIds?index_of('col34a')!=-1>checked</#if>>腾讯</label>
<label class="checkbox"><input type="checkbox" id="col35a" temp="35" rank="1" name="chkCol" f_name="districtSummary" <#if columnIds?index_of('col35a')!=-1>checked</#if>>小区备注</label>
<label class="checkbox"><input type="checkbox" id="col37a" temp="36" rank="1" name="chkCol" f_name="mappingAlias" <#if columnIds?index_of('col37a')!=-1>checked</#if>>映射别名</label>
<label class="checkbox"><input type="checkbox" id="col38a" temp="38" rank="1" name="chkCol" f_name="propertyLife" <#if columnIds?index_of('col38a')!=-1>checked</#if>>产权年限</label>
@@ -671,10 +670,7 @@
<td align="center" name="col33a" class="<#if columnIds?index_of('col33a')==-1>dn</#if>">
<#if district.lat??>${district.lng/10000000000},${district.lat/10000000000}</#if>
</td>
<td align="center" name="col34a" class="<#if columnIds?index_of('col34a')==-1>dn</#if>">
<#if district.lat3??>${district.lng3/10000000000},${district.lat3/10000000000}</#if>
</td>
<td align="center" name="col21a" class="<#if columnIds?index_of('col21a')==-1>dn</#if>">
<input class="cat" data-id="${district.id}" data-cat="office" data-name="${district.name!}" type="button" value="写字楼"/>
<input class="cat" data-id="${district.id}" data-cat="shop" data-name="${district.name!}" type="button" value="商铺"/>
@@ -89,23 +89,6 @@
<input name="district.lat" type="text" /> 注:填入整数,如393422150000
</td>
</tr>
<tr>
<td class = "t1">
经度(腾讯)<font color="red">*</font></td>
<td>
<input name="district.lng3" type="text" />
</td>
</tr>
<tr>
<td class = "t1">
纬度(腾讯)<font color="red">*</font></td>
<td>
<input name="district.lat3" type="text" /> 注:填入整数,如393422150000
</td>
</tr>
<tr>
<td class = "t1">地址<font color="red">*</font></td>
<td>
@@ -145,33 +145,6 @@
<input name="district.lat" type="text" value="${district.lat!0}" /> 注:填入整数,如393422150000
</td>
</tr>
<tr>
<td class = "t1">
<#if markMap?? && markMap["lng3"]?? && markMap["lng3"] == 1>
<i class="lock" name="lng3"></i>
<#else>
<i class="unlock" name="lng3"></i>
</#if>
经度(腾讯)<font color="red">*</font></td>
<td>
<input name="district.lng3" type="text" value="${district.lng3!0}" />
</td>
</tr>
<tr>
<td class = "t1">
<#if markMap?? && markMap["lat3"]?? && markMap["lat3"] == 1>
<i class="lock" name="lat3"></i>
<#else>
<i class="unlock" name="lat3"></i>
</#if>
纬度(腾讯)<font color="red">*</font></td>
<td>
<input name="district.lat3" type="text" value="${district.lat3!0}" /> 注:填入整数,如393422150000
</td>
</tr>
<tr>
<td class = "t1">
<#if markMap?? && markMap["address"]?? && markMap["address"] == 1>
@@ -25,8 +25,6 @@ import java.util.zip.GZIPOutputStream;
import com.hankcs.hanlp.HanLP;
import com.hankcs.hanlp.dictionary.py.Pinyin;
import com.sun.javafx.scene.transform.TransformUtils;
import net.sf.json.JSON;
import net.sf.json.JSONArray;
import net.sf.json.JSONNull;
import net.sf.json.JSONObject;
@@ -142,7 +142,6 @@
<div class="columnBox">
<p>CPDB大数据库,包含70多个维度的数据,各种行业及场景的应用</p>
<img src="${staticUrl}/images/fangjia/v5/cpdb_img01.png" class="mobile01">
<img src="${staticUrl}/images/fangjia/v5/cpdb_img01_m.png" class="mobile02" style="margin-bottom:3rem;">
<img src="${staticUrl}/images/fangjia/v5/column_bg01.png" class="column_bg" style="top:550px;">
</div>
@@ -65,8 +65,6 @@ public class District implements SeaweedEntity {
private Long lng;
private Long lat2;//°ÙÈ
private Long lng2;
private Long lat3;//ÌÚÑ
private Long lng3;
private String developer;
private String estateCompany;
private String allArea;
@@ -1075,21 +1073,6 @@ public class District implements SeaweedEntity {
this.lng2 = lng2;
}
public void setLat3(Long lat3) {
this.lat3 = lat3;
}
@Column(name = "lat3")
public Long getLat3() {
return lat3;
}
public void setLng3(Long lng3) {
this.lng3 = lng3;
}
@Column(name = "lng3")
public Long getLng3() {
return lng3;
}
@Column(name = "finish_building_date")
public String getFinishBuildingDate() {
return finishBuildingDate;
@@ -386,24 +386,24 @@ public class FshController {
Double oneMonthMarkup = null;
Evaluation evaluation = null;
if(model.isYouDa()){
if("上海".equals(model.getCity())){
YdDealPrice ydDealPrice = ydDealPriceService.findYdDealPrice(model.getCity(), model.getDistrict(), model.getName());
if(ydDealPrice == null){
mav.setModel(BasicModel.buildOKModel().setMsg("该小区暂不支持估价").markNoData());
return mav;
}else{
evaluation = evaluationService.ydEvaluate(district, buildYdEvaluation(model, request), model.getCalDate(),ydDealPrice.getPrice());
//近30天的涨跌幅:取最近1个月至3个月之间最近的那期价格和当前均价相比
Date end = DateUtil.getDateOfLastMonth(new Date(),-1);
Date start = DateUtil.getDateOfLastMonth(new Date(),-3);
List<YdDealPrice> ydDealPrices = ydDealPriceService.findYdDealPrices(model.getCity(), model.getDistrict(), model.getName(),start,end);
if(ydDealPrices!=null && ydDealPrices.size()>0){
oneMonthMarkup = (double)(ydDealPrices.get(0).getPrice()-ydDealPrice.getPrice())/ydDealPrice.getPrice();
}
}
}else{
// if("上海".equals(model.getCity())){
// YdDealPrice ydDealPrice = ydDealPriceService.findYdDealPrice(model.getCity(), model.getDistrict(), model.getName());
// if(ydDealPrice == null){
// mav.setModel(BasicModel.buildOKModel().setMsg("该小区暂不支持估价").markNoData());
// return mav;
// }else{
// evaluation = evaluationService.ydEvaluate(district, buildYdEvaluation(model, request), model.getCalDate(),ydDealPrice.getPrice());
// //近30天的涨跌幅:取最近1个月至3个月之间最近的那期价格和当前均价相比
// Date end = DateUtil.getDateOfLastMonth(new Date(),-1);
// Date start = DateUtil.getDateOfLastMonth(new Date(),-3);
// List<YdDealPrice> ydDealPrices = ydDealPriceService.findYdDealPrices(model.getCity(), model.getDistrict(), model.getName(),start,end);
// if(ydDealPrices!=null && ydDealPrices.size()>0){
// oneMonthMarkup = (double)(ydDealPrices.get(0).getPrice()-ydDealPrice.getPrice())/ydDealPrice.getPrice();
// }
// }
// }else{
evaluation = evaluationService.evaluate(district, buildYdEvaluation(model, request), model.getCalDate());
}
// }
}else{
evaluation = evaluationService.evaluate(district, buildEvaluation(model, request), model.getCalDate());
}
@@ -1163,18 +1163,18 @@ public class FshController {
private Map<String, Integer> buildDistrictAvgZouShiByMonth(DistrictModel model,DistrictKey districtKey, Date end, Date start) {
Map<String, List<Integer>> avgPriceDetailMap = new HashMap<String, List<Integer>>();
if(model.isYouDa() && "上海".equals(model.getCityName())){
List<YdDealPrice> list = ydDealPriceService.findYdDealPrices(districtKey.getCity(), districtKey.getRegion(), districtKey.getName(), start, end);
for (YdDealPrice stat : list) {
String ws = DateUtil.formatByPattern(stat.getWeekend(), "yyyy-MM");
List<Integer> prices = avgPriceDetailMap.get(ws);
if(null == prices){
prices = new ArrayList<Integer>();
avgPriceDetailMap.put(ws, prices);
}
prices.add(stat.getPrice());
}
}else{
// if(model.isYouDa() && "上海".equals(model.getCityName())){
// List<YdDealPrice> list = ydDealPriceService.findYdDealPrices(districtKey.getCity(), districtKey.getRegion(), districtKey.getName(), start, end);
// for (YdDealPrice stat : list) {
// String ws = DateUtil.formatByPattern(stat.getWeekend(), "yyyy-MM");
// List<Integer> prices = avgPriceDetailMap.get(ws);
// if(null == prices){
// prices = new ArrayList<Integer>();
// avgPriceDetailMap.put(ws, prices);
// }
// prices.add(stat.getPrice());
// }
// }else{
List<DistrictStat> list = districtStatService.findDistrictPrice(districtKey.getCity(), districtKey.getRegion(), districtKey.getName(), start, end);
for (DistrictStat stat : list) {
String ws = DateUtil.formatByPattern(stat.getWeekend(), "yyyy-MM");
@@ -1185,7 +1185,7 @@ public class FshController {
}
prices.add(stat.getAvgPrice());
}
}
// }
Map<String, Integer> districtMap = new HashMap<String, Integer>();
if(null != avgPriceDetailMap){
@@ -362,7 +362,7 @@ $('.formal_btn').on('click', function () {
});
function clickFormalReport(){
$.prompt('正式报告将尽快邮寄给您,请注意查收,谢谢!如有疑问可与我们联系:62392317', '请填写正式报告邮寄地址', function (value) {
$.prompt('正式报告将尽快邮寄给您,请注意查收,谢谢! ', '请填写正式报告邮寄地址', function (value) {
var address = $(".modal-text-input").val();
if(address==""){
$.alert('地址不能为空');
@@ -37,14 +37,13 @@
<p style="margin-top:-4px;"><i class="location"></i><small>${fjtEvaluation.address!} <#if fjtEvaluation.houseNum?? && fjtEvaluation.houseNum?contains("号")>${fjtEvaluation.houseNum!}<#else>${fjtEvaluation.houseNum!}号</#if>${fjtEvaluation.roomNum!}室</small> <a href="../district/weXinList?cityName=${fjtEvaluation.city!}" class="collectionLink" style="line-height: 32px;">重新评估&nbsp;></a></p>
<p style="color: #F4A460;font-size: 12px;line-height: 20px;margin-top: -5px;margin-left: 20px;">*请确认与产证地址一致</p>
<span></span>
<#if bohaiEvaluationAudit?? && bohaiEvaluationAudit.auditStatus!=0>
<p style="margin-top:14px;">评估总价:<#if fjtEvaluation??><strong class="<#if bohaiEvaluationAudit?? && bohaiEvaluationAudit.lastPrice??><#else>price_final</#if>">
${(bohaiEvaluationAudit.inquirePrice/10000)?int}</strong>&nbsp;万元</#if></p>
<p>评估单价:<#if bohaiEvaluationAudit.inquirePrice gt 0>${(bohaiEvaluationAudit.inquirePrice/fjtEvaluation.houseArea)?int}</#if>&nbsp;元/平米</p>
<#if bohaiEvaluationAudit?? && bohaiEvaluationAudit.auditStatus!=0 && bohaiEvaluationAudit.inquirePrice gt 0>
<p style="margin-top:14px;">评估总价:<#if fjtEvaluation??><strong class="<#if bohaiEvaluationAudit?? && (bohaiEvaluationAudit.lastPrice?? || bohaiEvaluationAudit.inquirePrice??)>price_final</#if>"><#if bohaiEvaluationAudit.lastPrice??>${(bohaiEvaluationAudit.lastPrice/10000)?int}<#elseif bohaiEvaluationAudit.inquirePrice??>${(bohaiEvaluationAudit.inquirePrice/10000)?int}</#if></strong>&nbsp;万元</#if></p>
<p>评估单价:<#if bohaiEvaluationAudit.lastPrice??>${(bohaiEvaluationAudit.lastPrice/fjtEvaluation.houseArea)?int}<#elseif bohaiEvaluationAudit.inquirePrice??>${(bohaiEvaluationAudit.inquirePrice/fjtEvaluation.houseArea)?int}</#if>&nbsp;元/平米</p>
<#if role?? && role==17 && loanInfoMap??>
<p style="margin-top:-12px;">所属分类:${loanInfoMap.type!}类</p>
<p style="margin-top:-10px;">可贷成数:${loanInfoMap.loan_ratio!}成</p>
<p style="margin-top:-22px;">可贷金额:<strong class="price_final">${(bohaiEvaluationAudit.inquirePrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}</strong>万元</p>
<p style="margin-top:-22px;">可贷金额:<strong class="price_final"><#if bohaiEvaluationAudit.lastPrice??>${(bohaiEvaluationAudit.lastPrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}<#elseif bohaiEvaluationAudit.inquirePrice??>${(bohaiEvaluationAudit.inquirePrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}</#if></strong>万元</p>
<p style="margin-top:-4px;">贷款利率:4.5%</p>
</#if>
<#else>
@@ -139,7 +139,7 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
query = new BasicDBObject()
query.append("weekend", weekend)
query.append("cat", "city")
cityCount = SQMETRE_RENT_COLL.count(query)
cityCount = SQMETRE_RENT_COLL.count(query)
//本月区域平米租金计算量
query = new BasicDBObject()
query.append("weekend", weekend)
@@ -23,7 +23,7 @@ class BigdataBuildingBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.citys_all;
def rows = []
def headers = []
for (def city : citys) {
@@ -69,7 +69,7 @@ class BigdataBuildingBatch extends BaseBatch{
q.put("city", city)
q.put("status", 1)
// q.put("name","辽源一村")
def dbcursor = LOU_DONG_STAT.find(q)
def dbcursor = LOU_DONG_STAT.find(q).batchSize(5000)
def map = [:]
def set = new HashSet()
for(def db : dbcursor){
@@ -28,7 +28,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
citys = citys.unique()
def rets = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
@@ -8,6 +8,7 @@ import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
@@ -27,7 +28,7 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataCommunityBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
runBatch(new BigdataCommunityBatch(), args);
@@ -35,7 +36,8 @@ class BigdataCommunityBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
@@ -43,18 +45,19 @@ class BigdataCommunityBatch extends BaseBatch{
//"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
println "starting 。。。。。。。。。"
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"])
println city +" Ð¡ÇøÁ¿£º"+dists.size()
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city))
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets)
println city +" "+rets.size()
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
}
@@ -105,7 +108,6 @@ class BigdataCommunityBatch extends BaseBatch{
if(ret){
rets << ret
}
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -173,38 +175,25 @@ class BigdataCommunityBatch extends BaseBatch{
ret << "pa101"
ret << ""
if(dist.address){
if(dist.address.toString().contains("Ū")){
String number = DataUtil.match(dist.address, Pattern.compile("\\d+Ū"))
if(number){
ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number))
ret << dist.address.toString().substring(dist.address.toString().indexOf(number), dist.address.toString().size())
def jsonObj = null
if(dist.address){
jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address)
}
if(jsonObj && jsonObj.result && jsonObj.result.street){
if(jsonObj.result.street){
ret << jsonObj.result.street
}else{
ret << ""
}
if(jsonObj.result.streetNumber){
ret << jsonObj.result.streetNumber
}else{
ret << ""
}
}else{
def jsonObj = null
if(dist.address){
// jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
jsonObj = OpenApiClient.standard1Action(dist.city, dist.address)
}
if(jsonObj && jsonObj.street){
if(jsonObj.street){
ret << jsonObj.street
}else{
ret << ""
}
if(jsonObj.streetNumber){
ret << jsonObj.streetNumber
}else{
ret << ""
}
}else{
ret << ""
ret << ""
}
ret << ""
ret << ""
}
}else{
ret << ""
@@ -220,10 +209,9 @@ class BigdataCommunityBatch extends BaseBatch{
ret << ""
ret << 0
}catch(Exception e){
println dist.city + " "+dist.name
println dist.city + " "+dist.name+" error"
e.printStackTrace()
}
return ret;
}
}
@@ -15,6 +15,7 @@ import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import groovy.sql.Sql
import java.util.concurrent.ConcurrentHashMap
@@ -32,8 +33,11 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataDealBatch extends BaseBatch{
// def startDate = 202206
// def endDate = 202208
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
@@ -41,6 +45,7 @@ class BigdataDealBatch extends BaseBatch{
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -52,18 +57,23 @@ class BigdataDealBatch extends BaseBatch{
@Override
public void process(Object args) {
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
def citys = CcbCommonUtils.citys;
// def ccbCitys = CcbCommonUtils.citys;
for(def city : citys){
// if(!ccbCitys.contains(city)){
addDealPriceMatch(city);
// }
// def citys = CcbCommonUtils.citys;
def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args)
if(args && args.size()>=2){
curDate = args[1].toString().toInteger()
}
// for(def city : citys){
//// if(!ccbCitys.contains(city)){
// addDealPriceMatch(city);
//// }
// }
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
@@ -79,7 +89,7 @@ class BigdataDealBatch extends BaseBatch{
def trades = getCityData(city, statMap, ppeDists)
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println city +" 成交共:"+rets.size()
println "【${index}】"+city +" 成交共:"+rets.size()
def cityDatas = []
cityDatas << headers
@@ -91,10 +101,8 @@ class BigdataDealBatch extends BaseBatch{
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
}
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成"
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", true);
println "txt 生成 " + rows.size()
}
def createData(def rets, cityCodeDatas, distCodeMap, source){
@@ -102,6 +110,9 @@ class BigdataDealBatch extends BaseBatch{
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
// def lianjiaPriceMap = findThirdPartyPrice("上海")
def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd')
def lianjiaPriceMap = getDistrictStatByCity("上海", startDate)
for(def i=0;i<rets.size();i++){
final def trade = rets[i]
fixedThreadPool.execute(new Runnable() {
@@ -109,7 +120,8 @@ class BigdataDealBatch extends BaseBatch{
try {
totalCount.addAndGet(1)
if(source == "youda"){
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap)
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap, lianjiaPriceMap)
if(ret){
result << ret
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
@@ -135,147 +147,176 @@ class BigdataDealBatch extends BaseBatch{
return result
}
def findThirdPartyPrice(def city) {
def q = new BasicDBObject()
q.put("city", city)
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap){
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte', endDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for (def db : dbcursor) {
def price = null
if (db.lianjia_esf_price && db.lianjia_esf_price > 3000) {
price = db.lianjia_esf_price
}
if (!price && db.beike_esf_price && db.beike_esf_price > 3000) {
price = db.beike_esf_price
}
if(price){
map.put(db.region+"##"+db.name, price)
}
}
return map
}
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap, def lianjiaPriceMap){
def ret = []
def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district")
if(!obj){
return ret;
}
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << trade.city
def data = distCodeMap.get(trade.fjRegion)
if(data){
ret << data.district_code
ret << data.district
}else{
ret << ""
ret << ""
}
def street = ""
def streetNumber = ""
if(trade.address){
if(trade.address.toString().contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
def price = lianjiaPriceMap.get(obj.region+"##"+obj.name)
if(price){
def mod = java.lang.Math.abs((trade.price-price)*100/price)
if(mod<=10){
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << trade.city
def data = distCodeMap.get(trade.fjRegion)
if(data){
ret << data.district_code
ret << data.district
}else{
ret << ""
ret << ""
}
}
if(!street){
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
if(jsonObj && jsonObj.searchAddress){
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){
def street = ""
def streetNumber = ""
if(trade.address){
if(trade.address.toString().contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
}
}else{
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
if(!street){
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
if(jsonObj && jsonObj.searchAddress){
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
}
}else{
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
}
}
}
}
}
if(!street){
if(obj.address){
if(obj.address.toString().contains("弄")){
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
if(number){
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
streetNumber = number
if(!street){
if(obj.address){
if(obj.address.toString().contains("弄")){
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
if(number){
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
streetNumber = number
}
}else{
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
if(jsonObj && jsonObj.searchAddress){
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
}
}
}
ret << street
ret << streetNumber
if(obj){
ret << obj._id.toString()
ret << trade.fjName
}
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
ret << ""//期数
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
if(roomNum && roomNum.contains("0")){
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
}else{
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
if(jsonObj && jsonObj.searchAddress){
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
ret << trade.floor
}
ret << ""
ret << trade.height
def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
def buildType = ''
if(arrs){
buildType = arrs[0]
}
ret << buildType
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
ret << trade.area
ret << ""
ret << getHouseRoom(buildType, trade.room, trade.hall)
ret << trade.room
ret << trade.hall
ret << ""
ret << trade.toilet
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
ret << ""
ret << CcbCodeUtils.getDirectionCode(trade.direction)
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
ret << ""
ret << trade.price
ret << trade.totalPrice * 10000
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
ret << "02"
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << "99"
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << trade.totalPrice * 10000
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
}
}
ret << street
ret << streetNumber
if(obj){
ret << obj._id.toString()
ret << trade.fjName
}
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
ret << ""//期数
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
if(roomNum && roomNum.contains("0")){
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
}else{
ret << trade.floor
}
ret << ""
ret << trade.height
def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
def buildType = ''
if(arrs){
buildType = arrs[0]
}
ret << buildType
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
ret << trade.area
ret << ""
ret << getHouseRoom(buildType, trade.room, trade.hall)
ret << trade.room
ret << trade.hall
ret << ""
ret << trade.toilet
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
ret << ""
ret << CcbCodeUtils.getDirectionCode(trade.direction)
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
ret << ""
ret << trade.price
ret << trade.totalPrice * 10000
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
ret << "02"
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << "99"
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << trade.totalPrice * 10000
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
return ret;
}
@@ -289,7 +330,7 @@ class BigdataDealBatch extends BaseBatch{
}
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << trade.city
ret << CcbCommonUtils.getCcbCity(trade.city)
if(data){
ret << data.district_code
ret << data.district
@@ -543,11 +584,14 @@ class BigdataDealBatch extends BaseBatch{
def getDistrictStatByCity(city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DIST_SELL_STAT.find(q)
q.put("weekend", new BasicDBObject('$gte', weekend))
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("weekend",-1))
def map = new ConcurrentHashMap()
for(def d : dbcursor){
map.put(d.region+"@"+d.district_name, d.avg_price)
def price = map.get(d.region+"##"+d.district_name)
if(!price && d.value_type==2){
map.put(d.region+"##"+d.district_name, d.avg_price)
}
}
return map
}
@@ -569,10 +613,10 @@ class BigdataDealBatch extends BaseBatch{
def addDealPriceMatch(city){
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20220601", "yyyyMMdd"))
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd"))
def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate)
final CountDownLatch latch = new CountDownLatch(trades.size())
@@ -10,6 +10,8 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
/**
* 建行模型估值项目
* 区域数据同步
@@ -17,33 +19,74 @@ import com.mongodb.DBCollection
@BatchOwner(Watcher.GaoJuan)
class BigdataDistrictBatch extends BaseBatch{
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
runBatch(new BigdataDistrictBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rows = []
for(def city : citys){
def datas = findDistrictCodeValue(city)
if(datas){
rows.addAll(datas)
}
println city+" "+ datas.size()
}
def allDatas = createDatas(rows)
def headers = ["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"]
def datas = readText("D:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.txt")
def rets = []
rets << headers
rets.addAll(allDatas)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\BIGDATA_DISTRICT_GJ.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.dat",headers, "utf-8",false);
def index = 0
for(def line : datas){
def arrs = line[0].split("\\|@\\|")
def sql = "select count(*) as count from BIGDATA_DISTRICT_GJ where distid='${arrs[0]}'";
def count = 0
sqlInstance.eachRow(sql, { r ->
count = r.count
})
if(count==0){
rets << line
}
}
writeTxt(rets, "D:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ_add.txt")
// def citys = CcbCommonUtils.citys_all;
// def rows = []
// for(def city : citys){
// def datas = findDistrictCodeValue(city)
// if(datas){
// rows.addAll(datas)
// }
// println city+" "+ datas.size()
// }
//
// def allDatas = createDatas(rows)
//
// def headers = ["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"]
//
// def rets = []
// rets << headers
// rets.addAll(allDatas)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\BIGDATA_DISTRICT_GJ.xlsx", rets, "数据")
// FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.dat",headers, "utf-8",false);
}
def writeTxt(def rows, def path){
StringBuilder builder = new StringBuilder()
for(def row : rows){
builder.append(row)
builder.append('\n')
}
org.apache.commons.io.FileUtils.writeStringToFile(new File(path), builder.toString(), "GBK")
}
private def readText(def fileName){
File file = new File(fileName)
def ret = [];
//对文件内容的每一行进行分割处理,比较常用在处理csv文件
file.splitEachLine(" ") {
def temp = it
ret << temp;
}
//转为Writable对象,可重定向输出
file.asWritable()
return ret;
}
def createDatas(rows){
def allRows = []
def index = 1
@@ -72,16 +115,16 @@ class BigdataDistrictBatch extends BaseBatch{
def findDistrictCodeValue(city){
def q = new BasicDBObject()
q.put("fangjia_city", city)
// q.put("district", new BasicDBObject('$ne',"市辖区"))
def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q)
q.put("fjw_city", city)
// def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q)
def dbcursor = CcbCommonUtils.FANGJIA_AREA_CODE.find(q)
def lists = []
for(def db : dbcursor){
if(db.district_code){
def m = [:]
m.id = db._id.toString()
m.dist_name = db.district
m.dist_spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(db.fangjia_district, "")
m.dist_spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(db.fjw_district, "")
m.city_id = db.city_code
m.dist_code = db.district_code
lists << m
@@ -67,20 +67,39 @@ class BigdataForsaleBatch extends BaseBatch{
@Override
public void process(Object args) {
def index = 0
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
statCity(citys)
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
index++
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220818, 20220919);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
}
}
def statCity(def citys){
def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\ÎÞ·¿Ô´³ÇÊÐ.xlsx",0,1)
def houseCitys = []
for(def d : datas){
houseCitys << d['C']
}
for(def city : citys){
if(!houseCitys.contains(city)){
def scity = CcbCommonUtils.specialCityMap.get(city)
if(!scity){
println city
}
}
}
}
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){
def totalCount = new AtomicLong()
@@ -429,7 +448,6 @@ class BigdataForsaleBatch extends BaseBatch{
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
@@ -538,7 +556,7 @@ class BigdataForsaleBatch extends BaseBatch{
}
rowMap.houseId = seawater.id
rowMap.city = city
rowMap.city = CcbCommonUtils.getCcbCity(city)
//rowMap.region =seawater.region
//rowMap.address = seawater.address
@@ -31,39 +31,40 @@ class BigdataPoiBatch extends BaseBatch{
@Override
public void process(Object args) {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def citys = CcbCommonUtils.citys;
// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx"
def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1)
def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
// def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1)
def citys = CcbCommonUtils.citys_252
// def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_丧葬设施_252.xlsx", 0,1)
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_医院_聚类2_252.xlsx", 0,1)
// def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_地铁_252.xlsx", 0,1)
// def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_公交_252.xlsx", 0,1)
// def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_银行_252.xlsx", 0,1)
def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_学校_252.xlsx", 0,1)
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\八城市医院(二次审核).xlsx", 0,1)
def rets = []
// def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
// rets.addAll(rows1)
def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
rets.addAll(rows1)
def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
rets.addAll(rows2)
//
// def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
// def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
// rets.addAll(rows2)
//
// def rows3 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
// rets.addAll(rows3)
def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
rets.addAll(rows4)
def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
rets.addAll(rows5)
////
// def rows4 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
// rets.addAll(rows4)
//
// def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
// rets.addAll(rows6)
def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
rets.addAll(rows6)
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"]
// def rows5 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
// rets.addAll(rows5)
//
def headers = []//["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"]
def fileName = "/home/goojia/ccb/BIGDATA_POI_GJ.dat"
if(CommonDatautils.isWindow()){
@@ -260,9 +261,9 @@ class BigdataPoiBatch extends BaseBatch{
}else if(type =="丧葬服务"){
ret << "ob101"
}else if(type =="学校"){
if(d['F'].contains("中学")){
if(d['B'].contains("中学")){
ret << "ed138"
}else if(d['F'].contains("小学")){
}else if(d['B'].contains("小学")){
ret << "ed114"
}
}
@@ -16,12 +16,16 @@ import groovy.sql.Sql
class CcbCommonUtils {
//
public static def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
// public static def citys = ["武汉"]
public static def citys_all = [ "长春","上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map_2020", "mongo.db.fangjia.com")
// public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map_2020", "mongo.db.fangjia.com")
public static final DBCollection FANGJIA_AREA_CODE = MongoUtils.getCollectionByName("business", "fangjia_area_code", "mongo.db.fangjia.com")
public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
@@ -34,39 +38,76 @@ class CcbCommonUtils {
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
public static def specialCityMap = ["阿坝州":"阿坝","阿拉善盟":"阿拉善","黄南州":"黄南","巴音郭楞":"巴州","兴安盟":"兴安","锡林郭勒盟":"锡林郭勒","凉山州":"凉山","西双版纳州":"西双版纳","德宏州":"德宏","迪庆州":"迪庆","海北州":"海北","海西州":"海西","博尔塔拉":"博州","甘孜州":"甘孜","玉树州":"玉树","果洛州":"果洛","文山州":"文山","怒江州":"怒江","克孜勒苏":"克州"]
public static DSCollection dataStoreConnection = null;
public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
public static def getRunningCitys(def args){
def citys = []
println "args 参数:"+args.toString()
if(args){
def countStr = args[0]
if(countStr=="344"){
citys = CcbCommonUtils.citys_all
}else if(countStr=="92"){
citys = CcbCommonUtils.citys_92
}else if(countStr=="252"){
citys = CcbCommonUtils.citys_252
}
}else{
citys = CcbCommonUtils.citys_92
}
println "运行城市数量:"+citys.size()
return citys;
}
public static def findCityCodeValue(){
def cityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"]
def cityCodeMap = [:]
FANGJIA_AREA_CODE.find().each {it->
def code = cityCodeMap.get(it.fjw_city)
if(!code){
cityCodeMap.put(it.fjw_city, it.city_code.toString())
}
}
// def oldCityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"]
return cityCodeMap;
}
public static def getCcbCity(def city){
if(city && specialCityMap.containsKey(city)){
return specialCityMap.get(city)
}
return city;
}
public static def findAllDistrictCodeValue(city){
def q = new BasicDBObject()
q.put("fangjia_city", city)
q.put("fjw_city", city)
//q.put("district", new BasicDBObject('$ne',"市辖区"))
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1))
def dbcursor = FANGJIA_AREA_CODE.find(q).sort(new BasicDBObject("_id", 1))
def map = [:]
for(def db : dbcursor){
map.put(db.fangjia_district, db)
map.put(db.fjw_district, db)
}
return map;
}
public static def findAllCityDistrictCodeValue(city){
def q = new BasicDBObject()
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1))
def dbcursor = FANGJIA_AREA_CODE.find(q).sort(new BasicDBObject("_id", 1))
def map = [:]
for(def db : dbcursor){
def distMap = map.get(db.fangjia_city)
def distMap = map.get(db.fjw_city)
if(!distMap){
distMap = [:]
map.put(db.fangjia_city, distMap)
map.put(db.fjw_city, distMap)
}
distMap.put(db.fangjia_district, db)
distMap.put(db.fjw_district, db)
}
return map;
}
@@ -376,7 +417,7 @@ class CcbCommonUtils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","辽源一村")
// q.put("name","云山尚城")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -11,7 +11,7 @@ class CcbFtpTestUtils {
static def ftpPort = 17495
static def ftpUser = "ccb"
static def ftpPassword = "ccb!@#"
static def ftpPath = "/home/ccb/output/"
static def ftpPath = "/home/ccb_evaluation/output/"
static def ftpPath_rh = "/home/ccb_rh/output/"
@@ -70,6 +70,22 @@ class CcbFtpTestUtils {
}
static def listFileNames(month){
//ÉÏ´«µ½ftp
sftpChannel = reConnect();
Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel);
return files
}
static def getFileLength(String path, String month){
sftpChannel = reConnect();
def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel)
return size
}
static main(args) {
@@ -3,11 +3,12 @@ package com.fangjia.script.ccbevaluation
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.jiangnanblank.SFTP
import com.jcraft.jsch.ChannelSftp
import com.jcraft.jsch.SftpATTRS
import org.apache.commons.codec.digest.DigestUtils
class CcbFtpUtils {
static def ftpServer = "192.168.0.45"
static def ftpServer = "192.168.0.45"//192.168.0.45
static def ftpPort = 17495
static def ftpUser = "ccb"
static def ftpPassword = "ccb!@#"
@@ -64,6 +65,13 @@ class CcbFtpUtils {
return files
}
static def getFileLength(String path, String month){
sftpChannel = reConnect();
def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel)
return size
}
static main(args) {
@@ -370,13 +370,12 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = ["上海","广州","东莞"]//CcbCommonUtils.citys
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def month = 202207
def citys = CcbCommonUtils.citys_92
def month = 202209
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys) {
def rets = findPeGteThirdParty(city, 202208)
def rets = findPeGteThirdParty(city, 202210)
def total = 0
println city +" "+rets.size()
for(def m : rets){
@@ -62,7 +62,7 @@ class DistrictGuidePriceBatch extends BaseBatch{
ret << dist._id.toString()
ret << dist.name
ret << cityCodeDatas.get(dist.city)
ret << dist.city
ret << CcbCommonUtils.getCcbCity(dist.city)
ret << areaData.district_code
ret << areaData.district
ret << ps.avg_price
@@ -15,38 +15,42 @@ class ExportOracleData {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def citys = null
static main(args) {
def task = new ExportOracleData()
task.run()
task.run(args)
}
def run() {
// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
def run(args) {
citys = CcbCommonUtils.getRunningCitys(args)
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
def new_dists = []
//不会变
//不会变 10.25
exportEvaluationRateData()
exportBigdataPoiData()
exportBigdataDistrictData()
exportBigdataPoiData()//
exportBigdataDistrictData()//
exportGuidedPrice()//随人工改变
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
exportGuidedPrice()//随人工改变//
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策//
// //处理小区情况
exportBigdataCommunity()
exportCommunityStaging()
exportCommunityBuileType()
exportBigdataCommunity()//
exportCommunityStaging()//没cityid
exportCommunityBuileType()//
exportThirdPartyPrice2(lastMonth, new_dists)
exportThirdPartyPrice2(lastMonth, new_dists)//
exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19
exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01
//第三方网站权重表
exportWebsiteRate();
//计算结果相关
//
//
// //计算结果相关
// exportAvmEvaluationGJNB(curMonth)
// exportAVMEvaluationItemGJNB(curMonth)
//
@@ -58,11 +62,11 @@ class ExportOracleData {
//
// exportCommunityPdescribeGj(curMonth)
// exportCommCaseDetailGj(curMonth)
//
// def curDate = curMonth.toString()+"28"
// SSHHelper jschCommand = new SSHHelper();
// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
//ok文件
//// ok文件
// exportLastFile()
}
@@ -129,24 +133,28 @@ class ExportOracleData {
def exportBigdataPoiData() {
def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.CITYID
data << r.DISTID
data << r.ID
data << r.POI_NAME
data << r.TYPE
data << r.TYPE_CODE
data << r.LATITUDE
data << r.LONGITUDE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj where CITYID='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.CITYID
data << r.DISTID
data << r.ID
data << r.POI_NAME
data << r.TYPE
data << r.TYPE_CODE
data << r.LATITUDE
data << r.LONGITUDE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
@@ -163,8 +171,9 @@ class ExportOracleData {
}
def exportCommunityStaging() {
def sql = "select ID,COMMID,MASTER_COMMID,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_staging_gj";
def allRows = []
def sql = "select ID,COMMID,MASTER_COMMID,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_staging_gj";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
@@ -193,9 +202,9 @@ class ExportOracleData {
// def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def allRows = []
// for(def city : citys){
// def cityid = cityCodeDataMap.get(city)
def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj";
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj where CITYID='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.CITYID
@@ -208,7 +217,8 @@ class ExportOracleData {
data << r.WORKDATE
allRows << data
});
// }
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat"
@@ -224,28 +234,31 @@ class ExportOracleData {
}
def exportBigdataDistrictData() {
def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.DISTID
data << r.DISTNAME
data << r.DISTSPELL
data << r.CITYID
data << r.DISTCODE
data << r.SEQ
data << r.DISTTYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.LJXZQH
data << r.LJXZQHDM
data << r.XZQHDM
data << r.WORKDATE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj where CITYID='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.DISTID
data << r.DISTNAME
data << r.DISTSPELL
data << r.CITYID
data << r.DISTCODE
data << r.SEQ
data << r.DISTTYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.LJXZQH
data << r.LJXZQHDM
data << r.XZQHDM
data << r.WORKDATE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
@@ -262,26 +275,30 @@ class ExportOracleData {
def exportGuidedPrice() {
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMID
data << r.COMMNAME
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.DEALDATE
data << r.AVG_PRICE
data << r.GUIDING_PERSON
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where CITYID='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMID
data << r.COMMNAME
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.DEALDATE
data << r.AVG_PRICE
data << r.GUIDING_PERSON
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
@@ -296,25 +313,29 @@ class ExportOracleData {
}
def exportCommunityGuidedPrice(def date) {
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where createtime>=to_date('${date}','yyyyMMdd')";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMID
data << r.COMMNAME
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.AVGPRICE
data << r.SOURCETYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' and createtime>=to_date('${date}','yyyyMMdd')";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMID
data << r.COMMNAME
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.AVGPRICE
data << r.SOURCETYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
@@ -331,30 +352,33 @@ class ExportOracleData {
def exportBigdataDeal(def dealDate, def new_dists) {
def allRows = []
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" +
"HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" +
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
"from bigdata_deal_gj where dealdate>=to_date('${dealDate}','yyyy-MM-dd') and distid not in ${newstr}"
def rets1 = dealData(sql1)
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" +
"HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" +
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
"from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd') and distid not in ${newstr}"
def rets1 = dealData(sql1)
def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" +
"HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" +
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
"from bigdata_deal_gj where distid in ${newstr}"
def rets2 = dealData(sql2)
allRows.addAll(rets1)
allRows.addAll(rets2)
}else{
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" +
"HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" +
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
"from bigdata_deal_gj where dealdate>=to_date('${dealDate}','yyyy-MM-dd')";
allRows = dealData(sql)
def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" +
"HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" +
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
"from bigdata_deal_gj where CITYID='${cityCode}' and distid in ${newstr}"
def rets2 = dealData(sql2)
allRows.addAll(rets1)
allRows.addAll(rets2)
}else{
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" +
"HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" +
"HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" +
"PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" +
"from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd')";
allRows.addAll(dealData(sql))
}
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
@@ -363,8 +387,6 @@ class ExportOracleData {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "BIGDATA_DEAL_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
@@ -497,25 +519,6 @@ class ExportOracleData {
def exportThirdPartyPrice2(price_date, new_dists) {
def allRows = []
if(new_dists) {
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " +
"COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " +
"price_date=${price_date} and distid not in ${newstr} ";
def rets1 = getThirdPartyDatas(sql1)
allRows.addAll(rets1)
def sql2 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " +
"COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " +
"distid in ${newstr} ";
def rets2 = getThirdPartyDatas(sql2)
allRows.addAll(rets2)
}else{
def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where price_date=${price_date} ";
allRows = getThirdPartyDatas(sql)
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
@@ -523,9 +526,37 @@ class ExportOracleData {
// fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
def index = 0
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def allRows = []
if(new_dists) {
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " +
"COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " +
"CITYID='${cityCode}' and price_date=${price_date} and distid not in ${newstr} ";
def rets1 = getThirdPartyDatas(sql1)
allRows.addAll(rets1)
def sql2 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " +
"COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " +
"CITYID='${cityCode}' and distid in ${newstr} ";
def rets2 = getThirdPartyDatas(sql2)
allRows.addAll(rets2)
}else{
def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where CITYID='${cityCode}' and price_date=${price_date} ";
allRows = getThirdPartyDatas(sql)
}
if(allRows && allRows.size()>0){
index = index + 1
if(index==1){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
}
}
}
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
}
@@ -552,53 +583,58 @@ class ExportOracleData {
}
def exportBigdataCommunity(){
def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" +
"\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj ";//where cityId in (350200,150100)
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DISTID
data << r.COMMID
data << r.DISTNAME
data << r.COMMNAME
data << r.ADDRESS
data << r.LATITUDE
data << r.LONGITUDE
data << r.LANDAREA
data << r.BUILDAREA
data << r.GREENINGRATE
data << r.PLOTRATIO
data << r.PARKINGCNT
data << r.BUILDYEAR
data << r.HOUSEUSE
data << r.ZTS
data << r.WORKDATE
data << r.UPDATEFLAG
data << r.UPDATETIME
data << r.COMMTYPE
data << r.COMMALIAS
data << r.STREET
data << r.STREETNO
data << r.IFNEW
data << ""
data << r.REMARK
data << r.CREATETIME
data << r.SOURCECODE
data << r.XZJDB
data << r.LJXZQHDM
data << r.LJXZQH
data << r.STATUS
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" +
"\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj where CITYID='${cityCode}' ";//where cityId in (350200,150100)
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DISTID
data << r.COMMID
data << r.DISTNAME
data << r.COMMNAME
data << r.ADDRESS
data << r.LATITUDE
data << r.LONGITUDE
data << r.LANDAREA
data << r.BUILDAREA
data << r.GREENINGRATE
data << r.PLOTRATIO
data << r.PARKINGCNT
data << r.BUILDYEAR
data << r.HOUSEUSE
data << r.ZTS
data << r.WORKDATE
data << r.UPDATEFLAG
data << r.UPDATETIME
data << r.COMMTYPE
data << r.COMMALIAS
data << r.STREET
data << r.STREETNO
data << r.IFNEW
data << ""
data << r.REMARK
data << r.CREATETIME
data << r.SOURCECODE
data << r.XZJDB
data << r.LJXZQHDM
data << r.LJXZQH
data << r.STATUS
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat"
}
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
//"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
//"乡镇/街道办","逻辑行政区划代码","逻辑行政区划","状态"]
//"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
//"乡镇/街道办","逻辑行政区划代码","逻辑行政区划","状态"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "BIGDATA_COMMUNITY_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
@@ -607,18 +643,15 @@ class ExportOracleData {
def exportBigdataForsale(publishDate, new_dists){
//["北京","上海"]
def citys = CcbCommonUtils.citys;
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
def index = 0
for(def city : citys){
def cityCode = cityCodeDataMap.get(city)
def cityCode = cityCodeMap.get(city)
def allRows = []
if(new_dists){
@@ -636,7 +669,13 @@ class ExportOracleData {
allRows = getHouseDatas(sql)
}
if(allRows.size()>0){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
index = index + 1
if(index == 1){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
}
}
}
@@ -778,8 +817,8 @@ class ExportOracleData {
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE",
//"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE",
//"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"]
//"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE",
//"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
println "AVM_EVALUATION_GJNB 数据文件输出完成......"
@@ -993,55 +1032,55 @@ class ExportOracleData {
//
// for(def city : citys) {
// def cityCode = cityCodeDataMap.get(city)
def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMUNITY_ID
data << r.CITYID
data << r.CITY_NAME
data << r.DISTRICT_CODE
data << r.DISTRICT_NAME
data << r.COMMUNITY_NAME
data << r.PQ_NAME
data << r.BASE_PRICE
data << r.CONFIDENCE_LEVEL
data << r.CNT
data << r.MIN_PRICE
data << r.MEDIANPRICE
data << r.MEANPRICE
data << r.MAX_PRICE
data << r.TIME
data << r.FIRSTDATAT
data << r.THIRDDATA
data << r.RELATIVEERROR
data << r.SKEWNESS
data << r.PQ_PRICE_RANGE
data << r.DISTRICT_PRICE_RANGE
data << r.DESCRIPTION
data << r.PRIV_BASE_PRICE
data << r.IS_ARTTFICIAL
data << r.PRIV_FINAL_PRICE
data << r.CREATE_DATE
data << r.PRICE
data << r.PRICE_SOURCE
data << r.CASE_MD_PRICE
data << r.CASE_AVG_PRICE
data << r.INDEX_LABEL
data << r.MAIN_PROPERTY
data << r.COMMTYPE
data << r.NEW_DEAL_PRICE
data << r.NEW_DEAL_DATE
data << r.PRICE_MOM_RATIO
data << r.MAX_MOM_RATIO
data << r.PRICE_YOY_RATIO
data << r.SATUR_RATIO
data << r.NEIGHBOR_PRICE
allRows << data
});
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMUNITY_ID
data << r.CITYID
data << r.CITY_NAME
data << r.DISTRICT_CODE
data << r.DISTRICT_NAME
data << r.COMMUNITY_NAME
data << r.PQ_NAME
data << r.BASE_PRICE
data << r.CONFIDENCE_LEVEL
data << r.CNT
data << r.MIN_PRICE
data << r.MEDIANPRICE
data << r.MEANPRICE
data << r.MAX_PRICE
data << r.TIME
data << r.FIRSTDATAT
data << r.THIRDDATA
data << r.RELATIVEERROR
data << r.SKEWNESS
data << r.PQ_PRICE_RANGE
data << r.DISTRICT_PRICE_RANGE
data << r.DESCRIPTION
data << r.PRIV_BASE_PRICE
data << r.IS_ARTTFICIAL
data << r.PRIV_FINAL_PRICE
data << r.CREATE_DATE
data << r.PRICE
data << r.PRICE_SOURCE
data << r.CASE_MD_PRICE
data << r.CASE_AVG_PRICE
data << r.INDEX_LABEL
data << r.MAIN_PROPERTY
data << r.COMMTYPE
data << r.NEW_DEAL_PRICE
data << r.NEW_DEAL_DATE
data << r.PRICE_MOM_RATIO
data << r.MAX_MOM_RATIO
data << r.PRICE_YOY_RATIO
data << r.SATUR_RATIO
data << r.NEIGHBOR_PRICE
allRows << data
});
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
// }
println "community_p_describe_gj 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
@@ -12,13 +12,14 @@ class ExportOracleEvaluationData {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def citys = null
static main(args) {
def task = new ExportOracleEvaluationData()
task.run()
task.run(args)
}
def run() {
def run(args) {
citys = CcbCommonUtils.getRunningCitys(args)
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
//¼ÆËã½á¹ûÏà¹Ø
@@ -369,7 +370,7 @@ class ExportOracleEvaluationData {
def exportCommCaseDetailGj(month){
def citys = CcbCommonUtils.citys;
// def citys = CcbCommonUtils.citys;
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
@@ -27,9 +27,46 @@ class FjwPricePkCcbPrice {
def task = new FjwPricePkCcbPrice()
// task.run()
// task.updatePrice()
task.addFjwPrice()
// task.addFjwPrice()
// task.addCcbPrice()
task.addMarketPrice()
}
def addMarketPrice(){
ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\拍卖南京.xlsx",{d->
if(d['D']) {
d['K'] = d['D']
}else if(d['H']){
d['K'] = d['H']
}
})
}
def addCcbPrice(){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\新老算法价格差异大小区.xlsx",{d->
if(d['B']){
def cityId = cityCodeDataMap.get(d['B'])
def dist = CommonDatautils.findDistrict(d['B'], d['C'], d['D'], "district")
if(dist){
def map = getCommunityBasePriceGjByCommid(cityId, dist._id.toString(), 202209)
if(map){
d['I'] = map.avg_price
d['J'] = map.calc_type
d['k'] = map.credit
}
}
// def dist = CommonDatautils.findDistrictById(d['B'])
// if(dist){
// d['J'] = dist.status
// }
}
})
}
def updatePrice(){
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\测试\\ccb.xlsx",{d->
@@ -174,6 +211,27 @@ class FjwPricePkCcbPrice {
}
def getCommunityBasePriceGjByCommid(cityid, commid, yearMonth) {
def sql = "select AVG_PRICE, CALC_TYPE,CONFIDENCE from community_base_price_gj where cityid='${cityid}' and year_month=${yearMonth} and commid='${commid}'";
def map = [:]
sqlInstance236.eachRow(sql, { r ->
map.avg_price = r.AVG_PRICE
// 计算类型,1:人工估值 2:样本计算 3:神经网络
if(r.CALC_TYPE){
if(r.CALC_TYPE == 1){
map.calc_type = "人工估值"
}else if(r.CALC_TYPE == 2){
map.calc_type = "样本计算"
}else if(r.CALC_TYPE == 3){
map.calc_type = "神经网络"
}
}
map.credit = r.CONFIDENCE
});
return map
}
def batchCreateFile(ret, fileName, headers, charset){
OutputStream out = new FileOutputStream(fileName);
def sb = new StringBuffer();
@@ -36,7 +36,7 @@ class GuiDingPriceBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
for (def city : citys) {
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2022-07-22", "2022-08-24")
def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-22", "2022-10-27")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -147,7 +147,7 @@ class GuiDingPriceBatch extends BaseBatch{
ret << dist._id.toString()
ret << dist.name
ret << cityCodeDatas.get(dist.city)
ret << dist.city
ret << CcbCommonUtils.getCcbCity(dist.city)
ret << areaData.district_code
ret << areaData.district
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
@@ -24,7 +24,7 @@ class StagingDistrictBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
for(def city : citys){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"])
File diff suppressed because it is too large Load Diff
@@ -22,10 +22,10 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202208//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202206
def endDate = 202208
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
def sDate= 202109//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202107
def endDate = 202109
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
@@ -42,7 +42,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def index = 0
def weekend = DateUtils.getFirstDayOfWeek(new Date())
@@ -67,7 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
}
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true);
}
@@ -73,15 +73,15 @@ class TradeStatLog {
}
def run(){
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.citys_92;
def rets = []
rets << ["城市","5月总量","6月总量","7月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
rets << ["城市","7月总量","8月总量","9月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2022-05-01"
def date = "2022-06-01"
def endDate = "2022-07-01"
def newDate = "2022-08-01"
def startDate = "2022-07-01"
def date = "2022-08-01"
def endDate = "2022-09-01"
def newDate = "2022-10-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -0,0 +1,146 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class AddressCut extends BaseBatch{
static main(args) {
runBatch(new AddressCut(), args);
}
@Override
public void process(Object args) {
sameAddr()
}
def sameAddr() {
def citys = CommonDatautils.findAllCity()
def dataMap = [:]
def index = 0
for (def c : citys) {
index++
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name", "address"])
def addrMap = [:]
for (def dist : dists) {
if (dist.address) {
def rets = addrMap.get(dist.address)
if (!rets) {
rets = []
}
rets << dist
addrMap.put(dist.address, rets)
}
}
dataMap.put(c, addrMap)
println index +" "+c
}
def sameAddrList = []
for(def c : citys) {
def cityMap = dataMap.get(c)
if(cityMap){
cityMap.each{addr,rets->
if(rets && rets.size()>1){
def keys = new HashSet()
for(def d : rets){
if(d.name){
def name = d.name.replace("一期","").replace("二期","").replace("三期","").replace("四期","").replace("五期","").replace("六期","").replace("七期","").replace("八期","").replace("九期","").replace("十期","").replace("一区","").replace("二区","").replace("三区","").replace("四区","").replace("五区","").replace("六区","").replace("七区","").replace("八区","").replace("九区","").replace("十区","").replace("A区","").replace("B区","").replace("C区","").replace("D区","")
.replace("十一街区","").replace("十二街区","").replace("十三街区","").replace("十四街区","").replace("十五街区","").replace("十六街区","").replace("十七街区","").replace("十八街区","").replace("十九街区","").replace("一街区","").replace("二街区","").replace("三街区","").replace("四街区","").replace("五街区","").replace("六街区","").replace("七街区","").replace("八街区","").replace("九街区","").replace("十街区","")
.replace("公寓","").replace("别墅","").replace("商住楼","").replace("花园","").replace("南区","").replace("北区","").replace("东区","").replace("西区","")
keys << d.region +"##"+ name
}
}
if(keys && keys.size()>1){
for(def d : rets) {
if (d.name) {
sameAddrList << [d.city, d.region, d.name, d.address]
}
}
}
}
}
}
}
//ExcelUtils.write("D:\\数据\\小区\\同地址小区.xlsx", sameAddrList, "相同地址小区")
FileUtils.batchCreateFile(sameAddrList, "D:\\数据\\小区\\同地址小区.txt", null, "utf-8", false);
println "完成"
}
def addrCut(){
def citys = CommonDatautils.findAllCity()
def index = 0
def fileName = "/home/goojia/gj/address_split.txt"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\address_split.txt"
}
for(def c : citys) {
index++
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name","address"])
def rets = cut(dists)
FileUtils.batchCreateFile(rets, fileName, null, "utf-8", true);
println index +" "+ c +" "+ rets.size()
}
}
def cut(rets){
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
final CountDownLatch latch = new CountDownLatch(rets.size())
def totalCount = new AtomicLong()
def rows = new CopyOnWriteArrayList()
for(def i=0;i<rets.size();i++){
final def d = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(d.address){
def jsonObj = FangjiaClientUtils.addressStructure(d.city, d.address)
if(jsonObj && jsonObj.result){
def udata = new BasicDBObject()
def street = null
def street_number = null
if(jsonObj.result.street){
street = jsonObj.result.street
}
if(jsonObj.result.streetNumber){
street_number = jsonObj.result.streetNumber
}
if(street || street_number){
println totalCount.get() +" "+d.city +" "+d.address +" "+street+" "+street_number
rows << [d.city,d.region, d.name, d.address, street, street_number]
// udata.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rows;
}
}
@@ -0,0 +1,26 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
DBCollection FANGJIA_AREA_CODE = MongoUtils.getCollectionByName("business", "fangjia_area_code", "mongo.db.fangjia.com")
def datas = ExcelUtils.read("D:\\数据\\建行\\二期\\建行最终区域编码.xlsx", 0, 1)
for(def d : datas){
def dbo = new BasicDBObject()
dbo.put("province",d['C'])
dbo.put("city", d['D'])
dbo.put("district", d['G'])
dbo.put("city_code", d['E'].toInteger())
dbo.put("district_code",d['F'].toInteger())
dbo.put("fjw_city", d['A'])
dbo.put("fjw_district", d['B'])
dbo.put("c_date", new Date())
FANGJIA_AREA_CODE.save(dbo)
}
@@ -42,7 +42,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.citys_all;
def rows = []
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// for (def city : citys) {
@@ -53,62 +53,62 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
// })
// }
// addBigdataDistrictFJW()
addBigdataDistrictFJW()
def citySeqnoMap = CcbCommonUtils.findDistrictSeqno();
def index = 0
for (def city : citys) {
index++
def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
// def citySeqnoMap = CcbCommonUtils.findDistrictSeqno();
//
// for(int i=0;i<10;i++){
// if(districts.size()==0){
// Thread.sleep(1000*60)
// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
// }else{
// break
// def index = 0
// for (def city : citys) {
// index++
// def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
////
//// for(int i=0;i<10;i++){
//// if(districts.size()==0){
//// Thread.sleep(1000*60)
//// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
//// }else{
//// break
//// }
//// }
//
// def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"])
//
// for(def dist : districts){
// def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"])
// if(obj){
// if(obj.alias && obj.alias.size()>0){
// dist.commalias = obj.alias.get(0)
// }else{
// dist.commalias = ""
// }
// dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure)
// dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2)
// dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date);
// dist.houseuse = "11"
// dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type)
// dist.compid = new ObjectId() //物业公司id
// dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2)
// }
// }
def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"])
for(def dist : districts){
def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"])
if(obj){
if(obj.alias && obj.alias.size()>0){
dist.commalias = obj.alias.get(0)
}else{
dist.commalias = ""
}
dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure)
dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2)
dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date);
dist.houseuse = "11"
dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type)
dist.compid = new ObjectId() //物业公司id
dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2)
}
}
println index +" "+city +" 小区加载完成 "+districts.size()
addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city))
println index +" "+city +" 1.小区表完成"
addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city))
println index +" "+city +" 2.小区建筑结构表"
addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city))
println index +" "+city +" 3.小区房屋类型表"
addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city))
println index +" "+city +" 4.小区建筑年代表"
addBigdataCommfeeGj(districts, cityCodeDataMap.get(city))
println index +" "+city +" 5.小区费用表"
addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city))
println index +" "+city +" 6.小区房屋用途表"
addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap)
println index +" "+ city + " 7.地理位置表"
// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行
}
// println index +" "+city +" 小区加载完成 "+districts.size()
// addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 1.小区表完成"
// addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 2.小区建筑结构表"
// addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 3.小区房屋类型表"
// addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 4.小区建筑年代表"
// addBigdataCommfeeGj(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 5.小区费用表"
// addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 6.小区房屋用途表"
// addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap)
// println index +" "+ city + " 7.地理位置表"
//
//// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行
//
// }
}
def addBigdataCommunityGJRH(def districts, def cityid){
@@ -557,14 +557,21 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
def addBigdataPositionGJ(districts, def cityid, citySeqnoMap){
def rets = []
def num = 0
for(def d : districts) {
num = num + 1
def ret = []
def detailMap = citySeqnoMap.get(d.cityid)
def seqno = findSeqno(d.cityid, d.commid, citySeqnoMap);
ret << d.cityid
ret << "5"
ret << d.commid
ret << "1"
ret << seqno
if(detailMap && seqno){
ret << seqno
}else{
ret << num
}
ret << d.lng
ret << d.lat
ret << ""
@@ -22,25 +22,29 @@ class ExportDistrictRongHe extends BaseBatch{
runBatch(new ExportDistrictRongHe(), args);
}
def citys = []
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
@Override
void process(Object args) {
citys = CcbCommonUtils.getRunningCitys(args)
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def new_dists = []//["350604","350605","410307","410308","450181"]
//按天导(每天一个文件夹)
//按天导(每天一个文件夹)
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
if(curDate.toString().endsWith("10")){
exportBigdataCommunityGJRH()
exportBigdataBuildingFJW()
exportBigdataCommfeeGj()
exportBigdataCommunityGJRH()//
exportBigdataBuildingFJW()//
exportBigdataCommfeeGj()//
exportBigdataCommbuildstruGJ()
exportBigdataCommhousetypeGJ()
exportBigdataCommbuildyearGj()
exportBigdataCommhouseuseGJ()
exportBigdataPositionGJ()
exportBigdataDistrictFJW()
exportBigdataAliasFJW()
exportBigdataCommbuildstruGJ()//
exportBigdataCommhousetypeGJ()//
exportBigdataCommbuildyearGj()//
exportBigdataCommhouseuseGJ()//
exportBigdataPositionGJ()//
exportBigdataDistrictFJW()//
exportBigdataAliasFJW()//
exportBigdataBuildprojFjw()
exportBigdataCommlayoutpicFjw()
@@ -59,14 +63,14 @@ class ExportDistrictRongHe extends BaseBatch{
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
//
//
//// //检查ok文件是否上传
//// //检查ok文件是否上传
//// exportAllBigdataDealFJW()
//// for(def city : citys) {
//// def cityCode = cityCodeDataMap.get(city)
////
// //按天导(一个文件夹下)
// //按天导(一个文件夹下)
// //exportBigdataDealFJW(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
// //按天导(每天一个文件夹)
// //按天导(每天一个文件夹)
// exportBigdataDealFJW2(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
//
//// exportBigdataCommunityGJRH(cityCode)
@@ -94,261 +98,308 @@ class ExportDistrictRongHe extends BaseBatch{
// exportOk()
// }
def exportBigdataCommunityGJRH(cityCode){
def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.COMMNAME
data << r.COMMALIAS
data << r.DISTID
data << r.STREET
data << r.STREETNO
data << r.ADDRESS
data << r.LANDAREA
data << r.BUILDAREA
data << r.COMMTYPE
data << r.IFNEW
data << r.GREENINGRATE
data << r.PLOTRATIO
data << r.PARKINGCNT
data << ''
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def exportBigdataCommunityGJRH(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......"
def index = 0
for(def city : citys){
def allRows = []
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW where CITYID='${cityCode}'";
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.COMMNAME
data << r.COMMALIAS
data << r.DISTID
data << r.STREET
data << r.STREETNO
data << r.ADDRESS
data << r.LANDAREA
data << r.BUILDAREA
data << r.COMMTYPE
data << r.IFNEW
data << r.GREENINGRATE
data << r.PLOTRATIO
data << r.PARKINGCNT
data << ''
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
if(allRows && allRows.size()>0){
index = index + 1
if(index == 1){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true);
}
}
}
println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......"
println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......"
}
def exportBigdataCommbuildstruGJ(){
def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.BUILDSTRU
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW where CITYID='${cityCode}'";
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.BUILDSTRU
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMBUILDSTRU_FJW 上传Ftp完成......"
println "BIGDATA_COMMBUILDSTRU_FJW 上传Ftp完成......"
}
def exportBigdataCommhousetypeGJ(cityCode){
def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW";// where CITYID='${cityCode}'
def exportBigdataCommhousetypeGJ(){
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.HOUSETYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW where CITYID='${cityCode}'";
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.HOUSETYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMHOUSETYPE_FJW 上传Ftp完成......"
println "BIGDATA_COMMHOUSETYPE_FJW 上传Ftp完成......"
}
def exportBigdataCommbuildyearGj(){
def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.BUILDYEAR
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW where CITYID='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.BUILDYEAR
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "BIGDATA_COMMBUILDYEAR_FJW 上传Ftp完成......"
println "BIGDATA_COMMBUILDYEAR_FJW 上传Ftp完成......"
}
def exportBigdataCommfeeGj(){
def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW";//where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.COMPID
data << r.FEE
data << r.HOUSEUSE
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.FEETYPE
data << r.NOTE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys){
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW where CITYID='${cityCode}'";
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.COMPID
data << r.FEE
data << r.HOUSEUSE
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.FEETYPE
data << r.NOTE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMFEE_FJW 上传Ftp完成......"
println "BIGDATA_COMMFEE_FJW 上传Ftp完成......"
}
def exportBigdataCommhouseuseGJ(){
def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.HOUSEUSE
data << r.HOUSECNT
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW where CITYID='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.HOUSEUSE
data << r.HOUSECNT
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMHOUSEUSE_FJW 上传Ftp完成......"
println "BIGDATA_COMMHOUSEUSE_FJW 上传Ftp完成......"
}
def exportBigdataPositionGJ(){
def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.OBJECTTYPE
data << r.OBJECTID
data << r.POSITIONTYPE
data << r.SEQNO
data << r.LONGITUDE
data << r.LATITUDE
data << r.ALTITUDE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW where CITYID='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.OBJECTTYPE
data << r.OBJECTID
data << r.POSITIONTYPE
data << r.SEQNO
data << r.LONGITUDE
data << r.LATITUDE
data << r.ALTITUDE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_POSITION_FJW 上传Ftp完成......"
println "BIGDATA_POSITION_FJW 上传Ftp完成......"
}
def exportBigdataDistrictFJW(){
def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.DISTID
data << r.DISTNAME
data << r.DISTSPELL
data << r.CITYID
data << r.DISTCODE
data << r.SEQ
data << r.DISTTYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW where CITYID='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.DISTID
data << r.DISTNAME
data << r.DISTSPELL
data << r.CITYID
data << r.DISTCODE
data << r.SEQ
data << r.DISTTYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "BIGDATA_DISTRICT_FJW 上传Ftp完成......"
println "BIGDATA_DISTRICT_FJW 上传Ftp完成......"
}
def exportBigdataAliasFJW(){
def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW";// where city_id='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.city_id
data << r.ca_id
data << r.address_id
data << r.address_name
data << r.status
data << r.exp_dt
data << r.version
data << r.is_key_name
data << r.data_source
data << r.description
data << r.alias_grade
data << r.addr_type_cd
data << r.ifexamine
data << r.ifdisplay
allRows << data
});
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW where city_id='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.city_id
data << r.ca_id
data << r.address_id
data << r.address_name
data << r.status
data << r.exp_dt
data << r.version
data << r.is_key_name
data << r.data_source
data << r.description
data << r.alias_grade
data << r.addr_type_cd
data << r.ifexamine
data << r.ifdisplay
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "ADDRESS_ALIAS_FJW 上传Ftp完成......"
println "ADDRESS_ALIAS_FJW 上传Ftp完成......"
}
def exportBigdataDealFJW(dealdate){
@@ -368,7 +419,7 @@ class ExportDistrictRongHe extends BaseBatch{
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
}
def allRows = []
@@ -453,7 +504,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}"
println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}"
def curEndStr = DateUtils.add(curStartStr, 1)
curStartDateStr = curEndStr
}
@@ -468,31 +519,36 @@ class ExportDistrictRongHe extends BaseBatch{
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
if(curDate.toString().endsWith("28")){
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " +
"where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd') and" +
" distid not in ${newstr} ";//CITYID='${cityCode}' and
def rets1 = createDeal(sql1)
def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " +
" where distid in ${newstr}"
def rets2 = createDeal(sql2)
allRows.addAll(rets1)
allRows.addAll(rets2)
}else{
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
allRows = createDeal(sql)
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
if(curDate.toString().endsWith("28")){
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " +
"where CITYID='${cityCode}' and dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd') and" +
" distid not in ${newstr} ";//CITYID='${cityCode}' and
def rets1 = createDeal(sql1)
def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " +
" where CITYID='${cityCode}' and distid in ${newstr}"
def rets2 = createDeal(sql2)
allRows.addAll(rets1)
allRows.addAll(rets2)
}else{
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}' and dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//
allRows = createDeal(sql)
}
}
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}"
println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}"
}
def createDeal(sql){
@@ -653,54 +709,68 @@ class ExportDistrictRongHe extends BaseBatch{
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "BIGDATA_DEAL_FJW 导出完成......"
println "BIGDATA_DEAL_FJW 导出完成......"
}
def exportBigdataBuildingFJW(cityCode){
def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW";//where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.BUILDALIAS
data << r.DISTID
data << r.COMMID
data << r.BUILDYEAR
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.BUILDSTRU
data << r.FLOORCNT
data << r.UNITCNT
data << r.HOUSECNT
data << r.IFELEVATOR
data << r.ELEVATORCNT
data << r.ELEVATORHOUSE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.VALID_FLAG
allRows << data
});
def exportBigdataBuildingFJW(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
def index = 0
for(def city : citys) {
def allRows = []
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW where CITYID='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.BUILDALIAS
data << r.DISTID
data << r.COMMID
data << r.BUILDYEAR
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.BUILDSTRU
data << r.FLOORCNT
data << r.UNITCNT
data << r.HOUSECNT
data << r.IFELEVATOR
data << r.ELEVATORCNT
data << r.ELEVATORHOUSE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.VALID_FLAG
allRows << data
});
if(allRows && allRows.size()>0){
index = index + 1
if(index == 1){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true);
}
}
}
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_BUILDING_FJW 上传Ftp完成......"
println "BIGDATA_BUILDING_FJW 上传Ftp完成......"
}
@@ -724,40 +794,40 @@ class ExportDistrictRongHe extends BaseBatch{
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "EVALUATE_RATE_FJW 上传Ftp完成......"
println "EVALUATE_RATE_FJW 上传Ftp完成......"
}
def exportBigdataBuildprojFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......"
println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......"
}
def exportBigdataCommlayoutpicFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......"
println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......"
}
def exportOk(curDate){
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
}
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",dataCodeFileName, curDate)
@@ -0,0 +1,112 @@
package com.fangjia.script.ccbevaluation.ronghe
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import com.jcraft.jsch.ChannelExec;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.Session
import com.jcraft.jsch.*;
class SSHHelperTest {
private Session session = null;
private Channel channel = null;
private String sftpHost = "192.168.0.86";
private int sftpPort = 17495;
private String sftpUserName = "ccb";
private String sftpPassword = "ccb!@#";
private int timeout = 30000;
/**
* 获取连接
* @return
*/
private ChannelExec getChannelExec() {
try {
if (channel != null && channel.isConnected()) {
return (ChannelExec) channel;
}
JSch jSch = new JSch();
if (session == null || !session.isConnected()) {
session = jSch.getSession(sftpUserName, sftpHost, sftpPort);
session.setPassword(sftpPassword);
Properties config = new Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.setTimeout(timeout);
session.connect();
}
channel = session.openChannel("exec");
} catch (Exception e) {
if (session != null) {
session.disconnect();
session = null;
}
channel = null;
}
return channel == null ? null : (ChannelExec) channel;
}
/**
* 关闭连接
*/
private void closeChannel() {
try {
if (channel != null) {
channel.disconnect();
channel = null;
}
if (session != null) {
session.disconnect();
session = null;
}
} catch (Exception e) {
System.out.println(e);
}
}
/**
* 执行服务器端命令
*/
public boolean executeCommand(String command) {
boolean flag = false;
ChannelExec channelExec = getChannelExec();
if (channelExec == null) {
return false;
}
try {
channelExec.setInputStream(null);
channelExec.setErrStream(System.err);
channelExec.setCommand(command);
InputStream ins = channelExec.getInputStream();
channelExec.connect();
BufferedReader reader = new BufferedReader(new InputStreamReader(ins, "utf-8"));
String line = null;
while ((line = reader.readLine()) != null) {
System.out.println(line);
}
reader.close();
closeChannel();
flag = true;
} catch (Exception e) {
System.out.println(e);
flag = false;
}
return flag;
}
public static void main(String[] args) {
SSHHelperTest jschCommand = new SSHHelperTest();
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/package.sh 20220110"));
}
}
@@ -473,6 +473,7 @@ class CommonDatautils {
if(dbo.mapping_alias){
detailMap.mapping_alias = dbo.mapping_alias;
}
detailMap.estate_type = dbo.estate_type2
}
}catch(Exception e){
@@ -754,7 +755,7 @@ class CommonDatautils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","и╜мШ╥╪╡щбл╟╤")
// q.put("name","ºþÅϼÑÔ·")
DBObject keys = new BasicDBObject();
if(fields){
@@ -11,6 +11,10 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.codec.digest.DigestUtils
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
@@ -26,7 +30,9 @@ class DistrictInfoExportBatch extends BaseBatch{
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection FZ_ADDR_COLL = MongoUtils.getCollectionByName2("business", "fz_district_address", lineUrl)
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName2("business", "fz_district", lineUrl)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
runBatch(new DistrictInfoExportBatch(), args)
@@ -87,7 +93,12 @@ class DistrictInfoExportBatch extends BaseBatch{
def startDate = DateUtils.getFirstDayOfMonth(lastDate);
def fzAddrMap = getFzAddress();
for(def rMap : districts){
//因为福州个性化问题,又不想影响自家数据库数据,所以另存一个福州小区库,用于手动添加数据
saveOrUpdateFzDist(districts)
def fzDistricts = getFzDistricts(city)
for(def rMap : fzDistricts){
//¼Û¸ñûÓÐÒ²Ìṩ
// def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
// if(dbo){
@@ -139,6 +150,51 @@ class DistrictInfoExportBatch extends BaseBatch{
return result
}
def saveOrUpdateFzDist(districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
for(def i=0;i<districts.size();i++){
final def d = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def dbo = FZ_DIST_COLL.findOne(new BasicDBObject("_id", d.get('_id')))
if(dbo){
def udate = new BasicDBObject()
d.each {k,v->
udate.put(k,v)
}
udate.removeField("_id")
udate.put("m_date", new Date())
FZ_DIST_COLL.update(new BasicDBObject('_id',d.get('_id')), new BasicDBObject('$set', udate));
}else{
FZ_DIST_COLL.save(d)
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def getFzDistricts(def city){
def q = new BasicDBObject()
q.put("city", city)
def rets = []
FZ_DIST_COLL.find(q).each{it->
rets << it
}
return rets
}
def processAlias(alias, def addr_alias){
def newAlias = []
if(alias){
@@ -29,7 +29,9 @@ class DistrictPriceExportBatch extends BaseBatch{
def charset = "GBK"
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com")
def specialDistsPriceMap = ["津泰新村7号、9-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580]
static main(args) {
runBatch(new DistrictPriceExportBatch(), args)
}
@@ -86,10 +88,21 @@ class DistrictPriceExportBatch extends BaseBatch{
return sb.toString()
}
def getFzDistricts(def city){
def q = new BasicDBObject()
q.put("city", city)
def rets = []
FZ_DIST_COLL.find(q).each{it->
rets << it
}
return rets
}
def createDatas(def city){
def districts = CommonDatautils.findAllDistricts(city, "district", null);
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
districts.addAll(removeDists)
// def districts = CommonDatautils.findAllDistricts(city, "district", null);
// def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
// districts.addAll(removeDists)
def districts = getFzDistricts(city)
def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
def dataStr = DateUtils.add(endDateStr, -30)
@@ -124,9 +137,15 @@ class DistrictPriceExportBatch extends BaseBatch{
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
}
}
}else{
curPrice = specialDistsPriceMap.get(rMap.name)
if(curPrice){
def randMarkup = nextDouble(0, 1)
curPrice = (int)(curPrice*randMarkup/100 + curPrice)
}
}
def m = []
if(curPrice){
if(curPrice && dbo){
m << dbo.get("_id").toString()//¾ù¼ÛID
}else{
m << new ObjectId().toString()
@@ -135,7 +154,11 @@ class DistrictPriceExportBatch extends BaseBatch{
m << rMap.city
m << rMap.region
m << rMap.name
m << curPrice
if(curPrice){
m << curPrice
}else{
m << 0
}
m << curDate.toInteger();
result << m
}
@@ -98,9 +98,11 @@ class FzDataResultStat {
def dist = CommonDatautils.findDistrictById(list[1])
def flag = "有效"
if(dist.status!=0){
if(dist && dist.status!=0){
flag = "失效"
}
rows << [keys[0], keys[1], keys[2], list[5].toString(), lastPrice.toString(), markup, flag]
}
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202208.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202209.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -18,9 +18,10 @@ class RegionUpdate {
}
def run1(){
def districts = CommonDatautils.findAllFiledDistrict("district","ÕÄÖÝ");
def city = "ËÄÆ½"
def districts = CommonDatautils.findAllFiledDistrict("district",city);
def regions = CommonDatautils.getRegionListByCity("ÕÄÖÝ")
def regions = CommonDatautils.getRegionListByCity(city)
def index = 0
def addIndex = 0
DistrictRegionTransfer transfer = new DistrictRegionTransfer()
@@ -28,6 +29,9 @@ class RegionUpdate {
if(districts){
//根据经纬度获取经纬度所在区域
for(def dist : districts){
if(dist.region!="¹«Ö÷Áë"){
continue
}
index++
if(!regions.contains(dist.region)){
@@ -35,28 +39,28 @@ class RegionUpdate {
println regionErrorIndex +" "+dist.name
}
// def region = getRegionByLocation(dist.lng/10000000000d+","+dist.lat/10000000000d);
// if(!region){
// def detailMap = getRegionByAddress("ÕÄÖÝ", dist.address);
// if(detailMap && detailMap.region && detailMap.region != dist.region){
// addIndex++
// dist.lng = (detailMap.lng.toDouble()*10000000000).longValue()
// dist.lat = (detailMap.lat.toDouble()*10000000000).longValue()
//
// def gmap = GeocodeUtils.getGaodeToBaidu(detailMap.lng.toDouble(), detailMap.lat.toDouble())
// dist.lng2 = (gmap.get("lng").toDouble()* 10000000000).longValue()
// dist.lat2 = (gmap.get("lat").toDouble()* 10000000000).longValue()
//
// println "[${addIndex}/${index}]"+dist.region +" "+detailMap.region+" "+ dist.name
// transfer.transfer(dist, dist.city, detailMap.region)
// }
// }else{
// if(region && region!=dist.region){
// addIndex++
// println "[${addIndex}/${index}]"+region +" "+dist.region+" "+ dist.name
// transfer.transfer(dist, dist.city, region)
// }
// }
def region = getRegionByLocation(city, dist.lng/10000000000d+","+dist.lat/10000000000d);
if(!region){
def detailMap = getRegionByAddress(city, dist.address);
if(detailMap && detailMap.region && detailMap.region != dist.region){
addIndex++
dist.lng = (detailMap.lng.toDouble()*10000000000).longValue()
dist.lat = (detailMap.lat.toDouble()*10000000000).longValue()
def gmap = GeocodeUtils.getGaodeToBaidu(detailMap.lng.toDouble(), detailMap.lat.toDouble())
dist.lng2 = (gmap.get("lng").toDouble()* 10000000000).longValue()
dist.lat2 = (gmap.get("lat").toDouble()* 10000000000).longValue()
println "[${addIndex}/${index}]"+dist.region +" "+detailMap.region+" "+ dist.name
transfer.transfer(dist, dist.city, detailMap.region)
}
}else{
if(region && region!=dist.region){
addIndex++
println "[${addIndex}/${index}]"+region +" "+dist.region+" "+ dist.name
transfer.transfer(dist, dist.city, region)
}
}
}
}
}
@@ -85,14 +89,16 @@ class RegionUpdate {
return detail
}
def getRegionByLocation(def location){
def getRegionByLocation(city, def location){
def json = GaoDeApiUtils.geocode(location);
def city = "ÕÄÖÝ"
def region = null;
if (json && json.info=="OK") {
def regeocode = json.regeocode
if(regeocode && regeocode.addressComponent){
region = regeocode.addressComponent.district
if(!region){
region = regeocode.addressComponent.township
}
}
}
if(region){
@@ -48,13 +48,13 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2022-07-01"
static def statDate = "2022-09-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser();//友达二手房存入mongo
// task.esfParser("D:\\数据\\友达\\2022年10月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
}
@@ -284,8 +284,8 @@ class YouDaDataParser {
}
def esfParser(){
def files = ["D:\\数据\\友达\\2022年8月\\查询1.DBF"]
def esfParser(path){
def files = [path]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -29,7 +29,30 @@ class DataMatch {
// task.run3()
// println task.findAddress("²ıÖݽֵÀÌÄÏã·")
task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
task.fzWeiyuanshuMatch("D:\\数据\\福州\\连江楼盘.xlsx")
}
def fzWeiyuanshuMatch(fileName){
ExcelUtils.multiThreadingModify(fileName,{d->
// if(!d['G']){
// def arr = OpenApiClient.search("福州", "连江", d['C'])
// if(arr){
// d['E'] = arr.city
// d['F'] = arr.district
// d['G'] = arr.name
// }
// }
def results = null
if(d['H'] && d['H']=="F" && d['C']){
results = OpenApiClient.addressMatch("福州",d['C'], "property", false, "连江", null)
}
if(results && results.precise>=0){
d['J'] = results.district
d['K'] = results.name
}
},0)
}
def run1(){
@@ -67,6 +67,7 @@ class FangjiaClientUtils {
//分词 省,市,区
public static def addressStand(def city, def address){
def jsonObj = null
def content = null
@@ -84,6 +85,22 @@ class FangjiaClientUtils {
return jsonObj;
}
//分词基础上的标准化,complete=false 不不全 short=true 省市区返回简拼
public static def addressStructure(def city, def address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def arroundPrice(city,address){
def jsonObj = null
def content = null
@@ -11,9 +11,10 @@ import org.apache.lucene.queryparser.classic.QueryParser
class DongGuanBankTest {
static main(args) {
def task = new DongGuanBankTest()
def path = "D:\\数据\\东莞银行\\东莞银行房价数据测试20220916 v2.xlsx"
def path = "D:\\数据\\东莞银行\\东莞银行测试_输出单价(2).xlsx"
// task.run(path)
task.addPrice(path)
// task.addPrice(path)
task.addMinMaxPrice(path)
}
def run(path){
@@ -92,7 +93,7 @@ class DongGuanBankTest {
def set = null
try{
def aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= 20210101 and region='${region}' and district_name in ('${aliasStr}')"
String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= 20210901 and region='${region}' and district_name in ('${aliasStr}') and root='anjuke.com'"
set = new HashSet()
ElasticUtils.executeSqlQuery(sql,{data->
@@ -132,4 +133,26 @@ class DongGuanBankTest {
return avg_price
}
def addMinMaxPrice(path){
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['V']){
def dist = CommonDatautils.findDistrict(d['H'], d['I'], d['J'], "district")
if(dist){
def alias = []
if(!dist.alias){
alias = []
}else{
alias = dist.alias
}
alias << dist.name
def map = findMaxAvgPrice(dist.city, dist.region, alias)
if(map){
d['V'] = map.max
d['W'] = map.min
}
}
}
})
}
}
@@ -19,9 +19,44 @@ class EsMonthHouseStat {
def task = new EsMonthHouseStat()
// task.run()
// task.tradeRun()
task.houseRun()
// task.houseRun()
// task.statHouseUpdateCount()
task.statDistHouse()
}
def statDistHouse(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\大连有效小区确认_2256--查好-1.xlsx",{d->
if(d['B']){
def alias = []
if(d['G']){
def s = d['G'].replace("[", "").replace("]", "").replace("'","").replace('"',"")
def arrs = null
if(s){
arrs = s.split(",")
}
if(arrs){
arrs = arrs as List
if(!arrs.contains(d['D'])){
arrs << d['D']
}
alias = arrs
}else{
alias << d['D']
}
}else{
alias << d['D']
}
def distCount = findHouseCount(d['B'], d['C'], alias, 20190101, 20221001)
if(distCount){
d['E'] = distCount
}
println d['B']+" "+d['C']+" "+d['D']+" "+distCount
}
})
}
def run(){
@@ -0,0 +1,55 @@
package com.fangjia.script.gj.es
import com.fangjia.script.util.ElasticUtils
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.25:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
findLogs()
def findLogs(){
def total = 0
try{
// String sql = "select * from fangjia-log* where request like '/jnBank%'"
// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
def q = QueryBuilders.fuzzyQuery("request", "/jnBank")
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.25", "fangjia-log*", queryBuilder);
println houses.size()
def set = new HashSet()
if(houses) {
for (def d : houses) {
def request = d.request
if(request){
def arrs = request.split("&")
if(arrs){
for(def a : arrs){
if(a.contains("CITY_CD")){
def list = a.split("=")
if(list && list.size()>=2){
set.add(list[1])
}
}
}
}
}
}
}
println set
// ElasticUtils.executeSqlQuery(sql,{ data->
// println data.request
// },"192.168.0.25")
}catch(Exception e){
e.printStackTrace()
}
return total
}
@@ -86,7 +86,7 @@ class DistrictIndexBatch extends BaseBatch{
city_avg_total_price = 531.2615913
}
if(city_avg_total_price){
calcute(city, dists, code, city_avg_total_price)
// calcute(city, dists, code, city_avg_total_price)
calcuteIndex(city, dists, code, city_avg_total_price)
}
@@ -49,7 +49,15 @@ public class BeiKeCrawlByChrome {
WebDriver driver = new ChromeDriver()
def citys = getAllCityUrls(driver)
citys = citys.reverse()
// citys = citys.reverse()
// def index = 0
// for(def c : citys){
// index++
// if(c.city=="ºÏ·Ê"){
// break
// }
// }
// citys = citys[index..citys.size()-1]
def start = 0
// for(def i=0;i<citys.size();i++){
// def m = citys[i]
@@ -190,7 +198,7 @@ public class BeiKeCrawlByChrome {
}
page++;
}else{
stop = true;
stop = true;
}
}
return detailUrls
@@ -390,7 +398,7 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
// if(e.text().contains("¸£ÖÝ")){
// if(e.text().contains("ºÏ·Ê")){
def cityMap = [:]
cityMap.city = e.text()
cityMap.url = "https:"+href
@@ -31,6 +31,7 @@ public class LianjiaCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
static main(args) {
def task = new LianjiaCrawlByChrome();
@@ -250,7 +251,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-01-01", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e, regionText)
@@ -469,11 +470,11 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
def citys= ["柯소鏤","뱍捺"]
def citys= ["¸£ÖÝ"]
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
if (!href.contains("fang.lianjia")){
cityMap.put(e.text(), href)
}
}
@@ -8,6 +8,7 @@ import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import com.rabbitmq.client.impl.AMQImpl
@@ -35,11 +36,12 @@ class PaimaiDataSend {
static main(args) {
def task = new PaimaiDataSend()
// task.jdRunByMongo()
// task.jdRun()
// task.aliZcRun()
task.aliZcRun2()
// task.aliFpRun()
// task.aliZcRun2()
task.aliFpRun()
// task.rmfysszcRun()
// task.jdRun()
// task.checkUrl()
// task.paimaiSendToEs()
@@ -66,7 +68,8 @@ class PaimaiDataSend {
def rmfysszcRun(){
def index = 0
def q = new BasicDBObject()
q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市", "苏州市"]))
// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市", "苏州市"]))
q.put("province","江苏省")
// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/88061.shtml")
def dbcursor = rmfysszc_coll.find(q).batchSize(3000)
for(def d : dbcursor){
@@ -149,10 +152,12 @@ class PaimaiDataSend {
def jdRun(){
def index = 0
def q = new BasicDBObject()
q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市","苏州市"]))
// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市","苏州市"]))
q.put("province","江苏")
// q.put("parser_status", new BasicDBObject('$ne',3))
// q.put("url","https://paimai.jd.com/115669796")
def dbcursor = jd_coll.find(q).batchSize(3000)
def rets = []
for(def d : dbcursor){
index++
println "jd:"+index
@@ -175,23 +180,57 @@ class PaimaiDataSend {
result.put("notice_detail_content", d.notice_detail_content.content)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
rets << json
if(rets.size()==10000){
println "共:"+rets.size()
sendJD(rets)
rets.clear()
}
}
def udata = new BasicDBObject()
udata.put("parser_status", 3)
udata.put("m_date", new Date())
jd_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
if(rets.size()>0){
println "共:"+rets.size()
sendJD(rets)
}
}
def sendJD(def rets){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def json = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 3)
udata.put("m_date", new Date())
jd_coll.update(new BasicDBObject("_id", json.get("url")), new BasicDBObject('$set', udata))
println "send: "+totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def aliZcRun(){
def pm_q = new BasicDBObject()
pm_q.put("city",new BasicDBObject('$in', ["南京", "宿迁","苏州"]))
// pm_q.put("city","苏州")
// pm_q.put("city",new BasicDBObject('$in', ["南京", "宿迁","苏州"]))
pm_q.put("province","江苏")
// pm_q.put("region","姑苏")
// pm_q.put("url","https://sf-item.taobao.com/sf_item/659117468505.htm")
pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")).append('$lt', DateUtils.parseFromTightYYYYMMDD("2022-09-15")))
// pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")).append('$lt', DateUtils.parseFromTightYYYYMMDD("2022-09-15")))
pm_q.put("mDate", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2022-10-01")))
Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
pm_q.put("url", new BasicDBObject('$regex', pattern))
// pm_q.put("mappingEstateType", null)
@@ -199,8 +238,8 @@ class PaimaiDataSend {
def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000)
println dbcursor.count()
for(def d : dbcursor){
def flag = sendAliZc(d.url)
// def flag = sendAli(d.url)
// def flag = sendAliZc(d.url)
def flag = sendAli(d.url)
if(!flag){
println "不存在:"+d.url
}
@@ -294,14 +333,16 @@ class PaimaiDataSend {
def aliFpRun(){
def index = 0
def q = new BasicDBObject()
q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
// q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
// q.put("province","江苏")
// q.put("sub_type","土地")
// q.put("parser_status", new BasicDBObject('$ne',2))
// q.put("url","https://sf-item.taobao.com/sf_item/631662596036.htm")
q.put("url","https://sf-item.taobao.com/sf_item/673646300374.htm")
//https://sf-item.taobao.com/sf_item/657131607486.htm
// q.put("parser_status", new BasicDBObject('$exists',false))
// q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01")))
// q.put("area", new BasicDBObject('$lt', 30))
def dbcursor = ali_fp_coll.find(q).batchSize(5000)
def dbcursor = ali_fp_coll.find(q).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def rets = []
@@ -314,8 +355,12 @@ class PaimaiDataSend {
JSON result = new JSONObject()
result.put("content", d.content)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("trades", bid_content)
try{
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("trades", bid_content)
}catch(Exception e){
println "trades结果格式问题 "+d.url
}
result.put("province", d.province)
result.put("city", d.city)
result.put("region", d.region)
@@ -328,7 +373,7 @@ class PaimaiDataSend {
json.put("result", result)
rets << json
if(rets.size()==10000){
if(rets.size()==5000){
println "共:"+rets.size()
send(rets)
rets.clear()
@@ -354,7 +399,7 @@ class PaimaiDataSend {
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 2)
udata.put("parser_status", 0)
udata.put("m_date", new Date())
ali_fp_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
println "send: "+totalCount.get()
@@ -374,11 +419,13 @@ class PaimaiDataSend {
def index = 0
def q = new BasicDBObject()
// q.put("city",new BasicDBObject('$in', ["苏州","宿迁","南京"]))
q.put("city","上海")
// q.put("city", "上海")
q.put("province","江苏")
q.put("sub_type","土地")
// q.put("parser_status", new BasicDBObject('$ne',1))
// q.put("region","姑苏")
q.put("url","https://susong-item.taobao.com/auction/652481594083.htm")
def dbcursor = ali_zc_coll.find(q).batchSize(5000)
q.put("url","https://susong-item.taobao.com/auction/671320162557.htm")
def dbcursor = ali_zc_coll.find(q).batchSize(1000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def rets = []
for(def d : dbcursor){
index++
@@ -393,6 +440,7 @@ class PaimaiDataSend {
JSON result = new JSONObject()
result.put("content", d.content)
result.put("sub_type", d.sub_type)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("trades", bid_content)
result.put("sub_count", d.sub_count)
@@ -408,7 +456,7 @@ class PaimaiDataSend {
json.put("result", result)
rets << json
if(rets.size()==10000){
if(rets.size()==1000){
println "共:"+rets.size()
send(rets)
rets.clear()
@@ -476,8 +524,9 @@ class PaimaiDataSend {
def getPaimaiDatas(){
def q = new BasicDBObject()
q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
// q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
q.put("province","江苏")
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-10-10")))
def dbcursor = paimai43_COLL.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
@@ -508,6 +557,7 @@ class PaimaiDataSend {
o.matchCity = d.matchCity
o.matchName = d.matchName
o.matchRegion = d.matchRegion
o.matchSeaweedId = d.matchSeaweedId
o.person = d.person
o.preemptivePerson = d.preemptivePerson
o.pricePkWeekly = d.pricePkWeekly
@@ -1,17 +1,79 @@
package com.fangjia.script.gj.region
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
class MapAreaUpdate {
static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new MapAreaUpdate()
task.run()
//task.run()
task.checkCcbRegion()
}
def checkCcbRegion(){
def datas = ExcelUtils.read("D:\\数据\\建行\\二期\\建行最终区域编码.xlsx",0,1)
def citys = CcbCommonUtils.citys
def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
def fjw_citys = []
for(def d : datas){
if(!fjw_citys.contains(d['A'])){
fjw_citys << d['A']
}
}
// for(def fjwCity : fjw_citys){
// if(!citys_252.contains(fjwCity) && !citys.contains(fjwCity)){
// println fjwCity
// }
// }
for(def newCity : citys_252){
if(!fjw_citys.contains(newCity)){
println fjwCity
}
}
// //是否全在之前92个城市中
// def ccb_92_citys = getCcb92City(sqlInstance)
// for(def m : ccb_92_citys){
// def isExist = false
// for(def d : datas){
// if(m.cityId==d['E'].toInteger().toString() && m.distId==d['F'].toInteger().toString()){
// isExist = true;
// break
// }
// }
// if(!isExist){
// println m.cityId +" "+ m.distId +" 不存在"
// }
// }
}
def getCcb92City(sqlInstance){
def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE from bigdata_district_gj"
def list = []
sqlInstance.eachRow(sql, { r ->
def m = [:]
m.region = r.DISTNAME
m.cityId = r.CITYID
m.distId = r.DISTID
list << m
})
return list;
}
def run(){
def rets = findAllCity()
for(def city : rets){
@@ -0,0 +1,94 @@
package com.fangjia.script.gj.test
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.core.RegionData
import groovy.sql.Sql
def newRegionCodeFile = "D:\\数据\\建行\\二期\\行政区处理_20220923.xlsx"
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def specialCityMap = ["内蒙古##兴安":"兴安盟","四川##甘孜":"甘孜州","云南##文山":"文山州","青海##玉树":"玉树州"]
//ExcelUtils.modify(newRegionCodeFile,{d->
// if(d['A'] && !d['I']){
// def city = specialCityMap.get(d["A"]+"##"+d["B"])
// if(!city){
// city = RegionData.formatCity(d['B'])
// }
// if(city){
// def region = RegionData.formatRegion(city, d['E'])
// if(region){
// d['I'] = city
// d['J'] = region
// }
// }
// }
//})
def newDatas = ExcelUtils.read(newRegionCodeFile,0,1)
def newDataMap = [:]
for(def d : newDatas){
def key = d['I']+"##"+d['J']
def rets = newDataMap.get(key)
if(!rets){
newDataMap.put(key, d)
}
}
def ccbMap = getCcb92City(sqlInstance)
ExcelUtils.modify("D:\\数据\\亲邻\\region_code.xlsx",{d->
if(!d['C'] && d['D']){
def dbo = CommonDatautils.findAlCityDetail(d['A'])
if(dbo){
d['C'] = dbo.province
}
}
// if(!d['D']){
// def m = newDataMap.get(d['A']+"##"+d['B'])
// if(m){
// d['C'] = m['A']
// d['D'] = m['B']
// d['E'] = m['C']
// d['F'] = m['D']
// d['G'] = m['E']
// }
// def m = ccbMap.get(d['A']+"##"+d['B'])
// if(m){
// d['D'] = m.city
// d['E'] = m.cityId
// d['F'] = m.distId
// d['G'] = m.region
// }
// }
})
def getCcb92City(sqlInstance){
def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE from bigdata_district_gj"
def map = [:]
sqlInstance.eachRow(sql, { r ->
def city = getCityName(r.CITYID)
def region = RegionData.formatRegion(city, r.DISTNAME)
def m = [:]
m.city = city
m.region = r.DISTNAME
m.cityId = r.CITYID
m.distId = r.DISTID
map.put(city+"##"+region, m)
})
return map;
}
def getCityName(cityId){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def city = null
cityCodeDataMap.each{cityName, id->
if(id == cityId){
city = cityName
}
}
return city;
}
@@ -25,7 +25,7 @@ ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
def q = new BasicDBObject()
// q.put("province",new BasicDBObject('$in', ["½­ËÕ","½­ËÕÊ¡"]))
q.put("status",new BasicDBObject('$ne', 1))
def dbcursor = school_coll.find(q).batchSize(5gdc,rfyhyerhtrbhjfxcs- vvv000)
def dbcursor = school_coll.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
JSON json = new JSONObject()
@@ -647,9 +647,9 @@ class CZDataBatch extends BaseBatch{
}
}
if(!priceMap){
priceMap = findRegionRentPrice(priceMap, city, region, estate_type2)
}
// if(!priceMap){
// priceMap = findRegionRentPrice(priceMap, city, region, estate_type2)
// }
//5室及以上价格小于2000去除
if(priceMap){
@@ -209,4 +209,20 @@ class SFTP {
return isDirExistFlag;
}
public long getFileSize(String srcSftpFilePath, ChannelSftp sftp) {
long fileSize;//文件大于等于0则存在
try {
SftpATTRS sftpATTRS = sftp.lstat(srcSftpFilePath);
fileSize = sftpATTRS.getSize();
} catch (Exception e) {
e.printStackTrace()
fileSize = -1;//获取文件大小异常
if (e.getMessage().toLowerCase().equals("no such file")) {
fileSize = -2;//文件不存在
}
}
return fileSize;
}
}
@@ -5,6 +5,9 @@ import com.fangjia.script.temp.shenjian.util.EsUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.util.ElasticUtils
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 补充学校经纬度
* @author shenjian
@@ -12,16 +15,25 @@ import com.fangjia.script.util.ElasticUtils
*/
class FillLngLat {
static ExecutorService executorService = Executors.newFixedThreadPool(4)
static void main(String[] args) {
ElasticUtils.executeSqlQuery("select * from dws_poi where location is null", { data ->
println(data)
def lngLat = GaoDeUtils.parserByGaode(data.city, data.region + data.name + data.address)
if (lngLat != null && lngLat.size() == 2 ) {
JSONObject jsonObject = new JSONObject()
jsonObject.put("lon", Double.valueOf(lngLat['lng']))
jsonObject.put("lat", Double.valueOf(lngLat['lat']))
EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id))
}
ElasticUtils.executeSqlQuery("select * from dws_poi where location is not null", { dataT ->
final def data = dataT
executorService.submit(new Runnable() {
@Override
void run() {
println(data)
def lngLat = GaoDeUtils.parserByGaode(data.city, data.region + data.name + data.address)
if (lngLat != null && lngLat.size() == 2 ) {
JSONObject jsonObject = new JSONObject()
jsonObject.put("lon", Double.valueOf(lngLat['lng']))
jsonObject.put("lat", Double.valueOf(lngLat['lat']))
EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id))
}
}
})
})
executorService.shutdown()
}
}
@@ -73,11 +73,11 @@ class SeaweedAroundPoi extends BaseBatch {
// void run() {
def city = obj.city
if (obj.lng) {
//def lng = SeaweedUtils.formatLatLng(obj.lng)
//def lat = SeaweedUtils.formatLatLng(obj.lat)
def lng = SeaweedUtils.formatLatLng(obj.lng)
def lat = SeaweedUtils.formatLatLng(obj.lat)
// {'lon': 121.810877, 'lat': 31.737642}
def lng = 121.810877
def lat = 31.737642
// def lng = 121.810877
// def lat = 31.737642
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
@@ -178,11 +178,11 @@ class SeaweedAroundPoi extends BaseBatch {
// put("写字楼", "120201");
// put("学校", "141200|141201|141202|141203|141204");
// put("银行", "1601|1603");
// put("大型超市", "0604");
put("大型超市", "0604");
// put("商场", "0601");
put("餐饮", "0501|0502|0503|0504");
// put("餐饮", "0501|0502|0503|0504");
// put("旅游景点", "110000");
// put("公园广场", "110100|110101|110102|110103|110104|110105")
put("公园广场", "110100|110101|110102|110103|110104|110105")
// put("图书馆", "140500");
// put("政府机关", "130100|130101|130102|130103|130104|130105|130106|130107");
// put("丧葬设施", "071900|071901|071902|071903")
@@ -0,0 +1,84 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.collections.CollectionUtils
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 小区周边POI数据获取
*
* @author shenjian
* @since 2021/5/24
*/
class SeaweedAroundPoiFromExcel {
static void main(String[] args) {
def types = listAllType()
dealPoi(new BasicDBObject(), types)
}
static dealPoi(DBObject query, def types) {
com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\建行\\审核小区\\东莞银行房价数据测试20220916-v3.xlsx", { List item ->
def city = item['H']
def lng = item['AQ']
def lat = item['AR']
if (lng) {
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi", jsonStr)
}
Thread.sleep(1000)
print(lng + " " + lat)
}
});
};
public static def listAllType() {
StringBuilder ret = new StringBuilder()
for (def val : KEYWORDS_MAP.values()) {
ret.append(val).append("|")
}
String result = ret.toString()
return result.substring(0, result.length() - 1)
};
public static final Map<String, String> KEYWORDS_MAP = new HashMap<String, String>() {
{
// put("餐饮", "05");
// put("地铁", "150500");
// put("公交", "1507");
// put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211");
// put("写字楼", "120201");
// put("学校", "141200|141201|141202|141203|141204");
// put("银行", "1601|1603");
put("大型超市", "0604");
// put("商场", "0601");
// put("餐饮", "0501|0502|0503|0504");
// put("旅游景点", "110000");
put("公园广场", "110100|110101|110102|110103|110104|110105")
// put("图书馆", "140500");
// put("政府机关", "130100|130101|130102|130103|130104|130105|130106|130107");
// put("丧葬设施", "071900|071901|071902|071903")
}
}
}
@@ -147,10 +147,12 @@ class GaoDeUtils {
params.page = "${page}".toString()
page++
params.key = this.getGaodeKey("v3/place/around")
//params.key = "25b2d459fd133e22f8abcebeac972692"
params.offset = "100"
JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params);
if (json == null) {
params.key = this.getGaodeKey("v3/place/around")
//params.key = "25b2d459fd133e22f8abcebeac972692"
params.offset = "100"
json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params);
}
@@ -1,5 +1,6 @@
package com.fangjia.script.thirdPriceCrawl
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
@@ -11,7 +12,7 @@ import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
@@ -77,6 +78,9 @@ class DataStandardBatch extends BaseBatch{
}
def rowIndex = 0;
DBCollection THIRD_MATCH_FILTER_COLL = MongoUtils.getCollectionByName("fangjia_craw", "third_match_filter", "192.168.0.235", 27018)
DBCollection THIRD_PRICE_FILTER_COLL = MongoUtils.getCollectionByName("fangjia_craw", "third_price_filter", "192.168.0.235", 27018)
static main(args) {
runBatch(new DataStandardBatch(), args)
}
@@ -87,9 +91,9 @@ class DataStandardBatch extends BaseBatch{
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
cityMap.each{city,alias->
println "开始加载【${city}】"
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"])
districtMap.putAll(detailMap)
println "开始加载【${city}】"
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
districtMap.putAll(detailMap)
}
while(flag){
@@ -116,9 +120,9 @@ class DataStandardBatch extends BaseBatch{
def final AtomicInteger count_ = new AtomicInteger(0)
def final AtomicInteger successIndex = new AtomicInteger(0)
datas.each { dbo ->
fixedThreadPool.execute(
new Runnable() {
void run() {
// fixedThreadPool.execute(
// new Runnable() {
// void run() {
try{
println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】"
if(dbo.name){
@@ -158,12 +162,12 @@ class DataStandardBatch extends BaseBatch{
}finally {
latch.countDown()
}
}
}
);
// }
// }
// );
}
latch.await()
// latch.await()
long end = System.currentTimeMillis()
println "第【${rowIndex}】次中匹配成功的:"+successCount+" 个,耗时:"+ (end - start)+" "+ DateUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss")
@@ -370,39 +374,44 @@ class DataStandardBatch extends BaseBatch{
if(data.name.contains("写字楼")||data.name.contains("办公楼")||data.name.contains("商业")){
district = null;
}else{
district = getMatchNameFromMap(data.city, data.region, data.name, data.address)
district = getMatchNameFromMap(data)
if(!district && data.name.length()==2){//没有匹配上,小区名只有2个字就把小区名加“小区”
district = getMatchNameFromMap(data.city, data.region, data.name+"小区", data.address)
def copyObj = data
copyObj.name = data.name+"小区"
district = getMatchNameFromMap(copyObj)
}
if(!district){
def newName = filterSpacialChar(data.name);//包含特殊符号
if(newName) {
district = getMatchNameFromMap(data.city, data.region, newName.replaceAll("_", ""), data.address)
def copyObj = data
copyObj.name = newName.replaceAll("_", "")
district = getMatchNameFromMap(copyObj)
if(!district){
def s1 = newName.substring(0, newName.indexOf("_"))
if (s1) {
//特殊符号前的内容匹配
//district = getMatchName(data.city, data.region, s1, data.address,"district")
district = getMatchNameFromMap(data.city, data.region, s1, data.address)
copyObj.name = s1
district = getMatchNameFromMap(copyObj)
if (!district) {
def s2 = newName.replace(s1, "").replaceAll("_", "")
def newName2 = null;
if (s2 && s2.length() >= 3) {
//特殊符号后的内容匹配
// district = getMatchName(data.city, data.region, s2, data.address,"district")
district = getMatchNameFromMap(data.city, data.region, s2, data.address)
copyObj.name = s2
district = getMatchNameFromMap(copyObj)
if (district) {
type = 3;
println "[匹配成功](后)[${district.city}-${district.region}-${district.name}-${district.address}]"
}
} else {
newName2 = newName.replaceAll("_", "")
copyObj.name = newName2
//district = getMatchName(data.city, data.region, newName2, data.address,"district")
district = getMatchNameFromMap(data.city, data.region, newName2, data.address)
district = getMatchNameFromMap(copyObj)
if (district) {
type = 4;
println "[匹配成功](All)[${district.city}-${district.region}-${district.name}-${district.address}]"
@@ -410,15 +419,20 @@ class DataStandardBatch extends BaseBatch{
}
if (!district) {
district = getMatchNameFromMap(data.city, null, newName2, data.address)
copyObj.name = newName2
copyObj.region = null
district = getMatchNameFromMap(copyObj)
if(!district){
district = getMatchNameFromMap(data.city, null, s1, data.address)
copyObj.name = s1
copyObj.region = null
district = getMatchNameFromMap(copyObj)
if (district) {
type = 5;
} else {
if (s2 && s2.length() >= 3) {
district = getMatchNameFromMap(data.city, null, s2, data.address)
copyObj.name = s2
copyObj.region = null
district = getMatchNameFromMap(copyObj)
if (district) {
type = 6;
}
@@ -446,11 +460,14 @@ class DataStandardBatch extends BaseBatch{
}catch (Exception e){
addr = data.address;
}
district = getMatchAddressFromMap(data.city, data.region, addr, data.name)
def copyObj = data
copyObj.address = addr
district = getMatchAddressFromMap(copyObj)
if(district){
type = 8;
}else{
district = getMatchAddressFromMap(data.city, null, addr, data.name)
copyObj.region = null
district = getMatchAddressFromMap(copyObj)
if(district){
type = 9;
}
@@ -460,25 +477,31 @@ class DataStandardBatch extends BaseBatch{
if(!district){
def districts = getMatchPingYing(data.city, data.region, data.name, addr);
district = checkMatchAddress(districts, addr);
district = checkMatchAddress(districts, addr, data);
if(district){
type = 10;
}else if(districts.size()==1){
district = districts[0];
type = 14
district = getFilerMatchDistrict(districts[0], data);
if(district){
type = 14
}
}
}
if(!district){
district = getOpenApiMatch(data.city, data.region, data.name, addr);
def copyObj = data
copyObj.address = addr
district = getOpenApiMatch(copyObj)
if(!district && addr){
district = getOpenApiMatch(data.city, data.region, addr, data.name);
district = getOpenApiMatch(copyObj)
}
if(district){
if(data.region != district.region){
type = 12;//区域不同
district = null;
}else{
def score = CosineSimilarAlgorithm.getSimilarity(district.name, copyObj.name)
district.score = score //此处用来做测试
type=15
}
}
@@ -522,7 +545,7 @@ class DataStandardBatch extends BaseBatch{
return null;
}
def checkMatchAddress(def districts, def address){
def checkMatchAddress(def districts, def address, def data){
def district = null;
if(districts.size()==1 && address && address.matches(".+?(路|街|大道).+(号|弄)")){
def tempDist = districts[0];
@@ -537,6 +560,10 @@ class DataStandardBatch extends BaseBatch{
}
}
}
if(district){
return getFilerMatchDistrict(district, data)
}
return district;
}
@@ -584,54 +611,154 @@ class DataStandardBatch extends BaseBatch{
}*/
def getMatchNameFromMap(def city, def region, def disName, String address){
//查询房价网是否存在该小区
def dbo = findSameNameFromMap(city, region, disName);
def getMatchNameFromMap(def data){
//查询房价网是否存在该小区 city, data.region, data.name, data.address, data.estate_type, avg_price
def dbo = findSameNameFromMap(data.city, data.region, data.name);
def addrFlag = null
if(address){
addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(data.address){
addrFlag = data.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(addrFlag){
dbo = addressCheck(dbo, region, address);
dbo = addressCheck(dbo, data.region, data.address);
}
}
if(!dbo){
//别名
def distrs = findAliasNameFromMap(city, region, disName);
def distrs = findAliasNameFromMap(data.city, data.region, data.name);
if(distrs && distrs.size()==1){
dbo = distrs[0]
}else if(distrs && distrs.size()>1 && address){
}else if(distrs && distrs.size()>1 && data.address){
for(def d : distrs){
if(addrFlag && (d.address.contains(address)||d.address==address || address.contains(d.address))){
if(addrFlag && (d.address.contains(data.address)||d.address==data.address || data.address.contains(d.address))){
dbo = d;
break;
}
}
}
if(dbo){
dbo = addressCheck(dbo, region, address);
dbo = addressCheck(dbo, data.region, data.address);
}
//地址
if(!dbo){
def result = findAddressFromMap(city, region, disName)
def result = findAddressFromMap(data.city, data.region, data.name)
if(result.size()==1){
dbo = result[0];
}else if(result && result.size()>1 && address){
}else if(result && result.size()>1 && data.address){
for(def d : result){
if(d.address.contains(address)||d.address==address || address.contains(d.address)){
if(d.address.contains(data.address)||d.address==data.address || data.address.contains(d.address)){
dbo = d;
break;
}
}
}
if(dbo){
dbo = addressCheck(dbo, region, address);
dbo = addressCheck(dbo, data.region, data.address);
}
}
}
if(dbo){
return getFilerMatchDistrict(dbo, data)
}
return dbo;
}
def getFilerMatchDistrict(dbo, data){
def types = []
if(data.estate_type){
types = getEstateTypes(data.estate_type)
}
if(types.size()>0 && types.size() == 1){
if(types.contains(dbo.estate_type)){
return dbo;
}else{
def ret = THIRD_MATCH_FILTER_COLL.findOne(data._id)
if(!ret){
data.put("msg","物业类型不匹配")
THIRD_MATCH_FILTER_COLL.save(data)
}
return null
}
}else{
def priceMap = findHousePrice(dbo.city, dbo.region, dbo.name, DateUtils.format(DateUtil.add(new Date(), -180),"yyyyMMdd").toInteger())
def price = priceMap.get(dbo.estate_type)
if(price && data.avg_price && data.avg_price.toString().isNumber()){
def pre = (data.avg_price-price)*100/price
if(pre>=-20 && pre<=20){
return dbo;
}else{
def ret = THIRD_PRICE_FILTER_COLL.findOne(data._id)
if(!ret){
data.put("msg","房源价格过滤")
THIRD_PRICE_FILTER_COLL.save(data)
}
return null
}
}
}
return dbo
}
def findHousePrice(def city, def region, def name, def startDate){
def typePriceMap = [:]
try{
String sql = "select avg_price, estate_type from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and region='${region}' and district_name='${name}'"
def houseMap = [:]
ElasticUtils.executeSqlQuery(sql,{ data->
if(data.estate_type){
def key = ""
if(data.estate_type.contains("住宅") || data.estate_type.contains("公寓")){
key = "普通住宅"
}else if(data.estate_type.contains("商住") || data.estate_type.contains("商务")){
key = "商住"
}else if( data.estate_type.contains("别墅")){
key = "别墅"
}else{
key = "其它"
}
def houses = null
if(key){
houses = houseMap.get(key)
if(!houses){
houses = []
}
}
if(data.avg_price){
houses << data.avg_price
houseMap.put(key, houses)
}
}
})
if(houseMap){
houseMap.each{k,v->
if(v && v.size()>=3){
def price = (int)(v.sum()/v.size())
typePriceMap.put(k, price)
}
}
}
}catch(Exception e){
e.printStackTrace()
}
return typePriceMap
}
def getEstateTypes(str){
def types = []
if(str){
if(str.contains("住宅") || str.contains("公寓")){
types << "普通住宅"
}
if(str.contains("商住") || str.contains("商务")){
types << "商住"
}
if(str.contains("别墅")){
types << "别墅"
}
}
return types;
}
/*def getMatchAddress(def city, def region, def address, def name, def cat){
def dbo = CommonDatautils.findSeaweed(city, region, address, cat);
@@ -664,22 +791,23 @@ class DataStandardBatch extends BaseBatch{
}
return dbo;
}*/
def getMatchAddressFromMap(def city, def region, def address, def name){
def getMatchAddressFromMap(def copyObj){
def dbo = null
if(address && isStandardRoad(address)){
dbo = findSameNameFromMap(city, region, address);
def name = copyObj.name
if(copyObj.address && isStandardRoad(copyObj.address)){
dbo = findSameNameFromMap(copyObj.city, copyObj.region, copyObj.address);
}
if(!dbo){
def result = null
if(address && isStandardRoad(address)){
result = findAddressFromMap(city, region, address);
if(copyObj.address && isStandardRoad(copyObj.address)){
result = findAddressFromMap(copyObj.city, copyObj.region, copyObj.address);
}
if(result && result.size()==1){
dbo = result[0];
}else if(result && result.size()>1 && name){
}else if(result && result.size()>1 && copyObj.name){
def score = null;
for(def d : result){
if(d.name == name){
if(d.name == copyObj.name){
dbo = d;
break;
}else{
@@ -706,7 +834,7 @@ class DataStandardBatch extends BaseBatch{
}
}
if(!dbo){
result = findAddrAliasFromMap(city, region, address);
result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address);
if(result.size()==1){
dbo = result[0];
}else if(result && result.size()>1 && name){
@@ -718,6 +846,11 @@ class DataStandardBatch extends BaseBatch{
}
}
}
if(dbo){
return getFilerMatchDistrict(dbo, copyObj)
}
}
return dbo;
}
@@ -756,7 +889,7 @@ class DataStandardBatch extends BaseBatch{
if(arrays[0] && arrays.size()==2){
def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(arrays[0], " ")
if(py1 == py2){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}
}
}
@@ -768,11 +901,15 @@ class DataStandardBatch extends BaseBatch{
}
def getOpenApiMatch(def city, def region, def name, def address){
if(StringUtils.isNotEmpty(name)){
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null, "fuzzy", false)
def getOpenApiMatch(def data){
if(StringUtils.isNotEmpty(data.name)){
def jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false)
if(jsonObject && jsonObject.credit>0 ){
return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address, credit:jsonObject.credit]
def dbo = [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address,estate_type:jsonObject.propertyType, credit:jsonObject.credit]
if(dbo){
return getFilerMatchDistrict(dbo, data)
}
return dbo
}
}
return null;
@@ -783,6 +920,9 @@ class DataStandardBatch extends BaseBatch{
def findAllNoMatchData(){
DBObject qdata = new BasicDBObject();
qdata.put("s_date", SHORT_DATE)
// qdata.put("city","北京")
// qdata.put("source","安居客")
// qdata.put("name",Pattern.compile("龚家宅.*"))
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
def ret = []
try{
@@ -925,6 +1065,9 @@ class DataStandardBatch extends BaseBatch{
udata2.put("match_region", district.region);
udata2.put("match_name", district.name);
udata2.put("match_address", district.address);
if(district.score){
udata2.put("score", district.score);
}
udata2.put("avg_price", data.avg_price);
udata2.put("type", type);
udata2.put("tid", data._id.toString());
@@ -948,9 +1091,9 @@ class DataStandardBatch extends BaseBatch{
def arrays = k.split("###")
if(arrays[0] && arrays[0]==keyword){
if(arrays.size()>=2){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:region, name:arrays[0]]
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
}
}
@@ -962,9 +1105,9 @@ class DataStandardBatch extends BaseBatch{
def arrays = k.split("###")
if(arrays[0] && arrays[0]==keyword){
if(arrays.size()>=2){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:region, name:arrays[0]]
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
}
}
}
@@ -990,18 +1133,18 @@ class DataStandardBatch extends BaseBatch{
def alias = v.alias;
if(alias.contains(keyword)){
if(arrays.size()>=2){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:region, name:arrays[0]]
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
}
}
}else if(v && v.mapping_alias){
def mappingAlias = v.mapping_alias
if(mappingAlias.contains(keyword)){
if(arrays.size()>=2){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:region, name:arrays[0]]
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
}
}
}
@@ -1016,18 +1159,18 @@ class DataStandardBatch extends BaseBatch{
def alias = v.alias;
if(alias.contains(keyword)){
if(arrays.size()>=2){
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:r, name:arrays[0]]
districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type]
}
}
}else if(v && v.mapping_alias){
def mappingAlias = v.mapping_alias
if(mappingAlias.contains(keyword)){
if(arrays.size()>=2){
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:r, name:arrays[0]]
districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type]
}
}
}
@@ -1052,9 +1195,9 @@ class DataStandardBatch extends BaseBatch{
def alias = v.addr_alias;
if(alias.contains(keyword)){
if(arrays.size()>=2){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:region, name:arrays[0]]
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
}
}
}
@@ -1068,9 +1211,9 @@ class DataStandardBatch extends BaseBatch{
def alias = v.addr_alias;
if(alias.contains(keyword)){
if(arrays.size()>=2){
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:r, name:arrays[0]]
districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type]
}
}
}
@@ -1093,9 +1236,9 @@ class DataStandardBatch extends BaseBatch{
def arrays = k.split("###")
if(arrays && arrays.size()==2 && arrays[1]==keyword){
if(arrays.size()>=2){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:region, name:arrays[0]]
districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type]
}
}
}
@@ -1106,9 +1249,9 @@ class DataStandardBatch extends BaseBatch{
def arrays = k.split("###")
if(arrays && arrays.size()==2 && arrays[1]==keyword){
if(arrays.size()>=2){
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type]
}else{
districts << [city:city, region:r, name:arrays[0]]
districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type]
}
}
}
@@ -40,11 +40,11 @@ class DataStat {
def run(){
def source = "我爱我家"//
def citys = findCrawlCity(source, 202207);
def source = "°²¾Ó¿Í"//
def citys = findCrawlCity(source, 202208);
for(def city : citys){
def count1 = findCount(source, city, 202207);
def count2 = findCount(source, city, 202208);
def count1 = findCount(source, city, 202208);
def count2 = findCount(source, city, 202209);
println city +" "+count1+" "+count2
// println count1
// println "--- ----"
@@ -1,5 +1,8 @@
package com.fangjia.script.thirdPriceCrawl
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
@@ -11,7 +14,8 @@ import org.bson.types.ObjectId
* 福州农商行
* 将抓取的小区均价同步到第三方均价表中
*/
class SynchDistrictPrice {
@BatchOwner(Watcher.GaoJuan)
class SynchDistrictPrice extends BaseBatch{
def static final SHORT_DATE = Utils.getLastMonth()
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
@@ -20,16 +24,22 @@ class SynchDistrictPrice {
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
//"365淘房":"365淘房","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","吉屋":"吉屋"
def esSources = ["搜房":"搜房","我爱我家":"我爱我家","365淘房":"365淘房"]
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"
def esSources = ["中原地产":"中原地产"]
static main(args) {
def task = new SynchDistrictPrice()
// task.run()
task.run1()
runBatch(new SynchDistrictPrice(), args);
}
@Override
public void process(Object args) {
run1()
}
def run1(){
println "starting......"
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def citys = CommonUtils.findAllCity()
@@ -47,7 +57,7 @@ class SynchDistrictPrice {
def names = new HashSet()
def rets = []
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='${mdate}' order by m_date", {d->
def s = d.city +"#"+d.region+"#"+d.name
def s = d.city +"#"+d.region+"#"+d.name+"#"+d.property_type
if(!names.contains(s)){
names << s
def data = new BasicDBObject()
@@ -57,6 +67,7 @@ class SynchDistrictPrice {
data.put("address", d.address)
data.put("avg_price", d.avg_price)
data.put("source", mgSource)
data.put("estate_type",d.property_type)
data.put("s_date", SHORT_DATE)
data.put("url", d.url)
data.put("c_date", new Date())
@@ -134,7 +145,7 @@ class SynchDistrictPrice {
println "["+source +"-"+city + "] add:"+ rets.size()+" 原抓取:"+count
}else{
for(def r : rets){
def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
def obj= findIsExists(r.city, r.region, r.name, r.estate_type, SHORT_DATE, source)
if(obj){
updateIndex++
// thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
@@ -142,6 +153,7 @@ class SynchDistrictPrice {
def udata = new BasicDBObject()
udata.put("avg_price", r.avg_price)
udata.put("estate_type", r.estate_type)
udata.put("m_date",new Date())
thirdPartyColl.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', udata))
@@ -171,13 +183,16 @@ class SynchDistrictPrice {
}
def findIsExists(def city, def region, def name, def sDate, def source){
def findIsExists(def city, def region, def name, estate_type, def sDate, def source){
def q = new BasicDBObject()
q.put("source", source)
q.put("s_date", sDate)
q.put("city", city)
q.put("region", region)
q.put("name", name)
if(estate_type){
q.put("estate_type",estate_type)
}
def obj = thirdPartyColl.findOne(q)
return obj
}
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
def count1 = findTotalCount(source, 202207);
def count2 = findTotalCount(source, 202208);
def count1 = findTotalCount(source, 202208);
def count2 = findTotalCount(source, 202209);
println source +" "+count1 +" "+count2
}
}
@@ -81,8 +81,8 @@ class ThirdDataToGatherBatch extends BaseBatch{
println "小区添加价格完成"
districts = addOtherWebsitePrice(districts);
println "小区添加第三方价格完成"
districts = addLevel(districts);
println "Ð¡ÇøÆÀ¼¶Íê³É"
// districts = addLevel(districts);
// println "Ð¡ÇøÆÀ¼¶Íê³É"
saveOrUpdate(districts);
println city + "保存修改完成"
toLineDB(city)
@@ -101,16 +101,16 @@ class ThirdDataToGatherBatch extends BaseBatch{
def tempPrice = d.get(field+"_price")
if(!tempPrice){
d.put(field+"_price", m.avg_price.toInteger())
if(d.fangjia_price && m.avg_price){
if(d.fangjia_price){
try{
d.put(field+"_diff", diffPercent(d.fangjia_price.toInteger(), m.avg_price.toInteger()))
}catch(Exception e){
println "dddd"
}
}
}
// if(d.fangjia_price && m.avg_price){
// if(d.fangjia_price){
// try{
// d.put(field+"_diff", diffPercent(d.fangjia_price.toInteger(), m.avg_price.toInteger()))
// }catch(Exception e){
// println "dddd"
// }
//
// }
// }
}
}else{
@@ -208,8 +208,9 @@ class ThirdDataToGatherBatch extends BaseBatch{
query.put("s_date", SHORT_DATE)
DBObject udata = new BasicDBObject()
udata.put("level",d.level);
udata.put("last_price", d.last_price);
// udata.put("level",d.level);
// udata.put("last_price", d.last_price);
udata.put("seaweed_id", d._id.toString())
udata.put("fangjia_price", d.fangjia_price);
fieldMap.each{k,v->
@@ -52,6 +52,22 @@ public class OpenApiClient {
return false
}
public static def search(def city, def district, def name){
Map<String, String> params = new HashMap<String, String>()
params.put("city", city);
params.put("keyword", name);
params.put("district", district)
params.put("token", TOKEN);
String content = new HttpRequest().get("http://open.fangjia.com/property/search", params, "utf-8")
JSONObject jsonObj = JSONObject.fromObject(content)
JSONArray array = jsonObj.get("result")
def matchObj = null
if(array){
matchObj = array[0]
}
return matchObj
}
public static List searchDistrict(double lng, double lat, int r, String city) {
DecimalFormat format = new DecimalFormat("#.0000000000");
try {
@@ -348,18 +364,20 @@ public class OpenApiClient {
}
//此接口地址标准化,带补全
public static JSONObject standard1Action(String city, String address) {
try {
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("address", address);
params.put("token", TOKEN);
params.put("__refresh_cache","1")
// params.put("__refresh_cache","1")
String content = new HttpRequest().get("https://open.fangjia.com/address/standard", params, "utf-8")
//192.168.0.100:30115
String content = new HttpRequest().get("http://open.fangjia.com/address/standard", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/address/standard", params, "utf-8")
result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/address/standard", params, "utf-8")
println("${city} ${address} ³ÌÐòµ÷ÓÃÒì³£")
Thread.sleep(6000)
}
@@ -369,7 +387,7 @@ public class OpenApiClient {
println("${city} ${address}")
}
} catch (Exception e) {
println("${city} ${address} ${e.getMessage()}")
println("[http error] ${city} ${address} ${e.getMessage()}")
e.printStackTrace();
}
return null
@@ -1,188 +1 @@
package com.fangjia.script.wenyan
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\国信达级联价格-上海.xlsx",{List item->
// if(item["AB"]){
// return
// }
// String region = null
// String name = item["F"]
// if(name.endsWith("小区")){
// name = name.replaceAll("小区\$", "")
// }
// region = RegionData.formatRegion(item["B"], item["D"])
// def results = []
// DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(item["B"], region, name)).each {
// if(!results.contains(it)){
// results.add(it)
// }
// }
// DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(item["B"], region).append("alias", name)).each {
// if(!results.contains(it)){
// results.add(it)
// }
// }
// if(results.size() == 1){
// item["Z"] = results[0]._id.toString()
// item["AA"] = results[0].region
// item["AB"] = results[0].name
// println(item)
// }
//})
import com.fangjia.core.ProvinceData
import com.fangjia.core.RegionData
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilders
//DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018)
//def count = 0
//List<List<String>> data = Collections.synchronizedList([])
//data << ["ID","城市","区域","小区名","经度","纬度","街道"]
//
//CommonUtils.findAllCity().each {def city->
// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item->
// if(StringUtils.isBlank(item.township) && item.lng){
// count++
// def lng = item.lng/10000000000d
// def lat = item.lat/10000000000d
// def location = "${lng},${lat}"
// JSONObject resp = GaoDeApiUtils.geocode(location)
// def row = []
// row << item._id.toString()
// row << item.city
// row << item.region
// row << item.name
// row << lng
// row << lat
// BasicDBObject dbo = new BasicDBObject()
// dbo.id = item._id.toString()
// dbo.city = item.city
// dbo.region = item.region
// dbo.name = item.name
// dbo.lng = lng
// dbo.lat = lat
// if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){
// row << resp.regeocode.addressComponent.township
// dbo.township = resp.regeocode.addressComponent.township
// }
// println(row)
// data << row
// SEAWEED_TOWNSHIP.insert(dbo)
// }
// })
//}
//ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据")
//ExcelUtils.modify("D:\\街道补充数据.xlsx",{List item ->
// if(item["G"] && item["H"] == null){
// DbUtils.seaweed.update(new BasicDBObject("_id", item["A"]), new BasicDBObject('$set', new BasicDBObject('township', item["G"])))
// }
// println(item)
//})
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\错误经纬度.xlsx",{List item->
// def location = "${item["A"]},${item["B"]}"
// JSONObject resp = GaoDeApiUtils.geocode(location)
// if(resp.regeocode && resp.regeocode.formatted_address){
// item['G'] = resp.regeocode.formatted_address
// println(item)
// }
//})
//def count = 0
//CommonUtils.findAllCity().each {def city->
// DbUtils.executeQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item->
// if(item.township){
// count++
// }
// }, new BasicDBObject("township", 1))
//}
//println(count)
//def GXD_CCB = [:]
//def CCB_FJW = [:]
//ExcelUtils.read("D:\\Users\\wenyan\\Downloads\\4月级联待优化小区_3个城市.xlsx",sheetIndex = 1).each {List item->
// if(item["V"] == "级联"){
// if(item["Y"] == "国信达"){
// GXD_CCB[item['J']] = item["E"]
// }else if(item["Y"] == "房价网"){
// CCB_FJW[item['E']] = item["J"]
// }
// }
//}
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\4月级联待优化小区_3个城市.xlsx",{List item->
// if(item["V"] == "级联" && item["Y"] == "国信达"){
// def ccbId = GXD_CCB.get(item['J'], null)
// def fjwId = CCB_FJW.get(ccbId, null)
// if(fjwId != null){
// item["AB"] = fjwId
//// BasicDBObject seaweed = DbUtils.seaweed.findById(fjwId)
//// List results = DbUtils.findAroundDistrict(seaweed)
//// long total = 0
//// long count = 0
//// for(BasicDBObject s : results){
//// def avg = DbUtils.findDistrictAveragePrice(seaweed.city, seaweed.region, seaweed.name, Date.parse('yyyyMMdd', '20220619'), seaweed.estate_type2)
//// if(avg){
//// total += avg.avg_price
//// count += 1
//// }
//// }
//// if(count > 0){
//// item['Z'] = (int)(total/count)
//// println(item)
//// }
// }
// }
//},sheetIndex = 1)
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\小区字段补充.xlsx",{List item->
BasicDBObject seaweed = DbUtils.seaweed.findById(item["A"])
if(StringUtils.isBlank(seaweed.create_year)){
def names = []
names.add(seaweed.name)
if(seaweed.alias){
for(def it in seaweed.alias){
if(StringUtils.isNotBlank(it) && !names.contains(it)){
names.add(it)
}
}
}
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery()
boolQueryBuilder.must(QueryBuilders.termQuery("city", seaweed.city))
boolQueryBuilder.must(QueryBuilders.termQuery("region", seaweed.region))
boolQueryBuilder.must(QueryBuilders.termsQuery("name", names));
item["B"] = seaweed.city
item["C"] = seaweed.region
item["D"] = seaweed.name
List result = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", boolQueryBuilder)
def buildYears = []
result.each {
if(StringUtils.isNotBlank(it.build_year) && !StringUtils.containsAny(it.build_year,"无")){
String build_year = it.build_year
build_year = build_year.replaceAll("((|\\().*()|\\))", "")
build_year = build_year.replaceAll("[年建]", "")
// property_fee = property_fee.replace("元/㎡·月", "元/平米/月")
buildYears.add(build_year)
}
}
buildYears = buildYears.unique()
if(buildYears.size() > 0){
DbUtils.seaweed.update(new BasicDBObject("_id", seaweed._id), new BasicDBObject('$set', new BasicDBObject('create_year', buildYears[0])))
println(item)
println(buildYears)
}
}
})
@@ -12,7 +12,7 @@ import org.jsoup.Jsoup
import java.util.zip.GZIPInputStream
class GaoDeApiUtils {
def static GAO_DE_AK = "4ccfe3c898a9489884bd2bcb1ef6dcde"
def static GAO_DE_AK = "90010d53a16ce9df544acab297645106"
/**
* 冪帠僅蛌華硊<br>
@@ -87,8 +87,6 @@ public class DistrictTriggerServiceImpl implements TriggerService {
newObj.put("lng", object.get("lng"));
newObj.put("lat2", object.get("lat2"));
newObj.put("lng2", object.get("lng2"));
newObj.put("lat3", object.get("lat3"));
newObj.put("lng3", object.get("lng3"));
newObj.put("estate_type", object.get("estate_type"));
newObj.put("estate_type2", object.get("estate_type2"));
newObj.put("build_type", object.get("build_type"));
@@ -138,15 +136,6 @@ public class DistrictTriggerServiceImpl implements TriggerService {
newObj.put("lat_lng", lat + "," + lng);
}
}
// if (null != object.get("lat3") && null != object.get("lng3")) {
// double lat3 = (double)((Long)object.get("lat3") / 10000000000.0);
// double lng3 = (double)((Long)object.get("lng3") / 10000000000.0);
// if (lat3 >= -90 && lat3 <= 90 && lng3 >= -180 && lng3 <= 180) {
// newObj.put("lat_lng3", lat3 + "," + lng3);
// }
// }
if (object.get("kind") != null) {
String[] kinds = ((String) object.get("kind")).split(",");
if (ArrayUtils.contains(kinds, DistrictKindEnum.bieshu.getValue())) {
@@ -2352,7 +2352,7 @@ public class AddressServiceImpl implements AddressService {
* @param city
* @param address
* @param checkBo
* @param time
* @param addressList
*/
private void checkAmapNew(String city, String address, AddressCheckBo checkBo, Set<String> addressList) {
try {
@@ -2704,17 +2704,6 @@ public class AddressServiceImpl implements AddressService {
return true;
}
}
} else if (null != district.getLat3() && null != district.getLng3()) { // ÌÚÑ×ø±ê gps
String locations = converTenxunToAmap(formatLatLng(district.getLng3()),
formatLatLng(district.getLat3()));
if (StringUtils.isNotEmpty(locations)) {
String[] los = locations.split(",");
double distance = GeoUtils.calculateDistance(Double.valueOf(los[0]), Double.valueOf(los[1]), lng,
lat, GaussSphere.WGS84);
if (distance <= DISTANCE_RANGE) {
return true;
}
}
}
} catch (Exception e) {
LOG.error("判断小区距离出错", e);
@@ -3571,7 +3560,7 @@ public class AddressServiceImpl implements AddressService {
*
* @param city
* @param address
* @param checkBo
* @param keywordsFormat
*/
private AddressCheckTruth checkTruthSplit(String city, String address, Map<String, String> keywordsFormat) {
// 1.拆分
@@ -4581,7 +4570,7 @@ public class AddressServiceImpl implements AddressService {
/**
* 地址评级具体逻辑
*
* @param checkBo
* @param checkTr
* @return
*/
private AddressCheckTruth gradeAddressTruth(AddressCheckTruth checkTr) {
@@ -4605,7 +4594,7 @@ public class AddressServiceImpl implements AddressService {
/**
* 小区评分
*
* @param checkBo
* @param check
* @return
*/
private boolean gradeDistrictTruth(AddressCheckTruth check,boolean isConsume) {
Binary file not shown.

Before

Width:  |  Height:  |  Size: 189 KiB

After

Width:  |  Height:  |  Size: 136 KiB

@@ -309,7 +309,7 @@ public class HouseToCController {
params.put("token", token);
params.put("size", query.getArea());
params.put("app", "youda");// open保存估价记录
params.put("youDa", true);
//params.put("youDa", true);
if (houseInfo.getFloor() != 0 && houseInfo.getHeight() != 0) {
params.put("floor", houseInfo.getFloor());
params.put("maxFloor", houseInfo.getHeight());
@@ -601,7 +601,7 @@ public class HouseToCController {
params.put("name", name);
params.put("type", type * (-1));
params.put("evalDate", DateUtils.date2Str(houseInfo.getEvalTime()));
params.put("youDa", true);
//params.put("youDa", true);
DistrictEvaluateSingleDto result = openApiProxyService.getProxyService(FshApiRemoteService.class)
.getFshDistPriceTrends(params);