小区修改
This commit is contained in:
+27
-3
@@ -12,6 +12,7 @@ import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
@@ -46,16 +47,19 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"])
|
||||
println city +" 小区量:"+dists.size()
|
||||
|
||||
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city))
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets)
|
||||
rows.addAll(rets)
|
||||
println city +" "+rets.size()
|
||||
|
||||
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/bigdata_community.dat"
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.dat"
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat"
|
||||
}
|
||||
|
||||
def allDatas = []
|
||||
@@ -66,8 +70,27 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
|
||||
}
|
||||
|
||||
def addDeleteDist(dists, ccbMaps, rets){
|
||||
def fangjiaDists = []
|
||||
for(def d : dists){
|
||||
fangjiaDists << d._id.toString()
|
||||
}
|
||||
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
|
||||
def ccbDists = ccbMaps.keySet()
|
||||
for(def ccbId : ccbDists){
|
||||
if(!fangjiaDists.contains(ccbId)){
|
||||
def datas = ccbMaps.get(ccbId)
|
||||
datas[datas.size()-1] = 1
|
||||
rets << datas
|
||||
}
|
||||
}
|
||||
println "添加删除状态小区:"+rets.size()
|
||||
return rets
|
||||
}
|
||||
|
||||
def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
//def rets = Collections.synchronizedList(new ArrayList());
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
@@ -182,6 +205,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << 0
|
||||
}catch(Exception e){
|
||||
println dist.city + " "+dist.name
|
||||
e.printStackTrace()
|
||||
|
||||
+49
-1
@@ -9,6 +9,7 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
|
||||
class CcbCommonUtils {
|
||||
|
||||
@@ -33,6 +34,9 @@ class CcbCommonUtils {
|
||||
|
||||
public static DSCollection dataStoreConnection = null;
|
||||
|
||||
public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
public static def findCityCodeValue(){
|
||||
def cityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"]
|
||||
return cityCodeMap;
|
||||
@@ -113,7 +117,7 @@ class CcbCommonUtils {
|
||||
if(alias){
|
||||
q.put("district_name", new BasicDBObject('$in', alias))
|
||||
}
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def total = dbcursor.count()
|
||||
def rets = new HashSet()
|
||||
@@ -272,4 +276,48 @@ class CcbCommonUtils {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
public static def findCcbDistrict(cityid){
|
||||
def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" +
|
||||
"\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj where cityId=${cityid}";
|
||||
def map = [:]
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.DISTID
|
||||
data << r.COMMID
|
||||
data << r.DISTNAME
|
||||
data << r.COMMNAME
|
||||
data << r.ADDRESS
|
||||
data << r.LATITUDE
|
||||
data << r.LONGITUDE
|
||||
data << r.LANDAREA
|
||||
data << r.BUILDAREA
|
||||
data << r.GREENINGRATE
|
||||
data << r.PLOTRATIO
|
||||
data << r.PARKINGCNT
|
||||
data << r.BUILDYEAR
|
||||
data << r.HOUSEUSE
|
||||
data << r.ZTS
|
||||
data << r.WORKDATE
|
||||
data << r.UPDATEFLAG
|
||||
data << r.UPDATETIME
|
||||
data << r.COMMTYPE
|
||||
data << r.COMMALIAS
|
||||
data << r.STREET
|
||||
data << r.STREETNO
|
||||
data << r.IFNEW
|
||||
data << ""
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.SOURCECODE
|
||||
data << r.XZJDB
|
||||
data << r.LJXZQHDM
|
||||
data << r.LJXZQH
|
||||
data << r.STATUS
|
||||
map.put(r.COMMID, data)
|
||||
});
|
||||
return map
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+11
-1
@@ -47,12 +47,22 @@ class SFTP {
|
||||
* @param sftp
|
||||
*/
|
||||
public void upload(String directory, String uploadFile, ChannelSftp sftp) {
|
||||
FileInputStream stream = null
|
||||
try {
|
||||
sftp.cd(directory);
|
||||
File file=new File(uploadFile);
|
||||
sftp.put(new FileInputStream(file), file.getName());
|
||||
stream = new FileInputStream(file)
|
||||
sftp.put(stream, file.getName());
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
if(stream!=null){
|
||||
stream.close();
|
||||
}
|
||||
// if (sftp != null) {
|
||||
// sftp.disconnect();
|
||||
// sftp.exit();
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+3
-3
@@ -15,9 +15,9 @@ class SynchDistrictPrice {
|
||||
def static final SHORT_DATE = Utils.getLastMonth()
|
||||
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
"anjuke.com":"安居客","cityhouse.cn":"城市房产"]
|
||||
// def sourceMap = ["lianjia.com":"链家"]
|
||||
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产"]
|
||||
def sourceMap = ["zhuge.com":"诸葛找房"]
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
task.run()
|
||||
|
||||
Reference in New Issue
Block a user