Merge branch 'script_gj' into 'master'

小区修改

See merge request !131
This commit is contained in:
2021-09-26 18:34:04 +08:00
4 changed files with 90 additions and 8 deletions
@@ -12,6 +12,7 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
@@ -46,16 +47,19 @@ class BigdataCommunityBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"])
println city +" 小区量:"+dists.size()
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city))
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets)
println city +" "+rets.size()
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
}
def fileName = "/home/goojia/ccb/bigdata_community.dat"
def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.dat"
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat"
}
def allDatas = []
@@ -66,8 +70,27 @@ class BigdataCommunityBatch extends BaseBatch{
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
}
def addDeleteDist(dists, ccbMaps, rets){
def fangjiaDists = []
for(def d : dists){
fangjiaDists << d._id.toString()
}
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
def ccbDists = ccbMaps.keySet()
for(def ccbId : ccbDists){
if(!fangjiaDists.contains(ccbId)){
def datas = ccbMaps.get(ccbId)
datas[datas.size()-1] = 1
rets << datas
}
}
println "添加删除状态小区:"+rets.size()
return rets
}
def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){
def rets = Collections.synchronizedList(new ArrayList());
//def rets = Collections.synchronizedList(new ArrayList());
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
@@ -182,6 +205,7 @@ class BigdataCommunityBatch extends BaseBatch{
ret << ""
ret << ""
ret << ""
ret << 0
}catch(Exception e){
println dist.city + " "+dist.name
e.printStackTrace()
@@ -9,6 +9,7 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.sql.Sql
class CcbCommonUtils {
@@ -33,6 +34,9 @@ class CcbCommonUtils {
public static DSCollection dataStoreConnection = null;
public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
public static def findCityCodeValue(){
def cityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"]
return cityCodeMap;
@@ -113,7 +117,7 @@ class CcbCommonUtils {
if(alias){
q.put("district_name", new BasicDBObject('$in', alias))
}
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = DEAL_PRICE_COLL.find(q)
def total = dbcursor.count()
def rets = new HashSet()
@@ -272,4 +276,48 @@ class CcbCommonUtils {
return null;
}
public static def findCcbDistrict(cityid){
def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" +
"\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj where cityId=${cityid}";
def map = [:]
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DISTID
data << r.COMMID
data << r.DISTNAME
data << r.COMMNAME
data << r.ADDRESS
data << r.LATITUDE
data << r.LONGITUDE
data << r.LANDAREA
data << r.BUILDAREA
data << r.GREENINGRATE
data << r.PLOTRATIO
data << r.PARKINGCNT
data << r.BUILDYEAR
data << r.HOUSEUSE
data << r.ZTS
data << r.WORKDATE
data << r.UPDATEFLAG
data << r.UPDATETIME
data << r.COMMTYPE
data << r.COMMALIAS
data << r.STREET
data << r.STREETNO
data << r.IFNEW
data << ""
data << r.REMARK
data << r.CREATETIME
data << r.SOURCECODE
data << r.XZJDB
data << r.LJXZQHDM
data << r.LJXZQH
data << r.STATUS
map.put(r.COMMID, data)
});
return map
}
}
@@ -47,12 +47,22 @@ class SFTP {
* @param sftp
*/
public void upload(String directory, String uploadFile, ChannelSftp sftp) {
FileInputStream stream = null
try {
sftp.cd(directory);
File file=new File(uploadFile);
sftp.put(new FileInputStream(file), file.getName());
stream = new FileInputStream(file)
sftp.put(stream, file.getName());
} catch (Exception e) {
e.printStackTrace();
} finally {
if(stream!=null){
stream.close();
}
// if (sftp != null) {
// sftp.disconnect();
// sftp.exit();
// }
}
}
@@ -15,9 +15,9 @@ class SynchDistrictPrice {
def static final SHORT_DATE = Utils.getLastMonth()
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
"anjuke.com":"安居客","cityhouse.cn":"城市房产"]
// def sourceMap = ["lianjia.com":"链家"]
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产"]
def sourceMap = ["zhuge.com":"诸葛找房"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()