update
This commit is contained in:
+16
-5
@@ -49,15 +49,25 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
def today = new Date().getDate()
|
||||
|
||||
println "starting 。。。。。。。。。"
|
||||
for(def city : citys){
|
||||
|
||||
index ++
|
||||
|
||||
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
|
||||
|
||||
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
|
||||
// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
|
||||
def ccbMaps = null
|
||||
if(today<=10){
|
||||
ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
|
||||
}else{
|
||||
ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
}
|
||||
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets)
|
||||
rows.addAll(rets)
|
||||
@@ -77,9 +87,10 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成->导Oracle"
|
||||
|
||||
delPgData()
|
||||
saveToPgSql(rows)
|
||||
|
||||
if(today > 10){
|
||||
delPgData()
|
||||
saveToPgSql(rows)
|
||||
}
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
|
||||
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
|
||||
// delPgData()
|
||||
|
||||
+8
-1
@@ -69,6 +69,10 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
|
||||
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM");
|
||||
def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM");
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataForsaleBatch(), args);
|
||||
}
|
||||
@@ -81,6 +85,9 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
def startDate = lastMonth+"18"
|
||||
def endDate = curMonth+"19"
|
||||
println "房源时间:"+startDate+" "+endDate
|
||||
|
||||
def houseIdsfilter = findAllHouseIdsByCity()
|
||||
|
||||
@@ -91,7 +98,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230518, 20230619, houseIdsfilter);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter);
|
||||
//测试通过匹配接口配房源效果
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||
}
|
||||
|
||||
+26
-5
@@ -20,6 +20,25 @@ class PocTest {
|
||||
static main(args) {
|
||||
def task = new PocTest();
|
||||
task.run();
|
||||
// task.export()
|
||||
}
|
||||
|
||||
def export(){
|
||||
ExcelUtils.modify("D:\\数据\\其它\\广州.xlsx", { row ->
|
||||
def dist = CommonDatautils.findDistrictById(row["B"])
|
||||
row['F'] = dist.address
|
||||
row['G'] = dist.lng/10000000000.0
|
||||
row['H'] = dist.lat/10000000000.0
|
||||
row['I'] = dist.estate_type2
|
||||
row['J'] = dist.build_type
|
||||
row['K'] = dist.schools
|
||||
row['L'] = dist.kindergartens
|
||||
row['M'] = dist.loudong_count
|
||||
row['N'] = dist.household_count
|
||||
row['O'] = dist.estate_charge
|
||||
row['P'] = dist.finish_building_date
|
||||
row['Q'] = dist.estate_company
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -32,9 +51,10 @@ class PocTest {
|
||||
* @return
|
||||
*/
|
||||
def run(){
|
||||
def inputfile = "D:\\数据\\重庆农商行\\poc\\test.xls"
|
||||
// def inputfile = "D:\\数据\\重庆农商行\\poc\\test.xls"
|
||||
def inputfile = "D:\\数据\\其它\\广州.xlsx"
|
||||
// match(inputfile)
|
||||
// addPrice(inputfile)
|
||||
addPrice(inputfile)
|
||||
// addArroundPrice(inputfile)
|
||||
// standard(inputfile)
|
||||
// addLoudongNum(inputfile)
|
||||
@@ -132,15 +152,16 @@ class PocTest {
|
||||
|
||||
def addPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def commid = row['R']
|
||||
def commid = row['B']
|
||||
if(commid){
|
||||
def dist = CommonDatautils.findDistrictById(commid.trim())
|
||||
if(dist){
|
||||
row['Q'] = dist.name
|
||||
// row['Q'] = dist.name
|
||||
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(priceObj){
|
||||
row['F'] = priceObj.avg_price
|
||||
row['R'] = priceObj.avg_price
|
||||
}
|
||||
row['S'] = CommonDatautils.findNewSqmetreRent(dist.city, dist.region, dist.name)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
+22
-22
@@ -252,7 +252,7 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name", "鲁能大厦")
|
||||
// q.put("name", "交发风景")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -899,28 +899,28 @@ class CommonDatautils {
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("_id", new ObjectId(id.toString().replace("[","").replace("]","")));
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
keys.put("alias", 1);
|
||||
keys.put("addr_alias", 1);
|
||||
keys.put("city", 1);
|
||||
keys.put("region", 1);
|
||||
keys.put("name", 1);
|
||||
keys.put("estate_type2", 1);
|
||||
keys.put("address", 1);
|
||||
keys.put("build_structure", 1);
|
||||
keys.put("out_pics", 1);
|
||||
keys.put("lat", 1);
|
||||
keys.put("lng", 1);
|
||||
keys.put("lat2", 1);
|
||||
keys.put("lng2", 1);
|
||||
keys.put("name_cat", 1);
|
||||
keys.put("finish_building_date", 1)
|
||||
keys.put("mapping_alias", 1)
|
||||
keys.put("sid", 1)
|
||||
keys.put("status", 1)
|
||||
keys.put("name_cat", 1)
|
||||
// DBObject keys = new BasicDBObject();
|
||||
// keys.put("alias", 1);
|
||||
// keys.put("addr_alias", 1);
|
||||
// keys.put("city", 1);
|
||||
// keys.put("region", 1);
|
||||
// keys.put("name", 1);
|
||||
// keys.put("estate_type2", 1);
|
||||
// keys.put("address", 1);
|
||||
// keys.put("build_structure", 1);
|
||||
// keys.put("out_pics", 1);
|
||||
// keys.put("lat", 1);
|
||||
// keys.put("lng", 1);
|
||||
// keys.put("lat2", 1);
|
||||
// keys.put("lng2", 1);
|
||||
// keys.put("name_cat", 1);
|
||||
// keys.put("finish_building_date", 1)
|
||||
// keys.put("mapping_alias", 1)
|
||||
// keys.put("sid", 1)
|
||||
// keys.put("status", 1)
|
||||
// keys.put("name_cat", 1)
|
||||
try{
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys);
|
||||
DSCursor cursor = getDataStoreConnection().find(q);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
return obj;
|
||||
|
||||
+3
-15
@@ -74,29 +74,17 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
|
||||
def createDatas(def city){
|
||||
def result = []
|
||||
def distKeys = []
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", null);
|
||||
for(def dist : districts){
|
||||
def key = dist.city +"##"+dist.region+"##"+dist.name
|
||||
if(!distKeys.contains(key)){
|
||||
distKeys << key
|
||||
}
|
||||
}
|
||||
|
||||
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
|
||||
// for(def dist : removeDists){
|
||||
// def key = dist.city +"##"+dist.region+"##"+dist.name
|
||||
// if(!distKeys.contains(key)){
|
||||
// distKeys << key
|
||||
// districts << dist
|
||||
// }
|
||||
// }
|
||||
|
||||
def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1)
|
||||
def startDate = DateUtils.getFirstDayOfMonth(lastDate);
|
||||
|
||||
def fzAddrMap = getFzAddress();
|
||||
|
||||
//因为福州个性化问题,又不想影响自家数据库数据,所以另存一个福州小区库,用于手动添加数据
|
||||
// saveOrUpdateFzDist(districts)
|
||||
saveOrUpdateFzDist(districts)
|
||||
remarkRemoveDist(removeDists)
|
||||
def fzDistricts = getFzDistricts(city)
|
||||
|
||||
|
||||
+1
-1
@@ -33,7 +33,7 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
|
||||
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||
|
||||
def specialDistsPriceMap = ["津泰新村7号、9-13号、15-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580]
|
||||
def specialDistsPriceMap = ["津泰新村9-13号、15-18号":54000,"津泰新村14号":72000,"津泰新村1-6号、8号":35000,"津泰新村7号":75000,"津泰新村19号":25000]
|
||||
static main(args) {
|
||||
runBatch(new DistrictPriceExportBatch(), args)
|
||||
}
|
||||
|
||||
+2
-2
@@ -48,11 +48,11 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2023-05-01"
|
||||
static def statDate = "2023-06-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2023年6月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2023年7月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
|
||||
+4
-4
@@ -435,7 +435,7 @@ class OfficeProcess {
|
||||
pstmt.setString(5, d.source)
|
||||
println count+" "+d.city+" "+d.name
|
||||
pstmt.addBatch()
|
||||
if(count % 1 ==0) {
|
||||
if(count % 100 ==0) {
|
||||
try {
|
||||
pstmt.executeBatch()
|
||||
} catch (Exception e) {
|
||||
@@ -480,7 +480,7 @@ class OfficeProcess {
|
||||
}
|
||||
|
||||
def checkRegion(){
|
||||
def citys = ["杭州","苏州"]//"上海","深圳","北京","广州","重庆","成都",
|
||||
def citys = ["上海"]//"上海","深圳","北京","广州","重庆","成都",
|
||||
for(def city : citys){
|
||||
def regionPointsMap = findRegionRange(city)
|
||||
def addDatas = findAddDatasByClickhouse(city)
|
||||
@@ -492,7 +492,7 @@ class OfficeProcess {
|
||||
if(data.lng){
|
||||
def flag = isInArea(data.lng.toString().toDouble(), data.lat.toString().toDouble(), regionPointsMap.get(data.region))
|
||||
def mark = flag?'T':'F'
|
||||
// rets << ["city":data.city, "region":data.region, "name":data.name, "source":data.source, "flag":mark]
|
||||
rets << ["city":data.city, "region":data.region, "name":data.name, "source":data.source, "flag":mark]
|
||||
updateOption(data.city, data.region, data.name, data.source, mark)
|
||||
println index+" "+data.region + " "+data.name+" "+ flag
|
||||
}
|
||||
@@ -502,7 +502,7 @@ class OfficeProcess {
|
||||
}
|
||||
|
||||
def findAddDatasByClickhouse(city){
|
||||
def sql = "select city,region, name,option,lng,lat,source from office where city='${city}' and option like '%添加%'"
|
||||
def sql = "select city,region, name,option,lng,lat,source from office where city='${city}' and region_fence is null"
|
||||
def datas = []
|
||||
sqlInstance.eachRow(sql, {r->
|
||||
datas << ["city":r.city,"region":r.region,"name":r.name, "lat":r.lat, "lng":r.lng, "option":r.option,"source":r.source]
|
||||
|
||||
+4
-2
@@ -490,9 +490,11 @@ public class BeiKeCrawlByChrome {
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
|
||||
elemUsername.sendKeys("15021384995");
|
||||
// elemUsername.sendKeys("15021384995");
|
||||
// elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
elemUsername.sendKeys("16283166371");
|
||||
elemPassword.sendKeys("Goojia123456");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(10000)
|
||||
|
||||
Reference in New Issue
Block a user