Merge branch 'script_gj' into 'master'

Script gj

See merge request !686
This commit is contained in:
2023-07-19 14:10:49 +08:00
11 changed files with 114 additions and 71 deletions
@@ -1215,6 +1215,7 @@ public class DistrictController {
params.add(new QueryParam("city", city));
params.add(new QueryParam("region", region));
params.add(new QueryParam("alias", name));
params.add(new QueryParam("visible", 0));
List<District> ds = districtService.findAllByParams(params);
if (CollectionUtils.isNotEmpty(ds)) {
@@ -2350,6 +2351,7 @@ public class DistrictController {
params = new ArrayList<QueryParam>(2);
params.add(new QueryParam("category", "district,new_district"));
params.add(new QueryParam("name", a));
params.add(new QueryParam("visible", 0));
ds = districtService.findAllByParams(params);
@@ -259,7 +259,19 @@
<#-- <input name="district.businessQuarter" type="text" style="width:300px" value="${district.businessQuarter!}" />-->
<#-- </td>-->
<#-- </tr>-->
<#-- <tr>-->
<tr>
<td class = "t1">
<#if markMap?? && markMap["township"]?? && markMap["township"] == 1>
<i class="lock" name="township"></i>
<#else>
<i class="unlock" name="township"></i>
</#if>
乡镇街道</td>
<td>
<input name="district.township" type="text" style="width:300px" value="${district.township!}" />
</td>
</tr>
<tr>
<td class = "t1">
<#if markMap?? && markMap["loop_line_info"]?? && markMap["loop_line_info"] == 1>
<i class="lock" name="loop_line_info"></i>
@@ -479,7 +491,7 @@
<#if markMap?? && markMap["estate_charge"]?? && markMap["estate_charge"] == 1>
<i class="lock" name="estate_charge"></i>
<#else>
<i class="unlock" name="estate_charge"></i>
<i class="unlock" name="得房率"></i>
</#if>
物 业 费</td>
<td>
@@ -838,18 +850,18 @@
</tr>
</table>
<table class = "saveTable">
<tr>
<td class = "t1" style="width:150px;">
<#if markMap?? && markMap["fitment"]?? && markMap["fitment"] == 1>
<i class="lock" name="fitment"></i>
<#else>
<i class="unlock" name="fitment"></i>
</#if>
装修情况</td>
<td>
<textarea alias="richarea" name="district.fitment" style="width:400px;height:100px;">${district.fitment!}</textarea>
</td>
</tr>
<#-- <tr>-->
<#-- <td class = "t1" style="width:150px;">-->
<#-- <#if markMap?? && markMap["fitment"]?? && markMap["fitment"] == 1>-->
<#-- <i class="lock" name="fitment"></i>-->
<#-- <#else>-->
<#-- <i class="unlock" name="fitment"></i>-->
<#-- </#if>-->
<#-- 装修情况</td>-->
<#-- <td>-->
<#-- <textarea alias="richarea" name="district.fitment" style="width:400px;height:100px;">${district.fitment!}</textarea>-->
<#-- </td>-->
<#-- </tr>-->
<#-- <tr>-->
<#-- <td class = "t1">-->
<#-- <#if markMap?? && markMap["open_date"]?? && markMap["open_date"] == 1>-->
@@ -49,15 +49,25 @@ class BigdataCommunityBatch extends BaseBatch{
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
def today = new Date().getDate()
println "starting 。。。。。。。。。"
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
def ccbMaps = null
if(today<=10){
ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
}else{
ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
}
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets)
@@ -77,9 +87,10 @@ class BigdataCommunityBatch extends BaseBatch{
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成->导Oracle"
delPgData()
saveToPgSql(rows)
if(today > 10){
delPgData()
saveToPgSql(rows)
}
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
// delPgData()
@@ -69,6 +69,10 @@ class BigdataForsaleBatch extends BaseBatch{
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM");
def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM");
static main(args) {
runBatch(new BigdataForsaleBatch(), args);
}
@@ -81,6 +85,9 @@ class BigdataForsaleBatch extends BaseBatch{
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def startDate = lastMonth+"18"
def endDate = curMonth+"19"
println "房源时间:"+startDate+" "+endDate
def houseIdsfilter = findAllHouseIdsByCity()
@@ -91,7 +98,7 @@ class BigdataForsaleBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230518, 20230619, houseIdsfilter);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
}
@@ -20,6 +20,25 @@ class PocTest {
static main(args) {
def task = new PocTest();
task.run();
// task.export()
}
def export(){
ExcelUtils.modify("D:\\数据\\其它\\广州.xlsx", { row ->
def dist = CommonDatautils.findDistrictById(row["B"])
row['F'] = dist.address
row['G'] = dist.lng/10000000000.0
row['H'] = dist.lat/10000000000.0
row['I'] = dist.estate_type2
row['J'] = dist.build_type
row['K'] = dist.schools
row['L'] = dist.kindergartens
row['M'] = dist.loudong_count
row['N'] = dist.household_count
row['O'] = dist.estate_charge
row['P'] = dist.finish_building_date
row['Q'] = dist.estate_company
})
}
@@ -32,9 +51,10 @@ class PocTest {
* @return
*/
def run(){
def inputfile = "D:\\数据\\重庆农商行\\poc\\test.xls"
// def inputfile = "D:\\数据\\重庆农商行\\poc\\test.xls"
def inputfile = "D:\\数据\\其它\\广州.xlsx"
// match(inputfile)
// addPrice(inputfile)
addPrice(inputfile)
// addArroundPrice(inputfile)
// standard(inputfile)
// addLoudongNum(inputfile)
@@ -132,15 +152,16 @@ class PocTest {
def addPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['R']
def commid = row['B']
if(commid){
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
row['Q'] = dist.name
// row['Q'] = dist.name
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
row['F'] = priceObj.avg_price
row['R'] = priceObj.avg_price
}
row['S'] = CommonDatautils.findNewSqmetreRent(dist.city, dist.region, dist.name)
}
}
})
@@ -252,7 +252,7 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name", "鲁能大厦")
// q.put("name", "交发风景")
DBObject keys = new BasicDBObject();
if(fields){
@@ -899,28 +899,28 @@ class CommonDatautils {
DBObject q = new BasicDBObject();
q.put("_id", new ObjectId(id.toString().replace("[","").replace("]","")));
DBObject keys = new BasicDBObject();
keys.put("alias", 1);
keys.put("addr_alias", 1);
keys.put("city", 1);
keys.put("region", 1);
keys.put("name", 1);
keys.put("estate_type2", 1);
keys.put("address", 1);
keys.put("build_structure", 1);
keys.put("out_pics", 1);
keys.put("lat", 1);
keys.put("lng", 1);
keys.put("lat2", 1);
keys.put("lng2", 1);
keys.put("name_cat", 1);
keys.put("finish_building_date", 1)
keys.put("mapping_alias", 1)
keys.put("sid", 1)
keys.put("status", 1)
keys.put("name_cat", 1)
// DBObject keys = new BasicDBObject();
// keys.put("alias", 1);
// keys.put("addr_alias", 1);
// keys.put("city", 1);
// keys.put("region", 1);
// keys.put("name", 1);
// keys.put("estate_type2", 1);
// keys.put("address", 1);
// keys.put("build_structure", 1);
// keys.put("out_pics", 1);
// keys.put("lat", 1);
// keys.put("lng", 1);
// keys.put("lat2", 1);
// keys.put("lng2", 1);
// keys.put("name_cat", 1);
// keys.put("finish_building_date", 1)
// keys.put("mapping_alias", 1)
// keys.put("sid", 1)
// keys.put("status", 1)
// keys.put("name_cat", 1)
try{
DSCursor cursor = getDataStoreConnection().find(q, keys);
DSCursor cursor = getDataStoreConnection().find(q);
if(cursor){
for(def obj : cursor){
return obj;
@@ -74,29 +74,17 @@ class DistrictInfoExportBatch extends BaseBatch{
def createDatas(def city){
def result = []
def distKeys = []
def districts = CommonDatautils.findAllDistricts(city, "district", null);
for(def dist : districts){
def key = dist.city +"##"+dist.region+"##"+dist.name
if(!distKeys.contains(key)){
distKeys << key
}
}
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
// for(def dist : removeDists){
// def key = dist.city +"##"+dist.region+"##"+dist.name
// if(!distKeys.contains(key)){
// distKeys << key
// districts << dist
// }
// }
def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1)
def startDate = DateUtils.getFirstDayOfMonth(lastDate);
def fzAddrMap = getFzAddress();
//因为福州个性化问题,又不想影响自家数据库数据,所以另存一个福州小区库,用于手动添加数据
// saveOrUpdateFzDist(districts)
saveOrUpdateFzDist(districts)
remarkRemoveDist(removeDists)
def fzDistricts = getFzDistricts(city)
@@ -33,7 +33,7 @@ class DistrictPriceExportBatch extends BaseBatch{
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
def specialDistsPriceMap = ["津泰新村7号、9-13号、15-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580]
def specialDistsPriceMap = ["津泰新村9-13号、15-18号":54000,"津泰新村14号":72000,"津泰新村1-6号、8号":35000,"津泰新村7号":75000,"津泰新村19号":25000]
static main(args) {
runBatch(new DistrictPriceExportBatch(), args)
}
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2023-05-01"
static def statDate = "2023-06-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年6月\\查询1.DBF");//友达二手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年7月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -435,7 +435,7 @@ class OfficeProcess {
pstmt.setString(5, d.source)
println count+" "+d.city+" "+d.name
pstmt.addBatch()
if(count % 1 ==0) {
if(count % 100 ==0) {
try {
pstmt.executeBatch()
} catch (Exception e) {
@@ -480,7 +480,7 @@ class OfficeProcess {
}
def checkRegion(){
def citys = ["杭州","苏州"]//"上海","深圳","北京","广州","重庆","成都",
def citys = ["上海"]//"上海","深圳","北京","广州","重庆","成都",
for(def city : citys){
def regionPointsMap = findRegionRange(city)
def addDatas = findAddDatasByClickhouse(city)
@@ -492,7 +492,7 @@ class OfficeProcess {
if(data.lng){
def flag = isInArea(data.lng.toString().toDouble(), data.lat.toString().toDouble(), regionPointsMap.get(data.region))
def mark = flag?'T':'F'
// rets << ["city":data.city, "region":data.region, "name":data.name, "source":data.source, "flag":mark]
rets << ["city":data.city, "region":data.region, "name":data.name, "source":data.source, "flag":mark]
updateOption(data.city, data.region, data.name, data.source, mark)
println index+" "+data.region + " "+data.name+" "+ flag
}
@@ -502,7 +502,7 @@ class OfficeProcess {
}
def findAddDatasByClickhouse(city){
def sql = "select city,region, name,option,lng,lat,source from office where city='${city}' and option like '%添加%'"
def sql = "select city,region, name,option,lng,lat,source from office where city='${city}' and region_fence is null"
def datas = []
sqlInstance.eachRow(sql, {r->
datas << ["city":r.city,"region":r.region,"name":r.name, "lat":r.lat, "lng":r.lng, "option":r.option,"source":r.source]
@@ -490,9 +490,11 @@ public class BeiKeCrawlByChrome {
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
elemUsername.sendKeys("15021384995");
// elemUsername.sendKeys("15021384995");
// elemPassword.sendKeys("12qwaszx");
elemPassword.sendKeys("12qwaszx");
elemUsername.sendKeys("16283166371");
elemPassword.sendKeys("Goojia123456");
btn.submit(); // 点击行为;
Thread.sleep(10000)