Merge branch 'script_gj' into 'master'

Script gj

See merge request !430
This commit is contained in:
2022-11-09 13:17:40 +08:00
9 changed files with 1331 additions and 21 deletions
@@ -68,6 +68,7 @@ class BigdataForsaleBatch extends BaseBatch{
public void process(Object args) {
def index = 0
def citys = CcbCommonUtils.getRunningCitys(args)
def citys = ["阿坝州"]//CcbCommonUtils.getRunningCitys(args)
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
index++
@@ -477,8 +478,14 @@ class BigdataForsaleBatch extends BaseBatch{
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
if(vallaFlag){
continue;
}
}
if(!seawater.total_price){
continue;
}
@@ -538,10 +545,6 @@ class BigdataForsaleBatch extends BaseBatch{
if(floor && floor.toInteger()<1){
floor = null
}
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
if(vallaFlag){
continue;
}
def rowMap = [:];
if(cat == "rent"){
@@ -976,9 +976,9 @@ class ExportOracleDataTest {
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_evaluation/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
def fileName = "/ccb_evaluation/MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
@@ -1007,9 +1007,9 @@ class ExportOracleDataTest {
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_evaluation/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
def fileName = "/ccb_evaluation/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-22", "2022-10-27")
def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-09")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -502,8 +502,18 @@ class ThirdPartyPriceBatch extends BaseBatch {
def findThirdPartyPrice(def city, def statMap){
def q = new BasicDBObject()
q.put("city", city)
<<<<<<< .mine
//q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
=======
q.put("s_date", endDate)
// q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
>>>>>>> .theirs
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
@@ -7,8 +7,12 @@ import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -16,13 +20,79 @@ import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class AddressCut extends BaseBatch{
final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
static main(args) {
runBatch(new AddressCut(), args);
}
@Override
public void process(Object args) {
sameAddr()
// sameAddr() //相同地址小区
// roadMatch()
addAnjukeRoad()
}
def addAnjukeRoad(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d ->
def roads = findThirdPartyPrice(d['A'], d['B'], d['C'], "链家")
if(roads){
d['O'] = roads[0]
}
});
}
def findThirdPartyPrice(city, region, name, source){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("source", source)
def dbcursor = thirdPartyPriceColl.find(q).sort(new BasicDBObject("s_date", -1)).limit(10)
def roads = []
for(def d : dbcursor){
if(d.address && !roads.contains(d.address)){
roads << d.address
}
}
return roads
}
def roadMatch(){
def cityRoadsMap = queryPoiMap()
println "道路加载完成....."
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d ->
def roads = cityRoadsMap.get(d['A'])
// if (roads && roads.contains(d['E'])) {
// d['L'] = "是"
// println d['A'] + " " + d['E']
// }
if(roads){
for(def road : roads){
if(!d['L'] && d['E'] && d['E'].toString().contains(road)){
d['L'] = "中间包含"
d['M'] = road
println d['A'] + " " + d['E']
break;
}
}
}
})
}
def sameAddr() {
@@ -143,4 +213,29 @@ class AddressCut extends BaseBatch{
latch.await()
return rows;
}
def queryPoiMap(){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,name,type FROM amap_poi WHERE 1=1")
// sqlBuffer.append(" and cityname = '" + city + "'")
// sqlBuffer.append(" and name = '" + name + "'")
sqlBuffer.append(" and type like '%道路名%'")
// sqlBuffer.append(" and MATCH(name,'${name}')")
// sqlBuffer.append(" and source = '高德' order by score() desc")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def cityMap = [:]
while (rs.next()) {
def city = rs.getString("cityname");
def road = rs.getString("name")
def roads = cityMap.get(city)
if(!roads){
roads = new HashSet()
}
roads << road
cityMap.put(city, roads)
}
return cityMap;
}
}
@@ -1,6 +1,7 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
@@ -19,7 +20,22 @@ class CcbDataStat {
static main(args) {
def task = new CcbDataStat()
// task.run()
task.tradeRun()
// task.tradeRun()
task.regionDistStat()
}
def regionDistStat(){
def citys = CcbCommonUtils.citys_252
def rows = []
for(def c : citys){
def regions = CommonDatautils.getRegionListByCity(c)
for(def r : regions){
def count = CommonDatautils.findRegionDistrictCount("district",c, r)
rows << [c, r, count]
println c +" "+r +" "+count
}
}
ExcelUtils.write("d:\\数据\\建行\\区域小区量统计.xlsx", rows, "统计")
}
def tradeRun(){
@@ -19,11 +19,11 @@ class EsMonthHouseStat {
def task = new EsMonthHouseStat()
// task.run()
// task.tradeRun()
// task.houseRun()
task.houseRun()
// task.statHouseUpdateCount()
task.statDistHouse()
// task.statDistHouse()
}
def statDistHouse(){