Merge branch 'script_gj' into 'master'

Script gj

See merge request !430
This commit is contained in:
2022-11-09 13:17:40 +08:00
9 changed files with 1331 additions and 21 deletions
-8
View File
@@ -242,13 +242,5 @@
<version>1.0.0</version> <version>1.0.0</version>
<classifier>jdk18</classifier> <classifier>jdk18</classifier>
</dependency> </dependency>
<!-- <dependency>-->
<!-- <groupId>org.springframework.boot</groupId>-->
<!-- <artifactId>spring-boot-devtools</artifactId>-->
<!-- &lt;!&ndash; optional 表示依赖是否向下传递 true表示不向下传递 默认值是false向下传递 &ndash;&gt;-->
<!-- <optional>true</optional>-->
<!-- </dependency>-->
</dependencies> </dependencies>
</project> </project>
@@ -68,6 +68,7 @@ class BigdataForsaleBatch extends BaseBatch{
public void process(Object args) { public void process(Object args) {
def index = 0 def index = 0
def citys = CcbCommonUtils.getRunningCitys(args) def citys = CcbCommonUtils.getRunningCitys(args)
def citys = ["阿坝州"]//CcbCommonUtils.getRunningCitys(args)
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){ for(def city : citys){
index++ index++
@@ -477,8 +478,14 @@ class BigdataForsaleBatch extends BaseBatch{
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue; continue;
} }
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
if(vallaFlag){
continue;
}
} }
if(!seawater.total_price){ if(!seawater.total_price){
continue; continue;
} }
@@ -538,10 +545,6 @@ class BigdataForsaleBatch extends BaseBatch{
if(floor && floor.toInteger()<1){ if(floor && floor.toInteger()<1){
floor = null floor = null
} }
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
if(vallaFlag){
continue;
}
def rowMap = [:]; def rowMap = [:];
if(cat == "rent"){ if(cat == "rent"){
@@ -976,9 +976,9 @@ class ExportOracleDataTest {
}); });
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_evaluation/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" def fileName = "/ccb_evaluation/MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
} }
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
@@ -1007,9 +1007,9 @@ class ExportOracleDataTest {
}); });
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_evaluation/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" def fileName = "/ccb_evaluation/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
} }
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价 //添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-22", "2022-10-27") def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-09")
if(prices){ if(prices){
def distMap = getDistrictMap(city) def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -502,8 +502,18 @@ class ThirdPartyPriceBatch extends BaseBatch {
def findThirdPartyPrice(def city, def statMap){ def findThirdPartyPrice(def city, def statMap){
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("city", city) q.put("city", city)
<<<<<<< .mine
//q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
=======
q.put("s_date", endDate) q.put("s_date", endDate)
// q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) // q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
>>>>>>> .theirs
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:] def map = [:]
for(def db : dbcursor){ for(def db : dbcursor){
@@ -7,8 +7,12 @@ import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService import java.util.concurrent.ExecutorService
@@ -16,13 +20,79 @@ import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong import java.util.concurrent.atomic.AtomicLong
class AddressCut extends BaseBatch{ class AddressCut extends BaseBatch{
final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
static main(args) { static main(args) {
runBatch(new AddressCut(), args); runBatch(new AddressCut(), args);
} }
@Override @Override
public void process(Object args) { public void process(Object args) {
sameAddr() // sameAddr() //相同地址小区
// roadMatch()
addAnjukeRoad()
}
def addAnjukeRoad(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d ->
def roads = findThirdPartyPrice(d['A'], d['B'], d['C'], "链家")
if(roads){
d['O'] = roads[0]
}
});
}
def findThirdPartyPrice(city, region, name, source){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("source", source)
def dbcursor = thirdPartyPriceColl.find(q).sort(new BasicDBObject("s_date", -1)).limit(10)
def roads = []
for(def d : dbcursor){
if(d.address && !roads.contains(d.address)){
roads << d.address
}
}
return roads
}
def roadMatch(){
def cityRoadsMap = queryPoiMap()
println "道路加载完成....."
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d ->
def roads = cityRoadsMap.get(d['A'])
// if (roads && roads.contains(d['E'])) {
// d['L'] = "是"
// println d['A'] + " " + d['E']
// }
if(roads){
for(def road : roads){
if(!d['L'] && d['E'] && d['E'].toString().contains(road)){
d['L'] = "中间包含"
d['M'] = road
println d['A'] + " " + d['E']
break;
}
}
}
})
} }
def sameAddr() { def sameAddr() {
@@ -143,4 +213,29 @@ class AddressCut extends BaseBatch{
latch.await() latch.await()
return rows; return rows;
} }
def queryPoiMap(){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,name,type FROM amap_poi WHERE 1=1")
// sqlBuffer.append(" and cityname = '" + city + "'")
// sqlBuffer.append(" and name = '" + name + "'")
sqlBuffer.append(" and type like '%道路名%'")
// sqlBuffer.append(" and MATCH(name,'${name}')")
// sqlBuffer.append(" and source = '高德' order by score() desc")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def cityMap = [:]
while (rs.next()) {
def city = rs.getString("cityname");
def road = rs.getString("name")
def roads = cityMap.get(city)
if(!roads){
roads = new HashSet()
}
roads << road
cityMap.put(city, roads)
}
return cityMap;
}
} }
@@ -1,6 +1,7 @@
package com.fangjia.script.ccbevaluation.data package com.fangjia.script.ccbevaluation.data
import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
@@ -19,7 +20,22 @@ class CcbDataStat {
static main(args) { static main(args) {
def task = new CcbDataStat() def task = new CcbDataStat()
// task.run() // task.run()
task.tradeRun() // task.tradeRun()
task.regionDistStat()
}
def regionDistStat(){
def citys = CcbCommonUtils.citys_252
def rows = []
for(def c : citys){
def regions = CommonDatautils.getRegionListByCity(c)
for(def r : regions){
def count = CommonDatautils.findRegionDistrictCount("district",c, r)
rows << [c, r, count]
println c +" "+r +" "+count
}
}
ExcelUtils.write("d:\\数据\\建行\\区域小区量统计.xlsx", rows, "统计")
} }
def tradeRun(){ def tradeRun(){
@@ -19,11 +19,11 @@ class EsMonthHouseStat {
def task = new EsMonthHouseStat() def task = new EsMonthHouseStat()
// task.run() // task.run()
// task.tradeRun() // task.tradeRun()
// task.houseRun() task.houseRun()
// task.statHouseUpdateCount() // task.statHouseUpdateCount()
task.statDistHouse() // task.statDistHouse()
} }
def statDistHouse(){ def statDistHouse(){