Merge branch 'script_gj' into 'master'
Script gj See merge request !430
This commit is contained in:
@@ -242,13 +242,5 @@
|
||||
<version>1.0.0</version>
|
||||
<classifier>jdk18</classifier>
|
||||
</dependency>
|
||||
|
||||
<!-- <dependency>-->
|
||||
<!-- <groupId>org.springframework.boot</groupId>-->
|
||||
<!-- <artifactId>spring-boot-devtools</artifactId>-->
|
||||
<!-- <!– optional 表示依赖是否向下传递 true表示不向下传递 默认值是false向下传递 –>-->
|
||||
<!-- <optional>true</optional>-->
|
||||
<!-- </dependency>-->
|
||||
|
||||
</dependencies>
|
||||
</project>
|
||||
|
||||
+7
-4
@@ -68,6 +68,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
public void process(Object args) {
|
||||
def index = 0
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def citys = ["阿坝州"]//CcbCommonUtils.getRunningCitys(args)
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
index++
|
||||
@@ -477,8 +478,14 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
|
||||
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
|
||||
if(vallaFlag){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(!seawater.total_price){
|
||||
continue;
|
||||
}
|
||||
@@ -538,10 +545,6 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
if(floor && floor.toInteger()<1){
|
||||
floor = null
|
||||
}
|
||||
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
|
||||
if(vallaFlag){
|
||||
continue;
|
||||
}
|
||||
|
||||
def rowMap = [:];
|
||||
if(cat == "rent"){
|
||||
|
||||
+1194
File diff suppressed because it is too large
Load Diff
+4
-4
@@ -976,9 +976,9 @@ class ExportOracleDataTest {
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_evaluation/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_evaluation/MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
@@ -1007,9 +1007,9 @@ class ExportOracleDataTest {
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_evaluation/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_evaluation/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
|
||||
+1
-1
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-22", "2022-10-27")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-09")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
|
||||
+10
@@ -502,8 +502,18 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
def findThirdPartyPrice(def city, def statMap){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
<<<<<<< .mine
|
||||
|
||||
//q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
=======
|
||||
q.put("s_date", endDate)
|
||||
// q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
|
||||
|
||||
>>>>>>> .theirs
|
||||
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
|
||||
+96
-1
@@ -7,8 +7,12 @@ import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
@@ -16,13 +20,79 @@ import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class AddressCut extends BaseBatch{
|
||||
|
||||
|
||||
final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
|
||||
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
static main(args) {
|
||||
runBatch(new AddressCut(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
sameAddr()
|
||||
// sameAddr() //相同地址小区
|
||||
// roadMatch()
|
||||
addAnjukeRoad()
|
||||
|
||||
}
|
||||
|
||||
def addAnjukeRoad(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d ->
|
||||
def roads = findThirdPartyPrice(d['A'], d['B'], d['C'], "链家")
|
||||
if(roads){
|
||||
d['O'] = roads[0]
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def findThirdPartyPrice(city, region, name, source){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("source", source)
|
||||
def dbcursor = thirdPartyPriceColl.find(q).sort(new BasicDBObject("s_date", -1)).limit(10)
|
||||
def roads = []
|
||||
for(def d : dbcursor){
|
||||
if(d.address && !roads.contains(d.address)){
|
||||
roads << d.address
|
||||
}
|
||||
}
|
||||
return roads
|
||||
}
|
||||
|
||||
def roadMatch(){
|
||||
def cityRoadsMap = queryPoiMap()
|
||||
println "道路加载完成....."
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d ->
|
||||
def roads = cityRoadsMap.get(d['A'])
|
||||
// if (roads && roads.contains(d['E'])) {
|
||||
// d['L'] = "是"
|
||||
// println d['A'] + " " + d['E']
|
||||
// }
|
||||
|
||||
if(roads){
|
||||
for(def road : roads){
|
||||
if(!d['L'] && d['E'] && d['E'].toString().contains(road)){
|
||||
d['L'] = "中间包含"
|
||||
d['M'] = road
|
||||
println d['A'] + " " + d['E']
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
}
|
||||
|
||||
def sameAddr() {
|
||||
@@ -143,4 +213,29 @@ class AddressCut extends BaseBatch{
|
||||
latch.await()
|
||||
return rows;
|
||||
}
|
||||
|
||||
def queryPoiMap(){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT cityname,name,type FROM amap_poi WHERE 1=1")
|
||||
// sqlBuffer.append(" and cityname = '" + city + "'")
|
||||
// sqlBuffer.append(" and name = '" + name + "'")
|
||||
sqlBuffer.append(" and type like '%道路名%'")
|
||||
// sqlBuffer.append(" and MATCH(name,'${name}')")
|
||||
// sqlBuffer.append(" and source = '高德' order by score() desc")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def cityMap = [:]
|
||||
while (rs.next()) {
|
||||
def city = rs.getString("cityname");
|
||||
def road = rs.getString("name")
|
||||
def roads = cityMap.get(city)
|
||||
if(!roads){
|
||||
roads = new HashSet()
|
||||
}
|
||||
roads << road
|
||||
cityMap.put(city, roads)
|
||||
|
||||
}
|
||||
return cityMap;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+17
-1
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
@@ -19,7 +20,22 @@ class CcbDataStat {
|
||||
static main(args) {
|
||||
def task = new CcbDataStat()
|
||||
// task.run()
|
||||
task.tradeRun()
|
||||
// task.tradeRun()
|
||||
task.regionDistStat()
|
||||
}
|
||||
|
||||
def regionDistStat(){
|
||||
def citys = CcbCommonUtils.citys_252
|
||||
def rows = []
|
||||
for(def c : citys){
|
||||
def regions = CommonDatautils.getRegionListByCity(c)
|
||||
for(def r : regions){
|
||||
def count = CommonDatautils.findRegionDistrictCount("district",c, r)
|
||||
rows << [c, r, count]
|
||||
println c +" "+r +" "+count
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\建行\\区域小区量统计.xlsx", rows, "统计")
|
||||
}
|
||||
|
||||
def tradeRun(){
|
||||
|
||||
+2
-2
@@ -19,11 +19,11 @@ class EsMonthHouseStat {
|
||||
def task = new EsMonthHouseStat()
|
||||
// task.run()
|
||||
// task.tradeRun()
|
||||
// task.houseRun()
|
||||
task.houseRun()
|
||||
|
||||
// task.statHouseUpdateCount()
|
||||
|
||||
task.statDistHouse()
|
||||
// task.statDistHouse()
|
||||
}
|
||||
|
||||
def statDistHouse(){
|
||||
|
||||
Reference in New Issue
Block a user