Merge branch 'script_gj' into 'master'
Script gj See merge request !430
This commit is contained in:
@@ -242,13 +242,5 @@
|
|||||||
<version>1.0.0</version>
|
<version>1.0.0</version>
|
||||||
<classifier>jdk18</classifier>
|
<classifier>jdk18</classifier>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
<!-- <dependency>-->
|
|
||||||
<!-- <groupId>org.springframework.boot</groupId>-->
|
|
||||||
<!-- <artifactId>spring-boot-devtools</artifactId>-->
|
|
||||||
<!-- <!– optional 表示依赖是否向下传递 true表示不向下传递 默认值是false向下传递 –>-->
|
|
||||||
<!-- <optional>true</optional>-->
|
|
||||||
<!-- </dependency>-->
|
|
||||||
|
|
||||||
</dependencies>
|
</dependencies>
|
||||||
</project>
|
</project>
|
||||||
|
|||||||
+7
-4
@@ -68,6 +68,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
|||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
def index = 0
|
def index = 0
|
||||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||||
|
def citys = ["阿坝州"]//CcbCommonUtils.getRunningCitys(args)
|
||||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
index++
|
index++
|
||||||
@@ -477,8 +478,14 @@ class BigdataForsaleBatch extends BaseBatch{
|
|||||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
|
||||||
|
if(vallaFlag){
|
||||||
|
continue;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
if(!seawater.total_price){
|
if(!seawater.total_price){
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -538,10 +545,6 @@ class BigdataForsaleBatch extends BaseBatch{
|
|||||||
if(floor && floor.toInteger()<1){
|
if(floor && floor.toInteger()<1){
|
||||||
floor = null
|
floor = null
|
||||||
}
|
}
|
||||||
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
|
|
||||||
if(vallaFlag){
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
def rowMap = [:];
|
def rowMap = [:];
|
||||||
if(cat == "rent"){
|
if(cat == "rent"){
|
||||||
|
|||||||
+1194
File diff suppressed because it is too large
Load Diff
+4
-4
@@ -976,9 +976,9 @@ class ExportOracleDataTest {
|
|||||||
});
|
});
|
||||||
|
|
||||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||||
def fileName = "/ccb_evaluation/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
def fileName = "/ccb_evaluation/MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||||
if(CommonDatautils.isWindow()){
|
if(CommonDatautils.isWindow()){
|
||||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||||
}
|
}
|
||||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||||
@@ -1007,9 +1007,9 @@ class ExportOracleDataTest {
|
|||||||
});
|
});
|
||||||
|
|
||||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||||
def fileName = "/ccb_evaluation/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
|
def fileName = "/ccb_evaluation/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||||
if(CommonDatautils.isWindow()){
|
if(CommonDatautils.isWindow()){
|
||||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
|
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||||
}
|
}
|
||||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||||
|
|||||||
+1
-1
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
|||||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||||
|
|
||||||
//添加自己库中的人工指导价
|
//添加自己库中的人工指导价
|
||||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-22", "2022-10-27")
|
def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-09")
|
||||||
if(prices){
|
if(prices){
|
||||||
def distMap = getDistrictMap(city)
|
def distMap = getDistrictMap(city)
|
||||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||||
|
|||||||
+10
@@ -502,8 +502,18 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
|||||||
def findThirdPartyPrice(def city, def statMap){
|
def findThirdPartyPrice(def city, def statMap){
|
||||||
def q = new BasicDBObject()
|
def q = new BasicDBObject()
|
||||||
q.put("city", city)
|
q.put("city", city)
|
||||||
|
<<<<<<< .mine
|
||||||
|
|
||||||
|
//q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||||
|
|
||||||
|
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||||
|
=======
|
||||||
q.put("s_date", endDate)
|
q.put("s_date", endDate)
|
||||||
// q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
// q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||||
|
|
||||||
|
|
||||||
|
>>>>>>> .theirs
|
||||||
|
|
||||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||||
def map = [:]
|
def map = [:]
|
||||||
for(def db : dbcursor){
|
for(def db : dbcursor){
|
||||||
|
|||||||
+96
-1
@@ -7,8 +7,12 @@ import com.fangjia.script.gj.district.FangjiaClientUtils
|
|||||||
import com.fangjia.script.lixue.util.DBUtils
|
import com.fangjia.script.lixue.util.DBUtils
|
||||||
import com.fangjia.script.util.DbUtils
|
import com.fangjia.script.util.DbUtils
|
||||||
import com.fangjia.script.util.ExcelUtils
|
import com.fangjia.script.util.ExcelUtils
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||||
|
|
||||||
|
import java.sql.ResultSet
|
||||||
import java.util.concurrent.CopyOnWriteArrayList
|
import java.util.concurrent.CopyOnWriteArrayList
|
||||||
import java.util.concurrent.CountDownLatch
|
import java.util.concurrent.CountDownLatch
|
||||||
import java.util.concurrent.ExecutorService
|
import java.util.concurrent.ExecutorService
|
||||||
@@ -16,13 +20,79 @@ import java.util.concurrent.Executors
|
|||||||
import java.util.concurrent.atomic.AtomicLong
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
|
|
||||||
class AddressCut extends BaseBatch{
|
class AddressCut extends BaseBatch{
|
||||||
|
|
||||||
|
|
||||||
|
final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
|
||||||
|
|
||||||
|
public static EsDataSource dataSource = new EsDataSource()
|
||||||
|
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||||
|
static {
|
||||||
|
dataSource.setUrl(address)
|
||||||
|
Properties connectionProperties = new Properties()
|
||||||
|
connectionProperties.put("user", "developer")
|
||||||
|
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||||
|
dataSource.setProperties(connectionProperties)
|
||||||
|
}
|
||||||
static main(args) {
|
static main(args) {
|
||||||
runBatch(new AddressCut(), args);
|
runBatch(new AddressCut(), args);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
sameAddr()
|
// sameAddr() //相同地址小区
|
||||||
|
// roadMatch()
|
||||||
|
addAnjukeRoad()
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
def addAnjukeRoad(){
|
||||||
|
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d ->
|
||||||
|
def roads = findThirdPartyPrice(d['A'], d['B'], d['C'], "链家")
|
||||||
|
if(roads){
|
||||||
|
d['O'] = roads[0]
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
def findThirdPartyPrice(city, region, name, source){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", city)
|
||||||
|
q.put("region", region)
|
||||||
|
q.put("name", name)
|
||||||
|
q.put("source", source)
|
||||||
|
def dbcursor = thirdPartyPriceColl.find(q).sort(new BasicDBObject("s_date", -1)).limit(10)
|
||||||
|
def roads = []
|
||||||
|
for(def d : dbcursor){
|
||||||
|
if(d.address && !roads.contains(d.address)){
|
||||||
|
roads << d.address
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return roads
|
||||||
|
}
|
||||||
|
|
||||||
|
def roadMatch(){
|
||||||
|
def cityRoadsMap = queryPoiMap()
|
||||||
|
println "道路加载完成....."
|
||||||
|
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d ->
|
||||||
|
def roads = cityRoadsMap.get(d['A'])
|
||||||
|
// if (roads && roads.contains(d['E'])) {
|
||||||
|
// d['L'] = "是"
|
||||||
|
// println d['A'] + " " + d['E']
|
||||||
|
// }
|
||||||
|
|
||||||
|
if(roads){
|
||||||
|
for(def road : roads){
|
||||||
|
if(!d['L'] && d['E'] && d['E'].toString().contains(road)){
|
||||||
|
d['L'] = "中间包含"
|
||||||
|
d['M'] = road
|
||||||
|
println d['A'] + " " + d['E']
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
def sameAddr() {
|
def sameAddr() {
|
||||||
@@ -143,4 +213,29 @@ class AddressCut extends BaseBatch{
|
|||||||
latch.await()
|
latch.await()
|
||||||
return rows;
|
return rows;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def queryPoiMap(){
|
||||||
|
final StringBuffer sqlBuffer = new StringBuffer()
|
||||||
|
sqlBuffer.append("SELECT cityname,name,type FROM amap_poi WHERE 1=1")
|
||||||
|
// sqlBuffer.append(" and cityname = '" + city + "'")
|
||||||
|
// sqlBuffer.append(" and name = '" + name + "'")
|
||||||
|
sqlBuffer.append(" and type like '%道路名%'")
|
||||||
|
// sqlBuffer.append(" and MATCH(name,'${name}')")
|
||||||
|
// sqlBuffer.append(" and source = '高德' order by score() desc")
|
||||||
|
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||||
|
def cityMap = [:]
|
||||||
|
while (rs.next()) {
|
||||||
|
def city = rs.getString("cityname");
|
||||||
|
def road = rs.getString("name")
|
||||||
|
def roads = cityMap.get(city)
|
||||||
|
if(!roads){
|
||||||
|
roads = new HashSet()
|
||||||
|
}
|
||||||
|
roads << road
|
||||||
|
cityMap.put(city, roads)
|
||||||
|
|
||||||
|
}
|
||||||
|
return cityMap;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
+17
-1
@@ -1,6 +1,7 @@
|
|||||||
package com.fangjia.script.ccbevaluation.data
|
package com.fangjia.script.ccbevaluation.data
|
||||||
|
|
||||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
import com.fangjia.script.common.DataHandleUtils
|
import com.fangjia.script.common.DataHandleUtils
|
||||||
import com.fangjia.script.util.DateUtils
|
import com.fangjia.script.util.DateUtils
|
||||||
import com.fangjia.script.util.ExcelUtils
|
import com.fangjia.script.util.ExcelUtils
|
||||||
@@ -19,7 +20,22 @@ class CcbDataStat {
|
|||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new CcbDataStat()
|
def task = new CcbDataStat()
|
||||||
// task.run()
|
// task.run()
|
||||||
task.tradeRun()
|
// task.tradeRun()
|
||||||
|
task.regionDistStat()
|
||||||
|
}
|
||||||
|
|
||||||
|
def regionDistStat(){
|
||||||
|
def citys = CcbCommonUtils.citys_252
|
||||||
|
def rows = []
|
||||||
|
for(def c : citys){
|
||||||
|
def regions = CommonDatautils.getRegionListByCity(c)
|
||||||
|
for(def r : regions){
|
||||||
|
def count = CommonDatautils.findRegionDistrictCount("district",c, r)
|
||||||
|
rows << [c, r, count]
|
||||||
|
println c +" "+r +" "+count
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ExcelUtils.write("d:\\数据\\建行\\区域小区量统计.xlsx", rows, "统计")
|
||||||
}
|
}
|
||||||
|
|
||||||
def tradeRun(){
|
def tradeRun(){
|
||||||
|
|||||||
+2
-2
@@ -19,11 +19,11 @@ class EsMonthHouseStat {
|
|||||||
def task = new EsMonthHouseStat()
|
def task = new EsMonthHouseStat()
|
||||||
// task.run()
|
// task.run()
|
||||||
// task.tradeRun()
|
// task.tradeRun()
|
||||||
// task.houseRun()
|
task.houseRun()
|
||||||
|
|
||||||
// task.statHouseUpdateCount()
|
// task.statHouseUpdateCount()
|
||||||
|
|
||||||
task.statDistHouse()
|
// task.statDistHouse()
|
||||||
}
|
}
|
||||||
|
|
||||||
def statDistHouse(){
|
def statDistHouse(){
|
||||||
|
|||||||
Reference in New Issue
Block a user