Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj
This commit is contained in:
+7
-7
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳6月份成交数据.xlsx", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳7月成交.xlsx", 0, 1)
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
@@ -77,14 +77,14 @@ for(def d : datas){
|
||||
map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
|
||||
map.avg_price = (int)(d['H'].toString().toDouble())
|
||||
if(d['I']){
|
||||
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
|
||||
map.sell_price = (int)(d['I'].toString().replace("万","").toDouble()*10000)
|
||||
}
|
||||
|
||||
// map.trade_date = d['J']
|
||||
def trade_date = d['J']
|
||||
if(trade_date){
|
||||
map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd')
|
||||
}
|
||||
map.trade_date = d['J']
|
||||
// def trade_date = d['J']
|
||||
// if(trade_date){
|
||||
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd')
|
||||
// }
|
||||
// else{
|
||||
// def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31")
|
||||
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
||||
|
||||
+27
-2
@@ -825,10 +825,31 @@ class CommonDatautils {
|
||||
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
if(!d.township){
|
||||
rets << d
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
public static def findAllRegionDistricts(def city, def region, def cat, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("status",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
for(def field : fields){
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
@@ -4056,6 +4077,9 @@ class CommonDatautils {
|
||||
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
||||
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
|
||||
|
||||
// rowMap.location = seawater.location
|
||||
rowMap.title = seawater.name
|
||||
|
||||
rets << rowMap
|
||||
}
|
||||
}
|
||||
@@ -4844,6 +4868,7 @@ class CommonDatautils {
|
||||
query.append("cat", "district")
|
||||
query.append("city", city)
|
||||
query.append("visible", 0)
|
||||
query.append("name", "长城大厦")
|
||||
DBObject sdq = new BasicDBObject();
|
||||
sdq.put("sd.point", lat + "," + lng);
|
||||
sdq.put("sd.sfield", "lat_lng");
|
||||
|
||||
+14
@@ -27,6 +27,20 @@ class FzDataResultStat {
|
||||
//// task.margeExcel()
|
||||
// task.saveCsvData()
|
||||
// task.testNewPrice()
|
||||
|
||||
// task.statHouse()
|
||||
}
|
||||
|
||||
def statHouse(){
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\福州待调查小区.xlsx",{d->
|
||||
def o = CommonDatautils.findDistrictById(d['A'])
|
||||
if(o){
|
||||
def count = CommonDatautils.findHouseCountByDate(o.city, o.region, o.name, o.alias, "sell", 20220818)
|
||||
d['K'] = count
|
||||
println o.name +" "+count
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def margeExcel(){
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202506.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202508.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+6
-7
@@ -33,17 +33,16 @@ class SuzouPocTest {
|
||||
|
||||
def test(){
|
||||
ExcelUtils.modify("D:\\数据\\上海联合产权交易所\\第1批拟处置房产清单(长城大厦、台山碧桂园).xlsx", { d ->
|
||||
if(d['O']){
|
||||
if(!d['P'] && d['I']){
|
||||
def floorAdds = d['I'].split("/")
|
||||
def ret1 = OpenApiClient.addressEvaluation(d['O'], null, d['D'], d['D'], d['K'], floorAdds[0], floorAdds[1])
|
||||
def ret1 = OpenApiClient.addressEvaluation(d['C'], null, d['D'], d['D'], d['K'], floorAdds[0], floorAdds[1])
|
||||
if (ret1) {
|
||||
d['N'] = ret1.totalPrice
|
||||
d['M'] = ret1.avgPrice
|
||||
d['P'] = ret1.district
|
||||
d['Q'] = ret1.name
|
||||
d['S'] = ret1.totalPrice
|
||||
d['T'] = ret1.avgPrice
|
||||
d['U'] = ret1.district
|
||||
d['V'] = ret1.name
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
+13
-1
@@ -12,6 +12,7 @@ import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import com.fangjia.script.gj.*
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
@@ -30,7 +31,18 @@ class DataMatch {
|
||||
// println task.findAddress("昌州街道棠香路")
|
||||
|
||||
// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
|
||||
task.fzWeiyuanshuMatch("D:\\数据\\福州\\连江楼盘.xlsx")
|
||||
// task.fzWeiyuanshuMatch("D:\\数据\\福州\\连江楼盘.xlsx")
|
||||
task.delRegionCode("D:\\数据\\需删除区域编码.xlsx")
|
||||
}
|
||||
|
||||
def delRegionCode(path){
|
||||
ExcelUtils.modify(path,{d->
|
||||
def q = new BasicDBObject("_id", new ObjectId(d['A']))
|
||||
def obj = FANGJIA_OFFICIAL_AREA_MAP.findOne(q)
|
||||
if(obj){
|
||||
FANGJIA_OFFICIAL_AREA_MAP.remove(q)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def fzWeiyuanshuMatch(fileName){
|
||||
|
||||
+1
-1
@@ -189,7 +189,7 @@ class AddDevShapeData {
|
||||
|
||||
|
||||
def run(){
|
||||
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\0812'
|
||||
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\0903'
|
||||
def dir = new File(directoryPath)
|
||||
if (dir.isDirectory()) {
|
||||
dir.eachFileRecurse(FileType.FILES) { file ->
|
||||
|
||||
+119
-7
@@ -1,10 +1,14 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
@@ -27,20 +31,21 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||
def fullCityMap = null//CommonDatautils.queryFullNameCity()
|
||||
def mcaDistMap = null//CommonDatautils.queryDistNameCityByMapping()
|
||||
|
||||
static main(args) {
|
||||
runBatch(new AutoAddDistrictBatch(), args)
|
||||
// runBatch(new AutoAddDistrictBatch(), args)
|
||||
|
||||
// def task = new AutoAddDistrictBatch();
|
||||
def task = new AutoAddDistrictBatch();
|
||||
// task.checkMap()
|
||||
task.addDistrict()
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
//查找第三方匹配没有匹配上的小区
|
||||
def citys = ["三明"]//CommonDatautils.findAllCity()
|
||||
def citys = ["泉州"]//CommonDatautils.findAllCity()
|
||||
def list = []
|
||||
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "百度类型","高德地图", "高德类型"]
|
||||
for(def c : citys){
|
||||
@@ -52,7 +57,7 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
|
||||
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\小区\\add_dist_info_1.xlsx"
|
||||
fileName = "D:\\数据\\数据建设\\小区\\泉州待新增小区.xlsx"
|
||||
}
|
||||
ExcelUtils.write(fileName, list, "待入库数据")
|
||||
|
||||
@@ -109,7 +114,7 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
gaode_name = gaode_obj.name
|
||||
}
|
||||
println "[ "+totalCount.get()+"/"+noMatchCount+" ]"+key +" trade_count:"+trade_count+" house_count:"+house_count+" baidu_count:"+baidu_count+" gaode_count:"+gaode_count
|
||||
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_type, gaode_count, gaode_type, gaode_name]
|
||||
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, baidu_type, gaode_count, gaode_type, gaode_name]
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
@@ -187,6 +192,7 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
q.put("match_id", new BasicDBObject('$exists',false))
|
||||
q.put("old_match_id", new BasicDBObject('$exists',false))
|
||||
q.put("city", c)
|
||||
q.put("district", "丰泽")
|
||||
def dbcursor = thirdSeaweedMappingColl.find(q).batchSize(5000)
|
||||
def rMap = [:]
|
||||
for(def d : dbcursor){
|
||||
@@ -293,4 +299,110 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
// d['O'] = score
|
||||
})
|
||||
}
|
||||
|
||||
def addDistrict(){
|
||||
def path = "D:\\数据\\数据建设\\小区\\新增价格.xlsx"
|
||||
ExcelUtils.modify(path, { d ->
|
||||
def dbo = new BasicDBObject()
|
||||
dbo.city = d['A'];
|
||||
dbo.region = d['B'];
|
||||
if(d['M']){
|
||||
dbo.name = d['M']
|
||||
def alias = []
|
||||
alias << d['C']
|
||||
dbo.alias = alias
|
||||
}else{
|
||||
dbo.name = d['C']
|
||||
}
|
||||
|
||||
if(d['D']){
|
||||
dbo.estate_type2 = d['D']
|
||||
}else{
|
||||
if(d['C'].contains("别墅")){
|
||||
dbo.estate_type2 = "别墅"
|
||||
}else if(d['C'].contains("商住")){
|
||||
dbo.estate_type2 = "商住"
|
||||
}else{
|
||||
dbo.estate_type2 = "普通住宅"
|
||||
}
|
||||
}
|
||||
|
||||
dbo.status = 0;
|
||||
dbo.cat = "district"
|
||||
dbo.c_date = new Date()
|
||||
dbo.m_date = new Date()
|
||||
|
||||
|
||||
// def rets = getEsDatas(d['A'], d['B'], d['C'])
|
||||
// for(def r : rets){
|
||||
// if(r.address && !dbo.address){
|
||||
// dbo.address = r.address
|
||||
// }
|
||||
// if(!dbo.lng){
|
||||
// if(r.location){
|
||||
// def lng = r.location.x
|
||||
// def lat = r.location.y
|
||||
// Long gaode_lng = lng*10000000000L
|
||||
// Long gaode_lat = lat*10000000000L
|
||||
// dbo.put("lng", gaode_lng)
|
||||
// dbo.put("lat", gaode_lat)
|
||||
// def bdpoint = FangjiaClientUtils.convert(lng+","+lat, 1)
|
||||
// if(bdpoint){
|
||||
// dbo.put("lng2", (bdpoint[0]*10000000000).longValue())
|
||||
// dbo.put("lat2", (bdpoint[1]*10000000000).longValue())
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// println dbo
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("city", dbo.city)
|
||||
// q.put("name", dbo.name)
|
||||
// q.put("cat", "district")
|
||||
// q.put("status", 1)
|
||||
// q.put("region", dbo.region)
|
||||
// def obj = DbUtils.seaweed.findOne(q)
|
||||
// if(obj){
|
||||
// println "【存在】"+dbo
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("status", 1)
|
||||
// udata.put("m_date", new Date())
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
|
||||
// }else{
|
||||
// DbUtils.seaweed.save(dbo);
|
||||
//
|
||||
// }
|
||||
|
||||
def stop = false
|
||||
// while (!stop){
|
||||
def dist = CommonDatautils.findDistrict(dbo.city, dbo.region, dbo.name, "district")
|
||||
if(dist){
|
||||
d['N'] = "已处理"
|
||||
//发重算均价命令
|
||||
addPrice(dist.city, dist.region, dist.name, dist.estate_type2, DateUtils.parseFromTightByPattern('2025-08-01','yyyy-MM-dd'),new Date())
|
||||
stop = true
|
||||
}
|
||||
// }
|
||||
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
def getEsDatas(city, region, name){
|
||||
String sql = "select * from community where city='${city}' and region='${region}' and name='${name}' "
|
||||
def datas = []
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
datas << data
|
||||
})
|
||||
return datas;
|
||||
}
|
||||
|
||||
|
||||
def addPrice(city, region, name, estateType2,startDate, endDate){
|
||||
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
|
||||
for(def weekend : weekends){
|
||||
CommonUtils.recalculationAverage(city, region, name, estateType2, weekend,false,0)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+6
-6
@@ -43,9 +43,9 @@ class BusinessQuarterProcess {
|
||||
}
|
||||
def run(){
|
||||
// synchBusinessData()
|
||||
// updateBusinessQuarter()
|
||||
updateBusinessQuarter()
|
||||
// addRegion()
|
||||
synchBusinessToEs()
|
||||
// synchBusinessToEs()
|
||||
}
|
||||
|
||||
def addRegion(){
|
||||
@@ -172,11 +172,11 @@ class BusinessQuarterProcess {
|
||||
bk_business_coll.update(new BasicDBObject("_id", row._id), new BasicDBObject('$set', row),true, false)
|
||||
}
|
||||
def updateBusinessQuarter(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def citys = ["上海"]//CommonDatautils.findAllCity()
|
||||
for(def c : citys){
|
||||
if(["上海","北京"].contains(c)){
|
||||
continue
|
||||
}
|
||||
// if(["上海","北京"].contains(c)){
|
||||
// continue
|
||||
// }
|
||||
def cityBusinessList = CommonDatautils.findBusinessByCity(c)
|
||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region","name", "business_quarter", "lat", "lng"])
|
||||
businessProcess(c, districts, cityBusinessList)
|
||||
|
||||
+2
-2
@@ -33,10 +33,10 @@ class ApiTest {
|
||||
|
||||
|
||||
// task.addressAvgPrice("D:\\数据\\北交所\\拍卖案例(-地址估值接口).xlsx")//地址出小区均价
|
||||
// task.addressEvaluate("D:\\数据\\北交所\\拍卖案例(-地址均价接口).xlsx")//地址估值
|
||||
task.addressEvaluate("D:\\数据\\北交所\\拍卖案例(-地址均价接口).xlsx")//地址估值
|
||||
// task.arroundFacilitiesByAddress("D:\\数据\\北交所\\拍卖案例(-周边poi接口).xlsx")//北交所周边poi 地址
|
||||
// task.jsAbcHouseCastTestByAddress("D:\\数据\\北交所\\拍卖案例(房源接口500条-价格控20%).xlsx")
|
||||
task.houseValueInformation("D:\\数据\\北交所\\北交所综合接口测试.xlsx")
|
||||
// task.houseValueInformation("D:\\数据\\北交所\\北交所综合接口测试.xlsx")
|
||||
|
||||
//五级地址标准化
|
||||
// task.addrStructure("D:\\数据\\地数\\地址标准化-测试用例.xlsx")
|
||||
|
||||
+30
-5
@@ -85,7 +85,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
public void process(Object args) {
|
||||
//目前查询库中是否已存在,用的是Oracle
|
||||
def index = 0
|
||||
def citys = CcbCommonUtils.getRunningCitys(["all"])
|
||||
def citys = CcbCommonUtils.getRunningCitys(["all"])
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
@@ -201,7 +201,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
@@ -492,7 +492,8 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
|
||||
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
|
||||
def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
|
||||
// def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
|
||||
def date = DateUtils.addMonth(endDate.toString(), -12, "yyyyMMdd")
|
||||
def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")
|
||||
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")//DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")
|
||||
|
||||
@@ -530,11 +531,15 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
|
||||
if(estate_type2 && estate_type2=="别墅"){
|
||||
if(estate_type2 && estate_type2=="别墅" && !name.contains("别墅")){
|
||||
def newName = name + "(别墅)"
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
newName = name + "别墅"
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
}
|
||||
|
||||
if(estate_type2 && estate_type2=="商住"){
|
||||
@@ -544,6 +549,21 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
|
||||
if(name.contains("别墅")){
|
||||
def newName = name.replace("别墅", "").replace("()", "").replace("()", "")
|
||||
def obj = CommonDatautils.findDistrict(city,region,newName,"district")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
if(obj && obj.alias){
|
||||
for(def s1 : alias){
|
||||
if(!alias.contains(s1)){
|
||||
alias << s1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
@@ -586,6 +606,11 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
if(vallaFlag){
|
||||
continue;
|
||||
}
|
||||
}else if (estate_type2 && estate_type2=="别墅"){
|
||||
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
|
||||
if(!vallaFlag){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -628,7 +653,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
if(!seawater.toilet){
|
||||
seawater.toilet = null
|
||||
}
|
||||
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
|
||||
if(seawater.toilet && (seawater.toilet>=10 || seawater.toilet<1)){
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user