Merge branch 'script_gj' into 'master'
Script gj See merge request !276
This commit is contained in:
+1
-1
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220118, 20220219);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220218, 20220319);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||
//测试通过匹配接口配房源效果
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||
|
||||
+8
-8
@@ -40,7 +40,7 @@ class TradeStatLog {
|
||||
|
||||
static main(args) {
|
||||
def task = new TradeStatLog()
|
||||
// task.run()//统计最近3个月抓取的成交数据量
|
||||
task.run()//统计最近3个月抓取的成交数据量
|
||||
// task.exportTrade()//导出未入库成交
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
@@ -49,7 +49,7 @@ class TradeStatLog {
|
||||
// task.deleteDealData()
|
||||
// task.deleteDealData2()
|
||||
// task.statTableCount()
|
||||
task.filterTrade()
|
||||
// task.filterTrade()
|
||||
|
||||
|
||||
}
|
||||
@@ -75,13 +75,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
rets << ["城市","11月总量","12月总量","1月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","12月总量","1月总量","2月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2021-11-01"
|
||||
def date = "2021-12-01"
|
||||
def endDate = "2022-01-01"
|
||||
def newDate = "2022-02-01"
|
||||
def startDate = "2021-12-01"
|
||||
def date = "2022-01-01"
|
||||
def endDate = "2022-02-01"
|
||||
def newDate = "2022-03-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
@@ -131,7 +131,7 @@ class TradeStatLog {
|
||||
}
|
||||
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-01成交统计.xlsx", rets, "数据")
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-02成交统计.xlsx", rets, "数据")
|
||||
println "表格生成完成"
|
||||
}
|
||||
|
||||
|
||||
+121
-23
@@ -1,25 +1,123 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import groovy.sql.Sql
|
||||
|
||||
|
||||
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
DBCollection CITY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "city_stat", "192.168.0.43")
|
||||
DBCollection CITY_INFO = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "192.168.0.43")
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
|
||||
def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", ZS_MONGODB_URL)
|
||||
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\city.xlsx",{d->
|
||||
if(d['B']){
|
||||
//def avg_price =CommonDatautils.queryCityPrice(d['B'])
|
||||
//d['C'] = avg_price
|
||||
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("area_name",new BasicDBObject( '$regex' , d['B'].replace("州","")+/.*$/))
|
||||
// def obj = COMMUNITY_CONN.findOne(q)
|
||||
// if(obj){
|
||||
// d['D'] = obj.area_code
|
||||
// println obj.area_name +" "+obj.area_code
|
||||
// }
|
||||
|
||||
//def obj = findDistrictPriceCount(d['B'], DateUtils.parseFromTightByPattern("2022-03-13","yyyy-MM-dd"), CITY_INFO)
|
||||
// d['D'] = CommonDatautils.findAllDistrictCount("district", d['B'])
|
||||
// d['E'] = findHouseCount(d['B'], 20210101, 20211231)
|
||||
//
|
||||
// def startDate = DateUtils.parseFromTightByPattern("2021-01-01", "yyyy-MM-dd")
|
||||
// def endDate = DateUtils.parseFromTightByPattern("2021-12-31", "yyyy-MM-dd")
|
||||
// def deals = findTradesDetailById(d['B'], startDate,endDate, DEAL_PRICE_COLL)
|
||||
// d['F'] = deals.size()
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
d['G'] = findCcbDistCount(cityCodeDataMap.get(d['B']), sqlInstance)
|
||||
}
|
||||
},0)
|
||||
|
||||
def findCcbDistCount(cityid, sqlInstance){
|
||||
|
||||
def sql = "select count(*) as count from bigdata_community_gj where cityId='${cityid}' and status=0 ";
|
||||
def total = null
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
total = r.count
|
||||
})
|
||||
return total
|
||||
}
|
||||
|
||||
|
||||
public static def findTradesDetailById(city, startDate, endDate, DEAL_PRICE_COLL){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("area", new BasicDBObject('$gte', 20))
|
||||
q.put("avg_price", new BasicDBObject('$gt', 0))
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def total = dbcursor.count()
|
||||
def rets = new HashSet()
|
||||
def lists = []
|
||||
for(def d : dbcursor){
|
||||
def key=d.city+"##"+d.match_region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
if(!rets.contains(key)){
|
||||
rets << key
|
||||
lists << d
|
||||
}
|
||||
}
|
||||
return lists
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHouseCount(def city, def startDate, def endDate){
|
||||
def aliasStr = null;
|
||||
def total = 0
|
||||
try{
|
||||
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} "
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
total = data.c
|
||||
})
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
def findDistrictPriceCount(def city, def weekend, def CITY_INFO){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
def obj = CITY_INFO.findOne(city)
|
||||
return obj
|
||||
}
|
||||
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工核实_8511.xlsx", { d ->
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(newRegion && !d['K']) {
|
||||
@@ -475,26 +573,26 @@ def city = "
|
||||
//
|
||||
//})
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\删除映射别名数据的副本.xlsx", { d ->
|
||||
if(d['A']){
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
def mappingAlias = dist.mapping_alias
|
||||
if(mappingAlias){
|
||||
if(mappingAlias.contains(d['B'])){
|
||||
mappingAlias.remove(d['B'])
|
||||
DBObject udata = new BasicDBObject();
|
||||
udata.put("mapping_alias", mappingAlias);
|
||||
udata.put("m_date", new Date());
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", d['A']), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println d['A']+"无映射别名"
|
||||
}
|
||||
}else{
|
||||
println d['A']+"映射别名为空"
|
||||
}
|
||||
}else{
|
||||
println d['A']+"没找到"
|
||||
}
|
||||
}
|
||||
})
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\删除映射别名数据的副本.xlsx", { d ->
|
||||
// if(d['A']){
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
// def mappingAlias = dist.mapping_alias
|
||||
// if(mappingAlias){
|
||||
// if(mappingAlias.contains(d['B'])){
|
||||
// mappingAlias.remove(d['B'])
|
||||
// DBObject udata = new BasicDBObject();
|
||||
// udata.put("mapping_alias", mappingAlias);
|
||||
// udata.put("m_date", new Date());
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", d['A']), new BasicDBObject('$set', udata))
|
||||
// }else{
|
||||
// println d['A']+"无映射别名"
|
||||
// }
|
||||
// }else{
|
||||
// println d['A']+"映射别名为空"
|
||||
// }
|
||||
// }else{
|
||||
// println d['A']+"没找到"
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
+49
-1
@@ -30,15 +30,63 @@ class MatchMapData {
|
||||
|
||||
static main(args) {
|
||||
def task = new MatchMapData()
|
||||
task.run()
|
||||
// task.run()
|
||||
// task.run1()
|
||||
// task.run2()
|
||||
// task.run3()
|
||||
// task.exportHouse()
|
||||
// task.addDistrictLevel()
|
||||
task.run4()
|
||||
|
||||
}
|
||||
|
||||
def run4() {
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\核实小区是否存在.xlsx", { d ->
|
||||
def regionCode = null
|
||||
if (d['A']) {
|
||||
def region = d['B']
|
||||
def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
if (newRegion) {
|
||||
regionCode = getRegionCode(newRegion)
|
||||
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
if (dist) {
|
||||
d['F'] = "完全一致"
|
||||
d['E'] = dist.name
|
||||
d['D'] = dist.name_cat
|
||||
d['G'] = dist.estate_type2
|
||||
} else {
|
||||
dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'], "district")
|
||||
if (dist) {
|
||||
d['F'] = "映射别名"
|
||||
d['E'] = dist.name
|
||||
d['D'] = dist.name_cat
|
||||
d['G'] = dist.estate_type2
|
||||
} else {
|
||||
def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district")
|
||||
if (rets) {
|
||||
if(rets.size()>1){
|
||||
d['F'] = "别名-多个"
|
||||
}else{
|
||||
d['F'] = "别名"
|
||||
}
|
||||
}
|
||||
if (rets) {
|
||||
dist = rets[0]
|
||||
}
|
||||
if (dist) {
|
||||
d['F'] = "完全一致"
|
||||
d['E'] = dist.name
|
||||
d['D'] = dist.name_cat
|
||||
d['G'] = dist.estate_type2
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}, 0)
|
||||
}
|
||||
|
||||
def run3(){
|
||||
|
||||
def files = ["合肥融合匹配_1215.xlsx"]
|
||||
|
||||
+198
@@ -0,0 +1,198 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
class TradeDistrictToDB {
|
||||
static main(args) {
|
||||
def task = new TradeDistrictToDB();
|
||||
// task.run()
|
||||
task.run1()
|
||||
}
|
||||
|
||||
def run1(){
|
||||
def index = 0
|
||||
def keys = []
|
||||
ExcelUtils.modify("D:\\数据\\建行\\建行成交\\成交未入库记录(12月)-新增.xlsx", { d ->
|
||||
if(d['D']){
|
||||
def key = d['A']+"##"+d['D']+"##"+d['C']
|
||||
if(!keys.contains(key)){
|
||||
def formatAddress = null
|
||||
if(d['G']){
|
||||
formatAddress = AddressUtil.formatAddress(d['A'], d['F'])
|
||||
}
|
||||
|
||||
Long glat = null
|
||||
Long glng = null
|
||||
Long blat = null
|
||||
Long blng = null
|
||||
if(d['G']){
|
||||
glat = d['H'].toString().toDouble() * 10000000000L
|
||||
glng = d['G'].toString().toDouble() * 10000000000L
|
||||
def baidu = BaiDuApiUtils.geoconv(d['G'].toString().toDouble(), d['H'].toString().toDouble(), 3)
|
||||
blng = baidu.x * 10000000000L
|
||||
blat = baidu.y * 10000000000L
|
||||
}
|
||||
|
||||
def name = d['C'].replace("·","").replace("II","二")
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
|
||||
keys << key
|
||||
if(name){
|
||||
def dist = CommonDatautils.findDistrict(d['A'], d['D'], name, "district")
|
||||
if(!dist){
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", d['A'])
|
||||
data.put("region",d['D'])
|
||||
data.put("name", name)
|
||||
data.put("estate_type2",estateType2)
|
||||
data.put("address", formatAddress)
|
||||
if(glat){
|
||||
data.put("lat",glat)
|
||||
data.put("lng",glng)
|
||||
data.put("lat2",blat)
|
||||
data.put("lng2",blng)
|
||||
}
|
||||
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
index++
|
||||
DbUtils.seaweed.save(data);
|
||||
FangjiaClientUtils.sendDistrictPriceCalculate(d['A'], d['D'], d['C'], 4);
|
||||
println index+" "+data
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
def run() {
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\成交未入库记录(12月)-新增.xlsx", { d ->
|
||||
def city = d['A']
|
||||
def region = d['B']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if (newRegion) {
|
||||
def map = findMapData(city, newRegion, d['C'])
|
||||
if(map){
|
||||
d['D'] = newRegion
|
||||
d['E'] = map.name
|
||||
d['F'] = map.address
|
||||
d['G'] = map.lng
|
||||
d['H'] = map.lat
|
||||
d['I'] = map.lng2
|
||||
d['J'] = map.lag2
|
||||
}
|
||||
}
|
||||
}, 0);
|
||||
}
|
||||
|
||||
def findMapData(def city, def region, name){
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name)
|
||||
def map = findStandardObject2(city, region, name, null, rets);
|
||||
if (!map || !map.lat) {
|
||||
rets = OpenApiUtils.getBiDuLocation(city, region, name);
|
||||
map = findStandardObject2(city, region, name, null, rets);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
def findStandardObject2(def city, def region, def name, def address, def rets) {
|
||||
if (rets) {
|
||||
def bestObj = [:]
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
def score = 0
|
||||
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
score = 0.2
|
||||
}
|
||||
if(mapObj.name && name){
|
||||
def newName = mapObj.name;
|
||||
def names = []
|
||||
if(mapObj.name && mapObj.name.indexOf("(")>-1){
|
||||
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
|
||||
names << name1
|
||||
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
|
||||
names << name2
|
||||
}else{
|
||||
names << newName
|
||||
}
|
||||
def maxNameScore = 0
|
||||
for(def n : names){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
|
||||
if(score1 && score1>0){
|
||||
def nameScore = (score1 * 4)/10
|
||||
if(nameScore > maxNameScore){
|
||||
maxNameScore = nameScore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(maxNameScore){
|
||||
try{
|
||||
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
|
||||
}catch(Exception e){
|
||||
maxNameScore = 0
|
||||
}
|
||||
|
||||
}
|
||||
score = maxNameScore + score;
|
||||
}
|
||||
|
||||
if(mapObj.address && address){
|
||||
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
|
||||
if(addressScore && addressScore>0){
|
||||
try{
|
||||
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
|
||||
}catch(Exception e){
|
||||
addressScore = 0
|
||||
}
|
||||
score = addressScore*4/10 + score
|
||||
}
|
||||
}
|
||||
|
||||
if(!bestObj){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}else{
|
||||
if(score > bestObj.score){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return bestObj
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+17
-1
@@ -218,7 +218,6 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","玉泉路甲46号院")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -918,6 +917,23 @@ class CommonDatautils {
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
public static def findDistrictsByCityAndName(String city, String name, String cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("city", city);
|
||||
q.put("cat", cat);
|
||||
q.put("name", name);
|
||||
q.put("visible", 0);
|
||||
|
||||
def list = []
|
||||
DSCursor cursor = getDataStoreConnection().find(q);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
list << obj;
|
||||
}
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
|
||||
public static def findDistrict(String city, String name, String cat){
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
package com.fangjia.script.fuzhou
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
|
||||
class ThirdTest {
|
||||
|
||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
|
||||
static main(args) {
|
||||
def task = new ThirdTest()
|
||||
// task.run()
|
||||
// task.run1()
|
||||
task.run3()
|
||||
// task.run4()
|
||||
|
||||
}
|
||||
|
||||
def run4(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房价网数据-公司方提供.xlsx", { d ->
|
||||
if (!d['C']) {
|
||||
// def name = d['A'].replace("(钢混)","").replace("(非钢混)","").replace("(混合)","")
|
||||
// def lists = CommonDatautils.findDistrictsByCityAndName("福州",name, "district")
|
||||
// if(lists){
|
||||
// for(def o : lists){
|
||||
//// if(d['B'] && d['B'].toString().contains(o.address)){
|
||||
// d['C'] = o.region
|
||||
// d['D'] = o.name
|
||||
// d['E'] = "名称去括号中的字"
|
||||
//// }
|
||||
// }
|
||||
// }
|
||||
def rMap = SeaweedDataMatchProcess.doMatch("福州", null, d['A'], d['B']);
|
||||
if(rMap){
|
||||
d['C'] = rMap.region
|
||||
d['D'] = rMap.name
|
||||
d['E'] = "匹配接口"
|
||||
}
|
||||
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def run3() {
|
||||
// ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房价网数据-公司方提供.xlsx", { d ->
|
||||
// if(!d['F']){
|
||||
// def price = CommonDatautils.queryDistrictPrice("福州",d['C'], d['D'])
|
||||
// d['F'] = price
|
||||
// }
|
||||
// },0)
|
||||
// List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202202.csv"))
|
||||
// def map1 = [:]
|
||||
// for(def s : list1){
|
||||
// def row = s.split("@\\|@")
|
||||
// def key = row[2]+"##"+row[3]+"##"+row[4]
|
||||
// map1.put(key, row[5])
|
||||
// }
|
||||
|
||||
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202202.csv"))
|
||||
def map1 = [:]
|
||||
for(def str : list1){
|
||||
def s = str.split("@\\|@")
|
||||
def key = s[2]+"##"+s[3]+"##"+s[4]
|
||||
def priceMap = [:]
|
||||
map1.put(key, priceMap)
|
||||
|
||||
if(s.size()>=7 && s[6]){
|
||||
priceMap.put("zhugezhaofang", s[6])
|
||||
}
|
||||
|
||||
if(s.size()>=8 && s[7]){
|
||||
priceMap.put("anjuke", s[7])
|
||||
}
|
||||
|
||||
if(s.size()>=9 && s[8]){
|
||||
priceMap.put("soufang", s[8])
|
||||
}
|
||||
|
||||
if(s.size()>=10 && s[9]){
|
||||
priceMap.put("lianjia", s[9])
|
||||
}
|
||||
|
||||
if(s.size()>=11 && s[10]){
|
||||
priceMap.put("cityhouse", s[10])
|
||||
}
|
||||
|
||||
if(s.size()>=12 && s[11]){
|
||||
priceMap.put("fangtan", s[11])
|
||||
}
|
||||
|
||||
if(s.size()>=13 && s[12]){
|
||||
priceMap.put("fangdoudou", s[12])
|
||||
}
|
||||
if(s.size()>=14 && s[13]){
|
||||
priceMap.put("maitian", s[13])
|
||||
}
|
||||
|
||||
try{
|
||||
if(s.size()>=15 && s[14]){
|
||||
priceMap.put("58tc", s[14])
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.getMessage()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房价网数据-公司方提供.xlsx", { d ->
|
||||
// if (!d['F']) {
|
||||
// def lists = CommonDatautils.findDistrictsByCityAndName("福州", d['A'], "district")
|
||||
// if(lists){
|
||||
// for(def o : lists){
|
||||
// if(o.address==d['B']){
|
||||
// d['C'] = o.region
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
def key = "福州##"+d['C']+"##"+d['D']
|
||||
def priceMap = map1.get(key)
|
||||
if(priceMap){
|
||||
if(priceMap.zhugezhaofang){
|
||||
d['F'] = priceMap.zhugezhaofang
|
||||
}
|
||||
if(priceMap.anjuke){
|
||||
d['G'] = priceMap.anjuke
|
||||
}
|
||||
if(priceMap.soufang){
|
||||
d['H'] = priceMap.soufang
|
||||
}
|
||||
if(priceMap.lianjia){
|
||||
d['I'] = priceMap.lianjia
|
||||
}
|
||||
if(priceMap.cityhouse){
|
||||
d['J'] = priceMap.cityhouse
|
||||
}
|
||||
if(priceMap.fangtan){
|
||||
d['K'] = priceMap.fangtan
|
||||
}
|
||||
if(priceMap.fangdoudou){
|
||||
d['L'] = priceMap.fangdoudou
|
||||
}
|
||||
if(priceMap.maitian){
|
||||
d['M'] = priceMap.maitian
|
||||
}
|
||||
|
||||
if(priceMap.get("58tc")){
|
||||
d['N'] = priceMap.get("58tc")
|
||||
}
|
||||
}
|
||||
}, 0)
|
||||
}
|
||||
|
||||
def run1(){
|
||||
ExcelUtils.modify("D:\\数据\\福州\\福州\\房屋评估数据-数据方公司提供-结果.xlsx",{d->
|
||||
if(d['B'] && !d['D']){
|
||||
if(d['B'].contains("号")){
|
||||
// def price = searchContent2("福州","https://fz.58.com", d['B'], 304)
|
||||
// if(price){
|
||||
// d['C'] = price
|
||||
// d['F'] = "58同城"
|
||||
// }
|
||||
def price = thirdPartyColl("福州", d['B'])
|
||||
if(price){
|
||||
d['D'] = price;
|
||||
d['F'] = "麦田"
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def thirdPartyColl(def city, def address){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("address", address)
|
||||
q.put("source","麦田")
|
||||
q.put("s_date",202202)
|
||||
def obj = thirdPartyColl.findOne(q)
|
||||
if(obj){
|
||||
return obj.avg_price
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
//查看此关键字是否有数据
|
||||
def searchContent2(def city,def url,def keyword, def cityid){
|
||||
def newUrl = url+"/esf-ajax/community/pc/autocomplete/?city_id=${cityid}&kw="+URLEncoder.encode(keyword, "utf-8")+"&type=2"
|
||||
def document = CrawalUtil.getDocumentByABuYun(newUrl, "访问过于频繁", "utf-8", null, true);
|
||||
def price = null
|
||||
if(document){
|
||||
println "【"+keyword +"】 查询"
|
||||
def jsonObj = JSONObject.fromObject(document)
|
||||
if(jsonObj && jsonObj.status=="ok"){
|
||||
if(jsonObj.data.size()>0){
|
||||
def datas = jsonObj.data
|
||||
def d = datas[0]
|
||||
price = d.price.toInteger()
|
||||
}
|
||||
}else{
|
||||
println "【"+keyword +"】 无结果"
|
||||
}
|
||||
}
|
||||
return price;
|
||||
}
|
||||
|
||||
def run(){
|
||||
def map = findThirdPrice("福州")
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房屋评估数据-数据方公司提供.xlsx",{d->
|
||||
if(!d['F']){
|
||||
def lists = CommonDatautils.findDistrictsByCityAndName("福州",d['A'], "district")
|
||||
if(!lists){
|
||||
lists = CommonDatautils.findDistrictByAlias("福州",null, d['A'], "district")
|
||||
}
|
||||
for(def o : lists){
|
||||
// if(o.address == d['B']){
|
||||
def priceMap = map.get(o.region+"##"+o.name)
|
||||
d['C'] = priceMap.anjuke_esf_price
|
||||
d['D'] = priceMap.maitian_esf_price
|
||||
d['E'] = priceMap.beike_esf_price
|
||||
d['F'] = o.region
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
def findThirdPrice(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("s_date", new BasicDBObject('$gte', 202109))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor) {
|
||||
def key = db.region+"##"+db.name
|
||||
def dataMap = map.get(key)
|
||||
if(!dataMap){
|
||||
dataMap = [:]
|
||||
map.put(key, dataMap)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){
|
||||
dataMap.put("maitian_esf_price", db.maitian_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
|
||||
dataMap.put("beike_esf_price", db.beike_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("beike_esf_price") && db.get("lianjia_esf_price") && db.lianjia_esf_price>3000){
|
||||
dataMap.put("beike_esf_price", db.lianjia_esf_price)
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
}
|
||||
+6
-5
@@ -37,7 +37,7 @@ class YouDaDataParser {
|
||||
public static final DBCollection youda_newhouse_coll = MongoUtils.getCollectionByName2("business", "res_newdealprice", offlineUrl)
|
||||
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
|
||||
//线下:228 线上:45
|
||||
//线下:228 线上:45 测试
|
||||
static def new_targetUrl = "http://192.168.0.228:9200/new_deal_price_lansi/doc"
|
||||
static def new_queryUrl = "http://192.168.0.228:9200/new_deal_price_lansi/doc/_search"
|
||||
|
||||
@@ -48,7 +48,7 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2022-01-01"
|
||||
static def statDate = "2022-02-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2022年2月\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2022年3月\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
@@ -490,8 +490,9 @@ class YouDaDataParser {
|
||||
if(s){
|
||||
def doc = new groovy.json.JsonSlurper().parseText(s)
|
||||
def hits = doc.hits
|
||||
if(hits){
|
||||
if(hits && hits.total instanceof Integer){
|
||||
total = hits.total
|
||||
|
||||
}
|
||||
}
|
||||
return total
|
||||
@@ -565,7 +566,7 @@ class YouDaDataParser {
|
||||
if(obj.premisesName && obj.premisesName!="null"){
|
||||
dbo.premisesName = obj.premisesName
|
||||
}
|
||||
dbo.price = obj.price
|
||||
dbo.price = obj.price.toInteger()
|
||||
dbo.region = obj.region
|
||||
dbo.resId = obj.ydId.toString()
|
||||
dbo.totalPrice = obj.totalPrice
|
||||
|
||||
+4
-4
@@ -36,7 +36,7 @@ public class BeiKeCrawlByChrome {
|
||||
def task = new BeiKeCrawlByChrome();
|
||||
task.run()
|
||||
// task.run1()
|
||||
// task.addDealRegion();
|
||||
task.addDealRegion();
|
||||
}
|
||||
|
||||
|
||||
@@ -142,7 +142,7 @@ public class BeiKeCrawlByChrome {
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
}else{
|
||||
@@ -342,9 +342,9 @@ public class BeiKeCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
// if(e.text().contains("天津")){
|
||||
if(e.text().contains("天津")){
|
||||
cityMap.put(e.text(), "https:"+href)
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
|
||||
+2
-1
@@ -514,8 +514,9 @@ class GaoDeUtils {
|
||||
// "2a1fb75990f2b39f6221e50892dd3988"
|
||||
// ]
|
||||
|
||||
//35925efd7333f1df52145e51b8e4e56a 15021384995
|
||||
def KeyList = [
|
||||
"10bcb421d2ee4e51ed9268d6dca58ad1"
|
||||
"10bcb421d2ee4e51ed9268d6dca58ad1"//17740818975
|
||||
]
|
||||
def count = RandomUtils.nextInt(KeyList.size())
|
||||
return KeyList.get(count);
|
||||
|
||||
+3
-4
@@ -187,7 +187,7 @@ public class LianjiaCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-30", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
}else{
|
||||
@@ -218,7 +218,7 @@ public class LianjiaCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-31", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
@@ -421,8 +421,7 @@ public class LianjiaCrawlByChrome {
|
||||
def href = e.attr("href")
|
||||
// if(!href.contains("fang.lianjia") && e.text().contains("福州")){
|
||||
cityMap.put(e.text(), href)
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
}
|
||||
|
||||
+52
-34
@@ -15,42 +15,60 @@ ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\֪ʶ
|
||||
def province = item['D']
|
||||
def city = item['E']
|
||||
def area = item['F']
|
||||
def township = item['G']
|
||||
def community = item['H']
|
||||
String township = item['G']
|
||||
String community = item['H']
|
||||
if (StringUtils.isBlank(city)) {
|
||||
return ;
|
||||
}
|
||||
if (ProvinceData.isGovernmentCity(city)) {
|
||||
townConn.find(new BasicDBObject("province", "重庆市")
|
||||
.append("region", area + "区")
|
||||
.append("street", township)
|
||||
.append("juweihui", community), new BasicDBObject()).limit(1).each {
|
||||
obj ->
|
||||
item['T'] = obj.juweihui_type
|
||||
}
|
||||
if (item['T'] == null) {
|
||||
townConn.find(new BasicDBObject("province", "重庆市")
|
||||
.append("region", area + "区")
|
||||
.append("juweihui", community), new BasicDBObject()).limit(1).each {
|
||||
obj ->
|
||||
item['T'] = obj.juweihui_type
|
||||
}
|
||||
}
|
||||
} else {
|
||||
townConn.find(new BasicDBObject("city", city + "市")
|
||||
.append("region", area + "区")
|
||||
.append("street", township)
|
||||
.append("juweihui", community), new BasicDBObject()).limit(1).each {
|
||||
obj ->
|
||||
item['T'] = obj.juweihui_type
|
||||
}
|
||||
if (item['T'] == null) {
|
||||
townConn.find(new BasicDBObject("city", city + "市")
|
||||
.append("region", area + "区")
|
||||
.append("juweihui", community), new BasicDBObject()).limit(1).each {
|
||||
obj ->
|
||||
item['T'] = obj.juweihui_type
|
||||
}
|
||||
}
|
||||
if (item['T'] != null) {
|
||||
return ;
|
||||
}
|
||||
// if (community == null) {
|
||||
// return ;
|
||||
// }
|
||||
if (township) {
|
||||
if (township.endsWith("镇") || township.endsWith("乡")) {
|
||||
item['T'] = 2
|
||||
} else if (township.endsWith("街道")) {
|
||||
item['T'] = 1
|
||||
}
|
||||
}
|
||||
// community = community.replaceAll("居委会", "")
|
||||
// community = community.replaceAll("居民委员会", "")
|
||||
// community = community.replaceAll("村民委员会", "")
|
||||
// community = community.replaceAll("村委会", "")
|
||||
//
|
||||
// if (ProvinceData.isGovernmentCity(city)) {
|
||||
// townConn.find(new BasicDBObject("province", "重庆市")
|
||||
// .append("region", area + "区")
|
||||
// .append("street", township)
|
||||
// .append("juweihui", new BasicDBObject('$regex', "^" + community)), new BasicDBObject()).limit(1).each {
|
||||
// obj ->
|
||||
// item['T'] = obj.juweihui_type
|
||||
// }
|
||||
// if (item['T'] == null) {
|
||||
// townConn.find(new BasicDBObject("province", "重庆市")
|
||||
// .append("region", area + "区")
|
||||
// .append("juweihui", new BasicDBObject('$regex', "^" + community)), new BasicDBObject()).limit(1).each {
|
||||
// obj ->
|
||||
// item['T'] = obj.juweihui_type
|
||||
// }
|
||||
// }
|
||||
// } else {
|
||||
// townConn.find(new BasicDBObject("city", city + "市")
|
||||
// .append("region", area + "区")
|
||||
// .append("street", township)
|
||||
// .append("juweihui", new BasicDBObject('$regex', "^" + community)), new BasicDBObject()).limit(1).each {
|
||||
// obj ->
|
||||
// item['T'] = obj.juweihui_type
|
||||
// }
|
||||
// if (item['T'] == null) {
|
||||
// townConn.find(new BasicDBObject("city", city + "市")
|
||||
// .append("region", area + "区")
|
||||
// .append("juweihui", new BasicDBObject('$regex', "^" + community)), new BasicDBObject()).limit(1).each {
|
||||
// obj ->
|
||||
// item['T'] = obj.juweihui_type
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
})
|
||||
|
||||
+2
-2
@@ -21,10 +21,10 @@ import java.util.concurrent.Executors
|
||||
class CommitteeLatLng {
|
||||
|
||||
static DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017)
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(20)
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(10)
|
||||
|
||||
static void main(String[] args) {
|
||||
dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0)).append("name", "席庄二村委会"))
|
||||
dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0)).append("year", 2021))
|
||||
//dealLocation()
|
||||
executorService.shutdown()
|
||||
}
|
||||
|
||||
+3
-3
@@ -40,10 +40,10 @@ class DataStat {
|
||||
|
||||
|
||||
def run(){
|
||||
def source = "58同城"//
|
||||
def citys = findCrawlCity(source, 202201);
|
||||
def source = "搜房"//
|
||||
def citys = findCrawlCity(source, 202112);
|
||||
for(def city : citys){
|
||||
def count1 = findCount(source, city, 202201);
|
||||
def count1 = findCount(source, city, 202112);
|
||||
def count2 = findCount(source, city, 202202);
|
||||
println city +" "+count1+" "+count2
|
||||
// println count1
|
||||
|
||||
+15
-15
File diff suppressed because one or more lines are too long
+2
-2
@@ -18,7 +18,7 @@ class SynchDistrictPrice {
|
||||
// def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
|
||||
//"fang.com":"搜房",
|
||||
def sourceMap = ["jiwu.com":"吉屋","ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
|
||||
def sourceMap = ["fang.com":"ËÑ·¿"]
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
task.run()
|
||||
@@ -26,7 +26,7 @@ class SynchDistrictPrice {
|
||||
|
||||
|
||||
def run(){
|
||||
def citys = CommonUtils.findAllCity()
|
||||
def citys = ["ÎÚ³ľÆë"]//CommonUtils.findAllCity()
|
||||
|
||||
sourceMap.each { k, v ->
|
||||
for(def city : citys){
|
||||
|
||||
Reference in New Issue
Block a user