This commit is contained in:
2022-05-27 15:15:33 +08:00
parent 9312b352b0
commit 24609324fb
27 changed files with 1736 additions and 240 deletions
@@ -97,7 +97,7 @@ class BigdataForsaleBatch extends BaseBatch{
totalCount.addAndGet(1)
def commId = rMap.get("_id").toString()
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2)
//def ret = findHousesByDistId(city, commId, "sell", startDate,endDate)
// if(ret && ret.size()<100){//导历史房源需要,以后不需要
// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
@@ -408,7 +408,7 @@ class BigdataForsaleBatch extends BaseBatch{
}
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
@@ -449,6 +449,13 @@ class BigdataForsaleBatch extends BaseBatch{
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(estate_type2 && estate_type2=="普通住宅"){
if(seawater.estate_type&& seawater.estate_type.contains("商住")){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
}
if(!seawater.total_price){
continue;
}
@@ -15,8 +15,8 @@ class DealPriceDataProcess {
def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
// def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -373,16 +373,17 @@ class DealPriceDataProcess {
def citys = CcbCommonUtils.citys
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def publishTime = '2021-10-01'
def month = 202203
def month = 202204
def rows = []
for(def city : citys) {
def rets = findPeGteThirdParty(city, 202204)
def rets = findPeGteThirdParty(city, 202205)
def total = 0
println city +" "+rets.size()
for(def m : rets){
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
sqlInstance236.executeUpdate(sql)
}
println city +" "+rets.size()
}
}
@@ -1,21 +1,29 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.fangjia.core.RegionData
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import groovy.sql.Sql
import org.bson.types.ObjectId
class LoadGuide {
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new LoadGuide()
task.run()
}
def run(){
addPersonGuided("D:\\数据\\建行\\人工指导价更新.xlsx")
// match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx")
@@ -26,9 +34,51 @@ class LoadGuide {
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx")
}
def addPersonGuided(filename){
ExcelUtils.modify(filename, { d ->
def city = d['B']
if(city){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
createSql( d, cityCodeDatas, distCodeMap)
}
})
}
def createSql(d, cityCodeDatas, distCodeMap){
def id = new ObjectId()
def commid = d['A']
def cityId = cityCodeDatas.get(d['B'])
def sampleRegion = d['C'].replace("街道","").replace("镇","")
def distRegion = distCodeMap.get(sampleRegion)
def cityName = d['B']
def distName = d['C']
def commName = d['D']
def price = d['E'].toDouble().toInteger()
def dist = CommonDatautils.findDistrict(cityName, sampleRegion, commName, "district")
if(dist){
def createtime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
def dealDate = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
def sql = "insert into guided_price_gj(ID,COMMID,COMMNAME,CITYID,CITYNAME,DISTID,DISTNAME,DEALDATE,AVG_PRICE,GUIDING_PERSON,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES " +
"('${id.toString()}','${dist._id.toString()}', '${commName}','${cityId}','${cityName}','${distRegion.district_code}','${distName}',to_date('${dealDate}','yyyy-mm-dd'),${price},'房价网','',to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),'FJW')"
println sql
sqlInstance.executeUpdate(sql)
}else{
println " 没查到 "+commName
}
}
def match(def filename){
def city = "西安"
ExcelUtils.modify(filename,{d->
@@ -7,6 +7,7 @@ import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.CommonUtils
@@ -26,9 +27,9 @@ class CcbDistrictToDB {
// task.run()
// task.run1()
// task.meituanRun()
// task.meituanSqrentRun()
task.meituanSqrentRun()
// task.updateDistrict()
task.updateDongguangRegion()
// task.updateDongguangRegion()
}
def updateDongguangRegion(){
@@ -80,13 +81,17 @@ class CcbDistrictToDB {
}
def meituanSqrentRun(){
ExcelUtils.modify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d ->
def index = 0
ExcelUtils.modify("D:\\数据\\美团\\美团待入库.xlsx", { d ->
index++
println index+" "+d['H']
def city = d['F']
def region = d['G']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion) {
def name = d['H']
def name = NameProcess.nameStandard(city, newRegion, d['H'])
name = nameFileter(name)
if(name){
def estateType2 = "普通住宅"
if(name.contains("别墅")){
@@ -103,7 +108,6 @@ class CcbDistrictToDB {
}
}
}
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
CommonUtils.recalculationSqRent(city, region, name, estateType2, lastMonthDay,false,0)
}
@@ -115,97 +119,100 @@ class CcbDistrictToDB {
def index = 0//美团需入库小区合并后.xlsx
def keys = new CopyOnWriteArrayList()
ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d ->
ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团待入库.xlsx", { d ->
index++
println index+" "+d['H']
def city = d['F']
def region = d['G']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion){
def name = d['H']
if(newRegion) {
def name = NameProcess.nameStandard(city, newRegion, d['H'])
name = nameFileter(name)
if (name) {
def formatAddress = null
if(d['E']){
formatAddress = AddressUtil.formatAddress(city, d['E'])
if (d['I']) {
formatAddress = AddressUtil.formatAddress(city, d['I'])
}
if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){
def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name)
def map = findStandardObject2(city,region, name, d['E'], rets);
if(map && map.address){
formatAddress = AddressUtil.formatAddress(city, map.address)
if(!formatAddress){
if(map.address.contains("交叉口")){
formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3)
}else if(map.address.contains("交叉")){
formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2)
}
}
}
}
// if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){
// def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name)
// def map = findStandardObject2(city,region, name, d['E'], rets);
// if(map && map.address){
// formatAddress = AddressUtil.formatAddress(city, map.address)
// if(!formatAddress){
// if(map.address.contains("交叉口")){
// formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3)
// }else if(map.address.contains("交叉")){
// formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2)
// }
// }
// }
// }
Long glat = null
Long glng = null
Long blat = null
Long blng = null
if(d['K']){
glat = d['L'].toString().toDouble() * 10000000000L
glng = d['K'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
if (d['L']) {
glat = d['M'].toString().toDouble() * 10000000000L
glng = d['L'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
def estateType2 = "普通住宅"
if(name.contains("别墅")){
if (name.contains("别墅")) {
estateType2 = "别墅"
}else if(name.contains("商住")){
} else if (name.contains("商住")) {
estateType2 = "商住"
}else{
def type = d['I']
if(type){
if(type.contains("别墅")){
} else {
def type = d['J']
if (type) {
if (type.contains("别墅")) {
estateType2 = "别墅"
}else if(type.contains("商住")){
} else if (type.contains("商住")) {
estateType2 = "商住"
}
}
}
def key = city+"##"+newRegion+"##"+name
if(name && !keys.contains(key)){
def key = city + "##" + newRegion + "##" + name
if (name && !keys.contains(key)) {
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
if(!dist){
if (!dist) {
keys << key
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("name_cat", 12)
data.put("status", 0)
data.put("cat", "district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",newRegion)
data.put("region", newRegion)
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("estate_type2", estateType2)
data.put("address", formatAddress)
if(glat){
data.put("lat",glat)
data.put("lng",glng)
data.put("lat2",blat)
data.put("lng2",blng)
if (glat) {
data.put("lat", glat)
data.put("lng", glng)
data.put("lat2", blat)
data.put("lng2", blng)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
data.put("mark_map", markMap)
index++
DbUtils.seaweed.save(data);
println index+" "+data
println index + " " + data
def stop = false
while (!stop){
if(data._id.toString()){
while (!stop) {
if (data._id.toString()) {
addPrice(data)
stop = true;
}
@@ -213,19 +220,17 @@ class CcbDistrictToDB {
}
}
}
}else{
println city +" "+region +" 区域无法标准化"
}
},0);
}
def addPrice(dist){
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-05-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-05-24")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
@@ -435,9 +440,12 @@ class CcbDistrictToDB {
}
def nameFileter(def name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
if(name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
}
return name;
}
@@ -1,10 +1,12 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
import com.fangjia.core.util.BaiDuApiUtils
@@ -13,7 +15,82 @@ class DistrictMarge {
static main(args) {
def task = new DistrictMarge()
//task.run()
task.repeatAddress()
// task.repeatAddress()
// task.removeDist()
task.updateAddress()
// task.addMappingAlias()
}
def updateAddress(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\更新小区地址.xlsx", { d ->
if (d['A']) {
def region = d['B'].trim()
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if (dist) {
DBObject udata = new BasicDBObject()
udata.put("address", d['D'])
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata))
}
}else{
println d['A'] + " 格式化失败 "+d['B']
}
}
});
}
def removeDist(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\删除小区.xlsx", { d ->
if (d['A']) {
def region = d['B']
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if (dist) {
DBObject udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata))
}
}else{
println d['A'] + " 格式化失败 "+d['B']
}
}
})
}
def addMappingAlias(){
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\新增映射别名.xlsx", { d ->
index ++
if (d['A']) {
def region = d['B'].trim()
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if (dist) {
def mapping_alias = dist.mapping_alias
if(!dist.mapping_alias){
mapping_alias = []
}
def alias = d['D']
def arrs = alias.split("、")
for(def s : arrs){
if(!mapping_alias.contains(s)){
mapping_alias << s
}
}
DBObject udata = new BasicDBObject()
udata.put("mapping_alias", mapping_alias)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata))
}
}else{
println d['A'] + " 格式化失败 "+d['B']
}
}
})
}
def repeatAddress(){
@@ -36,10 +36,37 @@ class MatchMapData {
// task.run3()
// task.exportHouse()
// task.addDistrictLevel()
task.run4()
// task.run4()
task.run5()
}
def run5(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\数据核实.xlsx", { d ->
if (d['A']) {
def region = d['B']
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if(dist){
if(dist.mapping_alias){
d['D'] = dist.mapping_alias.join(",")
}
d['E'] = dist.address
}
def dist1 = CommonDatautils.findDistrict(d['A'], newRegion, d['G'], "district")
if(dist1){
if(dist1.mapping_alias){
d['H'] = dist1.mapping_alias.join(",")
}
d['I'] = dist1.address
}
}
}
});
}
def run4() {
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\核实小区是否存在.xlsx", { d ->
def regionCode = null
@@ -0,0 +1,108 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject
import groovy.sql.Sql
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class OracleHouse {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
def task = new OracleHouse()
task.run()
}
def run(){
def rets = findAllHouse("2022-04-19")
def totalCount = new AtomicLong()
println rets.size()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
def arrs = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def flag = findHouseFlag(arrs[0])
println totalCount.get()+" "+ arrs[0] +" "+flag
if(flag){
def dist = CommonDatautils.findDistrictById(arrs[1])
if(dist && dist.estate_type2=="ÆÕͨסլ"){
removeHouse(arrs[0])
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
}
def findHouseFlag(def id){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("_id", id))
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "district_comp_potential", queryBuilder);
def delFlag = false
for(def h : houses){
if(h.estate_type && (h.estate_type.contains("ÉÌ") || h.estate_type.contains("°ì"))){
println id+" "+h.estate_type
delFlag = true
}
}
return delFlag
}
def findAllHouse(def publishDate) {
def sql = "select FORSALEID,COMMID from bigdata_forsale_gj where publishtime>=to_date('${publishDate}','yyyy-MM-dd')";
def allRows = []
def index = 0
sqlInstance.eachRow(sql, { r ->
index = index + 1
println index
def data = []
data << r.FORSALEID
data << r.COMMID
allRows << data
});
return allRows
}
def removeHouse(houseid){
def sql = "delete from bigdata_forsale_gj where FORSALEID='${houseid}' ";
sqlInstance.executeUpdate(sql)
}
}
@@ -2,28 +2,43 @@ package com.fangjia.script.chongqing
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DateUtil
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.paimai.DataUtil
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.google.gson.JsonObject
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.Sql
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CopyOnWriteArrayList
import java.util.zip.GZIPInputStream
class MatchEvaluateRegionTest {
static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/"
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def path = "D:\\数据\\重庆农商行\\2022-05-06\\估值限时测试-重庆农商行1.0.xlsx"
def path = "D:\\数据\\美团\\10城小区明细-V10.xlsx"
def total = 1500
static main(args) {
@@ -31,14 +46,26 @@ class MatchEvaluateRegionTest {
// task.run()//小区匹配
// task.run1()//地图匹配
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
task.addPrice()
// task.addPrice()
// task.addArroundPrice()
// task.addArroundPriceByLat()
// task.addRentRate()
// task.addAuthor()
// task.addLevel()
// task.addSorce()
// task.addArroundScore()
task.pkPrice()
}
def run(){
def index = 0
ExcelUtils.multiThreadingModify(path, { row ->
index ++
if(!row['F']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['C'], "property", false,null,null,)
println index+" "+row['E']
def jsonObject = OpenApiClient.addressMatch(row['A'], row['C'], "property", false,row['B'],null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['F'] = jsonObject.get("id")
row['G'] = jsonObject.get("city")
@@ -47,8 +74,8 @@ class MatchEvaluateRegionTest {
row['J'] = jsonObject.get("address")
row['K'] = jsonObject.get("propertyType")
row['L'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
row['M'] = jsonObject.get("lng")
row['N'] = jsonObject.get("lat")
}
}
});
@@ -56,8 +83,8 @@ class MatchEvaluateRegionTest {
def run1(){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['F']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['C'], null, true, null, null)
if(!row['F'] && row['A']){
def jsonObject = OpenApiClient.addressMatch(row['A'], row['E'], null, true, row['B'],row['C'].toString())
if(jsonObject != null && jsonObject.precise!=-1) {
row['F'] = jsonObject.get("id")
row['G'] = jsonObject.get("city")
@@ -66,42 +93,81 @@ class MatchEvaluateRegionTest {
row['J'] = jsonObject.get("address")
row['K'] = jsonObject.get("propertyType")
row['L'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
row['M'] = jsonObject.get("lng")
row['N'] = jsonObject.get("lat")
}
}
});
}
def pkPrice(){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['R'] && row['R'].toString().toDouble().toInteger()==1){
def obj = CommonDatautils.queryDistrictPrice(row['G'], row['H'], row['I'], DateUtils.parseFromTightByPattern("2022-05-22","yyyy-MM-dd"))
if(!obj){
row['R'] = ''
row['S'] = "无最新价"
}
}
});
}
def addPrice(){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['L']){
def priceObj = CommonDatautils.findDistrictPrice(row['G'], row['H'], row['I'])
if(priceObj){
row['M'] = priceObj.avg_price
row['N']= (int)(priceObj.avg_price * row['E'])
row['O'] = priceObj.avg_price
// row['N']= (int)(priceObj.avg_price * row['E'])
}
def avg_price = findAvgPrice(row['F'])
if(row['M'] && avg_price){
def weekendPrice = row['M'].toString().toDouble()
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(row['G'], row['H'], row['I'])
if(sqmetreRentPrice){
row['P'] = sqmetreRentPrice
}
// def avg_price = findAvgPrice(row['F'])
// if(row['M'] && avg_price){
// def weekendPrice = row['M'].toString().toDouble()
// def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
// row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
// }
}
});
}
def addRentRate(){
def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]
def radioMap = [:]
for(def city : citys) {
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay)
radioMap.put(city, avg_radio)
}
ExcelUtils.multiThreadingModify(path, { row ->
def city = row['G']
if (row['F'] && row['O'] && !row['P']) {
def avg_rent_radio = radioMap.get(city)
if(avg_rent_radio){
row['P'] = (int)(row['O']/avg_rent_radio)
row['S'] = "租金推算1"
}
}
})
}
def addArroundPrice(){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['M']) {
def arroundObj = FangjiaClientUtils.arroundPrice(row['B'], row['D'])
if(!row['O']) {
def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['E'])
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
def priceObj = arroundObj.result
row['M'] = priceObj.unitPrice
row['N'] = (int) (priceObj.unitPrice * row['E'])
row['Q'] = "周边价格"
row['O'] = priceObj.unitPrice
// row['N'] = (int) (priceObj.unitPrice * row['E'])
row['S'] = "周边价格"
}
}
});
@@ -252,4 +318,295 @@ class MatchEvaluateRegionTest {
});
return avg_price
}
def addSorce(){
ExcelUtils.multiThreadingModify(path, { d ->
if(d['F']){
def score = findDistrictScore(d['F'])
if(score){
d['R'] = score
}
}
});
}
def addArroundScore(){
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['R'] && d['F']){
def dist = null
if(d['F'].toString().size()>16){
dist = CommonDatautils.findDistrictById(d['F'])
}
if(!dist){
dist = new BasicDBObject()
dist.city = d['G']
dist.lng = d['M'].toString().toDouble()*10000000000L
dist.lat = d['N'].toString().toDouble()*10000000000L
}
if(dist.lat){
List<DBObject> list = DbUtils.findAroundDistrict(dist)
if (list) {
list = sortByDistance(dist, list);
if(list && list.size()>10){
list = list[0..9]
}
if(list){
def scores = []
for(def s : list){
def score = findDistrictScore(s._id.toString())
if(score){
scores << score
}
}
def arround_score =(int)(scores.sum()/scores.size())
d['R'] = arround_score
d['T'] = "周边小区评分2"
}
}
}
}
});
}
def addArroundPriceByLat(){
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['O'] && d['F'] && (d['K'].contains("住宅") || d['K'].contains("商住") ||d['K'].contains("别墅"))){
def dist = null
if(d['F'].toString().size()>16){
dist = CommonDatautils.findDistrictById(d['F'])
}
if(!dist){
dist = new BasicDBObject()
dist.city = d['G']
dist.lng = d['M'].toString().toDouble()*10000000000L
dist.lat = d['N'].toString().toDouble()*10000000000L
}
if(dist.lat){
List<DBObject> list = DbUtils.findAroundDistrict(dist)
if (list) {
list = sortByDistance(dist, list);
if(list && list.size()>10){
list = list[0..9]
}
if(list){
def prices = []
for(def s : list){
def priceObj = CommonDatautils.findDistrictPrice(s.city, s.region, s.name)
if(priceObj){
prices << priceObj.avg_price
}
}
if(prices){
def arround_price =(int)(prices.sum()/prices.size())
d['O'] = arround_price
d['S'] = "周边均价3"
}
}
}
}
}
});
}
def sortByDistance(d, list){
def newList = []
for(def nb : list){
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
nb.distance = distance
newList << nb
}
//排序
Collections.sort(newList, new Comparator() {
@Override
int compare(Object o1, Object o2) {
if(o1.distance > o2.distance)
return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1
if(o1.distance == o2.distance)
return 0;
return -1;
}
})
return newList;
}
def findDistrictScore(id){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("_id", id))
def indexObj = ElasticUtils.executeSimpleQuery("192.168.0.12", "district_comp_potential", queryBuilder);
def score = 0
for(def h : indexObj){
score = h.total_price_rate
if(score){
break;
}
}
return score
}
def addLevel() {
def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]//
def weekend = DateUtils.getFirstDayOfWeek(new Date());
def cityLevelMap = [:]
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
def datas = addPriceData(dists, priceMap);
datas = calcuteLevel(datas);
cityLevelMap.put(city, ["price":priceMap, "level":datas])
println city +" 楼盘评级同步完成"
}
ExcelUtils.multiThreadingModify(path, { row ->
def address = row['E']
def city = RegionData.formatCity(row['A'])
if(row['K'] && !row['K'].contains("住宅") && !row['K'].contains("商住") && !row['K'].contains("别墅")){
if(city){
def map = cityLevelMap.get(city)
def rMap = null
if(map){
rMap = getArroundDistrictPriceByAddress(city, address, map.level)
}
if(rMap){
if(rMap.rank){
row['Q'] = rMap.rank
}
}
}
}
})
}
def addPriceData(dists, priceMap){
def newLists = []
for(def d : dists){
def price = priceMap.get(d.city+"##"+d.region+"##"+d.name);
if(price){
d.avg_price = price;
newLists << d;
}
}
if(newLists){
newLists.sort {a,b->
def v1 = a.avg_price
def v2 = b.avg_price
return v1.compareTo(v2)
}
}
return newLists;
}
def calcuteLevel(def datas){
def total = datas.size()
def v = (int)(total/10)
def start = 0;
def end = 0;
def map = [:]
for(int i=1;i<=10;i++){
end = end + v;
if(i == 10){
map.put(i, [start, total])
}else{
map.put(i, [start, end])
}
start = end;
}
for(def d=0;d<datas.size();d++){
def dist = datas[d]
for (def o in map) {
def ranges = o.value
def m = ranges[0]
def n = ranges[1]
if(d==0){
if(d<=n){
dist.level = 10 - o.key + 1
break
}
}else{
if(d>m && d<=n){
dist.level = 10 - o.key + 1
break
}
}
}
}
return datas;
}
def getArroundDistrictPriceByAddress(def city, def address, def levels){
def json = GaoDeApiUtils.geo(city, address)
def location = null
def detail = new BasicDBObject()
def level = null
def rank = null
if(json && json.count>=1){
def m = json.geocodes[0]
if(m){
location = m.location
level = m.level
}
}
if(location){
def arrs = location.split(",")
detail.lng = arrs[0].toString().toDouble()*10000000000L
detail.lat = arrs[1].toString().toDouble()*10000000000L
}
if(detail.lng){
detail.city = city
List<DBObject> list = DbUtils.findAroundDistrictByRadius(detail, 3000)
if(list && list.size()>10){
list = sortByDistance(detail, list);
try{
list = list[0..9]
}catch(Exception e){
println list.size()
}
}
def arroundAvgPrice = getArroundAvgPrice(list)
if(arroundAvgPrice){
def mod = Integer.MAX_VALUE;
for(def d : levels){
def newMod = java.lang.Math.abs( d.avg_price - arroundAvgPrice)
if(newMod<mod){
mod = newMod
rank = d.level
}
}
}
}
def m = [:]
m.level = level
m.rank = rank
return m
}
def getArroundAvgPrice(def list){
def prices = []
for(def l : list){
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
if(priceObj){
prices << priceObj.avg_price
}
}
if(prices){
def avg_price = (prices.sum()/prices.size()).toInteger()
return avg_price;
}
return null
}
}
@@ -3,8 +3,13 @@ package com.fangjia.script.chongqing
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -13,17 +18,470 @@ import java.util.concurrent.atomic.AtomicLong
class PocStat {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl)
public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", lineUrl)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new PocStat();
task.regionDistStat();
// task.regionDistStat();
// task.sampleStat()
// task.tradeStat()
// task.paimaiStat()
// task.evalStat()
//
// task.fileldStat()
// task.removeDelete()
// task.loudongStat()
// task.houseLoudongStat()
task.houseLoudongStat2()
// task.updteHoledCount()
}
def loudongStat(){
def retMaps = getAllLouDong()
def total_count = retMaps.keySet().size()
def all_floorsCount = 0;
def all_buildYearCount = 0
def all_householdsCount = 0
def all_elevatorNumberCount = 0
def all_roomCount = 0
def f_all_floorsCount = 0;
def f_all_buildYearCount = 0
def f_all_householdsCount = 0
def f_all_elevatorNumberCount = 0
def f_all_roomCount = 0
def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
def z_count = 0
def f_count = 0
retMaps.each{key,lists->
def floorsCount = 0;
def buildYearCount = 0
def householdsCount = 0
def elevatorNumberCount = 0
def roomCount = 0
def f_floorsCount = 0;
def f_buildYearCount = 0
def f_householdsCount = 0
def f_elevatorNumberCount = 0
def f_roomCount = 0
def dist = CommonDatautils.findDistrictById(key)
if(zRegions.contains(dist.region)){
z_count++
}else{
f_count++
}
for(def row : lists){
if(zRegions.contains(dist.region)){
if(row.floors){
floorsCount++
}
if(dist.finish_building_date){
buildYearCount++
}
if(row.households){
householdsCount++
}else{
if(row.room_count && row.floors){
householdsCount++
}
}
if(row.elevatorNumber && row.elevatorNumber>0){
elevatorNumberCount++
}
if(row.room_count>0){
roomCount++
}
}else{
if(row.floors){
f_floorsCount++
}
if(dist.finish_building_date){
f_buildYearCount++
}
if(row.households){
f_householdsCount++
}else{
if(row.room_count && row.floors){
f_householdsCount++
}
}
if(row.elevatorNumber && row.elevatorNumber>0){
f_elevatorNumberCount++
}
if(row.room_count>0){
f_roomCount++
}
}
}
def dist_total_count = lists.size()*0.6
if(floorsCount >= dist_total_count){
all_floorsCount ++
}
if(buildYearCount >= dist_total_count){
all_buildYearCount ++
}
if(householdsCount >= dist_total_count){
all_householdsCount ++
}
if(elevatorNumberCount >= dist_total_count){
all_elevatorNumberCount ++
}
if(roomCount >= dist_total_count){
all_roomCount ++
}
if(f_floorsCount >= dist_total_count){
f_all_floorsCount ++
}
if(f_buildYearCount >= dist_total_count){
f_all_buildYearCount ++
}
if(f_householdsCount >= dist_total_count){
f_all_householdsCount ++
}
if(f_elevatorNumberCount >= dist_total_count){
f_all_elevatorNumberCount ++
}
if(f_roomCount >= dist_total_count){
f_all_roomCount ++
}
}
def floor_prec = DataHandleUtils.getDoubleRound(2, all_floorsCount*100/z_count)
def year_prec = DataHandleUtils.getDoubleRound(2, all_buildYearCount*100/z_count)
def households_prec = DataHandleUtils.getDoubleRound(2, all_householdsCount*100/z_count)
def elevatorNum_prec = DataHandleUtils.getDoubleRound(2, all_elevatorNumberCount*100/z_count)
def roomcount_prec = DataHandleUtils.getDoubleRound(2, all_roomCount*100/z_count)
def f_floor_prec = DataHandleUtils.getDoubleRound(2, f_all_floorsCount*100/f_count)
def f_year_prec = DataHandleUtils.getDoubleRound(2, f_all_buildYearCount*100/f_count)
def f_households_prec = DataHandleUtils.getDoubleRound(2, f_all_householdsCount*100/f_count)
def f_elevatorNum_prec = DataHandleUtils.getDoubleRound(2, f_all_elevatorNumberCount*100/f_count)
def f_roomcount_prec = DataHandleUtils.getDoubleRound(2, f_all_roomCount*100/f_count)
println "floor "+ floor_prec
println "year "+ year_prec
println "households "+ households_prec
println "elevator "+ elevatorNum_prec
def rets = []
rets << ["城区","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观"]
rets << ["主城区",100,100,100,0,floor_prec,0,year_prec,elevatorNum_prec,roomcount_prec,households_prec,0]
rets << ["非主城区",100,100,100,0,f_floor_prec,0,f_year_prec,f_elevatorNum_prec,f_roomcount_prec,f_households_prec,0]
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rets, "楼栋")
}
def houseLoudongStat2(){
def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
def dists = CommonDatautils.findAllDistrictByCity("重庆","district", ["city","region","name"])
def z_count = 0
def f_count = 0
def z_directionCount = 0
def f_directionCount = 0
def f_sightCount = 0
def z_sightCount = 0
def index = 0
for(def dist : dists){
index ++
def rets = getAllHouseLoudong2(dist._id.toString())
println index +" "+rets.size()
if(rets){
def directionCount = 0
def sightCount = 0
for(def d : rets){
if(d.direction){
directionCount ++
}
if(d.sight){
sightCount ++
}
}
def total = rets.size()*0.6
if(zRegions.contains(dist.region)){
z_count++
if(directionCount>=total){
z_directionCount++
}
if(sightCount >=total){
z_sightCount++
}
}else{
f_count++
if(directionCount>=total){
f_directionCount++
}
if(sightCount >=total){
f_sightCount++
}
}
}
}
def z_direct_prec = DataHandleUtils.getDoubleRound(2, z_directionCount*100/z_count)
def z_sight_prec = DataHandleUtils.getDoubleRound(2, z_sightCount*100/z_count)
def f_direct_prec = DataHandleUtils.getDoubleRound(2, f_directionCount*100/f_count)
def f_sight_prec = DataHandleUtils.getDoubleRound(2, f_sightCount*100/f_count)
def rets = []
rets << ["城区","户号","户ID","主朝向","景观","户型结构"]
rets << ["主城区",100,100,z_direct_prec, z_sight_prec, 100]
rets << ["非主城区",100,100, f_direct_prec, f_sight_prec, 100]
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rets, "户1")
}
def houseLoudongStat(){
def map = getAllHouseLoudong()
def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
def z_count = 0
def f_count = 0
def z_directionCount = 0
def f_directionCount = 0
def f_sightCount = 0
def z_sightCount = 0
map.each{key,list->
def dist = CommonDatautils.findDistrictById(key)
def directionCount = 0
def sightCount = 0
for(def d : list){
if(d.direction){
directionCount ++
}
if(d.sight){
sightCount ++
}
}
def total = list.size()*0.6
if(zRegions.contains(dist.region)){
z_count++
if(directionCount>=total){
z_directionCount++
}
if(sightCount >=total){
z_sightCount++
}
}else{
f_count++
if(directionCount>=total){
f_directionCount++
}
if(sightCount >=total){
f_sightCount++
}
}
}
def z_direct_prec = DataHandleUtils.getDoubleRound(2, z_directionCount*100/z_count)
def z_sight_prec = DataHandleUtils.getDoubleRound(2, z_sightCount*100/z_count)
def f_direct_prec = DataHandleUtils.getDoubleRound(2, f_directionCount*100/f_count)
def f_sight_prec = DataHandleUtils.getDoubleRound(2, f_sightCount*100/f_count)
def rets = []
rets << ["城区","户号","户ID","主朝向","景观","户型结构"]
rets << ["主城区",100,100,z_direct_prec, z_sight_prec, 100]
rets << ["非主城区",100,100, f_direct_prec, f_sight_prec, 100]
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rets, "户")
}
def getAllHouseLoudong(){
def q = new BasicDBObject()
q.put("city","重庆")
q.put("district_id", new BasicDBObject(QueryOperators.EXISTS, true))
def keys = new BasicDBObject()
keys.put("district_id",1)
keys.put("direction",1)
keys.put("sight",1)
def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000)
def retMap = [:]
for(def d : dbcursor){
def districtId = d.district_id
if(districtId){
def lists = retMap.get(districtId)
if(!lists){
lists = []
retMap.put(districtId, lists)
}
lists << d
}
}
return retMap;
}
def getAllHouseLoudong2(districtId){
def q = new BasicDBObject()
q.put("city","重庆")
q.put("district_id", districtId)
def keys = new BasicDBObject()
keys.put("district_id",1)
keys.put("direction",1)
keys.put("sight",1)
def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000)
def lists = []
for(def d : dbcursor){
lists << d
}
return lists;
}
def getAllLouDong(){
def q = new BasicDBObject()
q.put("city","重庆")
q.put("id_in_seaweed", new BasicDBObject(QueryOperators.EXISTS, true))
def dbcursor = LOUDONG_LINE_COLL.find(q).batchSize(5000)
def retMap = [:]
for(def d : dbcursor){
def districtId = d.id_in_seaweed
if(districtId){
def lists = retMap.get(districtId)
if(!lists){
lists = []
retMap.put(districtId, lists)
}
lists << d
}
}
return retMap;
}
def updteHoledCount(){
def datas = findAllLouDong()
def totalCount = new AtomicLong()
println datas.size()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
def arrs = datas[i].split("##")
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def max_room = findHouseLouDongRoom(arrs)
println totalCount.get() +" "+ max_room
if(max_room){
def q = new BasicDBObject()
q.put("id_in_seaweed", arrs[0])
q.put("house_num", arrs[1])
if(arrs.size()>2){
q.put("unit_num", arrs[2])
}
def udata = new BasicDBObject()
udata.put("room_count", max_room)
LOUDONG_LINE_COLL.update(q, new BasicDBObject('$set', udata), false, true);
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
}
def findAllLouDong(){
def q = new BasicDBObject()
q.put("city","重庆")
q.put("id_in_seaweed", new BasicDBObject(QueryOperators.EXISTS, true))
def dbcursor = LOUDONG_LINE_COLL.find(q)
def keys = new HashSet()
for(def d : dbcursor){
def districtId = d.id_in_seaweed
def key = districtId +"##"+d.house_num
if(d.unit_num && d.unit_num!="0"){
key = key + "##"+d.unit_num
}
if(!keys.contains(key)){
keys << key
}
}
return keys
}
def findHouseLouDongRoom(def arrs){
def q = new BasicDBObject()
q.put("city","重庆")
q.put("district_id", arrs[0])
q.put("house_num", arrs[1])
if(arrs.size()>2){
q.put("unit_num", arrs[2])
}
def dbcursor = HOUSE_LOUDONG_COLL.find(q)
def max_room = 0
for(def d : dbcursor){
def room = null
if(d.room_num && d.room_num.contains("-")){
room = d.room_num.substring(d.room_num.indexOf("-")+1, d.room_num.length())
}else if(d.room_num && d.room_num.contains("0")){
room = d.room_num.substring(d.room_num.toString().lastIndexOf("0")+1, d.room_num.length())
}
if(room && room.isNumber()){
if(room.toInteger()>max_room){
max_room = room.toInteger()
}
}
}
return max_room;
}
def removeDelete(){
def dists = CommonDatautils.findAllDistrictByCity("重庆","district", ["city","region","name", "name_cat"])
for(def obj : dists){
def name_cat = obj.name_cat;
if(name_cat==12){
def count = CommonDatautils.findDistrictCount(obj.city, obj.region, obj.name, "district")
if(count>1){
println obj.name +" "+count
DbUtils.seaweed.remove(new BasicDBObject("_id", obj._id))
}
}
}
}
def regionDistStat(){
@@ -36,25 +494,25 @@ class PocStat {
for(def region : set){
def count = CommonDatautils.findDistrictCountByRegionBlock(city, region, null, "district")
def regionCount = 0
def regionHouseLDCount = 0
for(def d : dists){
if(d.region == region){
def loudongCount = CommonDatautils.findLoudong(d._id.toString())
if(loudongCount && loudongCount>0){
regionCount = regionCount+ loudongCount
}
def houseLouDongCount = CommonDatautils.findHouseLoudong(d._id.toString())
if(houseLouDongCount && houseLouDongCount>0){
regionHouseLDCount = regionHouseLDCount + houseLouDongCount
}
}
}
rows << [region, count, regionCount, regionHouseLDCount]
// def regionCount = 0
// def regionHouseLDCount = 0
// for(def d : dists){
// if(d.region == region){
// def loudongCount = CommonDatautils.findLoudong(d._id.toString())
// if(loudongCount && loudongCount>0){
// regionCount = regionCount+ loudongCount
// }
//
// def houseLouDongCount = CommonDatautils.findHouseLoudong(d._id.toString())
// if(houseLouDongCount && houseLouDongCount>0){
// regionHouseLDCount = regionHouseLDCount + houseLouDongCount
// }
// }
// }
rows << [region, count]
println region +" 完成"
}
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rows, "各区楼盘量统计")
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rows, "各区楼盘量统计1")
}
def sampleStat(){
@@ -6,12 +6,17 @@ import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
class PocTest {
def total = 3259
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new PocTest();
task.run();
@@ -27,17 +32,21 @@ class PocTest {
* @return
*/
def run(){
def inputfile = "D:\\数据\\重庆农商行\\poc\\重庆农商行.xlsx"
match(inputfile)
addPrice(inputfile)
addArroundPrice(inputfile)
def inputfile = "D:\\数据\\重庆农商行\\poc\\test.xls"
// match(inputfile)
// addPrice(inputfile)
// addArroundPrice(inputfile)
// standard(inputfile)
// addLoudongNum(inputfile)
// addAuthor(inputfile)
// pkPrice(inputfile)
// addPpeValue(inputfile)
// matchMap(inputfile)
standard(inputfile)
addLoudongNum(inputfile)
// addDetail(inputfile, "D:\\数据\\重庆农商行\\poc\\楼盘详情.xlsx")
//
// createLoudong(inputfile,"D:\\数据\\重庆农商行\\poc\\底层数据-楼栋-户.xlsx")
@@ -47,6 +56,36 @@ class PocTest {
// createEvaluation(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
}
def addPpeValue(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { row ->
if(row['F'] && row['R']){
def avg_price = findAvgPrice(row['R'])
if(avg_price){
def ppe15 = Math.abs((row['F'] - avg_price) * 100 / avg_price)
row['Z'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
}
}
});
}
def pkPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['R']
if(commid){
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
def jsonObject = OpenApiClient.webankEvaluate(dist.city, dist.region, dist.name, row['E'].toString())
if(jsonObject != null && jsonObject.code==200) {
def result = jsonObject.result
if(result){
row['Y'] = result.get("unitPrice")
}
}
}
}
})
}
def match(path){
ExcelUtils.multiThreadingModify(path, { row ->
@@ -54,7 +93,7 @@ class PocTest {
def city = row['B']
def address = row['D']
def region = row['C']
println "city:"+city+" region:"+region+" address:"+address
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['Q'] = jsonObject.get("name")
@@ -95,8 +134,9 @@ class PocTest {
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['R']
if(commid){
def dist = CommonDatautils.findDistrictById(commid)
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
row['Q'] = dist.name
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
row['F'] = priceObj.avg_price
@@ -106,6 +146,8 @@ class PocTest {
})
}
def addArroundPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['F']) {
@@ -156,8 +198,8 @@ class PocTest {
if (jsonObject && jsonObject.result) {
if(jsonObject.result.size()>=1){
def obj = jsonObject.result[0]
row['T'] = obj.id
row['S'] = obj.houseNumber
row['T'] = obj.id
row['S'] = obj.houseNumber
}
}
@@ -172,6 +214,7 @@ class PocTest {
}
}
}
})
}
@@ -590,4 +633,34 @@ class PocTest {
}
ExcelUtils.write(output, rows, "成交案例")
}
def addAuthor(def path){
def authors = ["丁世菁","王文艳","董彦均","曹玉彦","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","穆良书","沈燕芳"]
def personTotal = authors.size()
def avg = Math.ceil(total/personTotal)
def index = 0
def count = 0;
ExcelUtils.modify(path, { row ->
if(row['B']){
if(index < avg){
row['X'] = authors[count]
}
index++
if(index >= avg){
count++
index = 0
}
}
});
}
def findAvgPrice(def commid){
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
def avg_price = null
sqlInstance.eachRow(sql, { r ->
avg_price = r.target_avg_price
});
return avg_price
}
}
@@ -3079,18 +3079,26 @@ class CommonDatautils {
queryCondition.put("region", region);
queryCondition.put("district_name", name);
queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte', endDate));
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)).limit(1)
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
DBObject dbo;
def prices = []
def index = 0
while(cur.hasNext()) {
dbo = cur.next()
def price = dbo.get("avg_price");
if(price){
break;
def obj = cur.next()
if(index == 0){
dbo = obj
}
prices << obj.get("avg_price");
index++
}
return dbo;
if(prices){
def avg_price = (int)(prices.sum()/prices.size())
dbo.avg_price = avg_price
return dbo;
}
return null;
}
@@ -6,10 +6,16 @@ import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.jusfoun.JusfounCommon
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
import java.text.DecimalFormat
import java.util.concurrent.TimeUnit
/**
@@ -19,8 +25,11 @@ import java.util.concurrent.TimeUnit
class DistrictPriceExportBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def charset = "GBK"
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
static main(args) {
runBatch(new DistrictPriceExportBatch(), args)
}
@@ -32,6 +41,9 @@ class DistrictPriceExportBatch extends BaseBatch{
def ret = createDatas("福州")
saveToDB(ret)
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
if(CommonDatautils.isWindow()){
@@ -76,10 +88,10 @@ class DistrictPriceExportBatch extends BaseBatch{
def createDatas(def city){
def districts = CommonDatautils.findAllDistricts(city, "district", null);
def lastDate = new Date()
def startDate = DateUtils.getFirstDayOfMonth(lastDate, -1)
def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
def dataStr = DateUtils.add(endDateStr, -30)
def startDate = DateUtils.parseFromTightYYYYMMDD(dataStr)
def lastDate = DateUtils.parseFromTightByPattern(endDateStr, "yyyy-MM-dd")
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv"))
// def map2 = [:]
// for(def s : list2) {
@@ -88,17 +100,34 @@ class DistrictPriceExportBatch extends BaseBatch{
// map2.put(key, row[5])
// }
def lastMonthPriceMap = findFzDistrictStat("福州",lastMonthDate)
def result = []
for(def rMap : districts){
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
if(dbo){
def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString())
def curPrice = dbo.get("avg_price")
if(lastMonthPrice && curPrice){
def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice
if(Math.abs(markup)>3){
def randMarkup = nextDouble(0, 3)
if(markup < 0){
markup = randMarkup*(-1)
}else{
markup = randMarkup
}
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
}
}
def m = []
m << dbo.get("_id").toString()//均价ID
m << rMap.get("_id").toString()//楼盘ID
m << rMap.city
m << rMap.region
m << rMap.name
m << dbo.get("avg_price")
m << curPrice
// def lastMonthPrice = map2.get(rMap.city + "##" + rMap.region+"##"+rMap.name)
// if(lastMonthPrice && dbo.get("avg_price")){
// lastMonthPrice = lastMonthPrice.toInteger()
@@ -125,6 +154,19 @@ class DistrictPriceExportBatch extends BaseBatch{
return result
}
def findFzDistrictStat(city,lastMonthDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("month", lastMonthDate)
def dbcursor = FZ_DISTRICT_STAT_COLL.find(q)
def priceMap = [:]
for(def d : dbcursor){
priceMap.put(d.comm_id, d.avg_price)
}
return priceMap
}
def export(def fileName, def datas){
def headers = ["id","comm_id","city","district","name","avg_price","year_month"]
writeCSV(headers, fileName, datas);
@@ -152,5 +194,44 @@ class DistrictPriceExportBatch extends BaseBatch{
file.append(System.getProperty("line.separator"));
}
}
//随机浮点数
public static double nextDouble(final double min, final double max) throws Exception {
DecimalFormat df = new DecimalFormat("#.00");
if (max < min) {
throw new Exception("min < max");
}
if (min == max) {
return min;
}
return Double.parseDouble(df.format(min + ((max - min) * new Random().nextDouble())));
}
def saveToDB(rets){
for(def r : rets){
def q = new BasicDBObject()
q.put("month", curDate.toInteger())
q.put("comm_id", r[1])
def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
if(obj){
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("avg_price", r[5])
FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata))
}else{
def udata = new BasicDBObject()
def o = new BasicDBObject()
o.put("_id", new ObjectId(r[0]))
o.put("comm_id", r[1])
o.put("city", r[2])
o.put("region", r[3])
o.put("name", r[4])
o.put("avg_price", r[5].toInteger())
o.put("month", r[6].toInteger())
FZ_DISTRICT_STAT_COLL.save(o)
}
}
}
}
@@ -8,7 +8,9 @@ import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.rabbitmq.client.impl.AMQImpl
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
/**
* 福州数据结果统计
@@ -17,16 +19,42 @@ class FzDataResultStat {
public static final DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new FzDataResultStat()
task.priceMarkupRun();
task.regionPriceTrendRun()
//// task.margeExcel()
// task.saveCsvData()
}
def margeExcel(){
def list2= ExcelUtils.read("D:\\数据\\福州\\价格比对_202204_新2.xlsx", 0 ,1)
def map2 = [:]
for(def s : list2){
def key = s["A"]+"##"+s["B"]+"##"+s["C"]
map2.put(key, s["D"])
}
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\价格比对_202204_新3.xlsx",{row->
if(row["A"]){
def key = row["A"]+"##"+row["B"]+"##"+row["C"]
def price = map2.get(key)
if(price){
row['L'] = price
}
}
})
}
def priceMarkupRun(){
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
def map1 = [:]
for(def s : list1){
def row = s.split("@\\|@")
@@ -137,16 +165,13 @@ class FzDataResultStat {
queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte',endDate));
DBCursor cur = REGION_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def price = 0;
def prices = [];
while(cur.hasNext()) {
DBObject dbo = cur.next()
price = dbo.get("avg_price");
if(price){
break;
}
prices << dbo.get("avg_price");
}
return price;
def avg_price = (int)(prices.sum()/prices.size())
return avg_price;
}
@@ -165,4 +190,45 @@ class FzDataResultStat {
weekends = weekends.reverse();
return weekends;
}
def saveCsvData(){
def files = getAllFiles("C:\\Users\\gaojuan\\福州数据(1)")
if(files){
for(def filepath : files){
List<String> list1= FileUtils.readLines(filepath)
println filepath
for(def s : list1){
def row = s.split("@\\|@")
def o = new BasicDBObject()
o.put("_id", new ObjectId(row[0]))
o.put("comm_id", row[1])
o.put("city", row[2])
o.put("region", row[3])
o.put("name", row[4])
o.put("avg_price", row[5].toInteger())
o.put("month", row[6].toInteger())
FZ_DISTRICT_STAT_COLL.save(o)
}
}
}
}
def static getAllFiles(def path) {
File file = new File(path);
File[] tempList = file.listFiles();
def files = [];
for (int i = 0; i < tempList.length; i++) {
if (tempList[i].isFile()) {
files << tempList[i];
}
if (tempList[i].isDirectory()) {
System.out.println("文件夹:" + tempList[i]);
}
}
return files;
}
}
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202203.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202204.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -137,7 +137,7 @@ ExcelUtils.write("d:\\
def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )"
println s
def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60"
DingDingMessageUtil.sendTextMessage(access_token, s)
//DingDingMessageUtil.sendTextMessage(access_token, s)
////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]);
@@ -112,22 +112,27 @@ class TradeExportBatch extends BaseBatch{
// q.put("trade_date", new BasicDBObject(QueryOperators.LT, endDate))
def cursor = DEAL_PRICE.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(1000);
def rets = []
def urls = []
if (cursor != null) {
while (cursor.hasNext()) {
DBObject dbo = cursor.next();
def district = districtMap.get(dbo.city + "_" + dbo.region + "_" + dbo.district_name);
if(district){
def ret = [];
ret << dbo._id.toString()
ret << district._id.toString()
ret << dbo.city
ret << dbo.region
ret << dbo.district_name
ret << dbo.avg_price
ret << dbo.area
ret << DateUtils.formatDateByPattern(dbo.trade_date, "yyyy-MM-dd")
ret << dbo.total_price
rets << ret;
if(!urls.contains(dbo.url)){
urls << dbo.url
def ret = [];
ret << dbo._id.toString()
ret << district._id.toString()
ret << dbo.city
ret << dbo.region
ret << dbo.district_name
ret << dbo.avg_price
ret << dbo.area
ret << DateUtils.formatDateByPattern(dbo.trade_date, "yyyy-MM-dd")
ret << dbo.total_price
rets << ret;
}
}
}
println "³É½»×ܼƣº"+rets.size()
@@ -1,10 +1,13 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.thirdPriceCrawl.Utils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import net.sf.json.JSONObject
import org.jsoup.Jsoup
import org.openqa.selenium.WebDriver
@@ -12,16 +15,21 @@ import org.openqa.selenium.chrome.ChromeDriver
class AnjukeAppCrawl {
public static final SHORT_DATE = Utils.getLastMonth();
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
public static def SOURCE = "安居客";
DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail")
static main(args) {
def task = new AnjukeAppCrawl();
task.run();
// task.run();
// task.run2()
task.addThirdPrice()
}
def run2(){
def url = "https://m.anjuke.com/fz/community/"
def url = "https://m.anjuke.com/sh/community/"
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
@@ -95,17 +103,28 @@ class AnjukeAppCrawl {
def run(){
//上海:11,北京:14
//"仓山":4071,
def map = ["쐈갛":4070,"뮬짜":4068,"츙븜":4073,"憩쉭":4069,"르헌":16378,"낀있":15880,"쯩棺":4072,"젯쉭":16380,"틱繭":16387,"쭈都":16382,"湛揭":16392,"츙헌":16384]
map.each{region, area_id->
// def map = ["晋安":4070,"鼓楼":4068,"闽侯":4073,"台江":4069,"福清":16378,"长乐":15880,"马尾":4072,"连江":16380,"平潭":16387,"罗源":16382,"永泰":16392,"闽清":16384]
// def regionMap = ["浦东":7,"闵行":11, "松江":15,"宝山":13,"嘉定":14,"徐汇":5,"青浦":19,"静安":2,"普陀":10,"杨浦":9,"奉贤":17,"黄浦":4,"虹口":8,"长宁":6,"金山":18,"崇明":20]
def regionMap = [ "朝阳":642,"海淀":601,"昌平":843,"丰台":753,"大兴":865,"通州":818,"房山":802,"顺义":837,"西城":704,"东城":690,"密云":856,"石景山":787,"怀柔":858,"门头沟":815,"延庆":860,"平谷":863,"北京周边":2678]
def cityMap = ["北京":regionMap]
def cityId = 14
regionMap.each{region, area_id->
for(def i=1;i<40;i++){
def stop = false;
def all_stop = false
def error_count=0
while(!stop){//渝北:2359 南岸:2368 沙坪坝:2380
//城口
def flagMap = getInfo(stop, i, area_id)
def flagMap = getInfo(stop, i, area_id, cityId, "上海")
stop = flagMap.stop
all_stop = flagMap.all_stop
error_count++
if(error_count>=20){
stop = true
all_stop = true
}
}
if(all_stop){
break;
@@ -114,17 +133,17 @@ class AnjukeAppCrawl {
}
}
def getInfo(def stop, def i, def area_id){
def getInfo(def stop, def i, def area_id, def cityId, def city){
def map = [:]
def url = "https://m.anjuke.com/esf-ajax/community/list?city_id=35&page_size=100&page=${i}&area_id=${area_id}&is_tw=1"
def url = "https://m.anjuke.com/esf-ajax/community/list?city_id=${cityId}&page_size=100&page=${i}&area_id=${area_id}&is_tw=1"
def header = [:]
def header = [:]
header.put("accept","application/json, text/plain, */*")
header.put("accept-encoding","gzip, deflate, br")
header.put("accept-language","zh-CN,zh;q=0.9")
header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117")
header.put("referer","https://m.anjuke.com/fz/community/")
header.put("referer","https://m.anjuke.com/sh/community/")
header.put("sec-fetch-dest","empty")
header.put("sec-fetch-mode","cors")
header.put("sec-fetch-site","same-origin")
@@ -134,6 +153,7 @@ class AnjukeAppCrawl {
def ret = ABuYunUtil.doGet(url, null, null, "utf-8")
def all_stop = false;
def error_count = 0
if (ret) {
JSONObject jsonObj = JSONObject.fromObject(ret);
if(jsonObj && jsonObj.status=="ok" ){
@@ -143,7 +163,7 @@ class AnjukeAppCrawl {
println "【${i}】 "+communities.size() +" "+url
if(communities.size()>0){
save("르鹵", communities)
save(city, communities)
Thread.sleep(10000)
stop = true;
}else{
@@ -154,6 +174,7 @@ class AnjukeAppCrawl {
}else{
stop = false;
error_count ++
println url +" 无数据"
}
@@ -195,4 +216,42 @@ class AnjukeAppCrawl {
q.put("jumpAction",url)
return detail.findOne(q);
}
def addThirdPrice(){
def q = new BasicDBObject()
def dbcursor = detail.find(q)
def index = 0
for(def d : dbcursor){
index++
def city = d.base.cityName
def region = d.base.areaName
def name = d.base.name
def obj = findByParams(city, region, name);
if(!obj){
def data = new BasicDBObject()
data.put("district_id",d.base.id)
data.put("city", city)
data.put("region", region )
data.put("name", name)
data.put("address",d.base.address)
data.put("avg_price", d.priceInfo.price)
data.put("source", SOURCE)
data.put("s_date", SHORT_DATE)
data.put("c_date", d.c_date)
thirdPartyColl.save(data)
println index +" "+data
}
}
}
def findByParams(city, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("source", SOURCE)
q.put("s_date", SHORT_DATE)
return thirdPartyColl.findOne(q);
}
}
@@ -22,9 +22,9 @@ class AnjukeToDB {
static main(args) {
def task = new AnjukeToDB();
// task.run();
task.run();
// task.addNoMatch()
task.updateEstateType()
// task.updateEstateType()
// task.gaodeRangeData()
}
@@ -315,16 +315,16 @@ class AnjukeToDB {
}
}
def run(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客重庆小区-芝芝.xlsx",0,1)
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\非人工处理.xlsx",0,1)
for(def data : datas){
if(data['K']=="芝芝"){
// if(data['K']=="镕杰"){
def flag = data['M']
if(flag && flag=="ÐÂÔö"){
add(data)
// add(data)
}else{
update(data)
}
}
// }
}
}
@@ -445,8 +445,6 @@ class AnjukeToDB {
}
if(!addr_alias.contains(obj.base.address) && obj.base.address.contains("ºÅ")){
addr_alias << obj.base.address
}
if(addr_alias!=dist.addr_alias && addr_alias.size()>0){
udata.put("addr_alias", addr_alias)
}
}
@@ -555,8 +553,13 @@ class AnjukeToDB {
def findAnjukeDistrictDetail(anjukeId){
def q = new BasicDBObject()
q.put("_id",new ObjectId(anjukeId))
return detail.findOne(q)
try{
q.put("_id",new ObjectId(anjukeId))
return detail.findOne(q)
}catch(Exception e){
print "[查询出错]anjukeId:"+anjukeId
}
return null
}
def nameFileter(def region, def name){
@@ -172,7 +172,8 @@ class NameProcess {
if((name.endsWith("村") || name.endsWith("街"))&& name.size()<=3){
return null
}
if(name.endsWith("电梯")||name.endsWith("大道")||name.endsWith("厂") || name.endsWith("组") || (name.endsWith("阁") && name.length()<=3) || name.endsWith("条")){
//|| name.endsWith("组")
if(name.endsWith("电梯")||name.endsWith("大道")||name.endsWith("厂") || name.endsWith("条")){
return null
}
if(name.contains("、")||name.contains("|")||name.contains(".")){
@@ -71,7 +71,8 @@ class DistrictIndexBatch extends BaseBatch{
def run(){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def citys = CommonDatautils.findAllCity();
def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]
//def citys = CommonDatautils.findAllCity();
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2","household_count"])
def code = cityCodeDatas.get(city)
@@ -86,7 +87,7 @@ class DistrictIndexBatch extends BaseBatch{
city_avg_total_price = 531.2615913
}
if(city_avg_total_price){
// calcute(city, dists, code, city_avg_total_price)
calcute(city, dists, code, city_avg_total_price)
calcuteIndex(city, dists, code, city_avg_total_price)
}
@@ -21,10 +21,58 @@ class MatchTest {
def task = new MatchTest()
// task.run()
// task.addNum()
task.dongguang()
// task.dongguang()
// task.meituanDel()
task.addLevel()
}
def addLevel() {
def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]//
def weekend = DateUtils.getFirstDayOfWeek(new Date());
def cityLevelMap = [:]
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
def datas = addPriceData(dists, priceMap);
datas = calcuteLevel(datas);
cityLevelMap.put(city, ["price":priceMap, "level":datas])
println city +" 楼盘评级同步完成"
}
def path = "D:\\数据\\美团\\10城小区明细-结果.xlsx"//美团金融小区测试list-所有数据-评级.xlsx
ExcelUtils.multiThreadingModify(path, { row ->
def name = row['D']
def address = row['E']
def city = RegionData.formatCity(row['A'])
if(name && !row['O']){
if(city){
def map = cityLevelMap.get(city)
def rMap = null
if(map){
rMap = getArroundDistrictPriceByAddress(city, address, map.level)
}
if(rMap){
if(rMap.rank){
row['Q'] = rMap.rank
}
// if(rMap.level){
// row['Q'] = rMap.level
// }
}
}
}
})
}
def meituanDel(){
def path = "D:\\数据\\美团\\美团需入库小区合并后.xlsx"
def index = 0
@@ -193,7 +193,7 @@ public class BeiKeCrawlByChrome {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
}
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü"))
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü")).toInteger()
}
}
@@ -215,7 +215,8 @@ public class BeiKeCrawlByChrome {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "贝壳找房"
def dbo = findTrade(detail)
def detailUrl = e.select(".title a").attr("href")
def dbo = findTrade(detailUrl)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
@@ -228,7 +229,7 @@ public class BeiKeCrawlByChrome {
detail.height = floorStr.toInteger()
}
}
detail.url = e.select(".title a").attr("href")
detail.url = detailUrl
detail.create_date = new Date()
detail.m_date = new Date()
println detail
@@ -313,8 +314,8 @@ public class BeiKeCrawlByChrome {
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
if (roomStr) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü"))
detail.toilet = roomStr.substring(roomStr.indexOf("ÎÀ") - 1, roomStr.indexOf("ÎÀ"))
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü")).toInteger()
detail.toilet = roomStr.substring(roomStr.indexOf("ÎÀ") - 1, roomStr.indexOf("ÎÀ")).toInteger()
}
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
@@ -336,13 +337,14 @@ public class BeiKeCrawlByChrome {
}
def findTrade(def detail){
def findTrade(def detailUrl){
def q = new BasicDBObject()
q.put("city", detail.city)
q.put("region", detail.region)
q.put("district_name", detail.district_name)
q.put("area", detail.area)
q.put("total_price", detail.total_price)
// q.put("region", detail.region)
// q.put("district_name", detail.district_name)
// q.put("area", detail.area)
// q.put("total_price", detail.total_price)
q.put("url", detailUrl)
q.put("trade_date", detail.trade_date)
q.put("source", detail.source)
def dbo = DEAL_PRICE_COLL.findOne(q)
@@ -34,11 +34,11 @@ public class LianjiaCrawlByChrome {
static main(args) {
def task = new LianjiaCrawlByChrome();
// task.run()
task.run()
// task.run1()
// task.addDealRegion();
// task.standardRegion()
task.updadteDealPriceField()
task.addDealRegion();
task.standardRegion()
// task.updadteDealPriceField()
}
def updadteDealPriceField(){
@@ -278,7 +278,7 @@ public class LianjiaCrawlByChrome {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
}
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü"))
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü")).toInteger()
}
}
@@ -398,8 +398,8 @@ public class LianjiaCrawlByChrome {
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
if (roomStr) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü"))
detail.toilet = roomStr.substring(roomStr.indexOf("ÎÀ") - 1, roomStr.indexOf("ÎÀ"))
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü")).toInteger()
detail.toilet = roomStr.substring(roomStr.indexOf("ÎÀ") - 1, roomStr.indexOf("ÎÀ")).toInteger()
}
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
@@ -16,9 +16,11 @@ import org.apache.poi.hssf.usermodel.HSSFWorkbook
import org.bson.types.ObjectId
import java.text.DecimalFormat
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
@@ -34,6 +36,9 @@ class CZDataBatch extends BaseBatch{
DBCollection DIST_COLL = MongoUtils.getCollectionByName("business", "jn_bank_district", "mongo.db.fangjia.com")
//final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls");
//final def czShopMap = readCZShop("d:\\常州数据结果\\原始数据\\常州江南银行商铺--更新ing(8).xls");
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
static main(args) {
runBatch(new CZDataBatch(), args);
}
@@ -113,25 +118,44 @@ class CZDataBatch extends BaseBatch{
}
def districtRun(def districts){
def total = districts.size();
def index = 0;
ExecutorService executorService = Executors.newFixedThreadPool(20)
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(total)
for(def i=0;i<total;i++){
final def district = districts[i]
executorService.submit(new Runnable() {
@Override
fixedThreadPool.execute(new Runnable() {
void run() {
index ++;
println index+"/"+total +" ["+district.city+"-"+district.region+"-"+district.name+"]";
converDistrict(district);
try{
totalCount.addAndGet(1)
converDistrict(district);
println totalCount.get()+"/"+total +" ["+district.city+"-"+district.region+"-"+district.name+"]";
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
executorService.shutdown()
executorService.awaitTermination(30, TimeUnit.MINUTES)
latch.await()
// def index = 0;
// ExecutorService executorService = Executors.newFixedThreadPool(20)
// for(def i=0;i<total;i++){
//
// final def district = districts[i]
// executorService.submit(new Runnable() {
// @Override
// void run() {
// index ++;
// println index+"/"+total +" ["+district.city+"-"+district.region+"-"+district.name+"]";
// converDistrict(district);
// }
// });
// }
// executorService.shutdown()
// executorService.awaitTermination(30, TimeUnit.MINUTES)
}
def converDistrict(def dist){
@@ -30,7 +30,7 @@ import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
import java.util.regex.Matcher
import java.util.regex.Pattern
import java.util.regex.Patter
/**
* 第三方数据标准化处理
@@ -40,7 +40,7 @@ class DataStat {
def run(){
def source = "±´¿ÇÕÒ·¿"//
def source = "58ͬ³Ç"//
def citys = findCrawlCity(source, 202202);
for(def city : citys){
def count1 = findCount(source, city, 202203);
@@ -192,38 +192,38 @@ class Esf58Crawl extends BaseBatch{
void process(Object args) {
//addContent("¸£ÖÝ","https://fz.58.com/xiaoqu/", 70);
def citys = ["上海","北京","广州","天津","武汉","南京","青岛","合肥","西安","沈阳","宁波","佛山","潍坊","济南","大连","兰州","石家庄","长春","温州","乌鲁木齐","太原","惠州","嘉兴","南通","南昌","烟台","临沂","呼和浩特","厦门","海口","常州","中山","绍兴","淄博","洛阳","珠海","金华","威海","德阳","唐山","泰安","襄阳","台州","镇江","银川","江门","荆州","衡阳","滨州","张家口","赣州","商丘","安庆","湖州","枣庄","聊城","阜阳","芜湖","吉林","遂宁","鞍山","东营","安阳","咸阳","蚌埠","九江","临汾","大同","六安","淮南","宿州","三亚","玉溪","六盘水","揭阳","汕尾","恩施","盘锦","伊犁","玉林","随州","抚州","景德镇","四平","丽水","安顺","毕节","天水","阜新","通化","攀枝花","吕梁","阿克苏","新余","松原","楚雄","鹤壁","忻州","本溪","拉萨","三门峡","黔东南","兴安盟","雅安","仙桃","乌海","贵港","安康","白城","平凉","庆阳","延安","潮州","白银","黔西南","丽江","吴忠","防城港","喀什","哈尔滨","钦州","鸡西","石嘴山","哈密","萍乡","鹰潭","武威","保山","铜仁","普洱","酒泉","宜昌","河池","朔州","百色","鹤岗","张家界","湘西","张掖","定西","昭通","济源","中卫","黑河","克拉玛依","铜川","辽源","云浮","博尔塔拉","固原","白山","广元","贺州","潜江","天门","伊春","石河子","临沧","榆林","临夏","来宾","成都","双鸭山","梧州","嘉峪关","海东","昌吉"]
// for(def city : citys){
// def str = cityList.get(city)
// for(def i=0;i<1;i++){
// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
// if(districts){
// def arrs = str.split("\\|")
// def url = "https://"+arrs[0]+".58.com/xiaoqu/"
// distMap.put(city, districts)
// addCrawlKeys(city);
// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
// }
// }
// }
cityList.each{city, str->
// "퀼꺼","뤘�","켓繫","緊憩","줄路","빌뵨뵈景","베왯","밤鹵",
def citys = ["鉗�","揭갛","躬捺","憩鹵","麗쉭","陵뇽","쉭쳔","쐽鹵","굇쑴","뷜捺","켓쑴","깝鹵","蠟소왯","�헴","머鹵","빤鹵","갛헤","좔냘","讀鏤","막捺","鮫갛","交빤","섈주","件퀼","같�","땜檀","갛捺","驅捺"]
for(def city : citys){
def str = cityList.get(city)
for(def i=0;i<1;i++){
if(citys.contains(city)){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
if(districts){
def arrs = str.split("\\|")
def url = "https://"+arrs[0]+".58.com/xiaoqu/"
distMap.put(city, districts)
addCrawlKeys(city);
addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
}
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
if(districts){
def arrs = str.split("\\|")
def url = "https://"+arrs[0]+".58.com/xiaoqu/"
distMap.put(city, districts)
addCrawlKeys(city);
addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
}
}
}
// cityList.each{city, str->
// for(def i=0;i<1;i++){
// if(citys.contains(city)){
// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
// if(districts){
// def arrs = str.split("\\|")
// def url = "https://"+arrs[0]+".58.com/xiaoqu/"
// distMap.put(city, districts)
//
// addCrawlKeys(city);
// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
// }
// }
// }
// }
}
@@ -266,7 +266,7 @@ public class OpenApiClient {
params.put("unitNumber", unitNum);
params.put("roomNumber", roomNum);
params.put("token", TOKEN);
String content = new HttpRequest().get("http://open.fangjia.com/property/searchHouseLoudong", params, "utf-8")
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchHouseLoudong", params, "utf-8")
@@ -288,6 +288,38 @@ public class OpenApiClient {
return null
}
public static JSONObject webankEvaluate(String city, String region, String address, String area){
try{
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("district", region);
params.put("address", address);
params.put("area", area.toString());
params.put("token", TOKEN);
String content = new HttpRequest().get("http://open.fangjia.com/webank/evaluate", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/webank/evaluate", params, "utf-8")
println("${city} ${address} 程序调用异常")
Thread.sleep(6000)
}
if(result){
if (result.getInt("code") == 200) {
return result;
} else if (result.getInt("code") !=0) {
println("${city} ${address} 程序调用异常")
}
}
} catch (Exception e) {
println("${city} ${address} ${e.getMessage()}")
e.printStackTrace();
}
return null
}
public static JSONObject standard2Action(String city, String address) {
try {
Map<String, String> params = new HashMap<String, String>();
@@ -706,7 +738,7 @@ public class OpenApiClient {
params.put("startDate", startDate)
params.put("limit", "100")
try {
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/auction/searchPaimais", params, "utf-8")
String content = new HttpRequest().get("http://open.fangjia.com/auction/searchPaimais", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
} catch (Exception e) {