This commit is contained in:
2022-09-30 14:15:48 +08:00
42 changed files with 2724 additions and 1206 deletions
@@ -177,8 +177,8 @@ public class BaseAvgImportServlet extends HttpServlet {
String realName) {
int totalNum = sheet.getLastRowNum();
System.out.println("上传文件共:" + totalNum + "个");
if (totalNum > 1000) {
totalNum = 1000;
if (totalNum > 5000) {
totalNum = 5000;
}
Integer totalLine = 0;// 总行数
@@ -257,4 +257,9 @@ public class GeoUtils {
ret += (150.0 * Math.sin(x / 12.0 * pi) + 300.0 * Math.sin(x / 30.0 * pi)) * 2.0 / 3.0;
return ret;
}
public static void main(String[] args){
Map<String, Double> location = gps84ToGcj02(31.218024629574998,121.36562884455);
System.out.println(location.toString());
}
}
@@ -1235,7 +1235,7 @@ public class WeiXinEvaluationController {
Point point = new Point(Double.valueOf(ll[0]), Double.valueOf(ll[1]));
points.add(point);
}
Point point = new Point(district.getLng()/10000000000.0, district.getLat()/10000000000.0);
Point point = new Point(district.getLng2()/10000000000.0, district.getLat2()/10000000000.0);
//boolean flag = InsidePolygon.aboutZone(point, points);
//boolean flag = InsidePolygon.isPtInPoly(point.getLon(), point.getLat(), points);
boolean flag = InsidePolygon2.checkIsInPolygon(point, points);
@@ -362,7 +362,7 @@ $('.formal_btn').on('click', function () {
});
function clickFormalReport(){
$.prompt('正式报告将尽快邮寄给您,请注意查收,谢谢!如有疑问可与我们联系:62392317', '请填写正式报告邮寄地址', function (value) {
$.prompt('正式报告将尽快邮寄给您,请注意查收,谢谢! ', '请填写正式报告邮寄地址', function (value) {
var address = $(".modal-text-input").val();
if(address==""){
$.alert('地址不能为空');
@@ -37,14 +37,13 @@
<p style="margin-top:-4px;"><i class="location"></i><small>${fjtEvaluation.address!} <#if fjtEvaluation.houseNum?? && fjtEvaluation.houseNum?contains("号")>${fjtEvaluation.houseNum!}<#else>${fjtEvaluation.houseNum!}号</#if>${fjtEvaluation.roomNum!}室</small> <a href="../district/weXinList?cityName=${fjtEvaluation.city!}" class="collectionLink" style="line-height: 32px;">重新评估&nbsp;></a></p>
<p style="color: #F4A460;font-size: 12px;line-height: 20px;margin-top: -5px;margin-left: 20px;">*请确认与产证地址一致</p>
<span></span>
<#if bohaiEvaluationAudit?? && bohaiEvaluationAudit.auditStatus!=0>
<p style="margin-top:14px;">评估总价:<#if fjtEvaluation??><strong class="<#if bohaiEvaluationAudit?? && bohaiEvaluationAudit.lastPrice??><#else>price_final</#if>">
${(bohaiEvaluationAudit.inquirePrice/10000)?int}</strong>&nbsp;万元</#if></p>
<p>评估单价:<#if bohaiEvaluationAudit.inquirePrice gt 0>${(bohaiEvaluationAudit.inquirePrice/fjtEvaluation.houseArea)?int}</#if>&nbsp;元/平米</p>
<#if bohaiEvaluationAudit?? && bohaiEvaluationAudit.auditStatus!=0 && bohaiEvaluationAudit.inquirePrice gt 0>
<p style="margin-top:14px;">评估总价:<#if fjtEvaluation??><strong class="<#if bohaiEvaluationAudit?? && (bohaiEvaluationAudit.lastPrice?? || bohaiEvaluationAudit.inquirePrice??)>price_final</#if>"><#if bohaiEvaluationAudit.lastPrice??>${(bohaiEvaluationAudit.lastPrice/10000)?int}<#elseif bohaiEvaluationAudit.inquirePrice??>${(bohaiEvaluationAudit.inquirePrice/10000)?int}</#if></strong>&nbsp;万元</#if></p>
<p>评估单价:<#if bohaiEvaluationAudit.lastPrice??>${(bohaiEvaluationAudit.lastPrice/fjtEvaluation.houseArea)?int}<#elseif bohaiEvaluationAudit.inquirePrice??>${(bohaiEvaluationAudit.inquirePrice/fjtEvaluation.houseArea)?int}</#if>&nbsp;元/平米</p>
<#if role?? && role==17 && loanInfoMap??>
<p style="margin-top:-12px;">所属分类:${loanInfoMap.type!}类</p>
<p style="margin-top:-10px;">可贷成数:${loanInfoMap.loan_ratio!}成</p>
<p style="margin-top:-22px;">可贷金额:<strong class="price_final">${(bohaiEvaluationAudit.inquirePrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}</strong>万元</p>
<p style="margin-top:-22px;">可贷金额:<strong class="price_final"><#if bohaiEvaluationAudit.lastPrice??>${(bohaiEvaluationAudit.lastPrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}<#elseif bohaiEvaluationAudit.inquirePrice??>${(bohaiEvaluationAudit.inquirePrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}</#if></strong>万元</p>
<p style="margin-top:-4px;">贷款利率:4.5%</p>
</#if>
<#else>
@@ -15,6 +15,7 @@ import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import groovy.sql.Sql
import java.util.concurrent.ConcurrentHashMap
@@ -32,6 +33,9 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataDealBatch extends BaseBatch{
def startDate = 202206
def endDate = 202208
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
@@ -41,6 +45,7 @@ class BigdataDealBatch extends BaseBatch{
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -102,6 +107,9 @@ class BigdataDealBatch extends BaseBatch{
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
// def lianjiaPriceMap = findThirdPartyPrice("上海")
def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd')
def lianjiaPriceMap = getDistrictStatByCity("上海", startDate)
for(def i=0;i<rets.size();i++){
final def trade = rets[i]
fixedThreadPool.execute(new Runnable() {
@@ -109,7 +117,8 @@ class BigdataDealBatch extends BaseBatch{
try {
totalCount.addAndGet(1)
if(source == "youda"){
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap)
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap, lianjiaPriceMap)
if(ret){
result << ret
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
@@ -135,147 +144,176 @@ class BigdataDealBatch extends BaseBatch{
return result
}
def findThirdPartyPrice(def city) {
def q = new BasicDBObject()
q.put("city", city)
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap){
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte', endDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for (def db : dbcursor) {
def price = null
if (db.lianjia_esf_price && db.lianjia_esf_price > 3000) {
price = db.lianjia_esf_price
}
if (!price && db.beike_esf_price && db.beike_esf_price > 3000) {
price = db.beike_esf_price
}
if(price){
map.put(db.region+"##"+db.name, price)
}
}
return map
}
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap, def lianjiaPriceMap){
def ret = []
def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district")
if(!obj){
return ret;
}
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << trade.city
def data = distCodeMap.get(trade.fjRegion)
if(data){
ret << data.district_code
ret << data.district
}else{
ret << ""
ret << ""
}
def street = ""
def streetNumber = ""
if(trade.address){
if(trade.address.toString().contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
def price = lianjiaPriceMap.get(obj.region+"##"+obj.name)
if(price){
def mod = java.lang.Math.abs((trade.price-price)*100/price)
if(mod<=10){
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << trade.city
def data = distCodeMap.get(trade.fjRegion)
if(data){
ret << data.district_code
ret << data.district
}else{
ret << ""
ret << ""
}
}
if(!street){
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
if(jsonObj && jsonObj.searchAddress){
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){
def street = ""
def streetNumber = ""
if(trade.address){
if(trade.address.toString().contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
}
}else{
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
if(!street){
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
if(jsonObj && jsonObj.searchAddress){
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
}
}else{
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
}
}
}
}
}
if(!street){
if(obj.address){
if(obj.address.toString().contains("弄")){
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
if(number){
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
streetNumber = number
if(!street){
if(obj.address){
if(obj.address.toString().contains("弄")){
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
if(number){
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
streetNumber = number
}
}else{
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
if(jsonObj && jsonObj.searchAddress){
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
}
}
}
ret << street
ret << streetNumber
if(obj){
ret << obj._id.toString()
ret << trade.fjName
}
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
ret << ""//期数
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
if(roomNum && roomNum.contains("0")){
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
}else{
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
if(jsonObj && jsonObj.searchAddress){
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
ret << trade.floor
}
ret << ""
ret << trade.height
def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
def buildType = ''
if(arrs){
buildType = arrs[0]
}
ret << buildType
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
ret << trade.area
ret << ""
ret << getHouseRoom(buildType, trade.room, trade.hall)
ret << trade.room
ret << trade.hall
ret << ""
ret << trade.toilet
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
ret << ""
ret << CcbCodeUtils.getDirectionCode(trade.direction)
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
ret << ""
ret << trade.price
ret << trade.totalPrice * 10000
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
ret << "02"
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << "99"
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << trade.totalPrice * 10000
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
}
}
ret << street
ret << streetNumber
if(obj){
ret << obj._id.toString()
ret << trade.fjName
}
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
ret << ""//期数
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
if(roomNum && roomNum.contains("0")){
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
}else{
ret << trade.floor
}
ret << ""
ret << trade.height
def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
def buildType = ''
if(arrs){
buildType = arrs[0]
}
ret << buildType
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
ret << trade.area
ret << ""
ret << getHouseRoom(buildType, trade.room, trade.hall)
ret << trade.room
ret << trade.hall
ret << ""
ret << trade.toilet
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
ret << ""
ret << CcbCodeUtils.getDirectionCode(trade.direction)
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
ret << ""
ret << trade.price
ret << trade.totalPrice * 10000
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
ret << "02"
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << "99"
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << trade.totalPrice * 10000
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
return ret;
}
@@ -543,11 +581,14 @@ class BigdataDealBatch extends BaseBatch{
def getDistrictStatByCity(city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DIST_SELL_STAT.find(q)
q.put("weekend", new BasicDBObject('$gte', weekend))
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("weekend",-1))
def map = new ConcurrentHashMap()
for(def d : dbcursor){
map.put(d.region+"@"+d.district_name, d.avg_price)
def price = map.get(d.region+"##"+d.district_name)
if(!price && d.value_type==2){
map.put(d.region+"##"+d.district_name, d.avg_price)
}
}
return map
}
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220718, 20220819);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220818, 20220919);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
@@ -77,8 +77,16 @@ class CcbCommonUtils {
q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate)));
def dbcursor = GUIDING_PRICE_COLL.find(q)
def rets = []
def sets = new HashSet()
for(def d : dbcursor){
rets << d
def guiding_date = DateUtils.formatDateByPattern(d.guiding_date, 'yyyy-MM-dd')
def key = d.city +"##"+d.region+"##"+d.name+"##" + guiding_date
if(!sets.contains(key)){
rets << d
sets << key
}else{
println key
}
}
return rets
}
@@ -370,24 +370,25 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys
def citys = ["上海","广州","东莞"]//CcbCommonUtils.citys
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def publishTime = '2021-10-01'
def month = 202206
def month = 202207
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys) {
def rets = findPeGteThirdParty(city, 202207)
def rets = findPeGteThirdParty(city, 202208)
def total = 0
println city +" "+rets.size()
for(def m : rets){
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
def cityId = cityCodeDatas.get(city)
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
sqlInstance236.executeUpdate(sql)
}
}
}
def findPeGteThirdParty(cityName, yearmonth){
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>30"
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>25"
def rets = []
sqlInstance236.eachRow(sql,{r->
def map = [:]
@@ -0,0 +1,209 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import groovy.sql.Sql
import org.apache.commons.lang.StringUtils
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class FjwPricePkCcbPrice {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new FjwPricePkCcbPrice()
// task.run()
// task.updatePrice()
task.addFjwPrice()
}
def updatePrice(){
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\测试\\ccb.xlsx",{d->
index++
println index
def dbo = CommonDatautils.queryDistrictPrice(d['B'], d['C'], d['D'], DateUtils.parseFromTightByPattern('2022-09-04','yyyy-MM-dd'))
if(dbo){
d['G'] = dbo.future_value
}
})
}
def addFjwPrice(){
ExcelUtils.modify("D:\\数据\\建行\\测试\\人工查价(7455)_0824_0913.xlsx",{d->
if(!d['M']){
def dist = CommonDatautils.findDistrictById(d['D'])
if(dist){
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2022-09-11","yyyy-MM-dd"))
if(dbo){
d['M'] = dbo.avg_price
}
}
}
})
}
def run(){
def ccbPrices = getCommunityBasePriceGj(202208)
println "ccb总量:"+ccbPrices.size()
def fjwPriceMap = CommonDatautils.queryAllDistrictPrice(DateUtils.parseFromTightByPattern('2022-09-04','yyyy-MM-dd'))
println "fjw总量:"+ccbPrices.size()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(ccbPrices.size())
def rets = new CopyOnWriteArrayList()
for(def i=0;i<ccbPrices.size();i++){
final def rMap = ccbPrices[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def row = []
def dist = CommonDatautils.findDistrictById(rMap.commid)
if(dist){
def priceMap = fjwPriceMap.get(dist.city+"##"+dist.region+"##"+dist.name)
def fjwPrice = null
def fjwCalPrice = null
if(priceMap){
fjwPrice = priceMap.avg_price
fjwCalPrice = priceMap.calc_value
}
row << rMap.commid
row << dist.city
row << dist.region
row << dist.name
row << fjwPrice
row << fjwCalPrice
if(priceMap && priceMap.value_type){
if(priceMap.value_type == 1){
row << "指导价"
}else if(priceMap.value_type == 2){
row << "自身样本"
}else if(priceMap.value_type == 3){
row << "第三方"
}else if(priceMap.value_type == 4){
row << "交易数据"
}else if(priceMap.value_type == 5){
row << "周边楼盘"
}else if(priceMap.value_type == 6){
row << "推导"
}else if(priceMap.value_type == 9){
row << "神经网络"
}else if(priceMap.value_type == 10){
row << "无方式"
}else{
row << ""
}
}else{
row << ""
}
row << rMap.avg_price
row << rMap.calc_type
if(fjwPrice && rMap.avg_price){
def precent = (fjwPrice - rMap.avg_price)*100/rMap.avg_price
row << DataHandleUtils.getDoubleRound(2, precent)
}else{
row << ""
}
rets << row
}else{
println rMap.commId + " 没找到小区"
}
println totalCount.get()+" "+ rMap.cityname +" "+rMap.commname
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await();
def headers = ["小区ID","城市","区域","小区名","房价网价格","房价网预期价格","房价网计算方式","建行价格","建行计算方式","价格差比(房价网价格-建行价格)*100/建行价格"]
batchCreateFile(rets, "D:\\数据\\建行\\测试\\房价网与建行均价比对.csv",headers,"GBK")
}
def getCommunityBasePriceGj(yearMonth) {
def sql = "select CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE from community_base_price_gj where year_month=${yearMonth} order by cityid";
def allRows = []
sqlInstance236.eachRow(sql, { r ->
def map = [:]
map.cityname = r.CITYNAME
map.distname = r.DISTNAME
map.commid = r.COMMID
map.commname = r.COMMNAME
map.year_month = r.YEAR_MONTH
map.calc_avg_price = r.CALC_AVG_PRICE
map.avg_price = r.AVG_PRICE
// 计算类型,1:人工估值 2:样本计算 3:神经网络
if(r.CALC_TYPE){
if(r.CALC_TYPE == 1){
map.calc_type = "人工估值"
}else if(r.CALC_TYPE == 2){
map.calc_type = "样本计算"
}else if(r.CALC_TYPE == 3){
map.calc_type = "神经网络"
}
}
allRows << map
});
return allRows
}
def batchCreateFile(ret, fileName, headers, charset){
OutputStream out = new FileOutputStream(fileName);
def sb = new StringBuffer();
// sb.append(headers.join("@|@"))
// sb.append(System.getProperty("line.separator"));
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
def index = 0
for(def row : ret){
index++
sb.append(StringUtils.join(row.toArray(), "@|@"))
sb.append(System.getProperty("line.separator"));
println index
if(index%10000==0){
println index+"开始写入CSV "+new Date().toLocaleString()
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
println index+"结束写入CSV "+new Date().toLocaleString()
}
}
if(sb.length()>0){
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
}
out.close();
}
}
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2022-07-22", "2022-08-21")
def prices = CcbCommonUtils.findPersonPrice(city, "2022-08-24", "2022-09-21")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
File diff suppressed because it is too large Load Diff
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202207//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202205
def endDate = 202207
def sDate= 202208//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202206
def endDate = 202208
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -75,13 +75,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys;
def rets = []
rets << ["城市","5月总量","6月总量","7月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","6月总量","7月总量","8月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2022-05-01"
def date = "2022-06-01"
def endDate = "2022-07-01"
def newDate = "2022-08-01"
def startDate = "2022-06-01"
def date = "2022-07-01"
def endDate = "2022-08-01"
def newDate = "2022-09-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -754,7 +754,7 @@ class CommonDatautils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","É­Íû·¼²ÝÂ̰¶")
// q.put("name","ºþÅϼÑÔ·")
DBObject keys = new BasicDBObject();
if(fields){
@@ -2049,6 +2049,20 @@ class CommonDatautils {
}
public static def queryAllDistrictPrice(startDate){
DBObject queryCondition = new BasicDBObject()
queryCondition.put('weekend', startDate);
DBCursor cur = DIST_STAT_COLL.find(queryCondition)
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def rMap = [:];
while(cur.hasNext()) {
DBObject dbo = cur.next()
rMap.put(dbo.get("city") + "##" + dbo.get("region")+"##"+ dbo.get("district_name"), dbo)
}
return rMap;
}
public static def queryDistrictPrices(city, startDate){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
@@ -3152,8 +3166,7 @@ class CommonDatautils {
DBObject dbo;
def prices = []
def index = 0
while(cur.hasNext()) {
def obj = cur.next()
for(def obj : cur) {
if(index == 0){
dbo = obj
}
@@ -0,0 +1,159 @@
package com.fangjia.script.crawl
import com.alibaba.fastjson.JSONArray
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.StringUtils
class SchoolLocation {
DBCollection school_parse_coll = MongoUtils.getCollectionByName("fangjia_craw", "school", "192.168.0.235", 27018)
static main(args) {
def task = new SchoolLocation()
task.run()
}
def run(){
def query = new BasicDBObject()
query.put("province","江苏")
//query.put("lat",null)
query.put("lat",new BasicDBObject('$exists',false))
def cursor = school_parse_coll.find(query)
def index = 0
for(def d : cursor){
index++
def datas = getSchoolLatAndLng(d.city, d.name, d.address, false)
if(datas){
for(def ret : datas){
def addr = d.address
if(d.address.contains(d.city+"市")){
def cityStr = d.city+"市"
addr = d.address.substring(d.address.indexOf(cityStr)+cityStr.size())
}
if(ret.name == d.name || (d.address.contains("号") && (ret.address.contains(addr) || addr.contains(ret.address)))){
def udata = new BasicDBObject()
udata.put("lat", ret.lat.toString().toDouble())
udata.put("lng", ret.lng.toString().toDouble())
udata.put("status", 0)
school_parse_coll.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
println index +" "+udata
break;
}
}
}
}
}
def getSchoolLatAndLng(city, name, address, isFullMatch){
if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){
return null;
}
address = address.trim().replace(" ","").replace("?","").replace(" ","")
String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=学校&city=" + city + "&output=json&offset=5&page=1&key=4e9592208336433ee1b7aa4d1d434418&extensions=all"
List<com.alibaba.fastjson.JSONObject> poiList = null;
com.alibaba.fastjson.JSONObject result = null;
try {
HttpRequest request = new HttpRequest();
Map<String, String> params = new HashMap<String, String>();
String response = request.get(url, params, "UTF-8");
com.alibaba.fastjson.JSONObject jsonResponse = com.alibaba.fastjson.JSONObject.parseObject(response)
if (jsonResponse != null) {
JSONArray ja = jsonResponse.getJSONArray("pois");
if(ja != null && ja.size() > 0) {
poiList = new ArrayList<>();
for(int i=0;i<ja.size();i++){
com.alibaba.fastjson.JSONObject jo_item = ja.getJSONObject(i);
if(jo_item.containsKey("cityname") && !jo_item.getString("cityname").contains(city)){
continue;
}
String item_address = jo_item.containsKey("address") ? jo_item.getString("address") : null;
String item_name = jo_item.containsKey("name") ? jo_item.getString("name") : null;
boolean getedData = false;
if(isFullMatch == true){
if(StringUtils.equalsIgnoreCase(item_address, address) || StringUtils.equalsIgnoreCase(item_name, address)){
getedData = true;
}
}else{
if(StringUtils.equalsIgnoreCase(item_address, address) || StringUtils.contains(item_address, address)
|| StringUtils.equalsIgnoreCase(item_name, address) || StringUtils.contains(item_name, address) || StringUtils.contains(address, item_name)){
getedData = true;
}else{
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(item_name, address);
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(item_address, address);
if(score1 || score2){
getedData = true;
}
if(!getedData){
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(address, " ")
def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(item_name, " ")
if(py1 == py2){
getedData = true;
}
}
}
}
def type = jo_item.getString("type");
if(getedData == true){
com.alibaba.fastjson.JSONObject jo = new com.alibaba.fastjson.JSONObject();
if(jo_item.containsKey("id")){
jo.put("id", jo_item.getString("id"))
}
if(jo_item.containsKey("name")){
jo.put("name", jo_item.getString("name"))
}
if(jo_item.containsKey("type")){
jo.put("type", type)
}
if(jo_item.containsKey("address")){
jo.put("address", jo_item.getString("address").replace("[]",""))
}
if(jo_item.containsKey("pname")){
jo.put("pname", jo_item.getString("pname"))
}
if(jo_item.containsKey("cityname")){
jo.put("cityname", jo_item.getString("cityname"))
}
if(jo_item.containsKey("adname")){
jo.put("adname", jo_item.getString("adname"))
}
if(jo_item.containsKey("location")){
def arrs = jo_item.getString("location").split(",")
jo.put("lat", arrs[1])
jo.put("lng", arrs[0])
}
if(type && type.contains("出入口")){
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+name+"-"+address+"】-${type}"
}else{
println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+name+"-"+address+"】"
poiList.add(jo);
}
}else{
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
}
}
}else{
println "X 高德地图 ( "+city+"-"+address+" )无结果!"
}
}
} catch (Exception e) {
e.printStackTrace();
}
return poiList;
}
}
@@ -11,6 +11,10 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.codec.digest.DigestUtils
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
@@ -26,7 +30,9 @@ class DistrictInfoExportBatch extends BaseBatch{
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection FZ_ADDR_COLL = MongoUtils.getCollectionByName2("business", "fz_district_address", lineUrl)
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName2("business", "fz_district", lineUrl)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
runBatch(new DistrictInfoExportBatch(), args)
@@ -87,7 +93,12 @@ class DistrictInfoExportBatch extends BaseBatch{
def startDate = DateUtils.getFirstDayOfMonth(lastDate);
def fzAddrMap = getFzAddress();
for(def rMap : districts){
//因为福州个性化问题,又不想影响自家数据库数据,所以另存一个福州小区库,用于手动添加数据
saveOrUpdateFzDist(districts)
def fzDistricts = getFzDistricts(city)
for(def rMap : fzDistricts){
//¼Û¸ñûÓÐÒ²Ìṩ
// def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
// if(dbo){
@@ -139,6 +150,50 @@ class DistrictInfoExportBatch extends BaseBatch{
return result
}
def saveOrUpdateFzDist(districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
for(def i=0;i<districts.size();i++){
final def d = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def dbo = FZ_DIST_COLL.findOne(new BasicDBObject("_id", d.get('_id')))
if(dbo){
def udate = new BasicDBObject()
d.each {k,v->
udate.put(k,v)
}
udate.removeField("_id")
FZ_DIST_COLL.update(new BasicDBObject('_id',d.get('_id')), new BasicDBObject('$set', udate));
}else{
FZ_DIST_COLL.save(d)
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def getFzDistricts(def city){
def q = new BasicDBObject()
q.put("city", city)
def rets = []
FZ_DIST_COLL.find(q).each{it->
rets << it
}
return rets
}
def processAlias(alias, def addr_alias){
def newAlias = []
if(alias){
@@ -29,7 +29,9 @@ class DistrictPriceExportBatch extends BaseBatch{
def charset = "GBK"
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com")
def specialDistsPriceMap = ["津泰新村7号、9-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580]
static main(args) {
runBatch(new DistrictPriceExportBatch(), args)
}
@@ -86,10 +88,21 @@ class DistrictPriceExportBatch extends BaseBatch{
return sb.toString()
}
def getFzDistricts(def city){
def q = new BasicDBObject()
q.put("city", city)
def rets = []
FZ_DIST_COLL.find(q).each{it->
rets << it
}
return rets
}
def createDatas(def city){
def districts = CommonDatautils.findAllDistricts(city, "district", null);
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
districts.addAll(removeDists)
// def districts = CommonDatautils.findAllDistricts(city, "district", null);
// def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
// districts.addAll(removeDists)
def districts = getFzDistricts(city)
def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
def dataStr = DateUtils.add(endDateStr, -30)
@@ -124,9 +137,15 @@ class DistrictPriceExportBatch extends BaseBatch{
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
}
}
}else{
curPrice = specialDistsPriceMap.get(rMap.name)
if(curPrice){
def randMarkup = nextDouble(0, 1)
curPrice = (int)(curPrice*randMarkup/100 + curPrice)
}
}
def m = []
if(curPrice){
if(curPrice && dbo){
m << dbo.get("_id").toString()//¾ù¼ÛID
}else{
m << new ObjectId().toString()
@@ -136,7 +136,7 @@ class FzDataResultStat {
def regionPriceTrendRun(){
def city = "¸£ÖÝ"
def add = -6;
def add = -9;
def months = []
while(add<0){
months << DateUtils.addMonth(new Date(), add)
@@ -325,8 +325,8 @@ class HouseExportBatch extends BaseBatch{
// int endDate = 20191201
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyyMMdd")
// def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd");
def endDate = "20220710"
def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd");
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
// QueryBuilder room_range = QueryBuilders.rangeQuery("room").from(1).to(5).includeLower(true).includeUpper(false);
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202207.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202208.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2022-07-01"
static def statDate = "2022-08-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser();//友达二手房存入mongo
// task.esfParser("D:\\数据\\友达\\2022年9月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -284,8 +284,8 @@ class YouDaDataParser {
}
def esfParser(){
def files = ["D:\\数据\\友达\\2022年8月\\查询1.DBF"]
def esfParser(path){
def files = [path]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -1,4 +1,4 @@
package com.fangjia.script.gj.test
package com.fangjia.script.gj.commStat
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
@@ -28,11 +28,30 @@ class CityStat {
def task = new CityStat()
// task.run()
// task.run1();
task.run3();
// task.run3();
// task.run4()
// task.run6()
// task.run7()
// task.run8()
task.dealPriceStat()
}
def dealPriceStat(){
//def citys = ["上海","北京","广州","合肥","南京","深圳","武汉","郑州","重庆","福州","厦门","青岛","太原","长春","苏州","南宁","沈阳","天津","西安","长沙","大连","南昌","东莞","佛山","济南","南通","镇江","宁波","烟台","哈尔滨"]
def citys = ["石家庄","贵阳"]
def rets = []
rets << ["城市","2022-4月总量"]
def startDate = DateUtils.parseFromTightByPattern('2022-01-01','yyyy-MM-dd')
def endDate = DateUtils.parseFromTightByPattern('2022-02-01','yyyy-MM-dd')
for(def c : citys){
def ret = []
ret << c
ret << findDealPriceCount(c, startDate, endDate)
rets << ret
println ret
}
ExcelUtils.write("D:\\数据\\2021成交数据.xlsx", rets, "月统计")
}
// def run8(){
@@ -464,6 +483,14 @@ class CityStat {
return count;
}
def findDealPriceCount(def city, def startDate, def endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt', endDate))
def count = DEAL_PRICE_COLL.count(q)
return count;
}
def calTradeDistCount(def city, def tradeDate, def districts){
def q = new BasicDBObject()
q.put("city", city)
@@ -1,4 +1,4 @@
package com.fangjia.script.gj.test
package com.fangjia.script.gj.commStat
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
@@ -29,24 +29,24 @@ DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "ave
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43")
def citys = ["乌鲁木齐","昌吉","阿克苏","伊犁","喀什"]
//def citys = ["乌鲁木齐","昌吉","阿克苏","伊犁","喀什"]
//for(def city : citys){
//
//}
ExcelUtils.multiThreadingModify("D:\\数据\\江苏\\新疆数据统计.xlsx",{d->
// def keys = getDealPrices(d['A'], DEAL_PRICE_COLL)
// if(keys){
// d['G'] = keys.size()
//ExcelUtils.multiThreadingModify("D:\\数据\\江苏\\新疆数据统计.xlsx",{d->
//// def keys = getDealPrices(d['A'], DEAL_PRICE_COLL)
//// if(keys){
//// d['G'] = keys.size()
//// }
//
//// def rets = findHousesByCity(d['A'], "sell", 20210701)
// if(d['A']){
// def count = findHouseCount(d['A'])
// d['H'] = count
// }
// def rets = findHousesByCity(d['A'], "sell", 20210701)
if(d['A']){
def count = findHouseCount(d['A'])
d['H'] = count
}
})
//
//})
def findHouseCount(city){
@@ -306,3 +306,23 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){
// rows << [c, districts.size(), alias_count, addr_alias_count]
//}
//ExcelUtils.write("D:\\数据\\江苏\\新疆数据统计.xlsx", rows, "结果")
//区域级统计
def city = "合肥"
def regions = CommonDatautils.getCityRegions(city)
def rows = []
for(def region : regions){
def districts = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias", "addr_alias"])
def alias_count = 0
def addr_alias_count = 0
for(def r : districts){
if(r.alias && r.alias.size()>0){
alias_count ++
}
if(r.addr_alias && r.addr_alias.size()>0){
addr_alias_count++
}
}
rows << [region, districts.size(), alias_count, addr_alias_count]
}
ExcelUtils.write("D:\\数据\\新安银行\\新安银行数据统计.xlsx", rows, "结果")
@@ -110,6 +110,7 @@ class FangjiaClientUtils {
public static def saveToEsPaimai(def data){
String jsonStr = com.alibaba.fastjson.JSONObject.toJSONString(data)
HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/paiMai", jsonStr)
// HttpUtils.postForJson("http://localhost:8084/FangJiaDataService/paiMai", jsonStr)
}
@@ -0,0 +1,135 @@
package com.fangjia.script.gj.dongguanbank
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import org.apache.lucene.queryparser.classic.QueryParser
class DongGuanBankTest {
static main(args) {
def task = new DongGuanBankTest()
def path = "D:\\数据\\东莞银行\\东莞银行房价数据测试20220916 v2.xlsx"
// task.run(path)
task.addPrice(path)
}
def run(path){
ExcelUtils.multiThreadingModify(path,{d->
def city = d['A'].replace("市","")
if(city){
def region = RegionData.formatRegion(city, d['B'])
def results = OpenApiClient.addressMatch(city, d['F'], "property", false, region, d['C'])
if(results && results.precise>=0){
d['H'] = results.city
d['I'] = results.district
d['J'] = results.name
d['Q'] = results.propertytype
}
}
})
}
def addPrice(path){
ExcelUtils.multiThreadingModify("D:\\数据\\东莞银行\\东莞银行房价数据测试20220916 v2.xlsx", { d ->
if(d['H']){
def dist = CommonDatautils.findDistrict(d['H'], d['I'], d['J'], "district")
def priceObj = CommonDatautils.queryDistrictPrice(d['H'], d['I'], d['J'], DateUtils.parseFromTightByPattern("2022-09-11","yyyy-MM-dd"))
if(priceObj){
d['M'] = priceObj.avg_price
d['N'] = (priceObj.avg_price * d['G']).toInteger()
d['O'] = ((int)(priceObj.avg_price/100))*100
if(dist){
d['R'] = dist.estate_type2
}
if(priceObj.value_type){
if(priceObj.value_type == 1){
d['S'] = "指导价"
}else if(priceObj.value_type == 2){
d['S'] = "自身样本"
}else if(priceObj.value_type == 3){
d['S'] = "第三方"
}else if(priceObj.value_type == 4){
d['S'] = "交易数据"
}else if(priceObj.value_type == 5){
d['S'] = "周边楼盘"
}else if(priceObj.value_type == 6){
d['S'] = "推导"
}else if(priceObj.value_type == 9){
d['S'] = "神经网络"
}else if(priceObj.value_type == 10){
d['S'] = "无方式"
}
}
}
if(!d['P']){
def alias = []
if(!dist.alias){
alias = []
}else{
alias = dist.alias
}
alias << dist.name
def map = findMaxAvgPrice(dist.city, dist.region, alias)
if(map){
d['P'] = map.max
d['Q'] = map.min
}
}
}
})
}
def findMaxAvgPrice(def city, def region, def alias){
def set = null
try{
def aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= 20210101 and region='${region}' and district_name in ('${aliasStr}')"
set = new HashSet()
ElasticUtils.executeSqlQuery(sql,{data->
set << data.avg_price
})
}catch(Exception e){
e.printStackTrace()
}
if(set && set.size()>=3){
def map = [:]
set = set.sort()
map.min = set[0]
map.max = set[set.size()-1]
return map
}
return null
}
/**
* 转义字符串中的特殊字符
* 仅过滤Lucene regexps, 其他的勿用
* @param keyword
* @return
*/
public static String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
def findMinPrice(city){
def avg_price = null
ElasticUtils.executeSqlQuery("select min(avg_price) as avg_price from seawater where p_date >= 20210901 and city='${city}'", {
avg_price = it.avg_price
})
return avg_price
}
}
@@ -86,7 +86,7 @@ class DistrictIndexBatch extends BaseBatch{
city_avg_total_price = 531.2615913
}
if(city_avg_total_price){
calcute(city, dists, code, city_avg_total_price)
// calcute(city, dists, code, city_avg_total_price)
calcuteIndex(city, dists, code, city_avg_total_price)
}
@@ -104,9 +104,23 @@ public class BeiKeCrawlByChrome {
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
for(def m : citys){
//def detailUrls = getPageUrls(driver, cityUrl)
def detailUrls = getPageDeail(driver, m.city, m.url)
println m.city +" ÏêÇéÁ´½Ó£º"+detailUrls.size()
driver.get(m.url+"/chengjiao/")
Thread.sleep(10000)
def doc = Jsoup.parse(driver.getPageSource());
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
if(regions){
for(def r : regions){
def regionText = r.text()
def href = m.url+ r.attr("href")
println "开始抓取【${m.city}】【${regionText}】【${href}】"
def detailUrls = getPageDeail(driver, m.city, href, regionText)
println m.url +" 详情链接:"+detailUrls.size()
}
}
// def detailUrls = getPageDeail(driver, m.city, m.url)
// println m.city +" 详情链接:"+detailUrls.size()
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
}
}
@@ -142,12 +156,12 @@ public class BeiKeCrawlByChrome {
}
def getPageDeail(driver, city, cityUrl){
def getPageDeail(driver, city, cityUrl, region){
def stop = false
def page = 1
def detailUrls = []
while(!stop){
def url = cityUrl + "/chengjiao/pg${page}/"
def url = cityUrl + "pg${page}/"
try{
driver.get(url)
}catch(Exception e){
@@ -164,7 +178,7 @@ public class BeiKeCrawlByChrome {
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
parse(city, e, region)
}else{
stop = true;
break;
@@ -182,7 +196,7 @@ public class BeiKeCrawlByChrome {
return detailUrls
}
def parse(city,e){
def parse(city,e, region){
try {
def title = e.select(".title a").text()
if (title) {
@@ -190,7 +204,7 @@ public class BeiKeCrawlByChrome {
def arrs = title.split(" ")
detail.city = city
detail.district_name = arrs[0]
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
detail.region = region//findRegionByThirdPary(detail.city, detail.district_name)
if(detail.region){
detail.region = RegionData.formatRegion(city, detail.region)
}
@@ -381,7 +395,7 @@ public class BeiKeCrawlByChrome {
cityMap.city = e.text()
cityMap.url = "https:"+href
list << cityMap
//cityMap.put(e.text(), "https:"+href)
// cityMap.put(e.text(), "https:"+href)
// }
}
}
@@ -7,6 +7,8 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.StringUtils
import org.apache.commons.lang.math.NumberUtils
import java.util.Map.Entry;
import java.util.regex.Pattern
@@ -175,22 +177,23 @@ class ExportPaimai {
def run(){
def city = "苏州"
def region = "姑苏"
def city = "合肥"
def region = null
def datas = findPaimaiData(city,region)
println "总量:"+datas.size()
def rows = []
def header = ["标地物地址","省","城市","区域","地址","标题提取地址","楼栋号","单元","室号","楼盘名称","建筑面积","竣工日期","房屋用途","总楼层",
"拍卖状态","是否成交","拍卖阶段","拍卖方式","成交价","拍卖开始时间","拍卖结束时间","起拍价","保证金","加价幅度","评估价","市场价",
"变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数","URL"]
"变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数", "折价率","映射类型","拍卖类型",
"映射资产类型","映射拍卖状态","URL"]
rows << header
// def set = new HashSet()
def set = new HashSet()
def index = 0
for(def d : datas){
// if(!set.contains(d.title)){
if(!set.contains(d.title)){
index++
println index
// set << d.title
set << d.title
def row = []
row << d.title.replace(",",",")
row << d.province
@@ -226,14 +229,19 @@ class ExportPaimai {
row << d.lookCount
row << d.callCount
row << d.delayCount
row << d.url
row << d.rateDiscount//折价率
row << d.mappingEstateType;//映射类型
row << d.zcType;//拍卖类型:资产拍卖 司法拍卖
row << d.mappingAgencyName//映射资产类型
row << d.mappingStatus//映射拍卖状态
row << d.url
rows << row
// }else{
// println d.title
// }
}else{
println d.title +" "+d.url
}
}
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\京东拍卖(${city}-姑苏-撤回中止).xlsx",rows,"数据")
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\合肥法拍.xlsx",rows,"数据")
// batchCreateFile(rows, "D:\\数据\\重庆农商行\\法拍\\淘宝拍卖(新).csv",null,"GBK")
println "生成完成........"
@@ -452,12 +460,12 @@ class ExportPaimai {
if(region){
pmquery.put("region",region)
}
pmquery.put("status", new BasicDBObject('$in', ["已撤回","已暂缓"]))
// pmquery.put("status", new BasicDBObject('$in', ["已撤回","已暂缓"]))
// pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-07-01")))
// Pattern pattern = Pattern.compile("rmfysszc.gov.cn", Pattern.MULTILINE);
Pattern pattern = Pattern.compile("jd.com", Pattern.MULTILINE);
// Pattern pattern = Pattern.compile("jd.com", Pattern.MULTILINE);
// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
// pmquery.put("url", new BasicDBObject('$regex', pattern))
def dbcursor = paimai43_COLL.find(pmquery).batchSize(2000)
@@ -465,6 +473,28 @@ class ExportPaimai {
for(def db : dbcursor){
rets << db
}
// Collections.sort(rets, new Comparator<BasicDBObject>() {
// @Override
// int compare(BasicDBObject o1, BasicDBObject o2) {
// if(o1.finishDate && o2.finishDate){
// return -o1.finishDate.compareTo(o2.finishDate)
// }else{
// return 0
// }
//
// }
// })
rets.sort {a,b->
if(a.finishDate && b.finishDate){
def v1 = a.finishDate
def v2 = b.finishDate
return v1.compareTo(v2)
}else{
return 0
}
}
rets = rets.reverse()
return rets
}
}
@@ -34,9 +34,9 @@ public class LianjiaCrawlByChrome {
static main(args) {
def task = new LianjiaCrawlByChrome();
// task.run()
task.run()
// task.run1()
task.addDealRegion();
// task.addDealRegion();
task.standardRegion()
// task.updadteDealPriceField()
}
@@ -129,7 +129,7 @@ public class LianjiaCrawlByChrome {
def standardRegion(){
def q = new BasicDBObject()
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")))
q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) )
def dbcursor = DEAL_PRICE_COLL.find(q)
for(def d : dbcursor){
@@ -152,8 +152,9 @@ public class LianjiaCrawlByChrome {
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
lastMonth = lastMonth +"-01"
def q = new BasicDBObject()
// q.put("city", "福州")
q.put("region", null)
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-05-01","yyyy-MM-dd")))
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")))
// q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
def dbcursor = DEAL_PRICE_COLL.find(q)
def keys = []
@@ -187,8 +188,19 @@ public class LianjiaCrawlByChrome {
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
cityMap.each{city, cityUrl->
//def detailUrls = getPageUrls(driver, cityUrl)
def detailUrls = getPageDeail(driver, city, cityUrl)
println cityUrl +" 详情链接:"+detailUrls.size()
driver.get(cityUrl+"chengjiao/")
def doc = Jsoup.parse(driver.getPageSource());
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
if(regions){
for(def r : regions){
def regionText = r.text()
def href = cityUrl.toString().substring(0, cityUrl.toString().length()-1) + r.attr("href")
println "开始抓取【${city}】【${regionText}】【${href}】"
def detailUrls = getPageDeail(driver, city, href, regionText)
println cityUrl +" 详情链接:"+detailUrls.size()
}
}
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
}
}
@@ -224,12 +236,12 @@ public class LianjiaCrawlByChrome {
}
def getPageDeail(driver, city, cityUrl){
def getPageDeail(driver, city, cityUrl, regionText){
def stop = false
def page = 1
def detailUrls = []
while(!stop){
def url = cityUrl + "chengjiao/pg${page}/"
def url = cityUrl + "pg${page}/"
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def elements = JQueryUtils.select(".listContent li", doc)
@@ -238,10 +250,10 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-06-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-07-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
parse(city, e, regionText)
}else{
stop = true;
break;
@@ -259,7 +271,7 @@ public class LianjiaCrawlByChrome {
return detailUrls
}
def parse(city,e){
def parse(city,e, regionText){
try {
def title = e.select(".title a").text()
if (title) {
@@ -267,7 +279,11 @@ public class LianjiaCrawlByChrome {
def arrs = title.split(" ")
detail.city = city
detail.district_name = arrs[0]
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
if(regionText){
detail.region = regionText
}else{
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
}
if(detail.region){
detail.region = RegionData.formatRegion(city, detail.region)
}
@@ -275,50 +291,61 @@ public class LianjiaCrawlByChrome {
def roomStr = arrs[1]
if (roomStr) {
if (roomStr.indexOf("室") > 0) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
def room = roomStr.substring(0, roomStr.indexOf("室"))
if(room && room.isNumber()){
detail.room = room.toInteger()
}
}
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger()
def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
if(hall && hall.isNumber()){
detail.hall = hall.toInteger()
}
}
}
def areaStr = arrs[2].replace("平米", "")
if (areaStr) {
detail.area = areaStr.toDouble()
}
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = price.toInteger() * 10000
}
if (detail.total_price && detail.area) {
detail.avg_price = (int) (detail.total_price / detail.area)
}
def tradeStr = e.select(".dealDate").text()
if (tradeStr) {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "链家在线"
detail.url = e.select(".title a").attr("href")
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
if("车位"!=roomStr){
if(arrs && arrs.size()>=3){
def areaStr = arrs[2].replace("平米", "")
if (areaStr) {
detail.area = areaStr.toDouble()
}
}
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = price.toInteger() * 10000
}
if (detail.total_price && detail.area) {
detail.avg_price = (int) (detail.total_price / detail.area)
}
def tradeStr = e.select(".dealDate").text()
if (tradeStr) {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "链家在线"
detail.url = e.select(".title a").attr("href")
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
}
}
}
}catch(Exception ex){
@@ -442,13 +469,13 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
def citys= ["福州"]
def citys= ["苏州"]
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
// if (!href.contains("fang.lianjia") && citys.contains(e.text())){
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
cityMap.put(e.text(), href)
// }
}
}
}
return cityMap
@@ -10,6 +10,7 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import com.rabbitmq.client.impl.AMQImpl
import java.text.SimpleDateFormat
import java.util.concurrent.CountDownLatch
@@ -29,18 +30,19 @@ class PaimaiDataSend {
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
def task = new PaimaiDataSend()
// task.jdRunByMongo()
// task.jdRun()
// task.aliZcRun()
task.aliZcRun2()
//// task.aliZcRun()
// task.aliZcRun2()
// task.aliFpRun()
// task.rmfysszcRun()
// task.checkUrl()
// task.paimaiSendToEs()
task.paimaiSendToEs()
}
@@ -64,7 +66,9 @@ class PaimaiDataSend {
def rmfysszcRun(){
def index = 0
def q = new BasicDBObject()
// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/76602.shtml")
// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市", "苏州市"]))
q.put("province","广东省")
// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/88061.shtml")
def dbcursor = rmfysszc_coll.find(q).batchSize(3000)
for(def d : dbcursor){
index++
@@ -95,11 +99,22 @@ class PaimaiDataSend {
}
}
def jdRun(){
def jdRunByMongo(){
def pm_q = new BasicDBObject()
pm_q.put("city","宿迁")
pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
pm_q.put("mappingStage", null)
def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000)
println dbcursor.count()
for(def d : dbcursor){
sendJd(d.url)
}
}
def sendJd(url){
def index = 0
def q = new BasicDBObject()
q.put("city","宿迁市")
// q.put("url","https://paimai.jd.com/108245329")
q.put("url", url)
def dbcursor = jd_coll.find(q).batchSize(3000)
for(def d : dbcursor){
index++
@@ -110,6 +125,7 @@ class PaimaiDataSend {
JSON result = new JSONObject()
result.put("content", d.desc_content)
result.put("region", d.region)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("bid_content", bid_content.toString())
@@ -131,20 +147,96 @@ class PaimaiDataSend {
jd_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
}
}
def jdRun(){
def index = 0
def q = new BasicDBObject()
// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市","苏州市"]))
q.put("province","广东")
// q.put("parser_status", new BasicDBObject('$ne',3))
// q.put("url","https://paimai.jd.com/115669796")
def dbcursor = jd_coll.find(q).batchSize(3000)
def rets = []
for(def d : dbcursor){
index++
println "jd:"+index
JSON json = new JSONObject()
json.put("project", "paimai_jd")
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.desc_content)
result.put("region", d.region)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("bid_content", bid_content.toString())
result.put("sub_count", d.sub_count)
def data_json = net.sf.json.JSONObject.fromObject(d.json)
result.put("data_json", data_json)
result.put("content_json", d.content)
result.put("item_notice_content", d.item_notice_content)
result.put("notice_detail_content", d.notice_detail_content.content)
json.put("result", result)
rets << json
if(rets.size()==10000){
println "共:"+rets.size()
sendJD(rets)
rets.clear()
}
}
if(rets.size()>0){
println "共:"+rets.size()
sendJD(rets)
}
}
def sendJD(def rets){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def json = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 3)
udata.put("m_date", new Date())
jd_coll.update(new BasicDBObject("_id", json.get("url")), new BasicDBObject('$set', udata))
println "send: "+totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def aliZcRun(){
def pm_q = new BasicDBObject()
pm_q.put("city","苏州")
pm_q.put("region","姑苏")
// pm_q.put("url","https://sf-item.taobao.com/sf_item/596573117068.htm")
pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
// pm_q.put("url", new BasicDBObject('$regex', pattern))
// pm_q.put("status", null)
pm_q.put("city",new BasicDBObject('$in', ["南京", "宿迁","苏州"]))
// pm_q.put("city","苏州")
// pm_q.put("region","姑苏")
// pm_q.put("url","https://sf-item.taobao.com/sf_item/659117468505.htm")
pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")).append('$lt', DateUtils.parseFromTightYYYYMMDD("2022-09-15")))
Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
pm_q.put("url", new BasicDBObject('$regex', pattern))
// pm_q.put("mappingEstateType", null)
// pm_q.put("area", new BasicDBObject('$lt', 30))
def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000)
println dbcursor.count()
for(def d : dbcursor){
def flag = sendAliZc(d.url)
// def flag = sendAli(d.url)
if(!flag){
println "²»´æÔÚ:"+d.url
}
@@ -185,7 +277,6 @@ class PaimaiDataSend {
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 1)
udata.put("m_date", new Date())
@@ -194,14 +285,60 @@ class PaimaiDataSend {
return flag
}
def sendAli(def url){
def index = 0
def q = new BasicDBObject()
q.put("url",url)
def dbcursor = ali_fp_coll.find(q).batchSize(5000)
def flag = false
for(def d : dbcursor){
index++
flag = true
println "ali-fp: "+index
JSON json = new JSONObject()
json.put("project", "paimai_taobao")
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.content)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("trades", bid_content)
result.put("province", d.province)
result.put("city", d.city)
result.put("region", d.region)
result.put("sub_count", d.sub_count)
def data_json = net.sf.json.JSONObject.fromObject(d.json)
result.put("data_json", data_json)
result.put("desc_content", d.desc_content)
result.put("item_notice_content", d.item_notice_content)
result.put("notice_detail_content", d.notice_detail_content)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 1)
udata.put("m_date", new Date())
ali_fp_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
}
return flag
}
def aliFpRun(){
def index = 0
def q = new BasicDBObject()
// q.put("city","宿迁")
q.put("url","https://sf-item.taobao.com/sf_item/649014193736.htm")
// q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
q.put("province","广东")
// q.put("parser_status", new BasicDBObject('$ne',2))
// q.put("url","https://sf-item.taobao.com/sf_item/631662596036.htm")
//https://sf-item.taobao.com/sf_item/657131607486.htm
// q.put("parser_status", new BasicDBObject('$exists',false))
def dbcursor = ali_fp_coll.find(q).batchSize(2000)
// q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01")))
// q.put("area", new BasicDBObject('$lt', 30))
def dbcursor = ali_fp_coll.find(q).batchSize(5000)
def rets = []
@@ -228,10 +365,20 @@ class PaimaiDataSend {
json.put("result", result)
rets << json
if(rets.size()==10000){
println "共:"+rets.size()
send(rets)
rets.clear()
}
}
println "共:"+rets.size()
if(rets.size()>0){
println "共:"+rets.size()
send(rets)
}
}
def send(def rets){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
@@ -244,7 +391,7 @@ class PaimaiDataSend {
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 1)
udata.put("parser_status", 2)
udata.put("m_date", new Date())
ali_fp_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
println "send: "+totalCount.get()
@@ -263,12 +410,19 @@ class PaimaiDataSend {
def aliZcRun2(){
def index = 0
def q = new BasicDBObject()
// q.put("city","宿迁")
// q.put("city",new BasicDBObject('$in', ["苏州","宿迁","南京"]))
q.put("province","广东")
// q.put("parser_status", new BasicDBObject('$ne',1))
// q.put("region","¹ÃËÕ")
q.put("url","https://zc-item.taobao.com/auction/672615219601.htm")
def dbcursor = ali_zc_coll.find(q).batchSize(1000)
// q.put("url","https://susong-item.taobao.com/auction/652481594083.htm")
def dbcursor = ali_zc_coll.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
index++
if(!d.url){
println d._id.toString()
continue
}
println "ali-zc: "+index
JSON json = new JSONObject()
json.put("project", "paimai_taobao_zc")
@@ -290,19 +444,52 @@ class PaimaiDataSend {
result.put("region", d.region)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 4)
udata.put("m_date", new Date())
ali_zc_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
rets << json
if(rets.size()==10000){
println "共:"+rets.size()
send(rets)
rets.clear()
}
}
if(rets.size()>0){
println "共:"+rets.size()
send(rets)
}
// println "共:"+rets.size()
//
// def totalCount = new AtomicLong()
// final CountDownLatch latch = new CountDownLatch(rets.size())
// for(def i=0;i<rets.size();i++){
// final def json = rets[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
// try{
// totalCount.addAndGet(1)
// byte[] packedMsg = MessageUtils.pack(json)
// CommonUtils.sendMessage("result2parse2dev", packedMsg)
//
//
// def udata = new BasicDBObject()
// udata.put("parser_status", 1)
// udata.put("m_date", new Date())
// ali_zc_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
// println "send: "+totalCount.get()
// } catch (Exception e) {
// e.printStackTrace()
// } finally {
// latch.countDown()
// }
// }
// });
// }
// latch.await()
}
def paimaiSendToEs(){
def rets = getPaimaiDatas()
println "共:"+rets.size()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
@@ -312,7 +499,7 @@ class PaimaiDataSend {
try{
totalCount.addAndGet(1)
def flag = FangjiaClientUtils.saveToEsPaimai(m)
println totalCount.get()+" "+m.city +" "+m.name +" "+flag
println totalCount.get()+" "+m.city +" "+m.title +" "+flag
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -326,56 +513,79 @@ class PaimaiDataSend {
def getPaimaiDatas(){
def q = new BasicDBObject()
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
def dbcursor = paimai43_COLL.find(q).batchSize(1000)
// q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-15")))
def dbcursor = paimai43_COLL.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
def o = [:]
o.addPrice = d.addPrice
o.agencyName = d.agencyName
o.area = d.area
o.buildYear = d.buildYear
o.callCount = d.callCount
o.city = d.city
o.court = d.court
o.delayCount = d.delayCount
o.delayWeekly = d.delayWeekly
o.depositPrice = d.depositPrice
o.enrollCount = d.enrollCount
o.estateType = d.estateType
o.evaluatePrice = d.evaluatePrice
o.finishDate = d.finishDate
o.height = d.height
o.houseNum = d.houseNum
o.id = d.id
o.introduce = d.introduce
if(d.city){
def o = [:]
o.addPrice = d.addPrice
o.agencyName = d.agencyName
o.area = d.area
o.buildYear = d.buildYear
o.callCount = d.callCount
o.city = d.city
o.court = d.court
o.delayCount = d.delayCount
o.delayWeekly = d.delayWeekly
o.depositPrice = d.depositPrice
o.enrollCount = d.enrollCount
o.estateType = d.estateType
o.evaluatePrice = d.evalatePrice
o.finishDate = d.finishDate
o.height = d.height
o.houseNum = d.houseNum
o.id = d.id
o.introduce = d.introduce
// o.location = d.location
o.lookCount = d.lookCount
o.marketPrice = d.marketPrice
o.matchAddress = d.matchAddress
o.matchCity = d.matchCity
o.matchName = d.matchName
o.matchRegion = d.matchRegion
o.person = d.person
o.preemptivePerson = d.preemptivePerson
o.pricePkWeekly = d.pricePkWeekly
o.province = d.province
o.rateDiscount = d.rateDiscount
o.region = d.region
o.roomNum = d.roomNum
o.sellOffAdvancePay = d.sellOffAdvancePay
o.source = d.source
o.stage = d.stage
o.startDate = d.startDate
o.startPrice = d.startPrice
o.status = d.status
o.tel = d.tel
o.title = d.title
o.tradePrice = d.tradePrice
o.type = d.type
o.unitNum = d.unitNum
o.url = d.url
rets << o
o.lookCount = d.lookCount
o.marketPrice = d.marketPrice
o.matchAddress = d.matchAddress
o.matchCity = d.matchCity
o.matchName = d.matchName
o.matchRegion = d.matchRegion
o.person = d.person
o.preemptivePerson = d.preemptivePerson
o.pricePkWeekly = d.pricePkWeekly
o.province = d.province
o.rateDiscount = d.rateDiscount
o.region = d.region
o.roomNum = d.roomNum
o.sellOffAdvancePay = d.sellOffAdvancePay
o.source = d.source
o.stage = d.stage
o.startDate = d.startDate
o.startPrice = d.startPrice
o.status = d.status
o.tel = d.tel
o.title = d.title
o.tradePrice = d.tradePrice
o.tradeAvgPrice = d.tradeAvgPrice
o.type = d.type
o.unitNum = d.unitNum
o.url = d.url
if(d.url.contains("taobao.com")){
o.source = "淘宝"
}else if(d.url.contains("jd.com")){
o.source = "京东"
}else if(d.url.contains("rmfysszc.gov.cn")){
o.source = "人民法院讼诉网"
}
o.rateDiscount = d.rateDiscount//折价率
o.mappingEstateType = d.mappingEstateType;//映射类型
o.paiMaiType = d.paiMaiType;//拍卖类型:资产拍卖 司法拍卖
o.mappingAgencyName = d.mappingAgencyName//映射资产类型
o.mappingStatus = d.mappingStatus//映射拍卖状态
o.mappingStage = d.mappingStage
if(d.startRateDiscount){
o.startRateDiscount = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, d.startRateDiscount);//起拍折价率 起拍折价率 = 起拍价 / 市场价或评估价
}
o.diffPrice = d.diffPrice;//差价 差价 = 成交价-市场价或评估价
rets << o
}
}
return rets;
}
@@ -8,6 +8,7 @@ import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.Mongo
@@ -15,10 +16,154 @@ import groovy.sql.Sql
import org.bson.types.ObjectId
import java.text.SimpleDateFormat
import java.util.regex.Pattern
def dist = CommonDatautils.findDistrictById("62ce233a3705890001dfa164")
DBUtils.seaweed.save(dist)
DBCollection coll = MongoUtils.getCollectionByName("seeds_db", "city_seeds", "192.168.0.235", 27018)
def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", ZS_MONGODB_URL)
//ExcelUtils.modify("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx",{d->
// def city = RegionData.formatCity(d['E'])
// if(city && !d['J']){
// def region = RegionData.formatRegion(city, d['G'])
// d['I'] = city
// d['J'] = region
// }
//})
//ExcelUtils.modify("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx",{d->
// def city = RegionData.formatCity(d['D'])
// if(!city){
// city = RegionData.formatCity(d['F'])
// }
// if(city && !d['H']){
// def region = RegionData.formatRegion(city, d['B'])
// d['G'] = city
// d['H'] = region
// }
//})
//def citys = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
//def datas = ExcelUtils.read("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx")
//def fjwCitys = []
//for(def d : datas){
// fjwCitys << d['G']
//}
//def rows = []
//for(def d : datas){
// if(citys.contains(d['G'])){
// rows << d
// }
//}
//ExcelUtils.write("D:\\数据\\建行\\二期\\统计局行政区划与房价网区域比对V2.xlsx", rows, "区域")
//for(def c : citys){
// if(!fjwCitys.contains(c)){
// println c
// }
//}
//
//ExcelUtils.write("D:\\数据\\建行\\二期\\(特殊城市)统计局行政区划与房价网区域比对.xlsx", rows, "区域")
//
//def map = [:]
//def datas1 = ExcelUtils.read("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx")
//for(def row : datas1){
// if(row['J']){
// def codeMap = [:]
// codeMap.prov = row['B']
// codeMap.city = row['F']
// codeMap.region = row['H']
// map.put(row['I']+"#"+row['J'], codeMap)
// }
//}
//def datas2 = ExcelUtils.read("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx")
//for(def row : datas2){
// if(row['H']){
// def codeMap = [:]
// codeMap.prov = row['E']
// codeMap.city = row['C']
// codeMap.region = row['A']
// map.put(row['G']+"#"+row['H'], codeMap)
// }
//}
//ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d->
// def key = d['B'] +"#"+d['C']
// def codeMap = map.get(key)
// if(codeMap && !d['D']){
// d['D'] = codeMap.prov
// d['E'] = codeMap.city
// d['F'] = codeMap.region
// }
//})
ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d->
if(!d['D']){
def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN)
if(infos && infos.size()==1){
def info = infos[0]
d['D'] = info.province_code
d['E'] = info.area_code
d['F'] = info.street_code
}
}
})
def findTownshipInfo(city, region, coll){
def q = new BasicDBObject()
q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE))
q.put("street_name", Pattern.compile('^.*'+region+'街道.*$', Pattern.CASE_INSENSITIVE))
def dbcursor = coll.find(q)
def rows = []
for(def d : dbcursor){
// if(d.street_name.contains(region)){
rows << d
// }
}
return rows
}
//def cityMap = CommonDatautils.findAllCityProvince()
//def rows = []
//rows << ["省", "市", "区域"]
//cityMap.each{c,p->
// def regions = CommonDatautils.getCityRegions(c)
// for(def r : regions){
// def row = []
// row << p
// row << c
// row << r
// rows << row
// }
//}
//ExcelUtils.write("D:\\数据\\亲邻\\房价网省市区.xlsx", rows, "区域")
//def q = new BasicDBObject()
//q.put("root", "anjuke.com")
//def dbcursor = coll.find(q)
//def map = [:]
//for(def d : dbcursor){
// map.put(d.city, d.city_short)
//}
//println map.size()
//println map
//def dist = CommonDatautils.findDistrictById("62ce233a3705890001dfa164")
//DBUtils.seaweed.save(dist)
//DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId("62ce233a3705890001dfa164")), new BasicDBObject('$set', dist))
//SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
//Date date = new Date(1617211269868);
@@ -0,0 +1,142 @@
package com.fangjia.script.gj.test
import com.alibaba.fastjson.JSON
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.util.MessageUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
DBCollection school_coll = MongoUtils.getCollectionByName("crawl_tmp", "school_ruyile_web", "192.168.0.235", 27018)
DBCollection sxue_school_coll = MongoUtils.getCollectionByName("crawl_tmp", "school_5sxue_web", "192.168.0.235", 27018)
DBCollection school_parse_coll = MongoUtils.getCollectionByName("fangjia_craw", "school", "192.168.0.235", 27018)
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
//Thread.sleep(60*60*1000)
//while (true){
def q = new BasicDBObject()
// q.put("province",new BasicDBObject('$in', ["江苏","江苏省"]))
q.put("status",new BasicDBObject('$ne', 1))
def dbcursor = school_coll.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
JSON json = new JSONObject()
json.put("project", d.project)
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.content)
def jsonStr = d.map_info
jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1)
if(jsonStr.contains("\n")){
jsonStr = jsonStr.replaceAll('\\n','')
}
result.put("data_json", net.sf.json.JSONObject.fromObject(jsonStr))
result.put("type", d.type)
json.put("result", result)
rets << json
if(rets.size()==5000){
println "共:"+rets.size()
send(rets, fixedThreadPool, school_coll)
rets.clear()
}
}
if(rets.size()>0){
println "共:"+rets.size()
send(rets, fixedThreadPool, school_coll)
rets.clear()
}
//}
def send(def rets, def fixedThreadPool, def school_coll){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def json = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
school_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
println "send: "+totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
//def rets = []
////def query = new BasicDBObject()
////query.put("source","搜学网")
//////query.put("url","http://xuexiao.51sxue.com/detail/id_26799.html")
//////query.put("lat",null)
////def cursor = school_parse_coll.find(query)
////for(def o : cursor){
// def q = new BasicDBObject()
//// q.put("province","江苏省")
//// q.put("url",o.url)
// def dbcursor = sxue_school_coll.find(q)
// def index = 0
// for(def d : dbcursor){
// index++
// println index
// JSON json = new JSONObject()
// json.put("project", d.project)
// json.put("url", d.url)
//
// JSON result = new JSONObject()
// result.put("content", d.content)
// result.put("name", d.name)
// result.put("list_item", d.list_item)
//
// def jsonStr = d.map_info
// jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1)
// if(jsonStr.contains("\n")){
// jsonStr = jsonStr.replaceAll('\\n','')
// }
// result.put("json_data", jsonStr)
//
// result.put("type", d.type)
// json.put("result", result)
//
// rets << json
// if(rets.size()==5000){
// println "共:"+rets.size()
// send(rets, fixedThreadPool, school_coll)
// rets.clear()
// }
// }
////}
//if(rets.size()>0){
// println "共:"+rets.size()
// send(rets, fixedThreadPool, school_coll)
// rets.clear()
//}
@@ -0,0 +1,39 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.util.EsUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.util.ElasticUtils
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* ²¹³äѧУ¾­Î³È
* @author shenjian
* @since 2022/9/8
*/
class FillLngLat {
static ExecutorService executorService = Executors.newFixedThreadPool(4)
static void main(String[] args) {
ElasticUtils.executeSqlQuery("select * from dws_poi where location is not null", { dataT ->
final def data = dataT
executorService.submit(new Runnable() {
@Override
void run() {
println(data)
def lngLat = GaoDeUtils.parserByGaode(data.city, data.region + data.name + data.address)
if (lngLat != null && lngLat.size() == 2 ) {
JSONObject jsonObject = new JSONObject()
jsonObject.put("lon", Double.valueOf(lngLat['lng']))
jsonObject.put("lat", Double.valueOf(lngLat['lat']))
EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id))
}
}
})
})
executorService.shutdown()
}
}
@@ -10,6 +10,7 @@ import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
@@ -27,15 +28,15 @@ import java.util.concurrent.Executors
@BatchOwner(Watcher.ShenJian)
class SeaweedAroundPoi extends BaseBatch {
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.235", 27018)
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi"
static ExecutorService executorService = Executors.newFixedThreadPool(30)
//static ExecutorService executorService = Executors.newFixedThreadPool(2)
static void main(String[] args) {
def types = listAllType()
//dealSpecialSeaweedAround(types)
dealPoi(new BasicDBObject(), types)
executorService.shutdown()
//executorService.shutdown()
}
/**
@@ -64,33 +65,40 @@ class SeaweedAroundPoi extends BaseBatch {
}
static dealPoi(DBObject query, def types) {
query.put("status", 1)
conn136.find(query).batchSize(5000).each {objTemp ->
query.put("canyin_status", 0)
conn136.find(query).batchSize(5000).limit(20000).noCursorTimeout(true).each {objTemp ->
final def obj = objTemp
executorService.submit(new Runnable() {
@Override
void run() {
// executorService.submit(new Runnable() {
// @Override
// void run() {
def city = obj.city
def lng = obj.lng
def lat = obj.lat
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
HttpUtils.postForJson(URL, jsonStr)
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
} else {
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空")))
if (obj.lng) {
//def lng = SeaweedUtils.formatLatLng(obj.lng)
//def lat = SeaweedUtils.formatLatLng(obj.lat)
// {'lon': 121.810877, 'lat': 31.737642}
def lng = 121.810877
def lat = 31.737642
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
HttpUtils.postForJson(URL, jsonStr)
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("canyin_status", 0).append("flag", "已获取周边POI")))
} else {
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("canyin_status", 2).append("flag", "周边POI为空")))
}
println("已处理小区:" + obj)
}
println("已处理小区:" + obj)
}
})
}
// })
// }
}
static getNewSeaweed() {
DBObject query = new BasicDBObject()
query.put("cat", "district")
query.put("visible", 0)
//query.put("city", new BasicDBObject('$in', ["连云港", "南通", "苏州", "无锡", "盐城", "扬州"]))
//query.put("city", "连云港")
//query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a"))
DBObject filter = new BasicDBObject()
@@ -163,17 +171,18 @@ class SeaweedAroundPoi extends BaseBatch {
public static final Map<String, String> KEYWORDS_MAP = new HashMap<String, String>() {
{
put("餐饮", "05");
// put("餐饮", "05");
// put("地铁", "150500");
// put("公交", "1507");
// put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211");
// put("写字楼", "120201");
// put("学校", "141200|141201|141202|141203|141204");
// put("银行", "1601|1603");
// put("大型超市", "0604");
put("大型超市", "0604");
// put("商场", "0601");
// put("餐饮", "0501|0502|0503|0504");
// put("旅游景点", "110000");
put("公园广场", "110100|110101|110102|110103|110104|110105")
// put("图书馆", "140500");
// put("政府机关", "130100|130101|130102|130103|130104|130105|130106|130107");
// put("丧葬设施", "071900|071901|071902|071903")
@@ -0,0 +1,84 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.collections.CollectionUtils
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 小区周边POI数据获取
*
* @author shenjian
* @since 2021/5/24
*/
class SeaweedAroundPoiFromExcel {
static void main(String[] args) {
def types = listAllType()
dealPoi(new BasicDBObject(), types)
}
static dealPoi(DBObject query, def types) {
com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\建行\\审核小区\\东莞银行房价数据测试20220916-v3.xlsx", { List item ->
def city = item['H']
def lng = item['AQ']
def lat = item['AR']
if (lng) {
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi", jsonStr)
}
Thread.sleep(1000)
print(lng + " " + lat)
}
});
};
public static def listAllType() {
StringBuilder ret = new StringBuilder()
for (def val : KEYWORDS_MAP.values()) {
ret.append(val).append("|")
}
String result = ret.toString()
return result.substring(0, result.length() - 1)
};
public static final Map<String, String> KEYWORDS_MAP = new HashMap<String, String>() {
{
// put("餐饮", "05");
// put("地铁", "150500");
// put("公交", "1507");
// put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211");
// put("写字楼", "120201");
// put("学校", "141200|141201|141202|141203|141204");
// put("银行", "1601|1603");
put("大型超市", "0604");
// put("商场", "0601");
// put("餐饮", "0501|0502|0503|0504");
// put("旅游景点", "110000");
put("公园广场", "110100|110101|110102|110103|110104|110105")
// put("图书馆", "140500");
// put("政府机关", "130100|130101|130102|130103|130104|130105|130106|130107");
// put("丧葬设施", "071900|071901|071902|071903")
}
}
}
@@ -146,11 +146,15 @@ class GaoDeUtils {
while (result.size() < count) {
params.page = "${page}".toString()
page++
params.key = this.getGaodeKey("v5/place/around")
JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params);
//params.key = this.getGaodeKey("v3/place/around")
params.key = "25b2d459fd133e22f8abcebeac972692"
params.offset = "100"
JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params);
if (json == null) {
params.key = this.getGaodeKey("v5/place/around")
json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params);
//params.key = this.getGaodeKey("v3/place/around")
params.key = "25b2d459fd133e22f8abcebeac972692"
params.offset = "100"
json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params);
}
if (json && json.info && json.info == "OK") {
count = json.getString("count").toInteger()
@@ -258,7 +262,7 @@ class GaoDeUtils {
}
if (json && json.info && json.info == "OK") {
try {
List<AmapPoiDetail> ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class)
List<AmapPoiDetail> ret = JSONObject.parseArray(json.pois.toString(), AmapPoiDetail.class)
return ret
} catch(Exception ex) {
ex.printStackTrace()
@@ -40,11 +40,11 @@ class DataStat {
def run(){
def source = "365ÌÔ·¿"//
def citys = findCrawlCity(source, 202206);
def source = "我爱我家"//
def citys = findCrawlCity(source, 202207);
for(def city : citys){
def count1 = findCount(source, city, 202206);
def count2 = findCount(source, city, 202207);
def count1 = findCount(source, city, 202207);
def count2 = findCount(source, city, 202208);
println city +" "+count1+" "+count2
// println count1
// println "--- ----"
@@ -16,15 +16,12 @@ class SynchDistrictPrice {
def static final SHORT_DATE = Utils.getLastMonth()
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
//"fang.com":"搜房",
// def sourceMap = ["58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
//"fang.com":"搜房",
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
//,"城市房产":"城市房产"
//"365淘房":"365淘房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房",
def esSources = ["C21":"C21","365淘房":"365淘房"]//"吉屋":"吉屋","365淘房":"365淘房"
//"365淘房":"365淘房","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","吉屋":"吉屋"
def esSources = ["搜房":"搜房","我爱我家":"我爱我家","365淘房":"365淘房"]
static main(args) {
def task = new SynchDistrictPrice()
// task.run()
@@ -32,13 +29,13 @@ class SynchDistrictPrice {
}
def run1(){
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def citys = CommonUtils.findAllCity()
// citys.remove("上海")
// citys.remove("广州")
esSources.each{esSource, mgSource->
for(def city : citys){
def rets = findAllEsData(city, esSource, mgSource)
def rets = findAllEsData(city, esSource, mgSource, mdate)
println city +" "+rets.size()
synchData(city, rets, mgSource)
}
@@ -46,10 +43,10 @@ class SynchDistrictPrice {
}
def findAllEsData(city,esSource, mgSource){
def findAllEsData(city,esSource, mgSource, mdate){
def names = new HashSet()
def rets = []
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='2022-07-01' order by m_date", {d->
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='${mdate}' order by m_date", {d->
def s = d.city +"#"+d.region+"#"+d.name
if(!names.contains(s)){
names << s
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
def count1 = findTotalCount(source, 202206);
def count2 = findTotalCount(source, 202207);
def count1 = findTotalCount(source, 202207);
def count2 = findTotalCount(source, 202208);
println source +" "+count1 +" "+count2
}
}
@@ -44,6 +44,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
void process(Object args) {
println SHORT_DATE
def citys = CommonDatautils.findAllCity();
def weekend = DateUtils.getFirstDayOfWeek(new Date());
//def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD("2021-07-15"));
def cityIndex = 0;
@@ -50,6 +50,21 @@ public class AmapPoiDetail {
private BizExtDTO bizExt;
@com.alibaba.fastjson.annotation.JSONField(name = "photos")
private List<PhotosDTO> photos;
@com.alibaba.fastjson.annotation.JSONField(name = "business")
private BusinessDTO business;
@lombok.NoArgsConstructor
@lombok.Data
public static class BusinessDTO {
@com.alibaba.fastjson.annotation.JSONField(name = "business_area")
private String businessArea;
@com.alibaba.fastjson.annotation.JSONField(name = "keytag")
private String keytag;
@com.alibaba.fastjson.annotation.JSONField(name = "opentime_today")
private String opentimeToday;
@com.alibaba.fastjson.annotation.JSONField(name = "opentime_week")
private String opentimeWeek;
}
@lombok.NoArgsConstructor
@lombok.Data
@@ -64,7 +79,7 @@ public class AmapPoiDetail {
@lombok.Data
public static class PhotosDTO {
@com.alibaba.fastjson.annotation.JSONField(name = "title")
private List<?> title;
private String title;
@com.alibaba.fastjson.annotation.JSONField(name = "url")
private String url;
}