Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
+2
-2
@@ -177,8 +177,8 @@ public class BaseAvgImportServlet extends HttpServlet {
|
||||
String realName) {
|
||||
int totalNum = sheet.getLastRowNum();
|
||||
System.out.println("上传文件共:" + totalNum + "个");
|
||||
if (totalNum > 1000) {
|
||||
totalNum = 1000;
|
||||
if (totalNum > 5000) {
|
||||
totalNum = 5000;
|
||||
}
|
||||
|
||||
Integer totalLine = 0;// 总行数
|
||||
|
||||
@@ -257,4 +257,9 @@ public class GeoUtils {
|
||||
ret += (150.0 * Math.sin(x / 12.0 * pi) + 300.0 * Math.sin(x / 30.0 * pi)) * 2.0 / 3.0;
|
||||
return ret;
|
||||
}
|
||||
|
||||
public static void main(String[] args){
|
||||
Map<String, Double> location = gps84ToGcj02(31.218024629574998,121.36562884455);
|
||||
System.out.println(location.toString());
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -1235,7 +1235,7 @@ public class WeiXinEvaluationController {
|
||||
Point point = new Point(Double.valueOf(ll[0]), Double.valueOf(ll[1]));
|
||||
points.add(point);
|
||||
}
|
||||
Point point = new Point(district.getLng()/10000000000.0, district.getLat()/10000000000.0);
|
||||
Point point = new Point(district.getLng2()/10000000000.0, district.getLat2()/10000000000.0);
|
||||
//boolean flag = InsidePolygon.aboutZone(point, points);
|
||||
//boolean flag = InsidePolygon.isPtInPoly(point.getLon(), point.getLat(), points);
|
||||
boolean flag = InsidePolygon2.checkIsInPolygon(point, points);
|
||||
|
||||
@@ -362,7 +362,7 @@ $('.formal_btn').on('click', function () {
|
||||
});
|
||||
|
||||
function clickFormalReport(){
|
||||
$.prompt('正式报告将尽快邮寄给您,请注意查收,谢谢!如有疑问可与我们联系:62392317', '请填写正式报告邮寄地址', function (value) {
|
||||
$.prompt('正式报告将尽快邮寄给您,请注意查收,谢谢! ', '请填写正式报告邮寄地址', function (value) {
|
||||
var address = $(".modal-text-input").val();
|
||||
if(address==""){
|
||||
$.alert('地址不能为空');
|
||||
|
||||
@@ -37,14 +37,13 @@
|
||||
<p style="margin-top:-4px;"><i class="location"></i><small>${fjtEvaluation.address!} <#if fjtEvaluation.houseNum?? && fjtEvaluation.houseNum?contains("号")>${fjtEvaluation.houseNum!}<#else>${fjtEvaluation.houseNum!}号</#if>${fjtEvaluation.roomNum!}室</small> <a href="../district/weXinList?cityName=${fjtEvaluation.city!}" class="collectionLink" style="line-height: 32px;">重新评估 ></a></p>
|
||||
<p style="color: #F4A460;font-size: 12px;line-height: 20px;margin-top: -5px;margin-left: 20px;">*请确认与产证地址一致</p>
|
||||
<span></span>
|
||||
<#if bohaiEvaluationAudit?? && bohaiEvaluationAudit.auditStatus!=0>
|
||||
<p style="margin-top:14px;">评估总价:<#if fjtEvaluation??><strong class="<#if bohaiEvaluationAudit?? && bohaiEvaluationAudit.lastPrice??><#else>price_final</#if>">
|
||||
${(bohaiEvaluationAudit.inquirePrice/10000)?int}</strong> 万元</#if></p>
|
||||
<p>评估单价:<#if bohaiEvaluationAudit.inquirePrice gt 0>${(bohaiEvaluationAudit.inquirePrice/fjtEvaluation.houseArea)?int}</#if> 元/平米</p>
|
||||
<#if bohaiEvaluationAudit?? && bohaiEvaluationAudit.auditStatus!=0 && bohaiEvaluationAudit.inquirePrice gt 0>
|
||||
<p style="margin-top:14px;">评估总价:<#if fjtEvaluation??><strong class="<#if bohaiEvaluationAudit?? && (bohaiEvaluationAudit.lastPrice?? || bohaiEvaluationAudit.inquirePrice??)>price_final</#if>"><#if bohaiEvaluationAudit.lastPrice??>${(bohaiEvaluationAudit.lastPrice/10000)?int}<#elseif bohaiEvaluationAudit.inquirePrice??>${(bohaiEvaluationAudit.inquirePrice/10000)?int}</#if></strong> 万元</#if></p>
|
||||
<p>评估单价:<#if bohaiEvaluationAudit.lastPrice??>${(bohaiEvaluationAudit.lastPrice/fjtEvaluation.houseArea)?int}<#elseif bohaiEvaluationAudit.inquirePrice??>${(bohaiEvaluationAudit.inquirePrice/fjtEvaluation.houseArea)?int}</#if> 元/平米</p>
|
||||
<#if role?? && role==17 && loanInfoMap??>
|
||||
<p style="margin-top:-12px;">所属分类:${loanInfoMap.type!}类</p>
|
||||
<p style="margin-top:-10px;">可贷成数:${loanInfoMap.loan_ratio!}成</p>
|
||||
<p style="margin-top:-22px;">可贷金额:<strong class="price_final">${(bohaiEvaluationAudit.inquirePrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}</strong>万元</p>
|
||||
<p style="margin-top:-22px;">可贷金额:<strong class="price_final"><#if bohaiEvaluationAudit.lastPrice??>${(bohaiEvaluationAudit.lastPrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}<#elseif bohaiEvaluationAudit.inquirePrice??>${(bohaiEvaluationAudit.inquirePrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}</#if></strong>万元</p>
|
||||
<p style="margin-top:-4px;">贷款利率:4.5%</p>
|
||||
</#if>
|
||||
<#else>
|
||||
|
||||
+164
-123
@@ -15,6 +15,7 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
@@ -32,6 +33,9 @@ import java.util.regex.Pattern
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def startDate = 202206
|
||||
def endDate = 202208
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
|
||||
@@ -41,6 +45,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
@@ -102,6 +107,9 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
|
||||
// def lianjiaPriceMap = findThirdPartyPrice("上海")
|
||||
def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd')
|
||||
def lianjiaPriceMap = getDistrictStatByCity("上海", startDate)
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def trade = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
@@ -109,7 +117,8 @@ class BigdataDealBatch extends BaseBatch{
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
if(source == "youda"){
|
||||
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap)
|
||||
|
||||
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap, lianjiaPriceMap)
|
||||
if(ret){
|
||||
result << ret
|
||||
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
|
||||
@@ -135,147 +144,176 @@ class BigdataDealBatch extends BaseBatch{
|
||||
return result
|
||||
}
|
||||
|
||||
def findThirdPartyPrice(def city) {
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
|
||||
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap){
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte', endDate))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for (def db : dbcursor) {
|
||||
def price = null
|
||||
if (db.lianjia_esf_price && db.lianjia_esf_price > 3000) {
|
||||
price = db.lianjia_esf_price
|
||||
}
|
||||
if (!price && db.beike_esf_price && db.beike_esf_price > 3000) {
|
||||
price = db.beike_esf_price
|
||||
}
|
||||
if(price){
|
||||
map.put(db.region+"##"+db.name, price)
|
||||
}
|
||||
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap, def lianjiaPriceMap){
|
||||
def ret = []
|
||||
def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district")
|
||||
if(!obj){
|
||||
return ret;
|
||||
}
|
||||
ret << cityCodeDatas.get(trade.city)
|
||||
ret << trade._id.toString()
|
||||
ret << trade.city
|
||||
def data = distCodeMap.get(trade.fjRegion)
|
||||
if(data){
|
||||
ret << data.district_code
|
||||
ret << data.district
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
def street = ""
|
||||
def streetNumber = ""
|
||||
if(trade.address){
|
||||
if(trade.address.toString().contains("弄")){
|
||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
def price = lianjiaPriceMap.get(obj.region+"##"+obj.name)
|
||||
if(price){
|
||||
def mod = java.lang.Math.abs((trade.price-price)*100/price)
|
||||
if(mod<=10){
|
||||
ret << cityCodeDatas.get(trade.city)
|
||||
ret << trade._id.toString()
|
||||
ret << trade.city
|
||||
def data = distCodeMap.get(trade.fjRegion)
|
||||
if(data){
|
||||
ret << data.district_code
|
||||
ret << data.district
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}
|
||||
if(!street){
|
||||
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){
|
||||
def street = ""
|
||||
def streetNumber = ""
|
||||
if(trade.address){
|
||||
if(trade.address.toString().contains("弄")){
|
||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
}
|
||||
}else{
|
||||
street = jsonObj.searchAddress.street
|
||||
streetNumber = jsonObj.searchAddress.streetNumber
|
||||
}
|
||||
if(!street){
|
||||
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){
|
||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
}
|
||||
}else{
|
||||
street = jsonObj.searchAddress.street
|
||||
streetNumber = jsonObj.searchAddress.streetNumber
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(!street){
|
||||
if(obj.address){
|
||||
if(obj.address.toString().contains("弄")){
|
||||
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
if(!street){
|
||||
if(obj.address){
|
||||
if(obj.address.toString().contains("弄")){
|
||||
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
}
|
||||
}else{
|
||||
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
street = jsonObj.searchAddress.street
|
||||
streetNumber = jsonObj.searchAddress.streetNumber
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ret << street
|
||||
ret << streetNumber
|
||||
|
||||
|
||||
if(obj){
|
||||
ret << obj._id.toString()
|
||||
ret << trade.fjName
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
|
||||
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
|
||||
ret << ""//期数
|
||||
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
|
||||
if(roomNum && roomNum.contains("0")){
|
||||
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
|
||||
}else{
|
||||
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
street = jsonObj.searchAddress.street
|
||||
streetNumber = jsonObj.searchAddress.streetNumber
|
||||
}
|
||||
ret << trade.floor
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << trade.height
|
||||
def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
|
||||
def buildType = ''
|
||||
if(arrs){
|
||||
buildType = arrs[0]
|
||||
}
|
||||
ret << buildType
|
||||
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
|
||||
ret << trade.area
|
||||
ret << ""
|
||||
ret << getHouseRoom(buildType, trade.room, trade.hall)
|
||||
ret << trade.room
|
||||
ret << trade.hall
|
||||
ret << ""
|
||||
ret << trade.toilet
|
||||
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.getDirectionCode(trade.direction)
|
||||
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
|
||||
ret << ""
|
||||
ret << trade.price
|
||||
ret << trade.totalPrice * 10000
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
|
||||
ret << "02"
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << "99"
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << trade.totalPrice * 10000
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
}
|
||||
}
|
||||
|
||||
ret << street
|
||||
ret << streetNumber
|
||||
|
||||
|
||||
if(obj){
|
||||
ret << obj._id.toString()
|
||||
ret << trade.fjName
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
|
||||
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
|
||||
ret << ""//期数
|
||||
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
|
||||
if(roomNum && roomNum.contains("0")){
|
||||
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
|
||||
}else{
|
||||
ret << trade.floor
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << trade.height
|
||||
def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
|
||||
def buildType = ''
|
||||
if(arrs){
|
||||
buildType = arrs[0]
|
||||
}
|
||||
ret << buildType
|
||||
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
|
||||
ret << trade.area
|
||||
ret << ""
|
||||
ret << getHouseRoom(buildType, trade.room, trade.hall)
|
||||
ret << trade.room
|
||||
ret << trade.hall
|
||||
ret << ""
|
||||
ret << trade.toilet
|
||||
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.getDirectionCode(trade.direction)
|
||||
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
|
||||
ret << ""
|
||||
ret << trade.price
|
||||
ret << trade.totalPrice * 10000
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
|
||||
ret << "02"
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << "99"
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << trade.totalPrice * 10000
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
return ret;
|
||||
}
|
||||
|
||||
@@ -543,11 +581,14 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def getDistrictStatByCity(city, weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
def dbcursor = DIST_SELL_STAT.find(q)
|
||||
q.put("weekend", new BasicDBObject('$gte', weekend))
|
||||
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("weekend",-1))
|
||||
def map = new ConcurrentHashMap()
|
||||
for(def d : dbcursor){
|
||||
map.put(d.region+"@"+d.district_name, d.avg_price)
|
||||
def price = map.get(d.region+"##"+d.district_name)
|
||||
if(!price && d.value_type==2){
|
||||
map.put(d.region+"##"+d.district_name, d.avg_price)
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
+1
-1
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220718, 20220819);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220818, 20220919);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||
//测试通过匹配接口配房源效果
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||
|
||||
+9
-1
@@ -77,8 +77,16 @@ class CcbCommonUtils {
|
||||
q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate)));
|
||||
def dbcursor = GUIDING_PRICE_COLL.find(q)
|
||||
def rets = []
|
||||
def sets = new HashSet()
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
def guiding_date = DateUtils.formatDateByPattern(d.guiding_date, 'yyyy-MM-dd')
|
||||
def key = d.city +"##"+d.region+"##"+d.name+"##" + guiding_date
|
||||
if(!sets.contains(key)){
|
||||
rets << d
|
||||
sets << key
|
||||
}else{
|
||||
println key
|
||||
}
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
+7
-6
@@ -370,24 +370,25 @@ class DealPriceDataProcess {
|
||||
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
def citys = CcbCommonUtils.citys
|
||||
def citys = ["上海","广州","东莞"]//CcbCommonUtils.citys
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def publishTime = '2021-10-01'
|
||||
def month = 202206
|
||||
def month = 202207
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty(city, 202207)
|
||||
def rets = findPeGteThirdParty(city, 202208)
|
||||
def total = 0
|
||||
println city +" "+rets.size()
|
||||
for(def m : rets){
|
||||
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
|
||||
def cityId = cityCodeDatas.get(city)
|
||||
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
|
||||
sqlInstance236.executeUpdate(sql)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findPeGteThirdParty(cityName, yearmonth){
|
||||
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>30"
|
||||
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>25"
|
||||
def rets = []
|
||||
sqlInstance236.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
|
||||
+209
@@ -0,0 +1,209 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import groovy.sql.Sql
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class FjwPricePkCcbPrice {
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new FjwPricePkCcbPrice()
|
||||
// task.run()
|
||||
// task.updatePrice()
|
||||
task.addFjwPrice()
|
||||
}
|
||||
|
||||
def updatePrice(){
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\测试\\ccb.xlsx",{d->
|
||||
index++
|
||||
println index
|
||||
def dbo = CommonDatautils.queryDistrictPrice(d['B'], d['C'], d['D'], DateUtils.parseFromTightByPattern('2022-09-04','yyyy-MM-dd'))
|
||||
if(dbo){
|
||||
d['G'] = dbo.future_value
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addFjwPrice(){
|
||||
ExcelUtils.modify("D:\\数据\\建行\\测试\\人工查价(7455)_0824_0913.xlsx",{d->
|
||||
if(!d['M']){
|
||||
def dist = CommonDatautils.findDistrictById(d['D'])
|
||||
if(dist){
|
||||
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2022-09-11","yyyy-MM-dd"))
|
||||
if(dbo){
|
||||
d['M'] = dbo.avg_price
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def run(){
|
||||
def ccbPrices = getCommunityBasePriceGj(202208)
|
||||
println "ccb总量:"+ccbPrices.size()
|
||||
def fjwPriceMap = CommonDatautils.queryAllDistrictPrice(DateUtils.parseFromTightByPattern('2022-09-04','yyyy-MM-dd'))
|
||||
println "fjw总量:"+ccbPrices.size()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(ccbPrices.size())
|
||||
|
||||
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
for(def i=0;i<ccbPrices.size();i++){
|
||||
final def rMap = ccbPrices[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def row = []
|
||||
def dist = CommonDatautils.findDistrictById(rMap.commid)
|
||||
|
||||
if(dist){
|
||||
def priceMap = fjwPriceMap.get(dist.city+"##"+dist.region+"##"+dist.name)
|
||||
def fjwPrice = null
|
||||
def fjwCalPrice = null
|
||||
if(priceMap){
|
||||
fjwPrice = priceMap.avg_price
|
||||
fjwCalPrice = priceMap.calc_value
|
||||
}
|
||||
row << rMap.commid
|
||||
row << dist.city
|
||||
row << dist.region
|
||||
row << dist.name
|
||||
row << fjwPrice
|
||||
row << fjwCalPrice
|
||||
if(priceMap && priceMap.value_type){
|
||||
if(priceMap.value_type == 1){
|
||||
row << "指导价"
|
||||
}else if(priceMap.value_type == 2){
|
||||
row << "自身样本"
|
||||
}else if(priceMap.value_type == 3){
|
||||
row << "第三方"
|
||||
}else if(priceMap.value_type == 4){
|
||||
row << "交易数据"
|
||||
}else if(priceMap.value_type == 5){
|
||||
row << "周边楼盘"
|
||||
}else if(priceMap.value_type == 6){
|
||||
row << "推导"
|
||||
}else if(priceMap.value_type == 9){
|
||||
row << "神经网络"
|
||||
}else if(priceMap.value_type == 10){
|
||||
row << "无方式"
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
row << rMap.avg_price
|
||||
row << rMap.calc_type
|
||||
if(fjwPrice && rMap.avg_price){
|
||||
def precent = (fjwPrice - rMap.avg_price)*100/rMap.avg_price
|
||||
row << DataHandleUtils.getDoubleRound(2, precent)
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
rets << row
|
||||
}else{
|
||||
println rMap.commId + " 没找到小区"
|
||||
}
|
||||
println totalCount.get()+" "+ rMap.cityname +" "+rMap.commname
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await();
|
||||
|
||||
|
||||
def headers = ["小区ID","城市","区域","小区名","房价网价格","房价网预期价格","房价网计算方式","建行价格","建行计算方式","价格差比(房价网价格-建行价格)*100/建行价格"]
|
||||
batchCreateFile(rets, "D:\\数据\\建行\\测试\\房价网与建行均价比对.csv",headers,"GBK")
|
||||
|
||||
}
|
||||
|
||||
|
||||
def getCommunityBasePriceGj(yearMonth) {
|
||||
def sql = "select CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE from community_base_price_gj where year_month=${yearMonth} order by cityid";
|
||||
def allRows = []
|
||||
sqlInstance236.eachRow(sql, { r ->
|
||||
def map = [:]
|
||||
map.cityname = r.CITYNAME
|
||||
map.distname = r.DISTNAME
|
||||
map.commid = r.COMMID
|
||||
map.commname = r.COMMNAME
|
||||
map.year_month = r.YEAR_MONTH
|
||||
map.calc_avg_price = r.CALC_AVG_PRICE
|
||||
map.avg_price = r.AVG_PRICE
|
||||
// 计算类型,1:人工估值 2:样本计算 3:神经网络
|
||||
if(r.CALC_TYPE){
|
||||
if(r.CALC_TYPE == 1){
|
||||
map.calc_type = "人工估值"
|
||||
}else if(r.CALC_TYPE == 2){
|
||||
map.calc_type = "样本计算"
|
||||
}else if(r.CALC_TYPE == 3){
|
||||
map.calc_type = "神经网络"
|
||||
}
|
||||
}
|
||||
allRows << map
|
||||
});
|
||||
return allRows
|
||||
}
|
||||
|
||||
|
||||
def batchCreateFile(ret, fileName, headers, charset){
|
||||
OutputStream out = new FileOutputStream(fileName);
|
||||
def sb = new StringBuffer();
|
||||
// sb.append(headers.join("@|@"))
|
||||
// sb.append(System.getProperty("line.separator"));
|
||||
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
|
||||
def index = 0
|
||||
for(def row : ret){
|
||||
index++
|
||||
sb.append(StringUtils.join(row.toArray(), "@|@"))
|
||||
sb.append(System.getProperty("line.separator"));
|
||||
|
||||
println index
|
||||
|
||||
if(index%10000==0){
|
||||
println index+"开始写入CSV "+new Date().toLocaleString()
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
println index+"结束写入CSV "+new Date().toLocaleString()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(sb.length()>0){
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
}
|
||||
out.close();
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-07-22", "2022-08-21")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-08-24", "2022-09-21")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
|
||||
+934
-825
File diff suppressed because it is too large
Load Diff
+3
-3
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= 202207//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202205
|
||||
def endDate = 202207
|
||||
def sDate= 202208//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202206
|
||||
def endDate = 202208
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
+5
-5
@@ -75,13 +75,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
rets << ["城市","5月总量","6月总量","7月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","6月总量","7月总量","8月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2022-05-01"
|
||||
def date = "2022-06-01"
|
||||
def endDate = "2022-07-01"
|
||||
def newDate = "2022-08-01"
|
||||
def startDate = "2022-06-01"
|
||||
def date = "2022-07-01"
|
||||
def endDate = "2022-08-01"
|
||||
def newDate = "2022-09-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
|
||||
+16
-3
@@ -754,7 +754,7 @@ class CommonDatautils {
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","ÉÍû·¼²ÝÂ̰¶")
|
||||
// q.put("name","ºþÅϼÑÔ·")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -2049,6 +2049,20 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
public static def queryAllDistrictPrice(startDate){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put('weekend', startDate);
|
||||
DBCursor cur = DIST_STAT_COLL.find(queryCondition)
|
||||
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def rMap = [:];
|
||||
while(cur.hasNext()) {
|
||||
DBObject dbo = cur.next()
|
||||
rMap.put(dbo.get("city") + "##" + dbo.get("region")+"##"+ dbo.get("district_name"), dbo)
|
||||
}
|
||||
return rMap;
|
||||
}
|
||||
|
||||
|
||||
public static def queryDistrictPrices(city, startDate){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
@@ -3152,8 +3166,7 @@ class CommonDatautils {
|
||||
DBObject dbo;
|
||||
def prices = []
|
||||
def index = 0
|
||||
while(cur.hasNext()) {
|
||||
def obj = cur.next()
|
||||
for(def obj : cur) {
|
||||
if(index == 0){
|
||||
dbo = obj
|
||||
}
|
||||
|
||||
+159
@@ -0,0 +1,159 @@
|
||||
package com.fangjia.script.crawl
|
||||
|
||||
import com.alibaba.fastjson.JSONArray
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
class SchoolLocation {
|
||||
|
||||
DBCollection school_parse_coll = MongoUtils.getCollectionByName("fangjia_craw", "school", "192.168.0.235", 27018)
|
||||
|
||||
static main(args) {
|
||||
def task = new SchoolLocation()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def query = new BasicDBObject()
|
||||
query.put("province","江苏")
|
||||
//query.put("lat",null)
|
||||
query.put("lat",new BasicDBObject('$exists',false))
|
||||
def cursor = school_parse_coll.find(query)
|
||||
def index = 0
|
||||
for(def d : cursor){
|
||||
index++
|
||||
def datas = getSchoolLatAndLng(d.city, d.name, d.address, false)
|
||||
if(datas){
|
||||
for(def ret : datas){
|
||||
def addr = d.address
|
||||
if(d.address.contains(d.city+"市")){
|
||||
def cityStr = d.city+"市"
|
||||
addr = d.address.substring(d.address.indexOf(cityStr)+cityStr.size())
|
||||
}
|
||||
if(ret.name == d.name || (d.address.contains("号") && (ret.address.contains(addr) || addr.contains(ret.address)))){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("lat", ret.lat.toString().toDouble())
|
||||
udata.put("lng", ret.lng.toString().toDouble())
|
||||
udata.put("status", 0)
|
||||
school_parse_coll.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
|
||||
println index +" "+udata
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getSchoolLatAndLng(city, name, address, isFullMatch){
|
||||
if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){
|
||||
return null;
|
||||
}
|
||||
address = address.trim().replace(" ","").replace("?","").replace(" ","")
|
||||
String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=学校&city=" + city + "&output=json&offset=5&page=1&key=4e9592208336433ee1b7aa4d1d434418&extensions=all"
|
||||
|
||||
List<com.alibaba.fastjson.JSONObject> poiList = null;
|
||||
com.alibaba.fastjson.JSONObject result = null;
|
||||
try {
|
||||
HttpRequest request = new HttpRequest();
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
String response = request.get(url, params, "UTF-8");
|
||||
com.alibaba.fastjson.JSONObject jsonResponse = com.alibaba.fastjson.JSONObject.parseObject(response)
|
||||
if (jsonResponse != null) {
|
||||
JSONArray ja = jsonResponse.getJSONArray("pois");
|
||||
if(ja != null && ja.size() > 0) {
|
||||
poiList = new ArrayList<>();
|
||||
for(int i=0;i<ja.size();i++){
|
||||
com.alibaba.fastjson.JSONObject jo_item = ja.getJSONObject(i);
|
||||
|
||||
if(jo_item.containsKey("cityname") && !jo_item.getString("cityname").contains(city)){
|
||||
continue;
|
||||
}
|
||||
|
||||
String item_address = jo_item.containsKey("address") ? jo_item.getString("address") : null;
|
||||
String item_name = jo_item.containsKey("name") ? jo_item.getString("name") : null;
|
||||
|
||||
boolean getedData = false;
|
||||
if(isFullMatch == true){
|
||||
if(StringUtils.equalsIgnoreCase(item_address, address) || StringUtils.equalsIgnoreCase(item_name, address)){
|
||||
getedData = true;
|
||||
}
|
||||
}else{
|
||||
if(StringUtils.equalsIgnoreCase(item_address, address) || StringUtils.contains(item_address, address)
|
||||
|| StringUtils.equalsIgnoreCase(item_name, address) || StringUtils.contains(item_name, address) || StringUtils.contains(address, item_name)){
|
||||
getedData = true;
|
||||
}else{
|
||||
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(item_name, address);
|
||||
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(item_address, address);
|
||||
if(score1 || score2){
|
||||
getedData = true;
|
||||
}
|
||||
|
||||
if(!getedData){
|
||||
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(address, " ")
|
||||
def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(item_name, " ")
|
||||
if(py1 == py2){
|
||||
getedData = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def type = jo_item.getString("type");
|
||||
if(getedData == true){
|
||||
com.alibaba.fastjson.JSONObject jo = new com.alibaba.fastjson.JSONObject();
|
||||
if(jo_item.containsKey("id")){
|
||||
jo.put("id", jo_item.getString("id"))
|
||||
}
|
||||
if(jo_item.containsKey("name")){
|
||||
jo.put("name", jo_item.getString("name"))
|
||||
}
|
||||
if(jo_item.containsKey("type")){
|
||||
jo.put("type", type)
|
||||
}
|
||||
if(jo_item.containsKey("address")){
|
||||
jo.put("address", jo_item.getString("address").replace("[]",""))
|
||||
}
|
||||
if(jo_item.containsKey("pname")){
|
||||
jo.put("pname", jo_item.getString("pname"))
|
||||
}
|
||||
if(jo_item.containsKey("cityname")){
|
||||
jo.put("cityname", jo_item.getString("cityname"))
|
||||
}
|
||||
if(jo_item.containsKey("adname")){
|
||||
jo.put("adname", jo_item.getString("adname"))
|
||||
}
|
||||
|
||||
if(jo_item.containsKey("location")){
|
||||
def arrs = jo_item.getString("location").split(",")
|
||||
jo.put("lat", arrs[1])
|
||||
jo.put("lng", arrs[0])
|
||||
}
|
||||
|
||||
if(type && type.contains("出入口")){
|
||||
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+name+"-"+address+"】-${type}"
|
||||
}else{
|
||||
println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+name+"-"+address+"】"
|
||||
poiList.add(jo);
|
||||
}
|
||||
|
||||
}else{
|
||||
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println "X 高德地图 ( "+city+"-"+address+" )无结果!"
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
|
||||
return poiList;
|
||||
}
|
||||
}
|
||||
+56
-1
@@ -11,6 +11,10 @@ import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
@@ -26,7 +30,9 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection FZ_ADDR_COLL = MongoUtils.getCollectionByName2("business", "fz_district_address", lineUrl)
|
||||
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName2("business", "fz_district", lineUrl)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictInfoExportBatch(), args)
|
||||
@@ -87,7 +93,12 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
def startDate = DateUtils.getFirstDayOfMonth(lastDate);
|
||||
|
||||
def fzAddrMap = getFzAddress();
|
||||
for(def rMap : districts){
|
||||
|
||||
//因为福州个性化问题,又不想影响自家数据库数据,所以另存一个福州小区库,用于手动添加数据
|
||||
saveOrUpdateFzDist(districts)
|
||||
def fzDistricts = getFzDistricts(city)
|
||||
|
||||
for(def rMap : fzDistricts){
|
||||
//¼Û¸ñûÓÐÒ²Ìṩ
|
||||
// def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
|
||||
// if(dbo){
|
||||
@@ -139,6 +150,50 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
return result
|
||||
}
|
||||
|
||||
def saveOrUpdateFzDist(districts){
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def d = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def dbo = FZ_DIST_COLL.findOne(new BasicDBObject("_id", d.get('_id')))
|
||||
if(dbo){
|
||||
def udate = new BasicDBObject()
|
||||
d.each {k,v->
|
||||
udate.put(k,v)
|
||||
}
|
||||
udate.removeField("_id")
|
||||
FZ_DIST_COLL.update(new BasicDBObject('_id',d.get('_id')), new BasicDBObject('$set', udate));
|
||||
}else{
|
||||
FZ_DIST_COLL.save(d)
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
|
||||
}
|
||||
|
||||
def getFzDistricts(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
def rets = []
|
||||
FZ_DIST_COLL.find(q).each{it->
|
||||
rets << it
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def processAlias(alias, def addr_alias){
|
||||
def newAlias = []
|
||||
if(alias){
|
||||
|
||||
+23
-4
@@ -29,7 +29,9 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
def charset = "GBK"
|
||||
|
||||
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
|
||||
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com")
|
||||
|
||||
def specialDistsPriceMap = ["津泰新村7号、9-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580]
|
||||
static main(args) {
|
||||
runBatch(new DistrictPriceExportBatch(), args)
|
||||
}
|
||||
@@ -86,10 +88,21 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
return sb.toString()
|
||||
}
|
||||
|
||||
def getFzDistricts(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
def rets = []
|
||||
FZ_DIST_COLL.find(q).each{it->
|
||||
rets << it
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def createDatas(def city){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", null);
|
||||
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
|
||||
districts.addAll(removeDists)
|
||||
// def districts = CommonDatautils.findAllDistricts(city, "district", null);
|
||||
// def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
|
||||
// districts.addAll(removeDists)
|
||||
def districts = getFzDistricts(city)
|
||||
|
||||
def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
|
||||
def dataStr = DateUtils.add(endDateStr, -30)
|
||||
@@ -124,9 +137,15 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
curPrice = specialDistsPriceMap.get(rMap.name)
|
||||
if(curPrice){
|
||||
def randMarkup = nextDouble(0, 1)
|
||||
curPrice = (int)(curPrice*randMarkup/100 + curPrice)
|
||||
}
|
||||
}
|
||||
def m = []
|
||||
if(curPrice){
|
||||
if(curPrice && dbo){
|
||||
m << dbo.get("_id").toString()//¾ù¼ÛID
|
||||
}else{
|
||||
m << new ObjectId().toString()
|
||||
|
||||
+1
-1
@@ -136,7 +136,7 @@ class FzDataResultStat {
|
||||
|
||||
def regionPriceTrendRun(){
|
||||
def city = "¸£ÖÝ"
|
||||
def add = -6;
|
||||
def add = -9;
|
||||
def months = []
|
||||
while(add<0){
|
||||
months << DateUtils.addMonth(new Date(), add)
|
||||
|
||||
+2
-2
@@ -325,8 +325,8 @@ class HouseExportBatch extends BaseBatch{
|
||||
// int endDate = 20191201
|
||||
|
||||
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyyMMdd")
|
||||
// def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd");
|
||||
def endDate = "20220710"
|
||||
def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd");
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
// QueryBuilder room_range = QueryBuilders.rangeQuery("room").from(1).to(5).includeLower(true).includeUpper(false);
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202207.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202208.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+4
-4
@@ -48,11 +48,11 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2022-07-01"
|
||||
static def statDate = "2022-08-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser();//友达二手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2022年9月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
@@ -284,8 +284,8 @@ class YouDaDataParser {
|
||||
}
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2022年8月\\查询1.DBF"]
|
||||
def esfParser(path){
|
||||
def files = [path]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
+29
-2
@@ -1,4 +1,4 @@
|
||||
package com.fangjia.script.gj.test
|
||||
package com.fangjia.script.gj.commStat
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
@@ -28,11 +28,30 @@ class CityStat {
|
||||
def task = new CityStat()
|
||||
// task.run()
|
||||
// task.run1();
|
||||
task.run3();
|
||||
// task.run3();
|
||||
// task.run4()
|
||||
// task.run6()
|
||||
// task.run7()
|
||||
// task.run8()
|
||||
task.dealPriceStat()
|
||||
}
|
||||
|
||||
def dealPriceStat(){
|
||||
//def citys = ["上海","北京","广州","合肥","南京","深圳","武汉","郑州","重庆","福州","厦门","青岛","太原","长春","苏州","南宁","沈阳","天津","西安","长沙","大连","南昌","东莞","佛山","济南","南通","镇江","宁波","烟台","哈尔滨"]
|
||||
def citys = ["石家庄","贵阳"]
|
||||
def rets = []
|
||||
rets << ["城市","2022-4月总量"]
|
||||
def startDate = DateUtils.parseFromTightByPattern('2022-01-01','yyyy-MM-dd')
|
||||
def endDate = DateUtils.parseFromTightByPattern('2022-02-01','yyyy-MM-dd')
|
||||
for(def c : citys){
|
||||
def ret = []
|
||||
ret << c
|
||||
ret << findDealPriceCount(c, startDate, endDate)
|
||||
rets << ret
|
||||
println ret
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\2021成交数据.xlsx", rets, "月统计")
|
||||
}
|
||||
|
||||
// def run8(){
|
||||
@@ -464,6 +483,14 @@ class CityStat {
|
||||
return count;
|
||||
}
|
||||
|
||||
def findDealPriceCount(def city, def startDate, def endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt', endDate))
|
||||
def count = DEAL_PRICE_COLL.count(q)
|
||||
return count;
|
||||
}
|
||||
|
||||
def calTradeDistCount(def city, def tradeDate, def districts){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
+34
-14
@@ -1,4 +1,4 @@
|
||||
package com.fangjia.script.gj.test
|
||||
package com.fangjia.script.gj.commStat
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
@@ -29,24 +29,24 @@ DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "ave
|
||||
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43")
|
||||
|
||||
|
||||
def citys = ["乌鲁木齐","昌吉","阿克苏","伊犁","喀什"]
|
||||
//def citys = ["乌鲁木齐","昌吉","阿克苏","伊犁","喀什"]
|
||||
//for(def city : citys){
|
||||
//
|
||||
//}
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\江苏\\新疆数据统计.xlsx",{d->
|
||||
// def keys = getDealPrices(d['A'], DEAL_PRICE_COLL)
|
||||
// if(keys){
|
||||
// d['G'] = keys.size()
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\江苏\\新疆数据统计.xlsx",{d->
|
||||
//// def keys = getDealPrices(d['A'], DEAL_PRICE_COLL)
|
||||
//// if(keys){
|
||||
//// d['G'] = keys.size()
|
||||
//// }
|
||||
//
|
||||
//// def rets = findHousesByCity(d['A'], "sell", 20210701)
|
||||
// if(d['A']){
|
||||
// def count = findHouseCount(d['A'])
|
||||
// d['H'] = count
|
||||
// }
|
||||
|
||||
// def rets = findHousesByCity(d['A'], "sell", 20210701)
|
||||
if(d['A']){
|
||||
def count = findHouseCount(d['A'])
|
||||
d['H'] = count
|
||||
}
|
||||
|
||||
})
|
||||
//
|
||||
//})
|
||||
|
||||
|
||||
def findHouseCount(city){
|
||||
@@ -306,3 +306,23 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){
|
||||
// rows << [c, districts.size(), alias_count, addr_alias_count]
|
||||
//}
|
||||
//ExcelUtils.write("D:\\数据\\江苏\\新疆数据统计.xlsx", rows, "结果")
|
||||
|
||||
//区域级统计
|
||||
def city = "合肥"
|
||||
def regions = CommonDatautils.getCityRegions(city)
|
||||
def rows = []
|
||||
for(def region : regions){
|
||||
def districts = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias", "addr_alias"])
|
||||
def alias_count = 0
|
||||
def addr_alias_count = 0
|
||||
for(def r : districts){
|
||||
if(r.alias && r.alias.size()>0){
|
||||
alias_count ++
|
||||
}
|
||||
if(r.addr_alias && r.addr_alias.size()>0){
|
||||
addr_alias_count++
|
||||
}
|
||||
}
|
||||
rows << [region, districts.size(), alias_count, addr_alias_count]
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\新安银行\\新安银行数据统计.xlsx", rows, "结果")
|
||||
+1
@@ -110,6 +110,7 @@ class FangjiaClientUtils {
|
||||
public static def saveToEsPaimai(def data){
|
||||
String jsonStr = com.alibaba.fastjson.JSONObject.toJSONString(data)
|
||||
HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/paiMai", jsonStr)
|
||||
// HttpUtils.postForJson("http://localhost:8084/FangJiaDataService/paiMai", jsonStr)
|
||||
}
|
||||
|
||||
|
||||
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
package com.fangjia.script.gj.dongguanbank
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
|
||||
class DongGuanBankTest {
|
||||
static main(args) {
|
||||
def task = new DongGuanBankTest()
|
||||
def path = "D:\\数据\\东莞银行\\东莞银行房价数据测试20220916 v2.xlsx"
|
||||
// task.run(path)
|
||||
task.addPrice(path)
|
||||
}
|
||||
|
||||
def run(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
def city = d['A'].replace("市","")
|
||||
if(city){
|
||||
def region = RegionData.formatRegion(city, d['B'])
|
||||
def results = OpenApiClient.addressMatch(city, d['F'], "property", false, region, d['C'])
|
||||
if(results && results.precise>=0){
|
||||
d['H'] = results.city
|
||||
d['I'] = results.district
|
||||
d['J'] = results.name
|
||||
d['Q'] = results.propertytype
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def addPrice(path){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\东莞银行\\东莞银行房价数据测试20220916 v2.xlsx", { d ->
|
||||
if(d['H']){
|
||||
def dist = CommonDatautils.findDistrict(d['H'], d['I'], d['J'], "district")
|
||||
def priceObj = CommonDatautils.queryDistrictPrice(d['H'], d['I'], d['J'], DateUtils.parseFromTightByPattern("2022-09-11","yyyy-MM-dd"))
|
||||
if(priceObj){
|
||||
d['M'] = priceObj.avg_price
|
||||
d['N'] = (priceObj.avg_price * d['G']).toInteger()
|
||||
d['O'] = ((int)(priceObj.avg_price/100))*100
|
||||
if(dist){
|
||||
d['R'] = dist.estate_type2
|
||||
}
|
||||
if(priceObj.value_type){
|
||||
if(priceObj.value_type == 1){
|
||||
d['S'] = "指导价"
|
||||
}else if(priceObj.value_type == 2){
|
||||
d['S'] = "自身样本"
|
||||
}else if(priceObj.value_type == 3){
|
||||
d['S'] = "第三方"
|
||||
}else if(priceObj.value_type == 4){
|
||||
d['S'] = "交易数据"
|
||||
}else if(priceObj.value_type == 5){
|
||||
d['S'] = "周边楼盘"
|
||||
}else if(priceObj.value_type == 6){
|
||||
d['S'] = "推导"
|
||||
}else if(priceObj.value_type == 9){
|
||||
d['S'] = "神经网络"
|
||||
}else if(priceObj.value_type == 10){
|
||||
d['S'] = "无方式"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(!d['P']){
|
||||
def alias = []
|
||||
if(!dist.alias){
|
||||
alias = []
|
||||
}else{
|
||||
alias = dist.alias
|
||||
}
|
||||
alias << dist.name
|
||||
def map = findMaxAvgPrice(dist.city, dist.region, alias)
|
||||
if(map){
|
||||
d['P'] = map.max
|
||||
d['Q'] = map.min
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def findMaxAvgPrice(def city, def region, def alias){
|
||||
def set = null
|
||||
try{
|
||||
def aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= 20210101 and region='${region}' and district_name in ('${aliasStr}')"
|
||||
|
||||
set = new HashSet()
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
set << data.avg_price
|
||||
})
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
if(set && set.size()>=3){
|
||||
def map = [:]
|
||||
set = set.sort()
|
||||
map.min = set[0]
|
||||
map.max = set[set.size()-1]
|
||||
return map
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 转义字符串中的特殊字符
|
||||
* 仅过滤Lucene regexps, 其他的勿用
|
||||
* @param keyword
|
||||
* @return
|
||||
*/
|
||||
public static String escapeExprSpecialWord(String keyword) {
|
||||
return QueryParser.escape(keyword);
|
||||
}
|
||||
|
||||
def findMinPrice(city){
|
||||
def avg_price = null
|
||||
ElasticUtils.executeSqlQuery("select min(avg_price) as avg_price from seawater where p_date >= 20210901 and city='${city}'", {
|
||||
avg_price = it.avg_price
|
||||
})
|
||||
return avg_price
|
||||
}
|
||||
|
||||
}
|
||||
+1
-1
@@ -86,7 +86,7 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
city_avg_total_price = 531.2615913
|
||||
}
|
||||
if(city_avg_total_price){
|
||||
calcute(city, dists, code, city_avg_total_price)
|
||||
// calcute(city, dists, code, city_avg_total_price)
|
||||
calcuteIndex(city, dists, code, city_avg_total_price)
|
||||
|
||||
}
|
||||
|
||||
+23
-9
@@ -104,9 +104,23 @@ public class BeiKeCrawlByChrome {
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
for(def m : citys){
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
def detailUrls = getPageDeail(driver, m.city, m.url)
|
||||
println m.city +" ÏêÇéÁ´½Ó£º"+detailUrls.size()
|
||||
driver.get(m.url+"/chengjiao/")
|
||||
Thread.sleep(10000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
def regionText = r.text()
|
||||
def href = m.url+ r.attr("href")
|
||||
|
||||
println "开始抓取【${m.city}】【${regionText}】【${href}】"
|
||||
def detailUrls = getPageDeail(driver, m.city, href, regionText)
|
||||
println m.url +" 详情链接:"+detailUrls.size()
|
||||
}
|
||||
}
|
||||
|
||||
// def detailUrls = getPageDeail(driver, m.city, m.url)
|
||||
// println m.city +" 详情链接:"+detailUrls.size()
|
||||
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
|
||||
}
|
||||
}
|
||||
@@ -142,12 +156,12 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
|
||||
|
||||
def getPageDeail(driver, city, cityUrl){
|
||||
def getPageDeail(driver, city, cityUrl, region){
|
||||
def stop = false
|
||||
def page = 1
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "/chengjiao/pg${page}/"
|
||||
def url = cityUrl + "pg${page}/"
|
||||
try{
|
||||
driver.get(url)
|
||||
}catch(Exception e){
|
||||
@@ -164,7 +178,7 @@ public class BeiKeCrawlByChrome {
|
||||
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd")
|
||||
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
parse(city, e, region)
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
@@ -182,7 +196,7 @@ public class BeiKeCrawlByChrome {
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
def parse(city,e){
|
||||
def parse(city,e, region){
|
||||
try {
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
@@ -190,7 +204,7 @@ public class BeiKeCrawlByChrome {
|
||||
def arrs = title.split(" ")
|
||||
detail.city = city
|
||||
detail.district_name = arrs[0]
|
||||
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
|
||||
detail.region = region//findRegionByThirdPary(detail.city, detail.district_name)
|
||||
if(detail.region){
|
||||
detail.region = RegionData.formatRegion(city, detail.region)
|
||||
}
|
||||
@@ -381,7 +395,7 @@ public class BeiKeCrawlByChrome {
|
||||
cityMap.city = e.text()
|
||||
cityMap.url = "https:"+href
|
||||
list << cityMap
|
||||
//cityMap.put(e.text(), "https:"+href)
|
||||
// cityMap.put(e.text(), "https:"+href)
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
+43
-13
@@ -7,6 +7,8 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.commons.lang.math.NumberUtils
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
@@ -175,22 +177,23 @@ class ExportPaimai {
|
||||
|
||||
|
||||
def run(){
|
||||
def city = "苏州"
|
||||
def region = "姑苏"
|
||||
def city = "合肥"
|
||||
def region = null
|
||||
def datas = findPaimaiData(city,region)
|
||||
println "总量:"+datas.size()
|
||||
def rows = []
|
||||
def header = ["标地物地址","省","城市","区域","地址","标题提取地址","楼栋号","单元","室号","楼盘名称","建筑面积","竣工日期","房屋用途","总楼层",
|
||||
"拍卖状态","是否成交","拍卖阶段","拍卖方式","成交价","拍卖开始时间","拍卖结束时间","起拍价","保证金","加价幅度","评估价","市场价",
|
||||
"变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数","URL"]
|
||||
"变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数", "折价率","映射类型","拍卖类型",
|
||||
"映射资产类型","映射拍卖状态","URL"]
|
||||
rows << header
|
||||
// def set = new HashSet()
|
||||
def set = new HashSet()
|
||||
def index = 0
|
||||
for(def d : datas){
|
||||
// if(!set.contains(d.title)){
|
||||
if(!set.contains(d.title)){
|
||||
index++
|
||||
println index
|
||||
// set << d.title
|
||||
set << d.title
|
||||
def row = []
|
||||
row << d.title.replace(",",",")
|
||||
row << d.province
|
||||
@@ -226,14 +229,19 @@ class ExportPaimai {
|
||||
row << d.lookCount
|
||||
row << d.callCount
|
||||
row << d.delayCount
|
||||
row << d.url
|
||||
|
||||
row << d.rateDiscount//折价率
|
||||
row << d.mappingEstateType;//映射类型
|
||||
row << d.zcType;//拍卖类型:资产拍卖 司法拍卖
|
||||
row << d.mappingAgencyName//映射资产类型
|
||||
row << d.mappingStatus//映射拍卖状态
|
||||
row << d.url
|
||||
rows << row
|
||||
// }else{
|
||||
// println d.title
|
||||
// }
|
||||
}else{
|
||||
println d.title +" "+d.url
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\京东拍卖(${city}-姑苏-撤回中止).xlsx",rows,"数据")
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\合肥法拍.xlsx",rows,"数据")
|
||||
|
||||
// batchCreateFile(rows, "D:\\数据\\重庆农商行\\法拍\\淘宝拍卖(新).csv",null,"GBK")
|
||||
println "生成完成........"
|
||||
@@ -452,12 +460,12 @@ class ExportPaimai {
|
||||
if(region){
|
||||
pmquery.put("region",region)
|
||||
}
|
||||
pmquery.put("status", new BasicDBObject('$in', ["已撤回","已暂缓"]))
|
||||
// pmquery.put("status", new BasicDBObject('$in', ["已撤回","已暂缓"]))
|
||||
// pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
|
||||
pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-07-01")))
|
||||
|
||||
// Pattern pattern = Pattern.compile("rmfysszc.gov.cn", Pattern.MULTILINE);
|
||||
Pattern pattern = Pattern.compile("jd.com", Pattern.MULTILINE);
|
||||
// Pattern pattern = Pattern.compile("jd.com", Pattern.MULTILINE);
|
||||
// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
|
||||
// pmquery.put("url", new BasicDBObject('$regex', pattern))
|
||||
def dbcursor = paimai43_COLL.find(pmquery).batchSize(2000)
|
||||
@@ -465,6 +473,28 @@ class ExportPaimai {
|
||||
for(def db : dbcursor){
|
||||
rets << db
|
||||
}
|
||||
|
||||
// Collections.sort(rets, new Comparator<BasicDBObject>() {
|
||||
// @Override
|
||||
// int compare(BasicDBObject o1, BasicDBObject o2) {
|
||||
// if(o1.finishDate && o2.finishDate){
|
||||
// return -o1.finishDate.compareTo(o2.finishDate)
|
||||
// }else{
|
||||
// return 0
|
||||
// }
|
||||
//
|
||||
// }
|
||||
// })
|
||||
rets.sort {a,b->
|
||||
if(a.finishDate && b.finishDate){
|
||||
def v1 = a.finishDate
|
||||
def v2 = b.finishDate
|
||||
return v1.compareTo(v2)
|
||||
}else{
|
||||
return 0
|
||||
}
|
||||
}
|
||||
rets = rets.reverse()
|
||||
return rets
|
||||
}
|
||||
}
|
||||
|
||||
+78
-51
@@ -34,9 +34,9 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
static main(args) {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
// task.run()
|
||||
task.run()
|
||||
// task.run1()
|
||||
task.addDealRegion();
|
||||
// task.addDealRegion();
|
||||
task.standardRegion()
|
||||
// task.updadteDealPriceField()
|
||||
}
|
||||
@@ -129,7 +129,7 @@ public class LianjiaCrawlByChrome {
|
||||
def standardRegion(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")))
|
||||
q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) )
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
for(def d : dbcursor){
|
||||
@@ -152,8 +152,9 @@ public class LianjiaCrawlByChrome {
|
||||
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
lastMonth = lastMonth +"-01"
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city", "福州")
|
||||
q.put("region", null)
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-05-01","yyyy-MM-dd")))
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")))
|
||||
// q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def keys = []
|
||||
@@ -187,8 +188,19 @@ public class LianjiaCrawlByChrome {
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
cityMap.each{city, cityUrl->
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
def detailUrls = getPageDeail(driver, city, cityUrl)
|
||||
println cityUrl +" 详情链接:"+detailUrls.size()
|
||||
driver.get(cityUrl+"chengjiao/")
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
def regionText = r.text()
|
||||
def href = cityUrl.toString().substring(0, cityUrl.toString().length()-1) + r.attr("href")
|
||||
|
||||
println "开始抓取【${city}】【${regionText}】【${href}】"
|
||||
def detailUrls = getPageDeail(driver, city, href, regionText)
|
||||
println cityUrl +" 详情链接:"+detailUrls.size()
|
||||
}
|
||||
}
|
||||
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
|
||||
}
|
||||
}
|
||||
@@ -224,12 +236,12 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
|
||||
|
||||
def getPageDeail(driver, city, cityUrl){
|
||||
def getPageDeail(driver, city, cityUrl, regionText){
|
||||
def stop = false
|
||||
def page = 1
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "chengjiao/pg${page}/"
|
||||
def url = cityUrl + "pg${page}/"
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
@@ -238,10 +250,10 @@ public class LianjiaCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-06-20", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-07-20", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
parse(city, e, regionText)
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
@@ -259,7 +271,7 @@ public class LianjiaCrawlByChrome {
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
def parse(city,e){
|
||||
def parse(city,e, regionText){
|
||||
try {
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
@@ -267,7 +279,11 @@ public class LianjiaCrawlByChrome {
|
||||
def arrs = title.split(" ")
|
||||
detail.city = city
|
||||
detail.district_name = arrs[0]
|
||||
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
|
||||
if(regionText){
|
||||
detail.region = regionText
|
||||
}else{
|
||||
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
|
||||
}
|
||||
if(detail.region){
|
||||
detail.region = RegionData.formatRegion(city, detail.region)
|
||||
}
|
||||
@@ -275,50 +291,61 @@ public class LianjiaCrawlByChrome {
|
||||
def roomStr = arrs[1]
|
||||
if (roomStr) {
|
||||
if (roomStr.indexOf("室") > 0) {
|
||||
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
|
||||
def room = roomStr.substring(0, roomStr.indexOf("室"))
|
||||
if(room && room.isNumber()){
|
||||
detail.room = room.toInteger()
|
||||
}
|
||||
|
||||
}
|
||||
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger()
|
||||
def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
|
||||
if(hall && hall.isNumber()){
|
||||
detail.hall = hall.toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def areaStr = arrs[2].replace("平米", "")
|
||||
if (areaStr) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = price.toInteger() * 10000
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
detail.avg_price = (int) (detail.total_price / detail.area)
|
||||
}
|
||||
def tradeStr = e.select(".dealDate").text()
|
||||
if (tradeStr) {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "链家在线"
|
||||
detail.url = e.select(".title a").attr("href")
|
||||
def dbo = findTrade(detail)
|
||||
if (!dbo) {
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
def houseInfos = houseInfo.split("\\|")
|
||||
detail.direction = houseInfos[0]
|
||||
detail.fitment = houseInfos[1]
|
||||
def floorStr = e.select(".positionInfo").text()
|
||||
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
|
||||
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorStr && floorStr.isNumber()) {
|
||||
detail.height = floorStr.toInteger()
|
||||
if("车位"!=roomStr){
|
||||
if(arrs && arrs.size()>=3){
|
||||
def areaStr = arrs[2].replace("平米", "")
|
||||
if (areaStr) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
}
|
||||
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = price.toInteger() * 10000
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
detail.avg_price = (int) (detail.total_price / detail.area)
|
||||
}
|
||||
def tradeStr = e.select(".dealDate").text()
|
||||
if (tradeStr) {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "链家在线"
|
||||
detail.url = e.select(".title a").attr("href")
|
||||
def dbo = findTrade(detail)
|
||||
if (!dbo) {
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
def houseInfos = houseInfo.split("\\|")
|
||||
detail.direction = houseInfos[0]
|
||||
detail.fitment = houseInfos[1]
|
||||
def floorStr = e.select(".positionInfo").text()
|
||||
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
|
||||
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorStr && floorStr.isNumber()) {
|
||||
detail.height = floorStr.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
}
|
||||
}
|
||||
}catch(Exception ex){
|
||||
@@ -442,13 +469,13 @@ public class LianjiaCrawlByChrome {
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityMap = [:]
|
||||
def citys= ["福州"]
|
||||
def citys= ["苏州"]
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
// if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
cityMap.put(e.text(), href)
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
|
||||
+290
-80
@@ -10,6 +10,7 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.AMQP
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -29,18 +30,19 @@ class PaimaiDataSend {
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
static main(args) {
|
||||
def task = new PaimaiDataSend()
|
||||
// task.jdRunByMongo()
|
||||
// task.jdRun()
|
||||
// task.aliZcRun()
|
||||
task.aliZcRun2()
|
||||
//// task.aliZcRun()
|
||||
// task.aliZcRun2()
|
||||
// task.aliFpRun()
|
||||
// task.rmfysszcRun()
|
||||
// task.checkUrl()
|
||||
|
||||
// task.paimaiSendToEs()
|
||||
task.paimaiSendToEs()
|
||||
}
|
||||
|
||||
|
||||
@@ -64,7 +66,9 @@ class PaimaiDataSend {
|
||||
def rmfysszcRun(){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/76602.shtml")
|
||||
// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市", "苏州市"]))
|
||||
q.put("province","广东省")
|
||||
// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/88061.shtml")
|
||||
def dbcursor = rmfysszc_coll.find(q).batchSize(3000)
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
@@ -95,11 +99,22 @@ class PaimaiDataSend {
|
||||
}
|
||||
}
|
||||
|
||||
def jdRun(){
|
||||
def jdRunByMongo(){
|
||||
def pm_q = new BasicDBObject()
|
||||
pm_q.put("city","宿迁")
|
||||
pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
|
||||
pm_q.put("mappingStage", null)
|
||||
def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000)
|
||||
println dbcursor.count()
|
||||
for(def d : dbcursor){
|
||||
sendJd(d.url)
|
||||
}
|
||||
}
|
||||
|
||||
def sendJd(url){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","宿迁市")
|
||||
// q.put("url","https://paimai.jd.com/108245329")
|
||||
q.put("url", url)
|
||||
def dbcursor = jd_coll.find(q).batchSize(3000)
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
@@ -110,6 +125,7 @@ class PaimaiDataSend {
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", d.desc_content)
|
||||
result.put("region", d.region)
|
||||
|
||||
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
|
||||
result.put("bid_content", bid_content.toString())
|
||||
@@ -131,20 +147,96 @@ class PaimaiDataSend {
|
||||
jd_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
def jdRun(){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市","苏州市"]))
|
||||
q.put("province","广东")
|
||||
// q.put("parser_status", new BasicDBObject('$ne',3))
|
||||
// q.put("url","https://paimai.jd.com/115669796")
|
||||
def dbcursor = jd_coll.find(q).batchSize(3000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
println "jd:"+index
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", "paimai_jd")
|
||||
json.put("url", d.url)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", d.desc_content)
|
||||
result.put("region", d.region)
|
||||
|
||||
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
|
||||
result.put("bid_content", bid_content.toString())
|
||||
|
||||
result.put("sub_count", d.sub_count)
|
||||
def data_json = net.sf.json.JSONObject.fromObject(d.json)
|
||||
result.put("data_json", data_json)
|
||||
result.put("content_json", d.content)
|
||||
result.put("item_notice_content", d.item_notice_content)
|
||||
result.put("notice_detail_content", d.notice_detail_content.content)
|
||||
json.put("result", result)
|
||||
|
||||
rets << json
|
||||
if(rets.size()==10000){
|
||||
println "共:"+rets.size()
|
||||
sendJD(rets)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
|
||||
if(rets.size()>0){
|
||||
println "共:"+rets.size()
|
||||
sendJD(rets)
|
||||
}
|
||||
}
|
||||
|
||||
def sendJD(def rets){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def json = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("parser_status", 3)
|
||||
udata.put("m_date", new Date())
|
||||
jd_coll.update(new BasicDBObject("_id", json.get("url")), new BasicDBObject('$set', udata))
|
||||
println "send: "+totalCount.get()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
|
||||
def aliZcRun(){
|
||||
def pm_q = new BasicDBObject()
|
||||
pm_q.put("city","苏州")
|
||||
pm_q.put("region","姑苏")
|
||||
// pm_q.put("url","https://sf-item.taobao.com/sf_item/596573117068.htm")
|
||||
pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
|
||||
// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
|
||||
// pm_q.put("url", new BasicDBObject('$regex', pattern))
|
||||
// pm_q.put("status", null)
|
||||
pm_q.put("city",new BasicDBObject('$in', ["南京", "宿迁","苏州"]))
|
||||
// pm_q.put("city","苏州")
|
||||
// pm_q.put("region","姑苏")
|
||||
// pm_q.put("url","https://sf-item.taobao.com/sf_item/659117468505.htm")
|
||||
pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")).append('$lt', DateUtils.parseFromTightYYYYMMDD("2022-09-15")))
|
||||
Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
|
||||
pm_q.put("url", new BasicDBObject('$regex', pattern))
|
||||
// pm_q.put("mappingEstateType", null)
|
||||
// pm_q.put("area", new BasicDBObject('$lt', 30))
|
||||
def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000)
|
||||
println dbcursor.count()
|
||||
for(def d : dbcursor){
|
||||
def flag = sendAliZc(d.url)
|
||||
// def flag = sendAli(d.url)
|
||||
if(!flag){
|
||||
println "²»´æÔÚ:"+d.url
|
||||
}
|
||||
@@ -185,7 +277,6 @@ class PaimaiDataSend {
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("parser_status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
@@ -194,14 +285,60 @@ class PaimaiDataSend {
|
||||
return flag
|
||||
}
|
||||
|
||||
def sendAli(def url){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
q.put("url",url)
|
||||
def dbcursor = ali_fp_coll.find(q).batchSize(5000)
|
||||
def flag = false
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
flag = true
|
||||
println "ali-fp: "+index
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", "paimai_taobao")
|
||||
json.put("url", d.url)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", d.content)
|
||||
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
|
||||
result.put("trades", bid_content)
|
||||
result.put("province", d.province)
|
||||
result.put("city", d.city)
|
||||
result.put("region", d.region)
|
||||
result.put("sub_count", d.sub_count)
|
||||
def data_json = net.sf.json.JSONObject.fromObject(d.json)
|
||||
result.put("data_json", data_json)
|
||||
result.put("desc_content", d.desc_content)
|
||||
result.put("item_notice_content", d.item_notice_content)
|
||||
result.put("notice_detail_content", d.notice_detail_content)
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("parser_status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
ali_fp_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
return flag
|
||||
}
|
||||
|
||||
def aliFpRun(){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city","宿迁")
|
||||
q.put("url","https://sf-item.taobao.com/sf_item/649014193736.htm")
|
||||
// q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
|
||||
q.put("province","广东")
|
||||
// q.put("parser_status", new BasicDBObject('$ne',2))
|
||||
// q.put("url","https://sf-item.taobao.com/sf_item/631662596036.htm")
|
||||
//https://sf-item.taobao.com/sf_item/657131607486.htm
|
||||
// q.put("parser_status", new BasicDBObject('$exists',false))
|
||||
def dbcursor = ali_fp_coll.find(q).batchSize(2000)
|
||||
// q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01")))
|
||||
// q.put("area", new BasicDBObject('$lt', 30))
|
||||
def dbcursor = ali_fp_coll.find(q).batchSize(5000)
|
||||
|
||||
def rets = []
|
||||
|
||||
@@ -228,10 +365,20 @@ class PaimaiDataSend {
|
||||
json.put("result", result)
|
||||
|
||||
rets << json
|
||||
if(rets.size()==10000){
|
||||
println "共:"+rets.size()
|
||||
send(rets)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
|
||||
println "共:"+rets.size()
|
||||
if(rets.size()>0){
|
||||
println "共:"+rets.size()
|
||||
send(rets)
|
||||
}
|
||||
}
|
||||
|
||||
def send(def rets){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
for(def i=0;i<rets.size();i++){
|
||||
@@ -244,7 +391,7 @@ class PaimaiDataSend {
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("parser_status", 1)
|
||||
udata.put("parser_status", 2)
|
||||
udata.put("m_date", new Date())
|
||||
ali_fp_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
|
||||
println "send: "+totalCount.get()
|
||||
@@ -263,12 +410,19 @@ class PaimaiDataSend {
|
||||
def aliZcRun2(){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city","宿迁")
|
||||
// q.put("city",new BasicDBObject('$in', ["苏州","宿迁","南京"]))
|
||||
q.put("province","广东")
|
||||
// q.put("parser_status", new BasicDBObject('$ne',1))
|
||||
// q.put("region","¹ÃËÕ")
|
||||
q.put("url","https://zc-item.taobao.com/auction/672615219601.htm")
|
||||
def dbcursor = ali_zc_coll.find(q).batchSize(1000)
|
||||
// q.put("url","https://susong-item.taobao.com/auction/652481594083.htm")
|
||||
def dbcursor = ali_zc_coll.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
if(!d.url){
|
||||
println d._id.toString()
|
||||
continue
|
||||
}
|
||||
println "ali-zc: "+index
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", "paimai_taobao_zc")
|
||||
@@ -290,19 +444,52 @@ class PaimaiDataSend {
|
||||
result.put("region", d.region)
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("parser_status", 4)
|
||||
udata.put("m_date", new Date())
|
||||
ali_zc_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
|
||||
rets << json
|
||||
if(rets.size()==10000){
|
||||
println "共:"+rets.size()
|
||||
send(rets)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
|
||||
if(rets.size()>0){
|
||||
println "共:"+rets.size()
|
||||
send(rets)
|
||||
}
|
||||
|
||||
// println "共:"+rets.size()
|
||||
//
|
||||
// def totalCount = new AtomicLong()
|
||||
// final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
// for(def i=0;i<rets.size();i++){
|
||||
// final def json = rets[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
// try{
|
||||
// totalCount.addAndGet(1)
|
||||
// byte[] packedMsg = MessageUtils.pack(json)
|
||||
// CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
//
|
||||
//
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("parser_status", 1)
|
||||
// udata.put("m_date", new Date())
|
||||
// ali_zc_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
|
||||
// println "send: "+totalCount.get()
|
||||
// } catch (Exception e) {
|
||||
// e.printStackTrace()
|
||||
// } finally {
|
||||
// latch.countDown()
|
||||
// }
|
||||
// }
|
||||
// });
|
||||
// }
|
||||
// latch.await()
|
||||
}
|
||||
|
||||
def paimaiSendToEs(){
|
||||
def rets = getPaimaiDatas()
|
||||
println "共:"+rets.size()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
for(def i=0;i<rets.size();i++){
|
||||
@@ -312,7 +499,7 @@ class PaimaiDataSend {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def flag = FangjiaClientUtils.saveToEsPaimai(m)
|
||||
println totalCount.get()+" "+m.city +" "+m.name +" "+flag
|
||||
println totalCount.get()+" "+m.city +" "+m.title +" "+flag
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
@@ -326,56 +513,79 @@ class PaimaiDataSend {
|
||||
|
||||
def getPaimaiDatas(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
|
||||
def dbcursor = paimai43_COLL.find(q).batchSize(1000)
|
||||
// q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
|
||||
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-15")))
|
||||
def dbcursor = paimai43_COLL.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
def o = [:]
|
||||
o.addPrice = d.addPrice
|
||||
o.agencyName = d.agencyName
|
||||
o.area = d.area
|
||||
o.buildYear = d.buildYear
|
||||
o.callCount = d.callCount
|
||||
o.city = d.city
|
||||
o.court = d.court
|
||||
o.delayCount = d.delayCount
|
||||
o.delayWeekly = d.delayWeekly
|
||||
o.depositPrice = d.depositPrice
|
||||
o.enrollCount = d.enrollCount
|
||||
o.estateType = d.estateType
|
||||
o.evaluatePrice = d.evaluatePrice
|
||||
o.finishDate = d.finishDate
|
||||
o.height = d.height
|
||||
o.houseNum = d.houseNum
|
||||
o.id = d.id
|
||||
o.introduce = d.introduce
|
||||
if(d.city){
|
||||
def o = [:]
|
||||
o.addPrice = d.addPrice
|
||||
o.agencyName = d.agencyName
|
||||
o.area = d.area
|
||||
o.buildYear = d.buildYear
|
||||
o.callCount = d.callCount
|
||||
o.city = d.city
|
||||
o.court = d.court
|
||||
o.delayCount = d.delayCount
|
||||
o.delayWeekly = d.delayWeekly
|
||||
o.depositPrice = d.depositPrice
|
||||
o.enrollCount = d.enrollCount
|
||||
o.estateType = d.estateType
|
||||
o.evaluatePrice = d.evalatePrice
|
||||
o.finishDate = d.finishDate
|
||||
o.height = d.height
|
||||
o.houseNum = d.houseNum
|
||||
o.id = d.id
|
||||
o.introduce = d.introduce
|
||||
// o.location = d.location
|
||||
o.lookCount = d.lookCount
|
||||
o.marketPrice = d.marketPrice
|
||||
o.matchAddress = d.matchAddress
|
||||
o.matchCity = d.matchCity
|
||||
o.matchName = d.matchName
|
||||
o.matchRegion = d.matchRegion
|
||||
o.person = d.person
|
||||
o.preemptivePerson = d.preemptivePerson
|
||||
o.pricePkWeekly = d.pricePkWeekly
|
||||
o.province = d.province
|
||||
o.rateDiscount = d.rateDiscount
|
||||
o.region = d.region
|
||||
o.roomNum = d.roomNum
|
||||
o.sellOffAdvancePay = d.sellOffAdvancePay
|
||||
o.source = d.source
|
||||
o.stage = d.stage
|
||||
o.startDate = d.startDate
|
||||
o.startPrice = d.startPrice
|
||||
o.status = d.status
|
||||
o.tel = d.tel
|
||||
o.title = d.title
|
||||
o.tradePrice = d.tradePrice
|
||||
o.type = d.type
|
||||
o.unitNum = d.unitNum
|
||||
o.url = d.url
|
||||
rets << o
|
||||
o.lookCount = d.lookCount
|
||||
o.marketPrice = d.marketPrice
|
||||
o.matchAddress = d.matchAddress
|
||||
o.matchCity = d.matchCity
|
||||
o.matchName = d.matchName
|
||||
o.matchRegion = d.matchRegion
|
||||
o.person = d.person
|
||||
o.preemptivePerson = d.preemptivePerson
|
||||
o.pricePkWeekly = d.pricePkWeekly
|
||||
o.province = d.province
|
||||
o.rateDiscount = d.rateDiscount
|
||||
o.region = d.region
|
||||
o.roomNum = d.roomNum
|
||||
o.sellOffAdvancePay = d.sellOffAdvancePay
|
||||
o.source = d.source
|
||||
o.stage = d.stage
|
||||
o.startDate = d.startDate
|
||||
o.startPrice = d.startPrice
|
||||
o.status = d.status
|
||||
o.tel = d.tel
|
||||
o.title = d.title
|
||||
o.tradePrice = d.tradePrice
|
||||
o.tradeAvgPrice = d.tradeAvgPrice
|
||||
o.type = d.type
|
||||
o.unitNum = d.unitNum
|
||||
o.url = d.url
|
||||
if(d.url.contains("taobao.com")){
|
||||
o.source = "淘宝"
|
||||
}else if(d.url.contains("jd.com")){
|
||||
o.source = "京东"
|
||||
}else if(d.url.contains("rmfysszc.gov.cn")){
|
||||
o.source = "人民法院讼诉网"
|
||||
}
|
||||
|
||||
o.rateDiscount = d.rateDiscount//折价率
|
||||
o.mappingEstateType = d.mappingEstateType;//映射类型
|
||||
o.paiMaiType = d.paiMaiType;//拍卖类型:资产拍卖 司法拍卖
|
||||
o.mappingAgencyName = d.mappingAgencyName//映射资产类型
|
||||
o.mappingStatus = d.mappingStatus//映射拍卖状态
|
||||
o.mappingStage = d.mappingStage
|
||||
|
||||
if(d.startRateDiscount){
|
||||
o.startRateDiscount = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, d.startRateDiscount);//起拍折价率 起拍折价率 = 起拍价 / 市场价或评估价
|
||||
}
|
||||
o.diffPrice = d.diffPrice;//差价 差价 = 成交价-市场价或评估价
|
||||
rets << o
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.Mongo
|
||||
@@ -15,10 +16,154 @@ import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.regex.Pattern
|
||||
|
||||
|
||||
def dist = CommonDatautils.findDistrictById("62ce233a3705890001dfa164")
|
||||
DBUtils.seaweed.save(dist)
|
||||
DBCollection coll = MongoUtils.getCollectionByName("seeds_db", "city_seeds", "192.168.0.235", 27018)
|
||||
|
||||
def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", ZS_MONGODB_URL)
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx",{d->
|
||||
// def city = RegionData.formatCity(d['E'])
|
||||
// if(city && !d['J']){
|
||||
// def region = RegionData.formatRegion(city, d['G'])
|
||||
// d['I'] = city
|
||||
// d['J'] = region
|
||||
// }
|
||||
//})
|
||||
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx",{d->
|
||||
// def city = RegionData.formatCity(d['D'])
|
||||
// if(!city){
|
||||
// city = RegionData.formatCity(d['F'])
|
||||
// }
|
||||
// if(city && !d['H']){
|
||||
// def region = RegionData.formatRegion(city, d['B'])
|
||||
// d['G'] = city
|
||||
// d['H'] = region
|
||||
// }
|
||||
//})
|
||||
|
||||
//def citys = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
|
||||
//def datas = ExcelUtils.read("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx")
|
||||
|
||||
//def fjwCitys = []
|
||||
//for(def d : datas){
|
||||
// fjwCitys << d['G']
|
||||
//}
|
||||
//def rows = []
|
||||
//for(def d : datas){
|
||||
// if(citys.contains(d['G'])){
|
||||
// rows << d
|
||||
// }
|
||||
//}
|
||||
//ExcelUtils.write("D:\\数据\\建行\\二期\\统计局行政区划与房价网区域比对V2.xlsx", rows, "区域")
|
||||
|
||||
//for(def c : citys){
|
||||
// if(!fjwCitys.contains(c)){
|
||||
// println c
|
||||
// }
|
||||
//}
|
||||
|
||||
//
|
||||
//ExcelUtils.write("D:\\数据\\建行\\二期\\(特殊城市)统计局行政区划与房价网区域比对.xlsx", rows, "区域")
|
||||
//
|
||||
|
||||
//def map = [:]
|
||||
//def datas1 = ExcelUtils.read("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx")
|
||||
//for(def row : datas1){
|
||||
// if(row['J']){
|
||||
// def codeMap = [:]
|
||||
// codeMap.prov = row['B']
|
||||
// codeMap.city = row['F']
|
||||
// codeMap.region = row['H']
|
||||
// map.put(row['I']+"#"+row['J'], codeMap)
|
||||
// }
|
||||
//}
|
||||
//def datas2 = ExcelUtils.read("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx")
|
||||
//for(def row : datas2){
|
||||
// if(row['H']){
|
||||
// def codeMap = [:]
|
||||
// codeMap.prov = row['E']
|
||||
// codeMap.city = row['C']
|
||||
// codeMap.region = row['A']
|
||||
// map.put(row['G']+"#"+row['H'], codeMap)
|
||||
// }
|
||||
//}
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d->
|
||||
// def key = d['B'] +"#"+d['C']
|
||||
// def codeMap = map.get(key)
|
||||
// if(codeMap && !d['D']){
|
||||
// d['D'] = codeMap.prov
|
||||
// d['E'] = codeMap.city
|
||||
// d['F'] = codeMap.region
|
||||
// }
|
||||
//})
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d->
|
||||
if(!d['D']){
|
||||
def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN)
|
||||
if(infos && infos.size()==1){
|
||||
def info = infos[0]
|
||||
d['D'] = info.province_code
|
||||
d['E'] = info.area_code
|
||||
d['F'] = info.street_code
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
def findTownshipInfo(city, region, coll){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE))
|
||||
q.put("street_name", Pattern.compile('^.*'+region+'街道.*$', Pattern.CASE_INSENSITIVE))
|
||||
|
||||
def dbcursor = coll.find(q)
|
||||
def rows = []
|
||||
for(def d : dbcursor){
|
||||
// if(d.street_name.contains(region)){
|
||||
rows << d
|
||||
// }
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
|
||||
|
||||
//def cityMap = CommonDatautils.findAllCityProvince()
|
||||
//def rows = []
|
||||
//rows << ["省", "市", "区域"]
|
||||
//cityMap.each{c,p->
|
||||
// def regions = CommonDatautils.getCityRegions(c)
|
||||
// for(def r : regions){
|
||||
// def row = []
|
||||
// row << p
|
||||
// row << c
|
||||
// row << r
|
||||
// rows << row
|
||||
// }
|
||||
//}
|
||||
//ExcelUtils.write("D:\\数据\\亲邻\\房价网省市区.xlsx", rows, "区域")
|
||||
|
||||
|
||||
|
||||
|
||||
//def q = new BasicDBObject()
|
||||
//q.put("root", "anjuke.com")
|
||||
//def dbcursor = coll.find(q)
|
||||
//def map = [:]
|
||||
//for(def d : dbcursor){
|
||||
// map.put(d.city, d.city_short)
|
||||
//}
|
||||
//println map.size()
|
||||
//println map
|
||||
|
||||
|
||||
//def dist = CommonDatautils.findDistrictById("62ce233a3705890001dfa164")
|
||||
//DBUtils.seaweed.save(dist)
|
||||
//DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId("62ce233a3705890001dfa164")), new BasicDBObject('$set', dist))
|
||||
//SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
||||
//Date date = new Date(1617211269868);
|
||||
|
||||
+142
@@ -0,0 +1,142 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.alibaba.fastjson.JSON
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.temp.shenjian.util.MessageUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
DBCollection school_coll = MongoUtils.getCollectionByName("crawl_tmp", "school_ruyile_web", "192.168.0.235", 27018)
|
||||
|
||||
DBCollection sxue_school_coll = MongoUtils.getCollectionByName("crawl_tmp", "school_5sxue_web", "192.168.0.235", 27018)
|
||||
|
||||
DBCollection school_parse_coll = MongoUtils.getCollectionByName("fangjia_craw", "school", "192.168.0.235", 27018)
|
||||
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
//Thread.sleep(60*60*1000)
|
||||
//while (true){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("province",new BasicDBObject('$in', ["江苏","江苏省"]))
|
||||
q.put("status",new BasicDBObject('$ne', 1))
|
||||
def dbcursor = school_coll.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", d.project)
|
||||
json.put("url", d.url)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", d.content)
|
||||
def jsonStr = d.map_info
|
||||
|
||||
jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1)
|
||||
if(jsonStr.contains("\n")){
|
||||
jsonStr = jsonStr.replaceAll('\\n','')
|
||||
}
|
||||
result.put("data_json", net.sf.json.JSONObject.fromObject(jsonStr))
|
||||
|
||||
|
||||
result.put("type", d.type)
|
||||
json.put("result", result)
|
||||
|
||||
|
||||
rets << json
|
||||
if(rets.size()==5000){
|
||||
println "共:"+rets.size()
|
||||
send(rets, fixedThreadPool, school_coll)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
|
||||
if(rets.size()>0){
|
||||
println "共:"+rets.size()
|
||||
send(rets, fixedThreadPool, school_coll)
|
||||
rets.clear()
|
||||
}
|
||||
//}
|
||||
|
||||
|
||||
|
||||
def send(def rets, def fixedThreadPool, def school_coll){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def json = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
school_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
|
||||
println "send: "+totalCount.get()
|
||||
} catch (Exception e) {
|
||||
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
//def rets = []
|
||||
////def query = new BasicDBObject()
|
||||
////query.put("source","搜学网")
|
||||
//////query.put("url","http://xuexiao.51sxue.com/detail/id_26799.html")
|
||||
//////query.put("lat",null)
|
||||
////def cursor = school_parse_coll.find(query)
|
||||
////for(def o : cursor){
|
||||
// def q = new BasicDBObject()
|
||||
//// q.put("province","江苏省")
|
||||
//// q.put("url",o.url)
|
||||
// def dbcursor = sxue_school_coll.find(q)
|
||||
// def index = 0
|
||||
// for(def d : dbcursor){
|
||||
// index++
|
||||
// println index
|
||||
// JSON json = new JSONObject()
|
||||
// json.put("project", d.project)
|
||||
// json.put("url", d.url)
|
||||
//
|
||||
// JSON result = new JSONObject()
|
||||
// result.put("content", d.content)
|
||||
// result.put("name", d.name)
|
||||
// result.put("list_item", d.list_item)
|
||||
//
|
||||
// def jsonStr = d.map_info
|
||||
// jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1)
|
||||
// if(jsonStr.contains("\n")){
|
||||
// jsonStr = jsonStr.replaceAll('\\n','')
|
||||
// }
|
||||
// result.put("json_data", jsonStr)
|
||||
//
|
||||
// result.put("type", d.type)
|
||||
// json.put("result", result)
|
||||
//
|
||||
// rets << json
|
||||
// if(rets.size()==5000){
|
||||
// println "共:"+rets.size()
|
||||
// send(rets, fixedThreadPool, school_coll)
|
||||
// rets.clear()
|
||||
// }
|
||||
// }
|
||||
////}
|
||||
//if(rets.size()>0){
|
||||
// println "共:"+rets.size()
|
||||
// send(rets, fixedThreadPool, school_coll)
|
||||
// rets.clear()
|
||||
//}
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
package com.fangjia.script.temp.shenjian.poi
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.temp.shenjian.util.EsUtils
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
/**
|
||||
* ²¹³äѧУ¾Î³¶È
|
||||
* @author shenjian
|
||||
* @since 2022/9/8
|
||||
*/
|
||||
class FillLngLat {
|
||||
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(4)
|
||||
static void main(String[] args) {
|
||||
ElasticUtils.executeSqlQuery("select * from dws_poi where location is not null", { dataT ->
|
||||
final def data = dataT
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
println(data)
|
||||
def lngLat = GaoDeUtils.parserByGaode(data.city, data.region + data.name + data.address)
|
||||
if (lngLat != null && lngLat.size() == 2 ) {
|
||||
JSONObject jsonObject = new JSONObject()
|
||||
jsonObject.put("lon", Double.valueOf(lngLat['lng']))
|
||||
jsonObject.put("lat", Double.valueOf(lngLat['lat']))
|
||||
EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id))
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
})
|
||||
executorService.shutdown()
|
||||
}
|
||||
}
|
||||
+31
-22
@@ -10,6 +10,7 @@ import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBList
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
@@ -27,15 +28,15 @@ import java.util.concurrent.Executors
|
||||
@BatchOwner(Watcher.ShenJian)
|
||||
class SeaweedAroundPoi extends BaseBatch {
|
||||
|
||||
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
|
||||
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.235", 27018)
|
||||
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi"
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(30)
|
||||
//static ExecutorService executorService = Executors.newFixedThreadPool(2)
|
||||
|
||||
static void main(String[] args) {
|
||||
def types = listAllType()
|
||||
//dealSpecialSeaweedAround(types)
|
||||
dealPoi(new BasicDBObject(), types)
|
||||
executorService.shutdown()
|
||||
//executorService.shutdown()
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -64,33 +65,40 @@ class SeaweedAroundPoi extends BaseBatch {
|
||||
}
|
||||
|
||||
static dealPoi(DBObject query, def types) {
|
||||
query.put("status", 1)
|
||||
conn136.find(query).batchSize(5000).each {objTemp ->
|
||||
query.put("canyin_status", 0)
|
||||
conn136.find(query).batchSize(5000).limit(20000).noCursorTimeout(true).each {objTemp ->
|
||||
final def obj = objTemp
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
// executorService.submit(new Runnable() {
|
||||
// @Override
|
||||
// void run() {
|
||||
def city = obj.city
|
||||
def lng = obj.lng
|
||||
def lat = obj.lat
|
||||
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
|
||||
if (CollectionUtils.isNotEmpty(pois)) {
|
||||
String jsonStr = JSONObject.toJSONString(pois)
|
||||
HttpUtils.postForJson(URL, jsonStr)
|
||||
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
|
||||
} else {
|
||||
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空")))
|
||||
if (obj.lng) {
|
||||
//def lng = SeaweedUtils.formatLatLng(obj.lng)
|
||||
//def lat = SeaweedUtils.formatLatLng(obj.lat)
|
||||
// {'lon': 121.810877, 'lat': 31.737642}
|
||||
def lng = 121.810877
|
||||
def lat = 31.737642
|
||||
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
|
||||
if (CollectionUtils.isNotEmpty(pois)) {
|
||||
String jsonStr = JSONObject.toJSONString(pois)
|
||||
HttpUtils.postForJson(URL, jsonStr)
|
||||
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("canyin_status", 0).append("flag", "已获取周边POI")))
|
||||
} else {
|
||||
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("canyin_status", 2).append("flag", "周边POI为空")))
|
||||
}
|
||||
println("已处理小区:" + obj)
|
||||
}
|
||||
println("已处理小区:" + obj)
|
||||
}
|
||||
})
|
||||
}
|
||||
// })
|
||||
// }
|
||||
}
|
||||
|
||||
static getNewSeaweed() {
|
||||
DBObject query = new BasicDBObject()
|
||||
query.put("cat", "district")
|
||||
query.put("visible", 0)
|
||||
//query.put("city", new BasicDBObject('$in', ["连云港", "南通", "苏州", "无锡", "盐城", "扬州"]))
|
||||
//query.put("city", "连云港")
|
||||
//query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a"))
|
||||
|
||||
DBObject filter = new BasicDBObject()
|
||||
@@ -163,17 +171,18 @@ class SeaweedAroundPoi extends BaseBatch {
|
||||
|
||||
public static final Map<String, String> KEYWORDS_MAP = new HashMap<String, String>() {
|
||||
{
|
||||
put("餐饮", "05");
|
||||
// put("餐饮", "05");
|
||||
// put("地铁", "150500");
|
||||
// put("公交", "1507");
|
||||
// put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211");
|
||||
// put("写字楼", "120201");
|
||||
// put("学校", "141200|141201|141202|141203|141204");
|
||||
// put("银行", "1601|1603");
|
||||
// put("大型超市", "0604");
|
||||
put("大型超市", "0604");
|
||||
// put("商场", "0601");
|
||||
// put("餐饮", "0501|0502|0503|0504");
|
||||
// put("旅游景点", "110000");
|
||||
put("公园广场", "110100|110101|110102|110103|110104|110105")
|
||||
// put("图书馆", "140500");
|
||||
// put("政府机关", "130100|130101|130102|130103|130104|130105|130106|130107");
|
||||
// put("丧葬设施", "071900|071901|071902|071903")
|
||||
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
package com.fangjia.script.temp.shenjian.poi
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.statistics.poi.AmapPoi
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBList
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.collections.CollectionUtils
|
||||
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
/**
|
||||
* 小区周边POI数据获取
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/5/24
|
||||
*/
|
||||
class SeaweedAroundPoiFromExcel {
|
||||
|
||||
static void main(String[] args) {
|
||||
def types = listAllType()
|
||||
dealPoi(new BasicDBObject(), types)
|
||||
}
|
||||
|
||||
static dealPoi(DBObject query, def types) {
|
||||
com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\建行\\审核小区\\东莞银行房价数据测试20220916-v3.xlsx", { List item ->
|
||||
def city = item['H']
|
||||
def lng = item['AQ']
|
||||
def lat = item['AR']
|
||||
if (lng) {
|
||||
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
|
||||
if (CollectionUtils.isNotEmpty(pois)) {
|
||||
String jsonStr = JSONObject.toJSONString(pois)
|
||||
HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi", jsonStr)
|
||||
}
|
||||
Thread.sleep(1000)
|
||||
print(lng + " " + lat)
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
|
||||
public static def listAllType() {
|
||||
StringBuilder ret = new StringBuilder()
|
||||
for (def val : KEYWORDS_MAP.values()) {
|
||||
ret.append(val).append("|")
|
||||
}
|
||||
String result = ret.toString()
|
||||
return result.substring(0, result.length() - 1)
|
||||
};
|
||||
|
||||
public static final Map<String, String> KEYWORDS_MAP = new HashMap<String, String>() {
|
||||
{
|
||||
// put("餐饮", "05");
|
||||
// put("地铁", "150500");
|
||||
// put("公交", "1507");
|
||||
// put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211");
|
||||
// put("写字楼", "120201");
|
||||
// put("学校", "141200|141201|141202|141203|141204");
|
||||
// put("银行", "1601|1603");
|
||||
put("大型超市", "0604");
|
||||
// put("商场", "0601");
|
||||
// put("餐饮", "0501|0502|0503|0504");
|
||||
// put("旅游景点", "110000");
|
||||
put("公园广场", "110100|110101|110102|110103|110104|110105")
|
||||
// put("图书馆", "140500");
|
||||
// put("政府机关", "130100|130101|130102|130103|130104|130105|130106|130107");
|
||||
// put("丧葬设施", "071900|071901|071902|071903")
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
+9
-5
@@ -146,11 +146,15 @@ class GaoDeUtils {
|
||||
while (result.size() < count) {
|
||||
params.page = "${page}".toString()
|
||||
page++
|
||||
params.key = this.getGaodeKey("v5/place/around")
|
||||
JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params);
|
||||
//params.key = this.getGaodeKey("v3/place/around")
|
||||
params.key = "25b2d459fd133e22f8abcebeac972692"
|
||||
params.offset = "100"
|
||||
JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params);
|
||||
if (json == null) {
|
||||
params.key = this.getGaodeKey("v5/place/around")
|
||||
json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params);
|
||||
//params.key = this.getGaodeKey("v3/place/around")
|
||||
params.key = "25b2d459fd133e22f8abcebeac972692"
|
||||
params.offset = "100"
|
||||
json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params);
|
||||
}
|
||||
if (json && json.info && json.info == "OK") {
|
||||
count = json.getString("count").toInteger()
|
||||
@@ -258,7 +262,7 @@ class GaoDeUtils {
|
||||
}
|
||||
if (json && json.info && json.info == "OK") {
|
||||
try {
|
||||
List<AmapPoiDetail> ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class)
|
||||
List<AmapPoiDetail> ret = JSONObject.parseArray(json.pois.toString(), AmapPoiDetail.class)
|
||||
return ret
|
||||
} catch(Exception ex) {
|
||||
ex.printStackTrace()
|
||||
|
||||
+4
-4
@@ -40,11 +40,11 @@ class DataStat {
|
||||
|
||||
|
||||
def run(){
|
||||
def source = "365ÌÔ·¿"//
|
||||
def citys = findCrawlCity(source, 202206);
|
||||
def source = "我爱我家"//
|
||||
def citys = findCrawlCity(source, 202207);
|
||||
for(def city : citys){
|
||||
def count1 = findCount(source, city, 202206);
|
||||
def count2 = findCount(source, city, 202207);
|
||||
def count1 = findCount(source, city, 202207);
|
||||
def count2 = findCount(source, city, 202208);
|
||||
println city +" "+count1+" "+count2
|
||||
// println count1
|
||||
// println "--- ----"
|
||||
|
||||
+7
-10
@@ -16,15 +16,12 @@ class SynchDistrictPrice {
|
||||
def static final SHORT_DATE = Utils.getLastMonth()
|
||||
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
//"fang.com":"搜房",
|
||||
// def sourceMap = ["58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
|
||||
//"fang.com":"搜房",
|
||||
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
|
||||
|
||||
//,"城市房产":"城市房产"
|
||||
//"365淘房":"365淘房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房",
|
||||
def esSources = ["C21":"C21","365淘房":"365淘房"]//"吉屋":"吉屋","365淘房":"365淘房"
|
||||
//"365淘房":"365淘房","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","吉屋":"吉屋"
|
||||
def esSources = ["搜房":"搜房","我爱我家":"我爱我家","365淘房":"365淘房"]
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
// task.run()
|
||||
@@ -32,13 +29,13 @@ class SynchDistrictPrice {
|
||||
}
|
||||
|
||||
def run1(){
|
||||
|
||||
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
|
||||
def citys = CommonUtils.findAllCity()
|
||||
// citys.remove("上海")
|
||||
// citys.remove("广州")
|
||||
|
||||
esSources.each{esSource, mgSource->
|
||||
for(def city : citys){
|
||||
def rets = findAllEsData(city, esSource, mgSource)
|
||||
def rets = findAllEsData(city, esSource, mgSource, mdate)
|
||||
println city +" "+rets.size()
|
||||
synchData(city, rets, mgSource)
|
||||
}
|
||||
@@ -46,10 +43,10 @@ class SynchDistrictPrice {
|
||||
}
|
||||
|
||||
|
||||
def findAllEsData(city,esSource, mgSource){
|
||||
def findAllEsData(city,esSource, mgSource, mdate){
|
||||
def names = new HashSet()
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='2022-07-01' order by m_date", {d->
|
||||
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='${mdate}' order by m_date", {d->
|
||||
def s = d.city +"#"+d.region+"#"+d.name
|
||||
if(!names.contains(s)){
|
||||
names << s
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202206);
|
||||
def count2 = findTotalCount(source, 202207);
|
||||
def count1 = findTotalCount(source, 202207);
|
||||
def count2 = findTotalCount(source, 202208);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
+1
@@ -44,6 +44,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
||||
void process(Object args) {
|
||||
println SHORT_DATE
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
||||
//def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD("2021-07-15"));
|
||||
def cityIndex = 0;
|
||||
|
||||
+16
-1
@@ -50,6 +50,21 @@ public class AmapPoiDetail {
|
||||
private BizExtDTO bizExt;
|
||||
@com.alibaba.fastjson.annotation.JSONField(name = "photos")
|
||||
private List<PhotosDTO> photos;
|
||||
@com.alibaba.fastjson.annotation.JSONField(name = "business")
|
||||
private BusinessDTO business;
|
||||
|
||||
@lombok.NoArgsConstructor
|
||||
@lombok.Data
|
||||
public static class BusinessDTO {
|
||||
@com.alibaba.fastjson.annotation.JSONField(name = "business_area")
|
||||
private String businessArea;
|
||||
@com.alibaba.fastjson.annotation.JSONField(name = "keytag")
|
||||
private String keytag;
|
||||
@com.alibaba.fastjson.annotation.JSONField(name = "opentime_today")
|
||||
private String opentimeToday;
|
||||
@com.alibaba.fastjson.annotation.JSONField(name = "opentime_week")
|
||||
private String opentimeWeek;
|
||||
}
|
||||
|
||||
@lombok.NoArgsConstructor
|
||||
@lombok.Data
|
||||
@@ -64,7 +79,7 @@ public class AmapPoiDetail {
|
||||
@lombok.Data
|
||||
public static class PhotosDTO {
|
||||
@com.alibaba.fastjson.annotation.JSONField(name = "title")
|
||||
private List<?> title;
|
||||
private String title;
|
||||
@com.alibaba.fastjson.annotation.JSONField(name = "url")
|
||||
private String url;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user