Merge branch 'script_gj' into 'master'
Script gj See merge request !379
This commit is contained in:
@@ -257,4 +257,9 @@ public class GeoUtils {
|
||||
ret += (150.0 * Math.sin(x / 12.0 * pi) + 300.0 * Math.sin(x / 30.0 * pi)) * 2.0 / 3.0;
|
||||
return ret;
|
||||
}
|
||||
|
||||
public static void main(String[] args){
|
||||
Map<String, Double> location = gps84ToGcj02(31.218024629574998,121.36562884455);
|
||||
System.out.println(location.toString());
|
||||
}
|
||||
}
|
||||
|
||||
+9
-1
@@ -77,8 +77,16 @@ class CcbCommonUtils {
|
||||
q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate)));
|
||||
def dbcursor = GUIDING_PRICE_COLL.find(q)
|
||||
def rets = []
|
||||
def sets = new HashSet()
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
def guiding_date = DateUtils.formatDateByPattern(d.guiding_date, 'yyyy-MM-dd')
|
||||
def key = d.city +"##"+d.region+"##"+d.name+"##" + guiding_date
|
||||
if(!sets.contains(key)){
|
||||
rets << d
|
||||
sets << key
|
||||
}else{
|
||||
println key
|
||||
}
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
+7
-6
@@ -370,24 +370,25 @@ class DealPriceDataProcess {
|
||||
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
def citys = CcbCommonUtils.citys
|
||||
def citys = ["上海","广州","东莞"]//CcbCommonUtils.citys
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def publishTime = '2021-10-01'
|
||||
def month = 202206
|
||||
def month = 202207
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty(city, 202207)
|
||||
def rets = findPeGteThirdParty(city, 202208)
|
||||
def total = 0
|
||||
println city +" "+rets.size()
|
||||
for(def m : rets){
|
||||
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
|
||||
def cityId = cityCodeDatas.get(city)
|
||||
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
|
||||
sqlInstance236.executeUpdate(sql)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findPeGteThirdParty(cityName, yearmonth){
|
||||
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>30"
|
||||
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>25"
|
||||
def rets = []
|
||||
sqlInstance236.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
|
||||
+193
@@ -0,0 +1,193 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import groovy.sql.Sql
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class FjwPricePkCcbPrice {
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new FjwPricePkCcbPrice()
|
||||
// task.run()
|
||||
task.updatePrice()
|
||||
}
|
||||
|
||||
def updatePrice(){
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\测试\\ccb.xlsx",{d->
|
||||
index++
|
||||
println index
|
||||
def dbo = CommonDatautils.queryDistrictPrice(d['B'], d['C'], d['D'], DateUtils.parseFromTightByPattern('2022-09-04','yyyy-MM-dd'))
|
||||
if(dbo){
|
||||
d['G'] = dbo.future_value
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def run(){
|
||||
def ccbPrices = getCommunityBasePriceGj(202208)
|
||||
println "ccb总量:"+ccbPrices.size()
|
||||
def fjwPriceMap = CommonDatautils.queryAllDistrictPrice(DateUtils.parseFromTightByPattern('2022-09-04','yyyy-MM-dd'))
|
||||
println "fjw总量:"+ccbPrices.size()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(ccbPrices.size())
|
||||
|
||||
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
for(def i=0;i<ccbPrices.size();i++){
|
||||
final def rMap = ccbPrices[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def row = []
|
||||
def dist = CommonDatautils.findDistrictById(rMap.commid)
|
||||
|
||||
if(dist){
|
||||
def priceMap = fjwPriceMap.get(dist.city+"##"+dist.region+"##"+dist.name)
|
||||
def fjwPrice = null
|
||||
def fjwCalPrice = null
|
||||
if(priceMap){
|
||||
fjwPrice = priceMap.avg_price
|
||||
fjwCalPrice = priceMap.calc_value
|
||||
}
|
||||
row << rMap.commid
|
||||
row << dist.city
|
||||
row << dist.region
|
||||
row << dist.name
|
||||
row << fjwPrice
|
||||
row << fjwCalPrice
|
||||
if(priceMap && priceMap.value_type){
|
||||
if(priceMap.value_type == 1){
|
||||
row << "指导价"
|
||||
}else if(priceMap.value_type == 2){
|
||||
row << "自身样本"
|
||||
}else if(priceMap.value_type == 3){
|
||||
row << "第三方"
|
||||
}else if(priceMap.value_type == 4){
|
||||
row << "交易数据"
|
||||
}else if(priceMap.value_type == 5){
|
||||
row << "周边楼盘"
|
||||
}else if(priceMap.value_type == 6){
|
||||
row << "推导"
|
||||
}else if(priceMap.value_type == 9){
|
||||
row << "神经网络"
|
||||
}else if(priceMap.value_type == 10){
|
||||
row << "无方式"
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
row << rMap.avg_price
|
||||
row << rMap.calc_type
|
||||
if(fjwPrice && rMap.avg_price){
|
||||
def precent = (fjwPrice - rMap.avg_price)*100/rMap.avg_price
|
||||
row << DataHandleUtils.getDoubleRound(2, precent)
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
rets << row
|
||||
}else{
|
||||
println rMap.commId + " 没找到小区"
|
||||
}
|
||||
println totalCount.get()+" "+ rMap.cityname +" "+rMap.commname
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await();
|
||||
|
||||
|
||||
def headers = ["小区ID","城市","区域","小区名","房价网价格","房价网预期价格","房价网计算方式","建行价格","建行计算方式","价格差比(房价网价格-建行价格)*100/建行价格"]
|
||||
batchCreateFile(rets, "D:\\数据\\建行\\测试\\房价网与建行均价比对.csv",headers,"GBK")
|
||||
|
||||
}
|
||||
|
||||
|
||||
def getCommunityBasePriceGj(yearMonth) {
|
||||
def sql = "select CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE from community_base_price_gj where year_month=${yearMonth} order by cityid";
|
||||
def allRows = []
|
||||
sqlInstance236.eachRow(sql, { r ->
|
||||
def map = [:]
|
||||
map.cityname = r.CITYNAME
|
||||
map.distname = r.DISTNAME
|
||||
map.commid = r.COMMID
|
||||
map.commname = r.COMMNAME
|
||||
map.year_month = r.YEAR_MONTH
|
||||
map.calc_avg_price = r.CALC_AVG_PRICE
|
||||
map.avg_price = r.AVG_PRICE
|
||||
// 计算类型,1:人工估值 2:样本计算 3:神经网络
|
||||
if(r.CALC_TYPE){
|
||||
if(r.CALC_TYPE == 1){
|
||||
map.calc_type = "人工估值"
|
||||
}else if(r.CALC_TYPE == 2){
|
||||
map.calc_type = "样本计算"
|
||||
}else if(r.CALC_TYPE == 3){
|
||||
map.calc_type = "神经网络"
|
||||
}
|
||||
}
|
||||
allRows << map
|
||||
});
|
||||
return allRows
|
||||
}
|
||||
|
||||
|
||||
def batchCreateFile(ret, fileName, headers, charset){
|
||||
OutputStream out = new FileOutputStream(fileName);
|
||||
def sb = new StringBuffer();
|
||||
// sb.append(headers.join("@|@"))
|
||||
// sb.append(System.getProperty("line.separator"));
|
||||
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
|
||||
def index = 0
|
||||
for(def row : ret){
|
||||
index++
|
||||
sb.append(StringUtils.join(row.toArray(), "@|@"))
|
||||
sb.append(System.getProperty("line.separator"));
|
||||
|
||||
println index
|
||||
|
||||
if(index%10000==0){
|
||||
println index+"开始写入CSV "+new Date().toLocaleString()
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
println index+"结束写入CSV "+new Date().toLocaleString()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(sb.length()>0){
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
}
|
||||
out.close();
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-07-22", "2022-08-21")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-07-22", "2022-08-24")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
|
||||
+119
-46
@@ -30,8 +30,8 @@ import java.util.regex.*;
|
||||
//def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
//def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
//def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
@@ -197,6 +197,79 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
|
||||
//println "共查到:"+index
|
||||
|
||||
|
||||
def index = 0
|
||||
//def commIdThirdMap = findAllThird(202207, sqlInstance236);
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
//cityCodeDatas.each{city,cityId->
|
||||
//
|
||||
// deleteThirdByCityId(cityId, 202207, sqlInstance236)
|
||||
// println city + "完成"
|
||||
//}
|
||||
|
||||
|
||||
def lines = org.apache.commons.io.FileUtils.readLines(new File("D:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat"))
|
||||
for(line in lines) {
|
||||
def arrs = line.toString().split("\\|@\\|")
|
||||
if(arrs){
|
||||
def sql = "delete from guided_price_gj where id='${arrs[0]}'"
|
||||
sqlInstance236.executeUpdate(sql)
|
||||
}
|
||||
}
|
||||
|
||||
def deleteThirdByCityId(def cityId,def price_date, def sqlInstance236){
|
||||
def sql = "delete from bigdata_third_party_price_gj where cityid='${cityId}' and price_date=${price_date}"
|
||||
sqlInstance236.executeUpdate(sql)
|
||||
}
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\删除小区2022-8-23.xlsx", { d ->
|
||||
// if (d['C']) {
|
||||
// index ++T
|
||||
// def cityId = cityCodeDatas.get(d['A'])
|
||||
// def sources = findThirdSource(cityId, d['C'], 202207, sqlInstance236)
|
||||
// if(sources.size()>1){
|
||||
// if(sources.contains("安居客") && sources.contains("58同城")){
|
||||
// sources.remove("58同城")
|
||||
// }else if(sources.contains("链家") && sources.contains("贝壳")){
|
||||
// sources.remove("贝壳")
|
||||
// }
|
||||
// }
|
||||
// if(sources.size()==1){
|
||||
// println index +" 删除 "+ d['C']+" ${d['A']}:"+ cityId
|
||||
// deleteThirdParty(cityId, d['C'], 202207, sqlInstance236)
|
||||
// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
def findAllThird(price_date, def sqlInstance236){
|
||||
def sql = "select commid, data_source from bigdata_third_party_price_gj where price_date=${price_date}"
|
||||
def map = [:]
|
||||
sqlInstance236.eachRow(sql,{r->
|
||||
def sources = map.get(r.commid)
|
||||
if(!sources){
|
||||
sources = []
|
||||
map.put(r.commid, sources)
|
||||
}
|
||||
sources << r.data_source
|
||||
});
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def deleteThirdParty(def cityId, def commid, def price_date, def sqlInstance236){
|
||||
def sql = "delete from bigdata_third_party_price_gj where cityid='${cityId}' and commid='${commid}' and price_date=${price_date}"
|
||||
sqlInstance236.executeUpdate(sql)
|
||||
}
|
||||
|
||||
def findThirdSource(def cityId, def commid, def price_date, def sqlInstance236){
|
||||
def sql = "select data_source from bigdata_third_party_price_gj where cityid='${cityId}' and commid='${commid}' and price_date=${price_date}"
|
||||
def sources = []
|
||||
sqlInstance236.eachRow(sql,{r->
|
||||
sources << r.data_source
|
||||
});
|
||||
return sources
|
||||
}
|
||||
|
||||
|
||||
def findAvmData(sqlInstance, def region, def name, def area, def price, def dealdate){
|
||||
def sql = "select count(*) as count from avm_evaluation_item_gj where year_month=202106 and data_type='成交' and cityname='上海' and distname='${region}' and commname='${name}' and buildarea=${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd') and target_avg_price=${price}"
|
||||
def count = 0
|
||||
@@ -326,52 +399,52 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
|
||||
//
|
||||
//},0)
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d->
|
||||
if(d['A']){
|
||||
// def region = d['B']
|
||||
// def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
// if(newRegion) {
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_cat", 11)
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date",new Date())
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d->
|
||||
// if(d['A']){
|
||||
//// def region = d['B']
|
||||
//// def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
//// if(newRegion) {
|
||||
//// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("name_cat", 11)
|
||||
// udata.put("status", 1)
|
||||
// udata.put("m_date",new Date())
|
||||
//
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("_id", dist._id)
|
||||
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
// }else{
|
||||
// println d
|
||||
// }
|
||||
//// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", dist._id)
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println d
|
||||
}
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d->
|
||||
// if(d['A']){
|
||||
//// def region = d['B']
|
||||
//// def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
//// if(newRegion) {
|
||||
//// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("name_cat", 11)
|
||||
// udata.put("status", 1)
|
||||
// udata.put("m_date",new Date())
|
||||
//
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("_id", dist._id)
|
||||
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
// }else{
|
||||
// println d
|
||||
// }
|
||||
}
|
||||
},0)
|
||||
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d->
|
||||
if(d['A']){
|
||||
// def region = d['B']
|
||||
// def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
// if(newRegion) {
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_cat", 11)
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", dist._id)
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println d
|
||||
}
|
||||
// }
|
||||
}
|
||||
},0)
|
||||
//// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
|
||||
|
||||
+14
@@ -2049,6 +2049,20 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
public static def queryAllDistrictPrice(startDate){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put('weekend', startDate);
|
||||
DBCursor cur = DIST_STAT_COLL.find(queryCondition)
|
||||
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def rMap = [:];
|
||||
while(cur.hasNext()) {
|
||||
DBObject dbo = cur.next()
|
||||
rMap.put(dbo.get("city") + "##" + dbo.get("region")+"##"+ dbo.get("district_name"), dbo)
|
||||
}
|
||||
return rMap;
|
||||
}
|
||||
|
||||
|
||||
public static def queryDistrictPrices(city, startDate){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
|
||||
+159
@@ -0,0 +1,159 @@
|
||||
package com.fangjia.script.crawl
|
||||
|
||||
import com.alibaba.fastjson.JSONArray
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
class SchoolLocation {
|
||||
|
||||
DBCollection school_parse_coll = MongoUtils.getCollectionByName("fangjia_craw", "school", "192.168.0.235", 27018)
|
||||
|
||||
static main(args) {
|
||||
def task = new SchoolLocation()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def query = new BasicDBObject()
|
||||
query.put("province","江苏")
|
||||
//query.put("lat",null)
|
||||
query.put("lat",new BasicDBObject('$exists',false))
|
||||
def cursor = school_parse_coll.find(query)
|
||||
def index = 0
|
||||
for(def d : cursor){
|
||||
index++
|
||||
def datas = getSchoolLatAndLng(d.city, d.name, d.address, false)
|
||||
if(datas){
|
||||
for(def ret : datas){
|
||||
def addr = d.address
|
||||
if(d.address.contains(d.city+"市")){
|
||||
def cityStr = d.city+"市"
|
||||
addr = d.address.substring(d.address.indexOf(cityStr)+cityStr.size())
|
||||
}
|
||||
if(ret.name == d.name || (d.address.contains("号") && (ret.address.contains(addr) || addr.contains(ret.address)))){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("lat", ret.lat.toString().toDouble())
|
||||
udata.put("lng", ret.lng.toString().toDouble())
|
||||
udata.put("status", 0)
|
||||
school_parse_coll.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
|
||||
println index +" "+udata
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getSchoolLatAndLng(city, name, address, isFullMatch){
|
||||
if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){
|
||||
return null;
|
||||
}
|
||||
address = address.trim().replace(" ","").replace("?","").replace(" ","")
|
||||
String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=学校&city=" + city + "&output=json&offset=5&page=1&key=4e9592208336433ee1b7aa4d1d434418&extensions=all"
|
||||
|
||||
List<com.alibaba.fastjson.JSONObject> poiList = null;
|
||||
com.alibaba.fastjson.JSONObject result = null;
|
||||
try {
|
||||
HttpRequest request = new HttpRequest();
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
String response = request.get(url, params, "UTF-8");
|
||||
com.alibaba.fastjson.JSONObject jsonResponse = com.alibaba.fastjson.JSONObject.parseObject(response)
|
||||
if (jsonResponse != null) {
|
||||
JSONArray ja = jsonResponse.getJSONArray("pois");
|
||||
if(ja != null && ja.size() > 0) {
|
||||
poiList = new ArrayList<>();
|
||||
for(int i=0;i<ja.size();i++){
|
||||
com.alibaba.fastjson.JSONObject jo_item = ja.getJSONObject(i);
|
||||
|
||||
if(jo_item.containsKey("cityname") && !jo_item.getString("cityname").contains(city)){
|
||||
continue;
|
||||
}
|
||||
|
||||
String item_address = jo_item.containsKey("address") ? jo_item.getString("address") : null;
|
||||
String item_name = jo_item.containsKey("name") ? jo_item.getString("name") : null;
|
||||
|
||||
boolean getedData = false;
|
||||
if(isFullMatch == true){
|
||||
if(StringUtils.equalsIgnoreCase(item_address, address) || StringUtils.equalsIgnoreCase(item_name, address)){
|
||||
getedData = true;
|
||||
}
|
||||
}else{
|
||||
if(StringUtils.equalsIgnoreCase(item_address, address) || StringUtils.contains(item_address, address)
|
||||
|| StringUtils.equalsIgnoreCase(item_name, address) || StringUtils.contains(item_name, address) || StringUtils.contains(address, item_name)){
|
||||
getedData = true;
|
||||
}else{
|
||||
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(item_name, address);
|
||||
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(item_address, address);
|
||||
if(score1 || score2){
|
||||
getedData = true;
|
||||
}
|
||||
|
||||
if(!getedData){
|
||||
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(address, " ")
|
||||
def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(item_name, " ")
|
||||
if(py1 == py2){
|
||||
getedData = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def type = jo_item.getString("type");
|
||||
if(getedData == true){
|
||||
com.alibaba.fastjson.JSONObject jo = new com.alibaba.fastjson.JSONObject();
|
||||
if(jo_item.containsKey("id")){
|
||||
jo.put("id", jo_item.getString("id"))
|
||||
}
|
||||
if(jo_item.containsKey("name")){
|
||||
jo.put("name", jo_item.getString("name"))
|
||||
}
|
||||
if(jo_item.containsKey("type")){
|
||||
jo.put("type", type)
|
||||
}
|
||||
if(jo_item.containsKey("address")){
|
||||
jo.put("address", jo_item.getString("address").replace("[]",""))
|
||||
}
|
||||
if(jo_item.containsKey("pname")){
|
||||
jo.put("pname", jo_item.getString("pname"))
|
||||
}
|
||||
if(jo_item.containsKey("cityname")){
|
||||
jo.put("cityname", jo_item.getString("cityname"))
|
||||
}
|
||||
if(jo_item.containsKey("adname")){
|
||||
jo.put("adname", jo_item.getString("adname"))
|
||||
}
|
||||
|
||||
if(jo_item.containsKey("location")){
|
||||
def arrs = jo_item.getString("location").split(",")
|
||||
jo.put("lat", arrs[1])
|
||||
jo.put("lng", arrs[0])
|
||||
}
|
||||
|
||||
if(type && type.contains("出入口")){
|
||||
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+name+"-"+address+"】-${type}"
|
||||
}else{
|
||||
println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+name+"-"+address+"】"
|
||||
poiList.add(jo);
|
||||
}
|
||||
|
||||
}else{
|
||||
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println "X 高德地图 ( "+city+"-"+address+" )无结果!"
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
|
||||
return poiList;
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -136,7 +136,7 @@ class FzDataResultStat {
|
||||
|
||||
def regionPriceTrendRun(){
|
||||
def city = "福州"
|
||||
def add = -6;
|
||||
def add = -9;
|
||||
def months = []
|
||||
while(add<0){
|
||||
months << DateUtils.addMonth(new Date(), add)
|
||||
@@ -149,7 +149,7 @@ class FzDataResultStat {
|
||||
|
||||
|
||||
def exportRegion(city, months){
|
||||
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
|
||||
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐","闽侯","福清","连江","闽清","罗源","永泰","平潭"];
|
||||
def datas =[]
|
||||
def header = ["城市", "区域"]
|
||||
for(def m : months){
|
||||
|
||||
+2
-2
@@ -325,8 +325,8 @@ class HouseExportBatch extends BaseBatch{
|
||||
// int endDate = 20191201
|
||||
|
||||
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyyMMdd")
|
||||
// def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd");
|
||||
def endDate = "20220710"
|
||||
def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd");
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
// QueryBuilder room_range = QueryBuilders.rangeQuery("room").from(1).to(5).includeLower(true).includeUpper(false);
|
||||
|
||||
+1
@@ -110,6 +110,7 @@ class FangjiaClientUtils {
|
||||
public static def saveToEsPaimai(def data){
|
||||
String jsonStr = com.alibaba.fastjson.JSONObject.toJSONString(data)
|
||||
HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/paiMai", jsonStr)
|
||||
// HttpUtils.postForJson("http://localhost:8084/FangJiaDataService/paiMai", jsonStr)
|
||||
}
|
||||
|
||||
|
||||
|
||||
+3
-3
@@ -376,13 +376,13 @@ public class BeiKeCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
// if(e.text().contains("福州")){
|
||||
if(e.text().contains("福州")){
|
||||
def cityMap = [:]
|
||||
cityMap.city = e.text()
|
||||
cityMap.url = "https:"+href
|
||||
list << cityMap
|
||||
//cityMap.put(e.text(), "https:"+href)
|
||||
// }
|
||||
// cityMap.put(e.text(), "https:"+href)
|
||||
}
|
||||
}
|
||||
}
|
||||
return list
|
||||
|
||||
+43
-13
@@ -7,6 +7,8 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.commons.lang.math.NumberUtils
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
@@ -175,22 +177,23 @@ class ExportPaimai {
|
||||
|
||||
|
||||
def run(){
|
||||
def city = "苏州"
|
||||
def region = "姑苏"
|
||||
def city = "南京"
|
||||
def region = null
|
||||
def datas = findPaimaiData(city,region)
|
||||
println "总量:"+datas.size()
|
||||
def rows = []
|
||||
def header = ["标地物地址","省","城市","区域","地址","标题提取地址","楼栋号","单元","室号","楼盘名称","建筑面积","竣工日期","房屋用途","总楼层",
|
||||
"拍卖状态","是否成交","拍卖阶段","拍卖方式","成交价","拍卖开始时间","拍卖结束时间","起拍价","保证金","加价幅度","评估价","市场价",
|
||||
"变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数","URL"]
|
||||
"变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数", "折价率","映射类型","拍卖类型",
|
||||
"映射资产类型","映射拍卖状态","URL"]
|
||||
rows << header
|
||||
// def set = new HashSet()
|
||||
def set = new HashSet()
|
||||
def index = 0
|
||||
for(def d : datas){
|
||||
// if(!set.contains(d.title)){
|
||||
if(!set.contains(d.title)){
|
||||
index++
|
||||
println index
|
||||
// set << d.title
|
||||
set << d.title
|
||||
def row = []
|
||||
row << d.title.replace(",",",")
|
||||
row << d.province
|
||||
@@ -226,14 +229,19 @@ class ExportPaimai {
|
||||
row << d.lookCount
|
||||
row << d.callCount
|
||||
row << d.delayCount
|
||||
row << d.url
|
||||
|
||||
row << d.rateDiscount//折价率
|
||||
row << d.mappingEstateType;//映射类型
|
||||
row << d.zcType;//拍卖类型:资产拍卖 司法拍卖
|
||||
row << d.mappingAgencyName//映射资产类型
|
||||
row << d.mappingStatus//映射拍卖状态
|
||||
row << d.url
|
||||
rows << row
|
||||
// }else{
|
||||
// println d.title
|
||||
// }
|
||||
}else{
|
||||
println d.title +" "+d.url
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\京东拍卖(${city}-姑苏-撤回中止).xlsx",rows,"数据")
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\南京法拍.xlsx",rows,"数据")
|
||||
|
||||
// batchCreateFile(rows, "D:\\数据\\重庆农商行\\法拍\\淘宝拍卖(新).csv",null,"GBK")
|
||||
println "生成完成........"
|
||||
@@ -452,12 +460,12 @@ class ExportPaimai {
|
||||
if(region){
|
||||
pmquery.put("region",region)
|
||||
}
|
||||
pmquery.put("status", new BasicDBObject('$in', ["已撤回","已暂缓"]))
|
||||
// pmquery.put("status", new BasicDBObject('$in', ["已撤回","已暂缓"]))
|
||||
// pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
|
||||
pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-07-01")))
|
||||
|
||||
// Pattern pattern = Pattern.compile("rmfysszc.gov.cn", Pattern.MULTILINE);
|
||||
Pattern pattern = Pattern.compile("jd.com", Pattern.MULTILINE);
|
||||
// Pattern pattern = Pattern.compile("jd.com", Pattern.MULTILINE);
|
||||
// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
|
||||
// pmquery.put("url", new BasicDBObject('$regex', pattern))
|
||||
def dbcursor = paimai43_COLL.find(pmquery).batchSize(2000)
|
||||
@@ -465,6 +473,28 @@ class ExportPaimai {
|
||||
for(def db : dbcursor){
|
||||
rets << db
|
||||
}
|
||||
|
||||
// Collections.sort(rets, new Comparator<BasicDBObject>() {
|
||||
// @Override
|
||||
// int compare(BasicDBObject o1, BasicDBObject o2) {
|
||||
// if(o1.finishDate && o2.finishDate){
|
||||
// return -o1.finishDate.compareTo(o2.finishDate)
|
||||
// }else{
|
||||
// return 0
|
||||
// }
|
||||
//
|
||||
// }
|
||||
// })
|
||||
rets.sort {a,b->
|
||||
if(a.finishDate && b.finishDate){
|
||||
def v1 = a.finishDate
|
||||
def v2 = b.finishDate
|
||||
return v1.compareTo(v2)
|
||||
}else{
|
||||
return 0
|
||||
}
|
||||
}
|
||||
rets = rets.reverse()
|
||||
return rets
|
||||
}
|
||||
}
|
||||
|
||||
+74
-47
@@ -129,7 +129,7 @@ public class LianjiaCrawlByChrome {
|
||||
def standardRegion(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")))
|
||||
q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) )
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
for(def d : dbcursor){
|
||||
@@ -152,8 +152,9 @@ public class LianjiaCrawlByChrome {
|
||||
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
lastMonth = lastMonth +"-01"
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", "福州")
|
||||
q.put("region", null)
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-05-01","yyyy-MM-dd")))
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")))
|
||||
// q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def keys = []
|
||||
@@ -187,8 +188,19 @@ public class LianjiaCrawlByChrome {
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
cityMap.each{city, cityUrl->
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
def detailUrls = getPageDeail(driver, city, cityUrl)
|
||||
println cityUrl +" 详情链接:"+detailUrls.size()
|
||||
driver.get(cityUrl+"chengjiao/")
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
def regionText = r.text()
|
||||
def href = cityUrl.toString().substring(0, cityUrl.toString().length()-1) + r.attr("href")
|
||||
|
||||
println "开始抓取【${city}】【${regionText}】【${href}】"
|
||||
def detailUrls = getPageDeail(driver, city, href, regionText)
|
||||
println cityUrl +" 详情链接:"+detailUrls.size()
|
||||
}
|
||||
}
|
||||
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
|
||||
}
|
||||
}
|
||||
@@ -224,12 +236,12 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
|
||||
|
||||
def getPageDeail(driver, city, cityUrl){
|
||||
def getPageDeail(driver, city, cityUrl, regionText){
|
||||
def stop = false
|
||||
def page = 1
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "chengjiao/pg${page}/"
|
||||
def url = cityUrl + "pg${page}/"
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
@@ -241,7 +253,7 @@ public class LianjiaCrawlByChrome {
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-06-20", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
parse(city, e, regionText)
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
@@ -259,7 +271,7 @@ public class LianjiaCrawlByChrome {
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
def parse(city,e){
|
||||
def parse(city,e, regionText){
|
||||
try {
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
@@ -267,7 +279,11 @@ public class LianjiaCrawlByChrome {
|
||||
def arrs = title.split(" ")
|
||||
detail.city = city
|
||||
detail.district_name = arrs[0]
|
||||
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
|
||||
if(regionText){
|
||||
detail.region = regionText
|
||||
}else{
|
||||
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
|
||||
}
|
||||
if(detail.region){
|
||||
detail.region = RegionData.formatRegion(city, detail.region)
|
||||
}
|
||||
@@ -275,50 +291,61 @@ public class LianjiaCrawlByChrome {
|
||||
def roomStr = arrs[1]
|
||||
if (roomStr) {
|
||||
if (roomStr.indexOf("室") > 0) {
|
||||
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
|
||||
def room = roomStr.substring(0, roomStr.indexOf("室"))
|
||||
if(room && room.isNumber()){
|
||||
detail.room = room.toInteger()
|
||||
}
|
||||
|
||||
}
|
||||
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger()
|
||||
def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
|
||||
if(hall && hall.isNumber()){
|
||||
detail.hall = hall.toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def areaStr = arrs[2].replace("平米", "")
|
||||
if (areaStr) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = price.toInteger() * 10000
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
detail.avg_price = (int) (detail.total_price / detail.area)
|
||||
}
|
||||
def tradeStr = e.select(".dealDate").text()
|
||||
if (tradeStr) {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "链家在线"
|
||||
detail.url = e.select(".title a").attr("href")
|
||||
def dbo = findTrade(detail)
|
||||
if (!dbo) {
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
def houseInfos = houseInfo.split("\\|")
|
||||
detail.direction = houseInfos[0]
|
||||
detail.fitment = houseInfos[1]
|
||||
def floorStr = e.select(".positionInfo").text()
|
||||
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
|
||||
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorStr && floorStr.isNumber()) {
|
||||
detail.height = floorStr.toInteger()
|
||||
if("车位"!=roomStr){
|
||||
if(arrs && arrs.size()>=3){
|
||||
def areaStr = arrs[2].replace("平米", "")
|
||||
if (areaStr) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
}
|
||||
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = price.toInteger() * 10000
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
detail.avg_price = (int) (detail.total_price / detail.area)
|
||||
}
|
||||
def tradeStr = e.select(".dealDate").text()
|
||||
if (tradeStr) {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "链家在线"
|
||||
detail.url = e.select(".title a").attr("href")
|
||||
def dbo = findTrade(detail)
|
||||
if (!dbo) {
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
def houseInfos = houseInfo.split("\\|")
|
||||
detail.direction = houseInfos[0]
|
||||
detail.fitment = houseInfos[1]
|
||||
def floorStr = e.select(".positionInfo").text()
|
||||
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
|
||||
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorStr && floorStr.isNumber()) {
|
||||
detail.height = floorStr.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
}
|
||||
}
|
||||
}catch(Exception ex){
|
||||
@@ -446,9 +473,9 @@ public class LianjiaCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
// if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
cityMap.put(e.text(), href)
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
|
||||
+241
-74
@@ -10,6 +10,7 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.AMQP
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -29,10 +30,11 @@ class PaimaiDataSend {
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
static main(args) {
|
||||
def task = new PaimaiDataSend()
|
||||
// task.jdRunByMongo()
|
||||
// task.jdRun()
|
||||
// task.aliZcRun()
|
||||
task.aliZcRun2()
|
||||
@@ -64,7 +66,8 @@ class PaimaiDataSend {
|
||||
def rmfysszcRun(){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/76602.shtml")
|
||||
// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市"]))
|
||||
q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/88061.shtml")
|
||||
def dbcursor = rmfysszc_coll.find(q).batchSize(3000)
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
@@ -95,11 +98,22 @@ class PaimaiDataSend {
|
||||
}
|
||||
}
|
||||
|
||||
def jdRun(){
|
||||
def jdRunByMongo(){
|
||||
def pm_q = new BasicDBObject()
|
||||
pm_q.put("city","宿迁")
|
||||
pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
|
||||
pm_q.put("mappingStage", null)
|
||||
def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000)
|
||||
println dbcursor.count()
|
||||
for(def d : dbcursor){
|
||||
sendJd(d.url)
|
||||
}
|
||||
}
|
||||
|
||||
def sendJd(url){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","宿迁市")
|
||||
// q.put("url","https://paimai.jd.com/108245329")
|
||||
q.put("url", url)
|
||||
def dbcursor = jd_coll.find(q).batchSize(3000)
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
@@ -110,6 +124,45 @@ class PaimaiDataSend {
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", d.desc_content)
|
||||
result.put("region", d.region)
|
||||
|
||||
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
|
||||
result.put("bid_content", bid_content.toString())
|
||||
|
||||
result.put("sub_count", d.sub_count)
|
||||
def data_json = net.sf.json.JSONObject.fromObject(d.json)
|
||||
result.put("data_json", data_json)
|
||||
result.put("content_json", d.content)
|
||||
result.put("item_notice_content", d.item_notice_content)
|
||||
result.put("notice_detail_content", d.notice_detail_content.content)
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("parser_status", 3)
|
||||
udata.put("m_date", new Date())
|
||||
jd_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
def jdRun(){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市"]))
|
||||
// q.put("parser_status", new BasicDBObject('$ne',3))
|
||||
q.put("url","https://paimai.jd.com/115669796")
|
||||
def dbcursor = jd_coll.find(q).batchSize(3000)
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
println "jd:"+index
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", "paimai_jd")
|
||||
json.put("url", d.url)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", d.desc_content)
|
||||
result.put("region", d.region)
|
||||
|
||||
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
|
||||
result.put("bid_content", bid_content.toString())
|
||||
@@ -134,17 +187,19 @@ class PaimaiDataSend {
|
||||
|
||||
def aliZcRun(){
|
||||
def pm_q = new BasicDBObject()
|
||||
pm_q.put("city","苏州")
|
||||
pm_q.put("region","姑苏")
|
||||
// pm_q.put("url","https://sf-item.taobao.com/sf_item/596573117068.htm")
|
||||
pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
|
||||
// pm_q.put("city","苏州")
|
||||
// pm_q.put("region","姑苏")
|
||||
pm_q.put("url","https://sf-item.taobao.com/sf_item/659117468505.htm")
|
||||
// pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
|
||||
// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
|
||||
// pm_q.put("url", new BasicDBObject('$regex', pattern))
|
||||
// pm_q.put("status", null)
|
||||
// pm_q.put("mappingEstateType", null)
|
||||
// pm_q.put("area", new BasicDBObject('$lt', 30))
|
||||
def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000)
|
||||
println dbcursor.count()
|
||||
for(def d : dbcursor){
|
||||
def flag = sendAliZc(d.url)
|
||||
// def flag = sendAliZc(d.url)
|
||||
def flag = sendAli(d.url)
|
||||
if(!flag){
|
||||
println "不存在:"+d.url
|
||||
}
|
||||
@@ -194,14 +249,59 @@ class PaimaiDataSend {
|
||||
return flag
|
||||
}
|
||||
|
||||
def sendAli(def url){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
q.put("url",url)
|
||||
def dbcursor = ali_fp_coll.find(q).batchSize(5000)
|
||||
def flag = false
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
flag = true
|
||||
println "ali-fp: "+index
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", "paimai_taobao")
|
||||
json.put("url", d.url)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", d.content)
|
||||
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
|
||||
result.put("trades", bid_content)
|
||||
result.put("province", d.province)
|
||||
result.put("city", d.city)
|
||||
result.put("region", d.region)
|
||||
result.put("sub_count", d.sub_count)
|
||||
def data_json = net.sf.json.JSONObject.fromObject(d.json)
|
||||
result.put("data_json", data_json)
|
||||
result.put("desc_content", d.desc_content)
|
||||
result.put("item_notice_content", d.item_notice_content)
|
||||
result.put("notice_detail_content", d.notice_detail_content)
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("parser_status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
ali_fp_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
return flag
|
||||
}
|
||||
|
||||
def aliFpRun(){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city","宿迁")
|
||||
q.put("url","https://sf-item.taobao.com/sf_item/649014193736.htm")
|
||||
q.put("city",new BasicDBObject('$in', ["南京","宿迁"]))
|
||||
// q.put("parser_status", new BasicDBObject('$ne',2))
|
||||
// q.put("url","https://sf-item.taobao.com/sf_item/631662596036.htm")
|
||||
//https://sf-item.taobao.com/sf_item/657131607486.htm
|
||||
// q.put("parser_status", new BasicDBObject('$exists',false))
|
||||
def dbcursor = ali_fp_coll.find(q).batchSize(2000)
|
||||
// q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01")))
|
||||
// q.put("area", new BasicDBObject('$lt', 30))
|
||||
def dbcursor = ali_fp_coll.find(q).batchSize(5000)
|
||||
|
||||
def rets = []
|
||||
|
||||
@@ -228,10 +328,20 @@ class PaimaiDataSend {
|
||||
json.put("result", result)
|
||||
|
||||
rets << json
|
||||
if(rets.size()==10000){
|
||||
println "共:"+rets.size()
|
||||
send(rets)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
|
||||
println "共:"+rets.size()
|
||||
if(rets.size()>0){
|
||||
println "共:"+rets.size()
|
||||
send(rets)
|
||||
}
|
||||
}
|
||||
|
||||
def send(def rets){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
for(def i=0;i<rets.size();i++){
|
||||
@@ -244,7 +354,7 @@ class PaimaiDataSend {
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("parser_status", 1)
|
||||
udata.put("parser_status", 2)
|
||||
udata.put("m_date", new Date())
|
||||
ali_fp_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
|
||||
println "send: "+totalCount.get()
|
||||
@@ -263,12 +373,18 @@ class PaimaiDataSend {
|
||||
def aliZcRun2(){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city","宿迁")
|
||||
q.put("city",new BasicDBObject('$in', ["上海"]))
|
||||
// q.put("parser_status", new BasicDBObject('$ne',1))
|
||||
// q.put("region","姑苏")
|
||||
q.put("url","https://zc-item.taobao.com/auction/672615219601.htm")
|
||||
def dbcursor = ali_zc_coll.find(q).batchSize(1000)
|
||||
// q.put("url","https://sf-item.taobao.com/sf_item/586415617368.htm")
|
||||
def dbcursor = ali_zc_coll.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
if(!d.url){
|
||||
println d._id.toString()
|
||||
continue
|
||||
}
|
||||
println "ali-zc: "+index
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", "paimai_taobao_zc")
|
||||
@@ -290,19 +406,52 @@ class PaimaiDataSend {
|
||||
result.put("region", d.region)
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("parser_status", 4)
|
||||
udata.put("m_date", new Date())
|
||||
ali_zc_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
|
||||
rets << json
|
||||
if(rets.size()==10000){
|
||||
println "共:"+rets.size()
|
||||
send(rets)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
|
||||
if(rets.size()>0){
|
||||
println "共:"+rets.size()
|
||||
send(rets)
|
||||
}
|
||||
|
||||
// println "共:"+rets.size()
|
||||
//
|
||||
// def totalCount = new AtomicLong()
|
||||
// final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
// for(def i=0;i<rets.size();i++){
|
||||
// final def json = rets[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
// try{
|
||||
// totalCount.addAndGet(1)
|
||||
// byte[] packedMsg = MessageUtils.pack(json)
|
||||
// CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
//
|
||||
//
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("parser_status", 1)
|
||||
// udata.put("m_date", new Date())
|
||||
// ali_zc_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
|
||||
// println "send: "+totalCount.get()
|
||||
// } catch (Exception e) {
|
||||
// e.printStackTrace()
|
||||
// } finally {
|
||||
// latch.countDown()
|
||||
// }
|
||||
// }
|
||||
// });
|
||||
// }
|
||||
// latch.await()
|
||||
}
|
||||
|
||||
def paimaiSendToEs(){
|
||||
def rets = getPaimaiDatas()
|
||||
println "共:"+rets.size()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
for(def i=0;i<rets.size();i++){
|
||||
@@ -312,7 +461,7 @@ class PaimaiDataSend {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def flag = FangjiaClientUtils.saveToEsPaimai(m)
|
||||
println totalCount.get()+" "+m.city +" "+m.name +" "+flag
|
||||
println totalCount.get()+" "+m.city +" "+m.title +" "+flag
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
@@ -326,56 +475,74 @@ class PaimaiDataSend {
|
||||
|
||||
def getPaimaiDatas(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
|
||||
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
|
||||
def dbcursor = paimai43_COLL.find(q).batchSize(1000)
|
||||
def dbcursor = paimai43_COLL.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
def o = [:]
|
||||
o.addPrice = d.addPrice
|
||||
o.agencyName = d.agencyName
|
||||
o.area = d.area
|
||||
o.buildYear = d.buildYear
|
||||
o.callCount = d.callCount
|
||||
o.city = d.city
|
||||
o.court = d.court
|
||||
o.delayCount = d.delayCount
|
||||
o.delayWeekly = d.delayWeekly
|
||||
o.depositPrice = d.depositPrice
|
||||
o.enrollCount = d.enrollCount
|
||||
o.estateType = d.estateType
|
||||
o.evaluatePrice = d.evaluatePrice
|
||||
o.finishDate = d.finishDate
|
||||
o.height = d.height
|
||||
o.houseNum = d.houseNum
|
||||
o.id = d.id
|
||||
o.introduce = d.introduce
|
||||
if(d.city){
|
||||
def o = [:]
|
||||
o.addPrice = d.addPrice
|
||||
o.agencyName = d.agencyName
|
||||
o.area = d.area
|
||||
o.buildYear = d.buildYear
|
||||
o.callCount = d.callCount
|
||||
o.city = d.city
|
||||
o.court = d.court
|
||||
o.delayCount = d.delayCount
|
||||
o.delayWeekly = d.delayWeekly
|
||||
o.depositPrice = d.depositPrice
|
||||
o.enrollCount = d.enrollCount
|
||||
o.estateType = d.estateType
|
||||
o.evaluatePrice = d.evalatePrice
|
||||
o.finishDate = d.finishDate
|
||||
o.height = d.height
|
||||
o.houseNum = d.houseNum
|
||||
o.id = d.id
|
||||
o.introduce = d.introduce
|
||||
// o.location = d.location
|
||||
o.lookCount = d.lookCount
|
||||
o.marketPrice = d.marketPrice
|
||||
o.matchAddress = d.matchAddress
|
||||
o.matchCity = d.matchCity
|
||||
o.matchName = d.matchName
|
||||
o.matchRegion = d.matchRegion
|
||||
o.person = d.person
|
||||
o.preemptivePerson = d.preemptivePerson
|
||||
o.pricePkWeekly = d.pricePkWeekly
|
||||
o.province = d.province
|
||||
o.rateDiscount = d.rateDiscount
|
||||
o.region = d.region
|
||||
o.roomNum = d.roomNum
|
||||
o.sellOffAdvancePay = d.sellOffAdvancePay
|
||||
o.source = d.source
|
||||
o.stage = d.stage
|
||||
o.startDate = d.startDate
|
||||
o.startPrice = d.startPrice
|
||||
o.status = d.status
|
||||
o.tel = d.tel
|
||||
o.title = d.title
|
||||
o.tradePrice = d.tradePrice
|
||||
o.type = d.type
|
||||
o.unitNum = d.unitNum
|
||||
o.url = d.url
|
||||
rets << o
|
||||
o.lookCount = d.lookCount
|
||||
o.marketPrice = d.marketPrice
|
||||
o.matchAddress = d.matchAddress
|
||||
o.matchCity = d.matchCity
|
||||
o.matchName = d.matchName
|
||||
o.matchRegion = d.matchRegion
|
||||
o.person = d.person
|
||||
o.preemptivePerson = d.preemptivePerson
|
||||
o.pricePkWeekly = d.pricePkWeekly
|
||||
o.province = d.province
|
||||
o.rateDiscount = d.rateDiscount
|
||||
o.region = d.region
|
||||
o.roomNum = d.roomNum
|
||||
o.sellOffAdvancePay = d.sellOffAdvancePay
|
||||
o.source = d.source
|
||||
o.stage = d.stage
|
||||
o.startDate = d.startDate
|
||||
o.startPrice = d.startPrice
|
||||
o.status = d.status
|
||||
o.tel = d.tel
|
||||
o.title = d.title
|
||||
o.tradePrice = d.tradePrice
|
||||
o.tradeAvgPrice = d.tradeAvgPrice
|
||||
o.type = d.type
|
||||
o.unitNum = d.unitNum
|
||||
o.url = d.url
|
||||
if(d.url.contains("taobao.com")){
|
||||
o.source = "淘宝"
|
||||
}else if(d.url.contains("jd.com")){
|
||||
o.source = "京东"
|
||||
}else if(d.url.contains("rmfysszc.gov.cn")){
|
||||
o.source = "人民法院讼诉网"
|
||||
}
|
||||
|
||||
o.rateDiscount = d.rateDiscount//折价率
|
||||
o.mappingEstateType = d.mappingEstateType;//映射类型
|
||||
o.paiMaiType = d.paiMaiType;//拍卖类型:资产拍卖 司法拍卖
|
||||
o.mappingAgencyName = d.mappingAgencyName//映射资产类型
|
||||
o.mappingStatus = d.mappingStatus//映射拍卖状态
|
||||
o.mappingStage = d.mappingStage
|
||||
rets << o
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
@@ -28,11 +28,29 @@ class CityStat {
|
||||
def task = new CityStat()
|
||||
// task.run()
|
||||
// task.run1();
|
||||
task.run3();
|
||||
// task.run3();
|
||||
// task.run4()
|
||||
// task.run6()
|
||||
// task.run7()
|
||||
// task.run8()
|
||||
task.dealPriceStat()
|
||||
}
|
||||
|
||||
def dealPriceStat(){
|
||||
def citys = ["上海","北京","广州","合肥","南京","深圳","武汉","郑州","重庆","福州","厦门","青岛","太原","长春","苏州","南宁","沈阳","天津","西安","长沙","大连","南昌","东莞","佛山","济南","南通","镇江","宁波","烟台","哈尔滨"]
|
||||
def rets = []
|
||||
rets << ["城市","2021-1月总量"]
|
||||
def startDate = DateUtils.parseFromTightByPattern('2021-12-01','yyyy-MM-dd')
|
||||
def endDate = DateUtils.parseFromTightByPattern('2022-01-01','yyyy-MM-dd')
|
||||
for(def c : citys){
|
||||
def ret = []
|
||||
ret << c
|
||||
ret << findDealPriceCount(c, startDate, endDate)
|
||||
rets << ret
|
||||
println ret
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\2021成交数据.xlsx", rets, "月统计")
|
||||
}
|
||||
|
||||
// def run8(){
|
||||
@@ -464,6 +482,14 @@ class CityStat {
|
||||
return count;
|
||||
}
|
||||
|
||||
def findDealPriceCount(def city, def startDate, def endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt', endDate))
|
||||
def count = DEAL_PRICE_COLL.count(q)
|
||||
return count;
|
||||
}
|
||||
|
||||
def calTradeDistCount(def city, def tradeDate, def districts){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
|
||||
@@ -8,6 +8,7 @@ import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.Mongo
|
||||
@@ -17,8 +18,19 @@ import org.bson.types.ObjectId
|
||||
import java.text.SimpleDateFormat
|
||||
|
||||
|
||||
def dist = CommonDatautils.findDistrictById("62ce233a3705890001dfa164")
|
||||
DBUtils.seaweed.save(dist)
|
||||
DBCollection coll = MongoUtils.getCollectionByName("seeds_db", "city_seeds", "192.168.0.235", 27018)
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("root", "anjuke.com")
|
||||
def dbcursor = coll.find(q)
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
map.put(d.city, d.city_short)
|
||||
}
|
||||
println map.size()
|
||||
println map
|
||||
//def dist = CommonDatautils.findDistrictById("62ce233a3705890001dfa164")
|
||||
//DBUtils.seaweed.save(dist)
|
||||
//DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId("62ce233a3705890001dfa164")), new BasicDBObject('$set', dist))
|
||||
//SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
||||
//Date date = new Date(1617211269868);
|
||||
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.alibaba.fastjson.JSON
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.temp.shenjian.util.MessageUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
DBCollection school_coll = MongoUtils.getCollectionByName("crawl_tmp", "school_ruyile_web", "192.168.0.235", 27018)
|
||||
|
||||
DBCollection sxue_school_coll = MongoUtils.getCollectionByName("crawl_tmp", "school_5sxue_web", "192.168.0.235", 27018)
|
||||
|
||||
DBCollection school_parse_coll = MongoUtils.getCollectionByName("fangjia_craw", "school", "192.168.0.235", 27018)
|
||||
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
//while (true){
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("province",new BasicDBObject('$in', ["江苏","江苏省"]))
|
||||
// q.put("status",new BasicDBObject('$ne', 1))
|
||||
// def dbcursor = school_coll.find(q).limit(1000)
|
||||
// def rets = []
|
||||
// for(def d : dbcursor){
|
||||
// JSON json = new JSONObject()
|
||||
// json.put("project", d.project)
|
||||
// json.put("url", d.url)
|
||||
//
|
||||
// JSON result = new JSONObject()
|
||||
// result.put("content", d.content)
|
||||
// def jsonStr = d.map_info
|
||||
//
|
||||
// jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1)
|
||||
// if(jsonStr.contains("\n")){
|
||||
// jsonStr = jsonStr.replaceAll('\\n','')
|
||||
// }
|
||||
// result.put("data_json", net.sf.json.JSONObject.fromObject(jsonStr))
|
||||
//
|
||||
//
|
||||
// result.put("type", d.type)
|
||||
// json.put("result", result)
|
||||
//
|
||||
//
|
||||
// rets << json
|
||||
//// if(rets.size()==5000){
|
||||
//// println "共:"+rets.size()
|
||||
//// send(rets, fixedThreadPool, school_coll)
|
||||
//// rets.clear()
|
||||
//// }
|
||||
// }
|
||||
//
|
||||
// if(rets.size()>0){
|
||||
// println "共:"+rets.size()
|
||||
// send(rets, fixedThreadPool, school_coll)
|
||||
// rets.clear()
|
||||
// }
|
||||
//}
|
||||
|
||||
|
||||
|
||||
def send(def rets, def fixedThreadPool, def school_coll){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def json = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
school_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
|
||||
println "send: "+totalCount.get()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def query = new BasicDBObject()
|
||||
query.put("source","搜学网")
|
||||
//query.put("url","http://xuexiao.51sxue.com/detail/id_26799.html")
|
||||
query.put("lat",null)
|
||||
def cursor = school_parse_coll.find(query)
|
||||
for(def o : cursor){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("province","江苏省")
|
||||
q.put("url",o.url)
|
||||
def dbcursor = sxue_school_coll.find(q)
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
println index
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", d.project)
|
||||
json.put("url", d.url)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", d.content)
|
||||
result.put("name", d.name)
|
||||
result.put("list_item", d.list_item)
|
||||
|
||||
def jsonStr = d.map_info
|
||||
jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1)
|
||||
if(jsonStr.contains("\n")){
|
||||
jsonStr = jsonStr.replaceAll('\\n','')
|
||||
}
|
||||
result.put("json_data", jsonStr)
|
||||
|
||||
|
||||
result.put("type", d.type)
|
||||
json.put("result", result)
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
sxue_school_coll.update(new BasicDBObject("url", d.get("url")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
package com.fangjia.script.temp.shenjian.poi
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.temp.shenjian.util.EsUtils
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
|
||||
/**
|
||||
* ²¹³äѧУ¾Î³¶È
|
||||
* @author shenjian
|
||||
* @since 2022/9/8
|
||||
*/
|
||||
class FillLngLat {
|
||||
|
||||
static void main(String[] args) {
|
||||
ElasticUtils.executeSqlQuery("select * from dws_poi where location is null", { data ->
|
||||
println(data)
|
||||
def lngLat = GaoDeUtils.parserByGaode(data.city, data.region + data.name + data.address)
|
||||
if (lngLat != null && lngLat.size() == 2 ) {
|
||||
JSONObject jsonObject = new JSONObject()
|
||||
jsonObject.put("lon", Double.valueOf(lngLat['lng']))
|
||||
jsonObject.put("lat", Double.valueOf(lngLat['lat']))
|
||||
EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+31
-22
@@ -10,6 +10,7 @@ import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBList
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
@@ -27,15 +28,15 @@ import java.util.concurrent.Executors
|
||||
@BatchOwner(Watcher.ShenJian)
|
||||
class SeaweedAroundPoi extends BaseBatch {
|
||||
|
||||
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
|
||||
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.235", 27018)
|
||||
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi"
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(30)
|
||||
//static ExecutorService executorService = Executors.newFixedThreadPool(2)
|
||||
|
||||
static void main(String[] args) {
|
||||
def types = listAllType()
|
||||
//dealSpecialSeaweedAround(types)
|
||||
dealPoi(new BasicDBObject(), types)
|
||||
executorService.shutdown()
|
||||
//executorService.shutdown()
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -64,33 +65,40 @@ class SeaweedAroundPoi extends BaseBatch {
|
||||
}
|
||||
|
||||
static dealPoi(DBObject query, def types) {
|
||||
query.put("status", 1)
|
||||
conn136.find(query).batchSize(5000).each {objTemp ->
|
||||
query.put("canyin_status", 0)
|
||||
conn136.find(query).batchSize(5000).limit(20000).noCursorTimeout(true).each {objTemp ->
|
||||
final def obj = objTemp
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
// executorService.submit(new Runnable() {
|
||||
// @Override
|
||||
// void run() {
|
||||
def city = obj.city
|
||||
def lng = obj.lng
|
||||
def lat = obj.lat
|
||||
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
|
||||
if (CollectionUtils.isNotEmpty(pois)) {
|
||||
String jsonStr = JSONObject.toJSONString(pois)
|
||||
HttpUtils.postForJson(URL, jsonStr)
|
||||
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
|
||||
} else {
|
||||
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空")))
|
||||
if (obj.lng) {
|
||||
//def lng = SeaweedUtils.formatLatLng(obj.lng)
|
||||
//def lat = SeaweedUtils.formatLatLng(obj.lat)
|
||||
// {'lon': 121.810877, 'lat': 31.737642}
|
||||
def lng = 121.810877
|
||||
def lat = 31.737642
|
||||
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
|
||||
if (CollectionUtils.isNotEmpty(pois)) {
|
||||
String jsonStr = JSONObject.toJSONString(pois)
|
||||
HttpUtils.postForJson(URL, jsonStr)
|
||||
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("canyin_status", 0).append("flag", "已获取周边POI")))
|
||||
} else {
|
||||
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("canyin_status", 2).append("flag", "周边POI为空")))
|
||||
}
|
||||
println("已处理小区:" + obj)
|
||||
}
|
||||
println("已处理小区:" + obj)
|
||||
}
|
||||
})
|
||||
}
|
||||
// })
|
||||
// }
|
||||
}
|
||||
|
||||
static getNewSeaweed() {
|
||||
DBObject query = new BasicDBObject()
|
||||
query.put("cat", "district")
|
||||
query.put("visible", 0)
|
||||
//query.put("city", new BasicDBObject('$in', ["连云港", "南通", "苏州", "无锡", "盐城", "扬州"]))
|
||||
//query.put("city", "连云港")
|
||||
//query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a"))
|
||||
|
||||
DBObject filter = new BasicDBObject()
|
||||
@@ -163,7 +171,7 @@ class SeaweedAroundPoi extends BaseBatch {
|
||||
|
||||
public static final Map<String, String> KEYWORDS_MAP = new HashMap<String, String>() {
|
||||
{
|
||||
put("餐饮", "05");
|
||||
// put("餐饮", "05");
|
||||
// put("地铁", "150500");
|
||||
// put("公交", "1507");
|
||||
// put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211");
|
||||
@@ -172,8 +180,9 @@ class SeaweedAroundPoi extends BaseBatch {
|
||||
// put("银行", "1601|1603");
|
||||
// put("大型超市", "0604");
|
||||
// put("商场", "0601");
|
||||
// put("餐饮", "0501|0502|0503|0504");
|
||||
put("餐饮", "0501|0502|0503|0504");
|
||||
// put("旅游景点", "110000");
|
||||
// put("公园广场", "110100|110101|110102|110103|110104|110105")
|
||||
// put("图书馆", "140500");
|
||||
// put("政府机关", "130100|130101|130102|130103|130104|130105|130106|130107");
|
||||
// put("丧葬设施", "071900|071901|071902|071903")
|
||||
|
||||
+7
-5
@@ -146,11 +146,13 @@ class GaoDeUtils {
|
||||
while (result.size() < count) {
|
||||
params.page = "${page}".toString()
|
||||
page++
|
||||
params.key = this.getGaodeKey("v5/place/around")
|
||||
JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params);
|
||||
params.key = this.getGaodeKey("v3/place/around")
|
||||
params.offset = "100"
|
||||
JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params);
|
||||
if (json == null) {
|
||||
params.key = this.getGaodeKey("v5/place/around")
|
||||
json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params);
|
||||
params.key = this.getGaodeKey("v3/place/around")
|
||||
params.offset = "100"
|
||||
json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params);
|
||||
}
|
||||
if (json && json.info && json.info == "OK") {
|
||||
count = json.getString("count").toInteger()
|
||||
@@ -258,7 +260,7 @@ class GaoDeUtils {
|
||||
}
|
||||
if (json && json.info && json.info == "OK") {
|
||||
try {
|
||||
List<AmapPoiDetail> ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class)
|
||||
List<AmapPoiDetail> ret = JSONObject.parseArray(json.pois.toString(), AmapPoiDetail.class)
|
||||
return ret
|
||||
} catch(Exception ex) {
|
||||
ex.printStackTrace()
|
||||
|
||||
+4
-4
@@ -40,11 +40,11 @@ class DataStat {
|
||||
|
||||
|
||||
def run(){
|
||||
def source = "365ÌÔ·¿"//
|
||||
def citys = findCrawlCity(source, 202206);
|
||||
def source = "我爱我家"//
|
||||
def citys = findCrawlCity(source, 202207);
|
||||
for(def city : citys){
|
||||
def count1 = findCount(source, city, 202206);
|
||||
def count2 = findCount(source, city, 202207);
|
||||
def count1 = findCount(source, city, 202207);
|
||||
def count2 = findCount(source, city, 202208);
|
||||
println city +" "+count1+" "+count2
|
||||
// println count1
|
||||
// println "--- ----"
|
||||
|
||||
+7
-10
@@ -16,15 +16,12 @@ class SynchDistrictPrice {
|
||||
def static final SHORT_DATE = Utils.getLastMonth()
|
||||
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
//"fang.com":"搜房",
|
||||
// def sourceMap = ["58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
|
||||
//"fang.com":"搜房",
|
||||
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
|
||||
|
||||
//,"城市房产":"城市房产"
|
||||
//"365淘房":"365淘房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房",
|
||||
def esSources = ["C21":"C21","365淘房":"365淘房"]//"吉屋":"吉屋","365淘房":"365淘房"
|
||||
//"365淘房":"365淘房","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","吉屋":"吉屋"
|
||||
def esSources = ["搜房":"搜房","我爱我家":"我爱我家","365淘房":"365淘房"]
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
// task.run()
|
||||
@@ -32,13 +29,13 @@ class SynchDistrictPrice {
|
||||
}
|
||||
|
||||
def run1(){
|
||||
|
||||
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
|
||||
def citys = CommonUtils.findAllCity()
|
||||
// citys.remove("上海")
|
||||
// citys.remove("广州")
|
||||
|
||||
esSources.each{esSource, mgSource->
|
||||
for(def city : citys){
|
||||
def rets = findAllEsData(city, esSource, mgSource)
|
||||
def rets = findAllEsData(city, esSource, mgSource, mdate)
|
||||
println city +" "+rets.size()
|
||||
synchData(city, rets, mgSource)
|
||||
}
|
||||
@@ -46,10 +43,10 @@ class SynchDistrictPrice {
|
||||
}
|
||||
|
||||
|
||||
def findAllEsData(city,esSource, mgSource){
|
||||
def findAllEsData(city,esSource, mgSource, mdate){
|
||||
def names = new HashSet()
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='2022-07-01' order by m_date", {d->
|
||||
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='${mdate}' order by m_date", {d->
|
||||
def s = d.city +"#"+d.region+"#"+d.name
|
||||
if(!names.contains(s)){
|
||||
names << s
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202206);
|
||||
def count2 = findTotalCount(source, 202207);
|
||||
def count1 = findTotalCount(source, 202207);
|
||||
def count2 = findTotalCount(source, 202208);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
+16
-1
@@ -50,6 +50,21 @@ public class AmapPoiDetail {
|
||||
private BizExtDTO bizExt;
|
||||
@com.alibaba.fastjson.annotation.JSONField(name = "photos")
|
||||
private List<PhotosDTO> photos;
|
||||
@com.alibaba.fastjson.annotation.JSONField(name = "business")
|
||||
private BusinessDTO business;
|
||||
|
||||
@lombok.NoArgsConstructor
|
||||
@lombok.Data
|
||||
public static class BusinessDTO {
|
||||
@com.alibaba.fastjson.annotation.JSONField(name = "business_area")
|
||||
private String businessArea;
|
||||
@com.alibaba.fastjson.annotation.JSONField(name = "keytag")
|
||||
private String keytag;
|
||||
@com.alibaba.fastjson.annotation.JSONField(name = "opentime_today")
|
||||
private String opentimeToday;
|
||||
@com.alibaba.fastjson.annotation.JSONField(name = "opentime_week")
|
||||
private String opentimeWeek;
|
||||
}
|
||||
|
||||
@lombok.NoArgsConstructor
|
||||
@lombok.Data
|
||||
@@ -64,7 +79,7 @@ public class AmapPoiDetail {
|
||||
@lombok.Data
|
||||
public static class PhotosDTO {
|
||||
@com.alibaba.fastjson.annotation.JSONField(name = "title")
|
||||
private List<?> title;
|
||||
private String title;
|
||||
@com.alibaba.fastjson.annotation.JSONField(name = "url")
|
||||
private String url;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user