This commit is contained in:
2022-09-13 10:25:14 +08:00
parent 36c932625b
commit 808cd95909
21 changed files with 1004 additions and 187 deletions
@@ -257,4 +257,9 @@ public class GeoUtils {
ret += (150.0 * Math.sin(x / 12.0 * pi) + 300.0 * Math.sin(x / 30.0 * pi)) * 2.0 / 3.0;
return ret;
}
public static void main(String[] args){
Map<String, Double> location = gps84ToGcj02(31.218024629574998,121.36562884455);
System.out.println(location.toString());
}
}
@@ -77,8 +77,16 @@ class CcbCommonUtils {
q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate)));
def dbcursor = GUIDING_PRICE_COLL.find(q)
def rets = []
def sets = new HashSet()
for(def d : dbcursor){
rets << d
def guiding_date = DateUtils.formatDateByPattern(d.guiding_date, 'yyyy-MM-dd')
def key = d.city +"##"+d.region+"##"+d.name+"##" + guiding_date
if(!sets.contains(key)){
rets << d
sets << key
}else{
println key
}
}
return rets
}
@@ -370,24 +370,25 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys
def citys = ["上海","广州","东莞"]//CcbCommonUtils.citys
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def publishTime = '2021-10-01'
def month = 202206
def month = 202207
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys) {
def rets = findPeGteThirdParty(city, 202207)
def rets = findPeGteThirdParty(city, 202208)
def total = 0
println city +" "+rets.size()
for(def m : rets){
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
def cityId = cityCodeDatas.get(city)
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
sqlInstance236.executeUpdate(sql)
}
}
}
def findPeGteThirdParty(cityName, yearmonth){
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>30"
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>25"
def rets = []
sqlInstance236.eachRow(sql,{r->
def map = [:]
@@ -0,0 +1,193 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import groovy.sql.Sql
import org.apache.commons.lang.StringUtils
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class FjwPricePkCcbPrice {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new FjwPricePkCcbPrice()
// task.run()
task.updatePrice()
}
def updatePrice(){
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\测试\\ccb.xlsx",{d->
index++
println index
def dbo = CommonDatautils.queryDistrictPrice(d['B'], d['C'], d['D'], DateUtils.parseFromTightByPattern('2022-09-04','yyyy-MM-dd'))
if(dbo){
d['G'] = dbo.future_value
}
})
}
def run(){
def ccbPrices = getCommunityBasePriceGj(202208)
println "ccb总量:"+ccbPrices.size()
def fjwPriceMap = CommonDatautils.queryAllDistrictPrice(DateUtils.parseFromTightByPattern('2022-09-04','yyyy-MM-dd'))
println "fjw总量:"+ccbPrices.size()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(ccbPrices.size())
def rets = new CopyOnWriteArrayList()
for(def i=0;i<ccbPrices.size();i++){
final def rMap = ccbPrices[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def row = []
def dist = CommonDatautils.findDistrictById(rMap.commid)
if(dist){
def priceMap = fjwPriceMap.get(dist.city+"##"+dist.region+"##"+dist.name)
def fjwPrice = null
def fjwCalPrice = null
if(priceMap){
fjwPrice = priceMap.avg_price
fjwCalPrice = priceMap.calc_value
}
row << rMap.commid
row << dist.city
row << dist.region
row << dist.name
row << fjwPrice
row << fjwCalPrice
if(priceMap && priceMap.value_type){
if(priceMap.value_type == 1){
row << "指导价"
}else if(priceMap.value_type == 2){
row << "自身样本"
}else if(priceMap.value_type == 3){
row << "第三方"
}else if(priceMap.value_type == 4){
row << "交易数据"
}else if(priceMap.value_type == 5){
row << "周边楼盘"
}else if(priceMap.value_type == 6){
row << "推导"
}else if(priceMap.value_type == 9){
row << "神经网络"
}else if(priceMap.value_type == 10){
row << "无方式"
}else{
row << ""
}
}else{
row << ""
}
row << rMap.avg_price
row << rMap.calc_type
if(fjwPrice && rMap.avg_price){
def precent = (fjwPrice - rMap.avg_price)*100/rMap.avg_price
row << DataHandleUtils.getDoubleRound(2, precent)
}else{
row << ""
}
rets << row
}else{
println rMap.commId + " 没找到小区"
}
println totalCount.get()+" "+ rMap.cityname +" "+rMap.commname
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await();
def headers = ["小区ID","城市","区域","小区名","房价网价格","房价网预期价格","房价网计算方式","建行价格","建行计算方式","价格差比(房价网价格-建行价格)*100/建行价格"]
batchCreateFile(rets, "D:\\数据\\建行\\测试\\房价网与建行均价比对.csv",headers,"GBK")
}
def getCommunityBasePriceGj(yearMonth) {
def sql = "select CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE from community_base_price_gj where year_month=${yearMonth} order by cityid";
def allRows = []
sqlInstance236.eachRow(sql, { r ->
def map = [:]
map.cityname = r.CITYNAME
map.distname = r.DISTNAME
map.commid = r.COMMID
map.commname = r.COMMNAME
map.year_month = r.YEAR_MONTH
map.calc_avg_price = r.CALC_AVG_PRICE
map.avg_price = r.AVG_PRICE
// 计算类型,1:人工估值 2:样本计算 3:神经网络
if(r.CALC_TYPE){
if(r.CALC_TYPE == 1){
map.calc_type = "人工估值"
}else if(r.CALC_TYPE == 2){
map.calc_type = "样本计算"
}else if(r.CALC_TYPE == 3){
map.calc_type = "神经网络"
}
}
allRows << map
});
return allRows
}
def batchCreateFile(ret, fileName, headers, charset){
OutputStream out = new FileOutputStream(fileName);
def sb = new StringBuffer();
// sb.append(headers.join("@|@"))
// sb.append(System.getProperty("line.separator"));
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
def index = 0
for(def row : ret){
index++
sb.append(StringUtils.join(row.toArray(), "@|@"))
sb.append(System.getProperty("line.separator"));
println index
if(index%10000==0){
println index+"开始写入CSV "+new Date().toLocaleString()
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
println index+"结束写入CSV "+new Date().toLocaleString()
}
}
if(sb.length()>0){
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
}
out.close();
}
}
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2022-07-22", "2022-08-21")
def prices = CcbCommonUtils.findPersonPrice(city, "2022-07-22", "2022-08-24")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -30,8 +30,8 @@ import java.util.regex.*;
//def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
//def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
//def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -197,6 +197,79 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
//println "共查到:"+index
def index = 0
//def commIdThirdMap = findAllThird(202207, sqlInstance236);
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
//cityCodeDatas.each{city,cityId->
//
// deleteThirdByCityId(cityId, 202207, sqlInstance236)
// println city + "完成"
//}
def lines = org.apache.commons.io.FileUtils.readLines(new File("D:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat"))
for(line in lines) {
def arrs = line.toString().split("\\|@\\|")
if(arrs){
def sql = "delete from guided_price_gj where id='${arrs[0]}'"
sqlInstance236.executeUpdate(sql)
}
}
def deleteThirdByCityId(def cityId,def price_date, def sqlInstance236){
def sql = "delete from bigdata_third_party_price_gj where cityid='${cityId}' and price_date=${price_date}"
sqlInstance236.executeUpdate(sql)
}
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\删除小区2022-8-23.xlsx", { d ->
// if (d['C']) {
// index ++T
// def cityId = cityCodeDatas.get(d['A'])
// def sources = findThirdSource(cityId, d['C'], 202207, sqlInstance236)
// if(sources.size()>1){
// if(sources.contains("安居客") && sources.contains("58同城")){
// sources.remove("58同城")
// }else if(sources.contains("链家") && sources.contains("贝壳")){
// sources.remove("贝壳")
// }
// }
// if(sources.size()==1){
// println index +" 删除 "+ d['C']+" ${d['A']}:"+ cityId
// deleteThirdParty(cityId, d['C'], 202207, sqlInstance236)
// }
// }
//},0)
def findAllThird(price_date, def sqlInstance236){
def sql = "select commid, data_source from bigdata_third_party_price_gj where price_date=${price_date}"
def map = [:]
sqlInstance236.eachRow(sql,{r->
def sources = map.get(r.commid)
if(!sources){
sources = []
map.put(r.commid, sources)
}
sources << r.data_source
});
return map
}
def deleteThirdParty(def cityId, def commid, def price_date, def sqlInstance236){
def sql = "delete from bigdata_third_party_price_gj where cityid='${cityId}' and commid='${commid}' and price_date=${price_date}"
sqlInstance236.executeUpdate(sql)
}
def findThirdSource(def cityId, def commid, def price_date, def sqlInstance236){
def sql = "select data_source from bigdata_third_party_price_gj where cityid='${cityId}' and commid='${commid}' and price_date=${price_date}"
def sources = []
sqlInstance236.eachRow(sql,{r->
sources << r.data_source
});
return sources
}
def findAvmData(sqlInstance, def region, def name, def area, def price, def dealdate){
def sql = "select count(*) as count from avm_evaluation_item_gj where year_month=202106 and data_type='成交' and cityname='上海' and distname='${region}' and commname='${name}' and buildarea=${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd') and target_avg_price=${price}"
def count = 0
@@ -326,52 +399,52 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
//
//},0)
ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d->
if(d['A']){
// def region = d['B']
// def newRegion = RegionData.formatRegion(d['A'], region)
// if(newRegion) {
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def udata = new BasicDBObject()
udata.put("name_cat", 11)
udata.put("status", 1)
udata.put("m_date",new Date())
//ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d->
// if(d['A']){
//// def region = d['B']
//// def newRegion = RegionData.formatRegion(d['A'], region)
//// if(newRegion) {
//// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// def udata = new BasicDBObject()
// udata.put("name_cat", 11)
// udata.put("status", 1)
// udata.put("m_date",new Date())
//
// def q = new BasicDBObject()
// q.put("_id", dist._id)
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// }else{
// println d
// }
//// }
// }
//},0)
def q = new BasicDBObject()
q.put("_id", dist._id)
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}else{
println d
}
//ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d->
// if(d['A']){
//// def region = d['B']
//// def newRegion = RegionData.formatRegion(d['A'], region)
//// if(newRegion) {
//// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// def udata = new BasicDBObject()
// udata.put("name_cat", 11)
// udata.put("status", 1)
// udata.put("m_date",new Date())
//
// def q = new BasicDBObject()
// q.put("_id", dist._id)
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// }else{
// println d
// }
}
},0)
ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d->
if(d['A']){
// def region = d['B']
// def newRegion = RegionData.formatRegion(d['A'], region)
// if(newRegion) {
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def udata = new BasicDBObject()
udata.put("name_cat", 11)
udata.put("status", 1)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", dist._id)
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}else{
println d
}
// }
}
},0)
//// }
// }
//},0)
@@ -2049,6 +2049,20 @@ class CommonDatautils {
}
public static def queryAllDistrictPrice(startDate){
DBObject queryCondition = new BasicDBObject()
queryCondition.put('weekend', startDate);
DBCursor cur = DIST_STAT_COLL.find(queryCondition)
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def rMap = [:];
while(cur.hasNext()) {
DBObject dbo = cur.next()
rMap.put(dbo.get("city") + "##" + dbo.get("region")+"##"+ dbo.get("district_name"), dbo)
}
return rMap;
}
public static def queryDistrictPrices(city, startDate){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
@@ -0,0 +1,159 @@
package com.fangjia.script.crawl
import com.alibaba.fastjson.JSONArray
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.StringUtils
class SchoolLocation {
DBCollection school_parse_coll = MongoUtils.getCollectionByName("fangjia_craw", "school", "192.168.0.235", 27018)
static main(args) {
def task = new SchoolLocation()
task.run()
}
def run(){
def query = new BasicDBObject()
query.put("province","江苏")
//query.put("lat",null)
query.put("lat",new BasicDBObject('$exists',false))
def cursor = school_parse_coll.find(query)
def index = 0
for(def d : cursor){
index++
def datas = getSchoolLatAndLng(d.city, d.name, d.address, false)
if(datas){
for(def ret : datas){
def addr = d.address
if(d.address.contains(d.city+"市")){
def cityStr = d.city+"市"
addr = d.address.substring(d.address.indexOf(cityStr)+cityStr.size())
}
if(ret.name == d.name || (d.address.contains("号") && (ret.address.contains(addr) || addr.contains(ret.address)))){
def udata = new BasicDBObject()
udata.put("lat", ret.lat.toString().toDouble())
udata.put("lng", ret.lng.toString().toDouble())
udata.put("status", 0)
school_parse_coll.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
println index +" "+udata
break;
}
}
}
}
}
def getSchoolLatAndLng(city, name, address, isFullMatch){
if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){
return null;
}
address = address.trim().replace(" ","").replace("?","").replace(" ","")
String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=学校&city=" + city + "&output=json&offset=5&page=1&key=4e9592208336433ee1b7aa4d1d434418&extensions=all"
List<com.alibaba.fastjson.JSONObject> poiList = null;
com.alibaba.fastjson.JSONObject result = null;
try {
HttpRequest request = new HttpRequest();
Map<String, String> params = new HashMap<String, String>();
String response = request.get(url, params, "UTF-8");
com.alibaba.fastjson.JSONObject jsonResponse = com.alibaba.fastjson.JSONObject.parseObject(response)
if (jsonResponse != null) {
JSONArray ja = jsonResponse.getJSONArray("pois");
if(ja != null && ja.size() > 0) {
poiList = new ArrayList<>();
for(int i=0;i<ja.size();i++){
com.alibaba.fastjson.JSONObject jo_item = ja.getJSONObject(i);
if(jo_item.containsKey("cityname") && !jo_item.getString("cityname").contains(city)){
continue;
}
String item_address = jo_item.containsKey("address") ? jo_item.getString("address") : null;
String item_name = jo_item.containsKey("name") ? jo_item.getString("name") : null;
boolean getedData = false;
if(isFullMatch == true){
if(StringUtils.equalsIgnoreCase(item_address, address) || StringUtils.equalsIgnoreCase(item_name, address)){
getedData = true;
}
}else{
if(StringUtils.equalsIgnoreCase(item_address, address) || StringUtils.contains(item_address, address)
|| StringUtils.equalsIgnoreCase(item_name, address) || StringUtils.contains(item_name, address) || StringUtils.contains(address, item_name)){
getedData = true;
}else{
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(item_name, address);
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(item_address, address);
if(score1 || score2){
getedData = true;
}
if(!getedData){
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(address, " ")
def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(item_name, " ")
if(py1 == py2){
getedData = true;
}
}
}
}
def type = jo_item.getString("type");
if(getedData == true){
com.alibaba.fastjson.JSONObject jo = new com.alibaba.fastjson.JSONObject();
if(jo_item.containsKey("id")){
jo.put("id", jo_item.getString("id"))
}
if(jo_item.containsKey("name")){
jo.put("name", jo_item.getString("name"))
}
if(jo_item.containsKey("type")){
jo.put("type", type)
}
if(jo_item.containsKey("address")){
jo.put("address", jo_item.getString("address").replace("[]",""))
}
if(jo_item.containsKey("pname")){
jo.put("pname", jo_item.getString("pname"))
}
if(jo_item.containsKey("cityname")){
jo.put("cityname", jo_item.getString("cityname"))
}
if(jo_item.containsKey("adname")){
jo.put("adname", jo_item.getString("adname"))
}
if(jo_item.containsKey("location")){
def arrs = jo_item.getString("location").split(",")
jo.put("lat", arrs[1])
jo.put("lng", arrs[0])
}
if(type && type.contains("出入口")){
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+name+"-"+address+"】-${type}"
}else{
println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+name+"-"+address+"】"
poiList.add(jo);
}
}else{
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
}
}
}else{
println "X 高德地图 ( "+city+"-"+address+" )无结果!"
}
}
} catch (Exception e) {
e.printStackTrace();
}
return poiList;
}
}
@@ -136,7 +136,7 @@ class FzDataResultStat {
def regionPriceTrendRun(){
def city = "福州"
def add = -6;
def add = -9;
def months = []
while(add<0){
months << DateUtils.addMonth(new Date(), add)
@@ -149,7 +149,7 @@ class FzDataResultStat {
def exportRegion(city, months){
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐","闽侯","福清","连江","闽清","罗源","永泰","平潭"];
def datas =[]
def header = ["城市", "区域"]
for(def m : months){
@@ -325,8 +325,8 @@ class HouseExportBatch extends BaseBatch{
// int endDate = 20191201
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyyMMdd")
// def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd");
def endDate = "20220710"
def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd");
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
// QueryBuilder room_range = QueryBuilders.rangeQuery("room").from(1).to(5).includeLower(true).includeUpper(false);
@@ -110,6 +110,7 @@ class FangjiaClientUtils {
public static def saveToEsPaimai(def data){
String jsonStr = com.alibaba.fastjson.JSONObject.toJSONString(data)
HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/paiMai", jsonStr)
// HttpUtils.postForJson("http://localhost:8084/FangJiaDataService/paiMai", jsonStr)
}
@@ -376,13 +376,13 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
// if(e.text().contains("福州")){
if(e.text().contains("福州")){
def cityMap = [:]
cityMap.city = e.text()
cityMap.url = "https:"+href
list << cityMap
//cityMap.put(e.text(), "https:"+href)
// }
// cityMap.put(e.text(), "https:"+href)
}
}
}
return list
@@ -7,6 +7,8 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.StringUtils
import org.apache.commons.lang.math.NumberUtils
import java.util.Map.Entry;
import java.util.regex.Pattern
@@ -175,22 +177,23 @@ class ExportPaimai {
def run(){
def city = "苏州"
def region = "姑苏"
def city = "南京"
def region = null
def datas = findPaimaiData(city,region)
println "总量:"+datas.size()
def rows = []
def header = ["标地物地址","省","城市","区域","地址","标题提取地址","楼栋号","单元","室号","楼盘名称","建筑面积","竣工日期","房屋用途","总楼层",
"拍卖状态","是否成交","拍卖阶段","拍卖方式","成交价","拍卖开始时间","拍卖结束时间","起拍价","保证金","加价幅度","评估价","市场价",
"变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数","URL"]
"变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数", "折价率","映射类型","拍卖类型",
"映射资产类型","映射拍卖状态","URL"]
rows << header
// def set = new HashSet()
def set = new HashSet()
def index = 0
for(def d : datas){
// if(!set.contains(d.title)){
if(!set.contains(d.title)){
index++
println index
// set << d.title
set << d.title
def row = []
row << d.title.replace(",",",")
row << d.province
@@ -226,14 +229,19 @@ class ExportPaimai {
row << d.lookCount
row << d.callCount
row << d.delayCount
row << d.url
row << d.rateDiscount//折价率
row << d.mappingEstateType;//映射类型
row << d.zcType;//拍卖类型:资产拍卖 司法拍卖
row << d.mappingAgencyName//映射资产类型
row << d.mappingStatus//映射拍卖状态
row << d.url
rows << row
// }else{
// println d.title
// }
}else{
println d.title +" "+d.url
}
}
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\京东拍卖(${city}-姑苏-撤回中止).xlsx",rows,"数据")
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\南京法拍.xlsx",rows,"数据")
// batchCreateFile(rows, "D:\\数据\\重庆农商行\\法拍\\淘宝拍卖(新).csv",null,"GBK")
println "生成完成........"
@@ -452,12 +460,12 @@ class ExportPaimai {
if(region){
pmquery.put("region",region)
}
pmquery.put("status", new BasicDBObject('$in', ["已撤回","已暂缓"]))
// pmquery.put("status", new BasicDBObject('$in', ["已撤回","已暂缓"]))
// pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-07-01")))
// Pattern pattern = Pattern.compile("rmfysszc.gov.cn", Pattern.MULTILINE);
Pattern pattern = Pattern.compile("jd.com", Pattern.MULTILINE);
// Pattern pattern = Pattern.compile("jd.com", Pattern.MULTILINE);
// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
// pmquery.put("url", new BasicDBObject('$regex', pattern))
def dbcursor = paimai43_COLL.find(pmquery).batchSize(2000)
@@ -465,6 +473,28 @@ class ExportPaimai {
for(def db : dbcursor){
rets << db
}
// Collections.sort(rets, new Comparator<BasicDBObject>() {
// @Override
// int compare(BasicDBObject o1, BasicDBObject o2) {
// if(o1.finishDate && o2.finishDate){
// return -o1.finishDate.compareTo(o2.finishDate)
// }else{
// return 0
// }
//
// }
// })
rets.sort {a,b->
if(a.finishDate && b.finishDate){
def v1 = a.finishDate
def v2 = b.finishDate
return v1.compareTo(v2)
}else{
return 0
}
}
rets = rets.reverse()
return rets
}
}
@@ -129,7 +129,7 @@ public class LianjiaCrawlByChrome {
def standardRegion(){
def q = new BasicDBObject()
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")))
q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) )
def dbcursor = DEAL_PRICE_COLL.find(q)
for(def d : dbcursor){
@@ -152,8 +152,9 @@ public class LianjiaCrawlByChrome {
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
lastMonth = lastMonth +"-01"
def q = new BasicDBObject()
q.put("city", "福州")
q.put("region", null)
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-05-01","yyyy-MM-dd")))
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")))
// q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
def dbcursor = DEAL_PRICE_COLL.find(q)
def keys = []
@@ -187,8 +188,19 @@ public class LianjiaCrawlByChrome {
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
cityMap.each{city, cityUrl->
//def detailUrls = getPageUrls(driver, cityUrl)
def detailUrls = getPageDeail(driver, city, cityUrl)
println cityUrl +" 详情链接:"+detailUrls.size()
driver.get(cityUrl+"chengjiao/")
def doc = Jsoup.parse(driver.getPageSource());
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
if(regions){
for(def r : regions){
def regionText = r.text()
def href = cityUrl.toString().substring(0, cityUrl.toString().length()-1) + r.attr("href")
println "开始抓取【${city}】【${regionText}】【${href}】"
def detailUrls = getPageDeail(driver, city, href, regionText)
println cityUrl +" 详情链接:"+detailUrls.size()
}
}
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
}
}
@@ -224,12 +236,12 @@ public class LianjiaCrawlByChrome {
}
def getPageDeail(driver, city, cityUrl){
def getPageDeail(driver, city, cityUrl, regionText){
def stop = false
def page = 1
def detailUrls = []
while(!stop){
def url = cityUrl + "chengjiao/pg${page}/"
def url = cityUrl + "pg${page}/"
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def elements = JQueryUtils.select(".listContent li", doc)
@@ -241,7 +253,7 @@ public class LianjiaCrawlByChrome {
def endDate = DateUtils.parseFromTightByPattern("2022-06-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
parse(city, e, regionText)
}else{
stop = true;
break;
@@ -259,7 +271,7 @@ public class LianjiaCrawlByChrome {
return detailUrls
}
def parse(city,e){
def parse(city,e, regionText){
try {
def title = e.select(".title a").text()
if (title) {
@@ -267,7 +279,11 @@ public class LianjiaCrawlByChrome {
def arrs = title.split(" ")
detail.city = city
detail.district_name = arrs[0]
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
if(regionText){
detail.region = regionText
}else{
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
}
if(detail.region){
detail.region = RegionData.formatRegion(city, detail.region)
}
@@ -275,50 +291,61 @@ public class LianjiaCrawlByChrome {
def roomStr = arrs[1]
if (roomStr) {
if (roomStr.indexOf("室") > 0) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
def room = roomStr.substring(0, roomStr.indexOf("室"))
if(room && room.isNumber()){
detail.room = room.toInteger()
}
}
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger()
def hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
if(hall && hall.isNumber()){
detail.hall = hall.toInteger()
}
}
}
def areaStr = arrs[2].replace("平米", "")
if (areaStr) {
detail.area = areaStr.toDouble()
}
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = price.toInteger() * 10000
}
if (detail.total_price && detail.area) {
detail.avg_price = (int) (detail.total_price / detail.area)
}
def tradeStr = e.select(".dealDate").text()
if (tradeStr) {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "链家在线"
detail.url = e.select(".title a").attr("href")
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
if("车位"!=roomStr){
if(arrs && arrs.size()>=3){
def areaStr = arrs[2].replace("平米", "")
if (areaStr) {
detail.area = areaStr.toDouble()
}
}
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = price.toInteger() * 10000
}
if (detail.total_price && detail.area) {
detail.avg_price = (int) (detail.total_price / detail.area)
}
def tradeStr = e.select(".dealDate").text()
if (tradeStr) {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "链家在线"
detail.url = e.select(".title a").attr("href")
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
}
}
}
}catch(Exception ex){
@@ -446,9 +473,9 @@ public class LianjiaCrawlByChrome {
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
// if (!href.contains("fang.lianjia") && citys.contains(e.text())){
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
cityMap.put(e.text(), href)
// }
}
}
}
return cityMap
@@ -10,6 +10,7 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import com.rabbitmq.client.impl.AMQImpl
import java.text.SimpleDateFormat
import java.util.concurrent.CountDownLatch
@@ -33,6 +34,7 @@ class PaimaiDataSend {
static main(args) {
def task = new PaimaiDataSend()
// task.jdRunByMongo()
// task.jdRun()
// task.aliZcRun()
task.aliZcRun2()
@@ -64,7 +66,8 @@ class PaimaiDataSend {
def rmfysszcRun(){
def index = 0
def q = new BasicDBObject()
// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/76602.shtml")
// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市"]))
q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/88061.shtml")
def dbcursor = rmfysszc_coll.find(q).batchSize(3000)
for(def d : dbcursor){
index++
@@ -95,11 +98,22 @@ class PaimaiDataSend {
}
}
def jdRun(){
def jdRunByMongo(){
def pm_q = new BasicDBObject()
pm_q.put("city","宿迁")
pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
pm_q.put("mappingStage", null)
def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000)
println dbcursor.count()
for(def d : dbcursor){
sendJd(d.url)
}
}
def sendJd(url){
def index = 0
def q = new BasicDBObject()
q.put("city","宿迁市")
// q.put("url","https://paimai.jd.com/108245329")
q.put("url", url)
def dbcursor = jd_coll.find(q).batchSize(3000)
for(def d : dbcursor){
index++
@@ -110,6 +124,45 @@ class PaimaiDataSend {
JSON result = new JSONObject()
result.put("content", d.desc_content)
result.put("region", d.region)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("bid_content", bid_content.toString())
result.put("sub_count", d.sub_count)
def data_json = net.sf.json.JSONObject.fromObject(d.json)
result.put("data_json", data_json)
result.put("content_json", d.content)
result.put("item_notice_content", d.item_notice_content)
result.put("notice_detail_content", d.notice_detail_content.content)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 3)
udata.put("m_date", new Date())
jd_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
}
}
def jdRun(){
def index = 0
def q = new BasicDBObject()
q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市"]))
// q.put("parser_status", new BasicDBObject('$ne',3))
q.put("url","https://paimai.jd.com/115669796")
def dbcursor = jd_coll.find(q).batchSize(3000)
for(def d : dbcursor){
index++
println "jd:"+index
JSON json = new JSONObject()
json.put("project", "paimai_jd")
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.desc_content)
result.put("region", d.region)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("bid_content", bid_content.toString())
@@ -134,17 +187,19 @@ class PaimaiDataSend {
def aliZcRun(){
def pm_q = new BasicDBObject()
pm_q.put("city","苏州")
pm_q.put("region","姑苏")
// pm_q.put("url","https://sf-item.taobao.com/sf_item/596573117068.htm")
pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
// pm_q.put("city","苏州")
// pm_q.put("region","姑苏")
pm_q.put("url","https://sf-item.taobao.com/sf_item/659117468505.htm")
// pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
// pm_q.put("url", new BasicDBObject('$regex', pattern))
// pm_q.put("status", null)
// pm_q.put("mappingEstateType", null)
// pm_q.put("area", new BasicDBObject('$lt', 30))
def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000)
println dbcursor.count()
for(def d : dbcursor){
def flag = sendAliZc(d.url)
// def flag = sendAliZc(d.url)
def flag = sendAli(d.url)
if(!flag){
println "不存在:"+d.url
}
@@ -194,15 +249,59 @@ class PaimaiDataSend {
return flag
}
def sendAli(def url){
def index = 0
def q = new BasicDBObject()
q.put("url",url)
def dbcursor = ali_fp_coll.find(q).batchSize(5000)
def flag = false
for(def d : dbcursor){
index++
flag = true
println "ali-fp: "+index
JSON json = new JSONObject()
json.put("project", "paimai_taobao")
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.content)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("trades", bid_content)
result.put("province", d.province)
result.put("city", d.city)
result.put("region", d.region)
result.put("sub_count", d.sub_count)
def data_json = net.sf.json.JSONObject.fromObject(d.json)
result.put("data_json", data_json)
result.put("desc_content", d.desc_content)
result.put("item_notice_content", d.item_notice_content)
result.put("notice_detail_content", d.notice_detail_content)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 1)
udata.put("m_date", new Date())
ali_fp_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
}
return flag
}
def aliFpRun(){
def index = 0
def q = new BasicDBObject()
// q.put("city","宿迁")
q.put("url","https://sf-item.taobao.com/sf_item/665877683620.htm")
q.put("city",new BasicDBObject('$in', ["南京","宿迁"]))
// q.put("parser_status", new BasicDBObject('$ne',2))
// q.put("url","https://sf-item.taobao.com/sf_item/631662596036.htm")
//https://sf-item.taobao.com/sf_item/657131607486.htm
// q.put("parser_status", new BasicDBObject('$exists',false))
// q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01")))
def dbcursor = ali_fp_coll.find(q).batchSize(2000)
// q.put("area", new BasicDBObject('$lt', 30))
def dbcursor = ali_fp_coll.find(q).batchSize(5000)
def rets = []
@@ -229,10 +328,20 @@ class PaimaiDataSend {
json.put("result", result)
rets << json
if(rets.size()==10000){
println "共:"+rets.size()
send(rets)
rets.clear()
}
}
println "共:"+rets.size()
if(rets.size()>0){
println "共:"+rets.size()
send(rets)
}
}
def send(def rets){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
@@ -245,7 +354,7 @@ class PaimaiDataSend {
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 1)
udata.put("parser_status", 2)
udata.put("m_date", new Date())
ali_fp_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
println "send: "+totalCount.get()
@@ -264,13 +373,18 @@ class PaimaiDataSend {
def aliZcRun2(){
def index = 0
def q = new BasicDBObject()
q.put("city","苏州")
q.put("city",new BasicDBObject('$in', ["上海"]))
// q.put("parser_status", new BasicDBObject('$ne',1))
// q.put("region","姑苏")
// q.put("url","https://zc-item.taobao.com/auction/636884262340.htm")
def dbcursor = ali_zc_coll.find(q).batchSize(1000)
// q.put("url","https://sf-item.taobao.com/sf_item/586415617368.htm")
def dbcursor = ali_zc_coll.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
index++
if(!d.url){
println d._id.toString()
continue
}
println "ali-zc: "+index
JSON json = new JSONObject()
json.put("project", "paimai_taobao_zc")
@@ -292,35 +406,47 @@ class PaimaiDataSend {
result.put("region", d.region)
json.put("result", result)
rets << result
rets << json
if(rets.size()==10000){
println "共:"+rets.size()
send(rets)
rets.clear()
}
}
println "共:"+rets.size()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def json = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)//result2parse2dev
def udata = new BasicDBObject()
udata.put("parser_status", 4)
udata.put("m_date", new Date())
ali_zc_coll.update(new BasicDBObject("_id", json.get("_id")), new BasicDBObject('$set', udata))
println "send: "+totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
if(rets.size()>0){
println "共:"+rets.size()
send(rets)
}
latch.await()
// println "共:"+rets.size()
//
// def totalCount = new AtomicLong()
// final CountDownLatch latch = new CountDownLatch(rets.size())
// for(def i=0;i<rets.size();i++){
// final def json = rets[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
// try{
// totalCount.addAndGet(1)
// byte[] packedMsg = MessageUtils.pack(json)
// CommonUtils.sendMessage("result2parse2dev", packedMsg)
//
//
// def udata = new BasicDBObject()
// udata.put("parser_status", 1)
// udata.put("m_date", new Date())
// ali_zc_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
// println "send: "+totalCount.get()
// } catch (Exception e) {
// e.printStackTrace()
// } finally {
// latch.countDown()
// }
// }
// });
// }
// latch.await()
}
def paimaiSendToEs(){
@@ -349,8 +475,9 @@ class PaimaiDataSend {
def getPaimaiDatas(){
def q = new BasicDBObject()
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-07-01")))
def dbcursor = paimai43_COLL.find(q).batchSize(1000)
q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
def dbcursor = paimai43_COLL.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
if(d.city){
@@ -367,7 +494,7 @@ class PaimaiDataSend {
o.depositPrice = d.depositPrice
o.enrollCount = d.enrollCount
o.estateType = d.estateType
o.evaluatePrice = d.evaluatePrice
o.evaluatePrice = d.evalatePrice
o.finishDate = d.finishDate
o.height = d.height
o.houseNum = d.houseNum
@@ -396,6 +523,7 @@ class PaimaiDataSend {
o.tel = d.tel
o.title = d.title
o.tradePrice = d.tradePrice
o.tradeAvgPrice = d.tradeAvgPrice
o.type = d.type
o.unitNum = d.unitNum
o.url = d.url
@@ -406,6 +534,13 @@ class PaimaiDataSend {
}else if(d.url.contains("rmfysszc.gov.cn")){
o.source = "人民法院讼诉网"
}
o.rateDiscount = d.rateDiscount//折价率
o.mappingEstateType = d.mappingEstateType;//映射类型
o.paiMaiType = d.paiMaiType;//拍卖类型:资产拍卖 司法拍卖
o.mappingAgencyName = d.mappingAgencyName//映射资产类型
o.mappingStatus = d.mappingStatus//映射拍卖状态
o.mappingStage = d.mappingStage
rets << o
}
}
@@ -28,11 +28,29 @@ class CityStat {
def task = new CityStat()
// task.run()
// task.run1();
task.run3();
// task.run3();
// task.run4()
// task.run6()
// task.run7()
// task.run8()
task.dealPriceStat()
}
def dealPriceStat(){
def citys = ["上海","北京","广州","合肥","南京","深圳","武汉","郑州","重庆","福州","厦门","青岛","太原","长春","苏州","南宁","沈阳","天津","西安","长沙","大连","南昌","东莞","佛山","济南","南通","镇江","宁波","烟台","哈尔滨"]
def rets = []
rets << ["城市","2021-1月总量"]
def startDate = DateUtils.parseFromTightByPattern('2021-12-01','yyyy-MM-dd')
def endDate = DateUtils.parseFromTightByPattern('2022-01-01','yyyy-MM-dd')
for(def c : citys){
def ret = []
ret << c
ret << findDealPriceCount(c, startDate, endDate)
rets << ret
println ret
}
ExcelUtils.write("D:\\数据\\2021成交数据.xlsx", rets, "月统计")
}
// def run8(){
@@ -464,6 +482,14 @@ class CityStat {
return count;
}
def findDealPriceCount(def city, def startDate, def endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt', endDate))
def count = DEAL_PRICE_COLL.count(q)
return count;
}
def calTradeDistCount(def city, def tradeDate, def districts){
def q = new BasicDBObject()
q.put("city", city)
@@ -8,6 +8,7 @@ import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.Mongo
@@ -17,8 +18,19 @@ import org.bson.types.ObjectId
import java.text.SimpleDateFormat
def dist = CommonDatautils.findDistrictById("62ce233a3705890001dfa164")
DBUtils.seaweed.save(dist)
DBCollection coll = MongoUtils.getCollectionByName("seeds_db", "city_seeds", "192.168.0.235", 27018)
def q = new BasicDBObject()
q.put("root", "anjuke.com")
def dbcursor = coll.find(q)
def map = [:]
for(def d : dbcursor){
map.put(d.city, d.city_short)
}
println map.size()
println map
//def dist = CommonDatautils.findDistrictById("62ce233a3705890001dfa164")
//DBUtils.seaweed.save(dist)
//DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId("62ce233a3705890001dfa164")), new BasicDBObject('$set', dist))
//SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
//Date date = new Date(1617211269868);
@@ -0,0 +1,136 @@
package com.fangjia.script.gj.test
import com.alibaba.fastjson.JSON
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.util.MessageUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
DBCollection school_coll = MongoUtils.getCollectionByName("crawl_tmp", "school_ruyile_web", "192.168.0.235", 27018)
DBCollection sxue_school_coll = MongoUtils.getCollectionByName("crawl_tmp", "school_5sxue_web", "192.168.0.235", 27018)
DBCollection school_parse_coll = MongoUtils.getCollectionByName("fangjia_craw", "school", "192.168.0.235", 27018)
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
//while (true){
// def q = new BasicDBObject()
// q.put("province",new BasicDBObject('$in', ["江苏","江苏省"]))
// q.put("status",new BasicDBObject('$ne', 1))
// def dbcursor = school_coll.find(q).limit(1000)
// def rets = []
// for(def d : dbcursor){
// JSON json = new JSONObject()
// json.put("project", d.project)
// json.put("url", d.url)
//
// JSON result = new JSONObject()
// result.put("content", d.content)
// def jsonStr = d.map_info
//
// jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1)
// if(jsonStr.contains("\n")){
// jsonStr = jsonStr.replaceAll('\\n','')
// }
// result.put("data_json", net.sf.json.JSONObject.fromObject(jsonStr))
//
//
// result.put("type", d.type)
// json.put("result", result)
//
//
// rets << json
//// if(rets.size()==5000){
//// println "共:"+rets.size()
//// send(rets, fixedThreadPool, school_coll)
//// rets.clear()
//// }
// }
//
// if(rets.size()>0){
// println "共:"+rets.size()
// send(rets, fixedThreadPool, school_coll)
// rets.clear()
// }
//}
def send(def rets, def fixedThreadPool, def school_coll){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def json = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
school_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
println "send: "+totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def query = new BasicDBObject()
query.put("source","搜学网")
//query.put("url","http://xuexiao.51sxue.com/detail/id_26799.html")
query.put("lat",null)
def cursor = school_parse_coll.find(query)
for(def o : cursor){
def q = new BasicDBObject()
// q.put("province","江苏省")
q.put("url",o.url)
def dbcursor = sxue_school_coll.find(q)
def index = 0
for(def d : dbcursor){
index++
println index
JSON json = new JSONObject()
json.put("project", d.project)
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.content)
result.put("name", d.name)
result.put("list_item", d.list_item)
def jsonStr = d.map_info
jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1)
if(jsonStr.contains("\n")){
jsonStr = jsonStr.replaceAll('\\n','')
}
result.put("json_data", jsonStr)
result.put("type", d.type)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
sxue_school_coll.update(new BasicDBObject("url", d.get("url")), new BasicDBObject('$set', udata))
}
}
@@ -40,11 +40,11 @@ class DataStat {
def run(){
def source = "365淘房"//
def citys = findCrawlCity(source, 202206);
def source = "我爱我家"//
def citys = findCrawlCity(source, 202207);
for(def city : citys){
def count1 = findCount(source, city, 202206);
def count2 = findCount(source, city, 202207);
def count1 = findCount(source, city, 202207);
def count2 = findCount(source, city, 202208);
println city +" "+count1+" "+count2
// println count1
// println "--- ----"
@@ -16,15 +16,12 @@ class SynchDistrictPrice {
def static final SHORT_DATE = Utils.getLastMonth()
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
//"fang.com":"搜房",
// def sourceMap = ["58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
//"fang.com":"搜房",
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
//,"城市房产":"城市房产"
//"365淘房":"365淘房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房",
def esSources = ["C21":"C21","365淘房":"365淘房"]//"吉屋":"吉屋","365淘房":"365淘房"
//"365淘房":"365淘房","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","吉屋":"吉屋"
def esSources = ["搜房":"搜房","我爱我家":"我爱我家","365淘房":"365淘房"]
static main(args) {
def task = new SynchDistrictPrice()
// task.run()
@@ -32,13 +29,13 @@ class SynchDistrictPrice {
}
def run1(){
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def citys = CommonUtils.findAllCity()
// citys.remove("上海")
// citys.remove("广州")
esSources.each{esSource, mgSource->
for(def city : citys){
def rets = findAllEsData(city, esSource, mgSource)
def rets = findAllEsData(city, esSource, mgSource, mdate)
println city +" "+rets.size()
synchData(city, rets, mgSource)
}
@@ -46,10 +43,10 @@ class SynchDistrictPrice {
}
def findAllEsData(city,esSource, mgSource){
def findAllEsData(city,esSource, mgSource, mdate){
def names = new HashSet()
def rets = []
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='2022-07-01' order by m_date", {d->
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='${mdate}' order by m_date", {d->
def s = d.city +"#"+d.region+"#"+d.name
if(!names.contains(s)){
names << s
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
def count1 = findTotalCount(source, 202206);
def count2 = findTotalCount(source, 202207);
def count1 = findTotalCount(source, 202207);
def count2 = findTotalCount(source, 202208);
println source +" "+count1 +" "+count2
}
}