Merge branch 'script_gj' into 'master'
Script gj See merge request !259
This commit is contained in:
+1
-1
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211218, 20220119);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220118, 20220219);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||
//测试通过匹配接口配房源效果
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||
|
||||
+340
@@ -0,0 +1,340 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
class ZsNoExistsToDB {
|
||||
|
||||
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new ZsNoExistsToDB()
|
||||
task.run()
|
||||
// task.addDistrictToDB()
|
||||
}
|
||||
|
||||
def addDistrictToDB(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\浙商\\不存在小区(结果V1).xlsx",0,1)
|
||||
def keys = []
|
||||
def addIndex = 0
|
||||
for(def d : datas){
|
||||
if(d['J'] && d['J']=="新增"){
|
||||
def city = d['A']
|
||||
def region = d['B']
|
||||
def name = d['F'].toString().replace("·","")
|
||||
def address = d['G']
|
||||
def dist = CommonDatautils.findDistrict(city,region,name,"district")
|
||||
if(!dist){
|
||||
dist = CommonDatautils.findDistrict(city,region,d['C'],"district")
|
||||
}
|
||||
if(!dist){
|
||||
keys << city+"##"+region+"##"+name
|
||||
def alias = []
|
||||
def str = d['H']
|
||||
if(str){
|
||||
def arrs = str.split(",")
|
||||
for(def s : arrs){
|
||||
if(!alias.contains(s)){
|
||||
alias << s
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(address){
|
||||
address = AddressUtil.formatAddress(city, address)
|
||||
}
|
||||
def lng = d['M']
|
||||
def lat = d['L']
|
||||
def lng2 = null
|
||||
def lat2 = null
|
||||
long lng_new = 0
|
||||
long lat_new = 0
|
||||
long lng2_new = 0
|
||||
long lat2_new = 0
|
||||
if(lng){
|
||||
lng = lng.toString().toDouble()
|
||||
lat = lat.toString().toDouble()
|
||||
// 高德转百度
|
||||
def baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
|
||||
lng2 = baidu.x
|
||||
lat2 = baidu.y
|
||||
lng_new = lng *10000000000L
|
||||
lat_new = lat *10000000000L
|
||||
if(lng2){
|
||||
lng2_new = lng2 *10000000000L
|
||||
lat2_new = lat2 *10000000000L
|
||||
}
|
||||
}
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", city)
|
||||
data.put("region",region)
|
||||
data.put("name", name)
|
||||
data.put("estate_type2",estateType2)
|
||||
data.put("address", address)
|
||||
if(lng_new){
|
||||
data.put("lat",lat_new)
|
||||
data.put("lng",lng_new)
|
||||
data.put("lat2",lat2_new)
|
||||
data.put("lng2",lng2_new)
|
||||
}
|
||||
if(alias){
|
||||
data.put("alias",alias)
|
||||
}
|
||||
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
DbUtils.seaweed.save(data)
|
||||
FangjiaClientUtils.sendDistrictPriceCalculate(city,region, name, 12);
|
||||
addIndex++
|
||||
println "[新增] "+addIndex +" "+ data
|
||||
|
||||
}else{
|
||||
println dist.city +" "+dist.region+" "+dist.name+" 小区已存在"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def run(){
|
||||
def rets = findOpenLog2()
|
||||
ExcelUtils.write("D:\\数据\\浙商\\不存在小区2.xlsx", rets, "数据")
|
||||
println rets.size()
|
||||
|
||||
// matchMap2()
|
||||
}
|
||||
|
||||
def matchMap2() {
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\浙商\\不存在小区.xlsx", { d ->
|
||||
if(!d['E']){
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C'])
|
||||
def map = findStandardObject(d['A'],d['B'],d['C'], rets);
|
||||
|
||||
def locationMap = [:]
|
||||
if (!map || !map.lat) {
|
||||
rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
|
||||
map = findStandardObject(d['A'],d['B'],d['C'], rets);
|
||||
}
|
||||
if(map){
|
||||
println map
|
||||
if(map.adname){
|
||||
d['E'] = map.adname
|
||||
}else if(map.region){
|
||||
d['E'] = map.region
|
||||
}
|
||||
d['F'] = map.name
|
||||
d['G'] = map.address
|
||||
d['K'] = map.estateType
|
||||
d['L'] = map.lat
|
||||
d['M'] = map.lng
|
||||
d['N'] = map.lat2
|
||||
d['O'] = map.lng2
|
||||
d['P'] = map.source
|
||||
d['Q'] = RegionData.formatRegion(d['A'], d['E'])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def findStandardObject(def city, def region, def name, def rets) {
|
||||
def map;
|
||||
if (rets) {
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){
|
||||
mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
|
||||
map = mapObj;
|
||||
break;
|
||||
}else if(mapObj.address && mapObj.address.contains(name)){
|
||||
map = mapObj;
|
||||
break;
|
||||
}else if(mapObj.name && name){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name);
|
||||
if(score1){
|
||||
map = mapObj;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
def matchMap(){
|
||||
ExcelUtils.modify("D:\\数据\\浙商\\不存在小区.xlsx",{data->
|
||||
def gdData = FangjiaClientUtils.getMapData(data['A'], data['B'], data['C'], data['D'],"", "高德")
|
||||
if(!gdData){
|
||||
gdData = FangjiaClientUtils.getMapData(data['A'], "", data['C'], data['D'],"", "高德")
|
||||
}
|
||||
if(gdData){
|
||||
data['E'] = gdData.district
|
||||
data['F'] = gdData.name
|
||||
data['G'] = gdData.address
|
||||
data['H'] = gdData.type
|
||||
data['I'] = gdData.lng
|
||||
data['J'] = gdData.lat
|
||||
data['K'] = gdData.score
|
||||
}
|
||||
|
||||
def bdData = FangjiaClientUtils.getMapData(data['A'], data['B'], data['C'], data['D'],"", "百度")
|
||||
if(!bdData){
|
||||
bdData = FangjiaClientUtils.getMapData(data['A'], "", data['C'], data['D'],"", "百度")
|
||||
}
|
||||
if(bdData){
|
||||
data['L'] = bdData.district
|
||||
data['M'] = bdData.name
|
||||
data['N'] = bdData.address
|
||||
data['O'] = bdData.type
|
||||
data['P'] = bdData.lng
|
||||
data['Q'] = bdData.lat
|
||||
data['R'] = bdData.score
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findOpenLog2(){
|
||||
def time = DateUtils.parseFromTightYYYYMMDD("2022-01-01")
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-02-18")
|
||||
QueryBuilder date_range_cdate = QueryBuilders.rangeQuery("c_date").from(time.getTime()).to(endDate).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007569')).must(QueryBuilders.termQuery("request_uri", '/czbank/evaluation')).must(date_range_cdate)
|
||||
def rets = []
|
||||
rets << ["城市", "区域", "名称","地址", "高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","DB存在"]
|
||||
def datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
|
||||
def index = 0
|
||||
def count = 0
|
||||
def keys = []
|
||||
for(def d : datas){
|
||||
if(d.code == 400){
|
||||
index++
|
||||
def msg = d.msg
|
||||
if(msg.contains("该楼盘不存在")){
|
||||
def arrs = msg.split(" ")
|
||||
if(arrs){
|
||||
def rMap = [:]
|
||||
for(def s : arrs){
|
||||
if(s.contains(":")){
|
||||
def ars = s.split(":")
|
||||
if(ars.size()==2){
|
||||
rMap.put(ars[0], ars[1])
|
||||
}else{
|
||||
rMap.put(ars[0], null)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(rMap){
|
||||
count++
|
||||
def key = rMap.city+"#"+rMap.region+"#"+rMap.name
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
def params = d.parameter_map
|
||||
def address = null
|
||||
if(params && params.address){
|
||||
address = params.address
|
||||
}
|
||||
def r = [rMap.city, rMap.region, rMap.name, address]
|
||||
if(!rets.contains(r)){
|
||||
rets << r
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println msg
|
||||
}
|
||||
}
|
||||
}
|
||||
println "400: "+index+" 该楼盘不存在:"+count+" 去重后:"+rets.size()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findOpenLog(){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT * FROM open_uri_log WHERE 1=1")
|
||||
sqlBuffer.append(" and user_id = '1007569'")
|
||||
sqlBuffer.append(" and request_uri = '/czbank/evaluation'")
|
||||
sqlBuffer.append(" and c_date >= '2022-01-01'")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def rets = []
|
||||
rets << ["城市", "区域", "名称","地址", "高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","DB存在"]
|
||||
def index = 0
|
||||
def count = 0
|
||||
while (rs.next()){
|
||||
def code = rs.getInt("code")
|
||||
if(code == 400){
|
||||
index++
|
||||
def msg = rs.getString("msg")
|
||||
if(msg && msg.contains("该楼盘不存在")){
|
||||
def arrs = msg.split(" ")
|
||||
if(arrs){
|
||||
def rMap = [:]
|
||||
for(def s : arrs){
|
||||
if(s.contains(":")){
|
||||
def ars = s.split(":")
|
||||
if(ars.size()==2){
|
||||
rMap.put(ars[0], ars[1])
|
||||
}else{
|
||||
rMap.put(ars[0], null)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(rMap){
|
||||
count++
|
||||
def params = rs.getObject("parameter_map")
|
||||
println params
|
||||
def r = [rMap.city, rMap.region, rMap.name]
|
||||
if(!rets.contains(r)){
|
||||
rets << r
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println msg
|
||||
}
|
||||
}
|
||||
}
|
||||
println "400: "+index+" 该楼盘不存在:"+count+" 去重后:"+rets.size()
|
||||
return rets;
|
||||
}
|
||||
}
|
||||
+383
@@ -0,0 +1,383 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.gj.paimai.ChromeDriverPool
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.WebElement
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
import org.openqa.selenium.support.ui.ExpectedConditions
|
||||
import org.openqa.selenium.support.ui.WebDriverWait
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
|
||||
public class BeiKeCrawlByChrome {
|
||||
|
||||
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
def task = new BeiKeCrawlByChrome();
|
||||
task.run()
|
||||
// task.run1()
|
||||
// task.addDealRegion();
|
||||
}
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def cityMap = getAllCityUrls(driver)
|
||||
getLogin(driver)
|
||||
crawlData(driver, cityMap)
|
||||
|
||||
|
||||
}
|
||||
|
||||
def addDealRegion(){
|
||||
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
lastMonth = lastMonth +"-01"
|
||||
def q = new BasicDBObject()
|
||||
q.put("region", null)
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
|
||||
// q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def keys = []
|
||||
for(def d : dbcursor){
|
||||
keys << ["city":d.city, "name":d.district_name]
|
||||
}
|
||||
for(def m : keys){
|
||||
//def list = CommonDatautils.findDistrictList(m.city, null, m.name, "district")
|
||||
def list = CommonDatautils.findDistrictByAlias(m.city, null, m.name, "district")
|
||||
if(list && list.size()==1){
|
||||
def dist = list[0]
|
||||
|
||||
def query = new BasicDBObject()
|
||||
query.put("city", m.city)
|
||||
query.put("region", null)
|
||||
query.put("district_name", m.name)
|
||||
// query.put("source", "链家在线")
|
||||
query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern(lastMonth,"yyyy-MM-dd")))
|
||||
|
||||
def data = new BasicDBObject()
|
||||
data.put("region", dist.region)
|
||||
def ds = DEAL_PRICE_COLL.count(query)
|
||||
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def crawlData(driver, cityMap){
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
cityMap.each{city, cityUrl->
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
def detailUrls = getPageDeail(driver, city, cityUrl)
|
||||
println cityUrl +" 详情链接:"+detailUrls.size()
|
||||
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
|
||||
}
|
||||
}
|
||||
|
||||
def getPageUrls(driver, cityUrl){
|
||||
def stop = false
|
||||
def page = 1
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "chengjiao/pg${page}/"
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
if(elements){
|
||||
for(def e : elements){
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
page++;
|
||||
}else{
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
|
||||
def getPageDeail(driver, city, cityUrl){
|
||||
def stop = false
|
||||
def page = 1
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "/chengjiao/pg${page}/"
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
if(elements){
|
||||
for(def e : elements){
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2021-12-31", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
page++;
|
||||
}else{
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
def parse(city,e){
|
||||
try {
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
def arrs = title.split(" ")
|
||||
detail.city = city
|
||||
detail.district_name = arrs[0]
|
||||
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
|
||||
if(detail.region){
|
||||
detail.region = RegionData.formatRegion(city, detail.region)
|
||||
}
|
||||
|
||||
def roomStr = arrs[1]
|
||||
if (roomStr) {
|
||||
if (roomStr.indexOf("室") > 0) {
|
||||
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
|
||||
}
|
||||
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
|
||||
}
|
||||
}
|
||||
|
||||
def areaStr = arrs[2].replace("平米", "")
|
||||
if (areaStr) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = price.toInteger() * 10000
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
detail.avg_price = (int) (detail.total_price / detail.area)
|
||||
}
|
||||
def tradeStr = e.select(".dealDate").text()
|
||||
if (tradeStr) {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "贝壳找房"
|
||||
def dbo = findTrade(detail)
|
||||
if (!dbo) {
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
def houseInfos = houseInfo.split("\\|")
|
||||
detail.direction = houseInfos[0]
|
||||
detail.fitment = houseInfos[1]
|
||||
def floorStr = e.select(".positionInfo").text()
|
||||
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
|
||||
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorStr && floorStr.isNumber()) {
|
||||
detail.height = floorStr.toInteger()
|
||||
}
|
||||
}
|
||||
detail.url = e.select(".title a").attr("href")
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
}
|
||||
}catch(Exception ex){
|
||||
ex.printStackTrace()
|
||||
println e.select(".title a").attr("href")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findRegionByThirdPary(city, district_name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("name", district_name)
|
||||
q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
|
||||
def dbcursor = thirdPartyColl.find(q).sort(new BasicDBObject('s_date', -1)).limit(1)
|
||||
def dbo = null;
|
||||
for(def d : dbcursor){
|
||||
dbo = d
|
||||
}
|
||||
if(dbo){
|
||||
return dbo.region
|
||||
}
|
||||
return null;
|
||||
}
|
||||
def getContent(driver, detailUrl){
|
||||
driver.get(detailUrl)
|
||||
def str = driver.getPageSource()
|
||||
if(str && str.contains("<title>登录</title>")){
|
||||
getLogin(driver)
|
||||
return getContent(driver, detailUrl)
|
||||
}
|
||||
return str;
|
||||
}
|
||||
def crawlDetail(driver,DEAL_PRICE_COLL, detailUrls){
|
||||
if(detailUrls){
|
||||
for(def i=0;i<detailUrls.size();i++){
|
||||
def detailUrl = detailUrls[i]
|
||||
try{
|
||||
def detail = crawling(driver, DEAL_PRICE_COLL, detailUrl)
|
||||
println ${i}+" "+detail.city + " " + detail.region + " " + detail.district_name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def crawling(driver, DEAL_PRICE_COLL, detailUrl){
|
||||
def content = getContent(driver, detailUrl)
|
||||
def doc = Jsoup.parse(content);
|
||||
def eles = JQueryUtils.select(".deal-bread a", doc)
|
||||
def detail = new BasicDBObject()
|
||||
if(eles && eles.size()>=4){
|
||||
detail.city = eles[0].text().replace("房产网","")
|
||||
detail.region = eles[2].text().replace("二手房成交","")
|
||||
detail.district_name = eles[eles.size()-1].text().replace("二手房成交","")
|
||||
}
|
||||
if(detail.district_name) {
|
||||
def price = JQueryUtils.selectText(".dealTotalPrice i", doc)
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = price.toInteger() * 10000
|
||||
}
|
||||
def areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("建筑面积", "").replace("㎡", "")
|
||||
if (areaStr) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
detail.avg_price = (int) (detail.total_price / detail.area)
|
||||
}
|
||||
def tradeStr = JQueryUtils.selectText(".house-title span", doc).replace("成交", "").trim()
|
||||
if (tradeStr) {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "链家在线"
|
||||
def dbo = findTrade(DEAL_PRICE_COLL, detail)
|
||||
if (!dbo) {
|
||||
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
|
||||
if (roomStr) {
|
||||
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
|
||||
detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫"))
|
||||
}
|
||||
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
|
||||
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
|
||||
def floorStr = JQueryUtils.selectText("li:contains(所在楼层)", doc).replace("所在楼层", "")
|
||||
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
|
||||
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorStr && floorStr.isNumber()) {
|
||||
detail.height = floorStr.toInteger()
|
||||
}
|
||||
}
|
||||
detail.url = detailUrl
|
||||
detail.c_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
}
|
||||
return detail;
|
||||
}
|
||||
|
||||
|
||||
def findTrade(def detail){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", detail.city)
|
||||
q.put("region", detail.region)
|
||||
q.put("district_name", detail.district_name)
|
||||
q.put("area", detail.area)
|
||||
q.put("total_price", detail.total_price)
|
||||
q.put("trade_date", detail.trade_date)
|
||||
q.put("source", detail.source)
|
||||
def dbo = DEAL_PRICE_COLL.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
|
||||
def getAllCityUrls(driver){
|
||||
driver.get("https://www.ke.com/city/")
|
||||
Thread.sleep(8000)
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityMap = [:]
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
// if(e.text().contains("天津")){
|
||||
cityMap.put(e.text(), "https:"+href)
|
||||
// }
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
}
|
||||
|
||||
def getLogin(driver){
|
||||
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
Thread.sleep(10000)
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
|
||||
|
||||
|
||||
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
|
||||
|
||||
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
|
||||
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
|
||||
elemUsername.sendKeys("15021384995");
|
||||
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+2
@@ -19,6 +19,8 @@ class SnapshotMonitor extends BaseBatch {
|
||||
|
||||
// 运维组 运维警报
|
||||
static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
|
||||
// 房金部自动检测机器人
|
||||
// https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736
|
||||
|
||||
static void main(String[] args) {
|
||||
runBatch(new SnapshotMonitor(), args)
|
||||
|
||||
+5
-5
@@ -19,18 +19,18 @@ class GetPoiFromExcel {
|
||||
|
||||
static void main(String[] args) {
|
||||
AtomicLong atomicLong = new AtomicLong(0)
|
||||
List<File> fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\POI"), null, false);
|
||||
//List<File> fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\POI"), null, false);
|
||||
List<File> fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\高德反查无ID"), null, false);
|
||||
for (File file : fileList) {
|
||||
List<List<String>> seaweeds = ExcelUtils.read(file.getPath())
|
||||
final cityTemp = file.getName().substring(0, file.getName().indexOf("高德"))
|
||||
for (List<String> infoTemp : seaweeds) {
|
||||
final List<String> info = infoTemp
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
String city = cityTemp
|
||||
String region = info.get(2)
|
||||
String seaweedName = info.get(3)
|
||||
String city = info.get(0)
|
||||
String region = info.get(1)
|
||||
String seaweedName = info.get(2)
|
||||
List<AmapPoi> pois = GaoDeUtils.searchPoi(city, seaweedName, region, "120300|120301|120302");
|
||||
if (CollectionUtils.isNotEmpty(pois)) {
|
||||
String jsonStr = JSONObject.toJSONString(pois)
|
||||
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
package com.fangjia.script.temp.shenjian.qianhai
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.fangjia.script.xingyebank.CityUtil
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\前海\\前海征信测试数据test.xlsx",{List item->
|
||||
|
||||
if(StringUtils.isBlank(item['B'])){
|
||||
return;
|
||||
}
|
||||
HttpRequest request = new HttpRequest()
|
||||
def params = [:]
|
||||
params["address"] = item['B']
|
||||
params["city"] = getCityFormAddress(item['B'])
|
||||
params["token"] = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
JSONObject json = request.getJsonContent("http://open.fangjia.com/qhzx/propertyDetail", params, "UTF-8")
|
||||
|
||||
if(json.code == 200 && json.result){
|
||||
JSONObject result = json.result
|
||||
item['C'] = CityUtil.getProvinceByCity(result.city)
|
||||
item['D'] = result.city
|
||||
item['E'] = result.district
|
||||
item['F'] = result.name
|
||||
if (result.name) {
|
||||
item['O'] = '住宅地址'
|
||||
} else {
|
||||
item['O'] = '非住宅地址'
|
||||
}
|
||||
item['G'] = result.address
|
||||
item['H'] = result.lng
|
||||
item['I'] = result.lat
|
||||
|
||||
// 小区均价查询
|
||||
int avgPrice = OpenApiClient.queryDistrictAvgPrice(result.city,result.district,result.name)
|
||||
if(avgPrice > 0){
|
||||
item['M'] = avgPrice
|
||||
}
|
||||
params.clear()
|
||||
params["city"] = result.city
|
||||
params["district"] = result.district
|
||||
params["name"] = result.name
|
||||
params["token"] = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
// 小区周边一公里均价
|
||||
json = request.getJsonContent("http://open.fangjia.com/property/aroundAverage", params, "UTF-8")
|
||||
if(json.result){
|
||||
item['J'] = json.getJSONObject("result").averagePrice
|
||||
}
|
||||
}
|
||||
params.clear()
|
||||
params["city"] = getCityFormAddress(item['B'])
|
||||
params["address"] = item['B']
|
||||
params["token"] = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
|
||||
// 地址认证接口
|
||||
json = request.getJsonContent("http://open.fangjia.com/address/auth", params, "UTF-8")
|
||||
if(json && json.result){
|
||||
JSONObject result = json.result
|
||||
item['K'] = result.formattedAddress
|
||||
item['L'] = result.authLevel
|
||||
}
|
||||
// 小区详情接口
|
||||
params.clear()
|
||||
params["token"] = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
params["city"] = getCityFormAddress(item['B'])
|
||||
params["name"] = item['F']
|
||||
params["region"] = item['E']
|
||||
|
||||
json = request.getJsonContent("http://open.fangjia.com/district/show", params, "UTF-8")
|
||||
if(json){
|
||||
item['N'] = json.buildType
|
||||
}
|
||||
|
||||
println(item)
|
||||
})
|
||||
|
||||
def getCityFormAddress(String addr){
|
||||
List<String> cities = CommonUtils.findAllCity()
|
||||
for(String city : cities){
|
||||
if(addr.contains(city)){
|
||||
return city
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
Reference in New Issue
Block a user