Merge branch 'script_gj' into 'master'

Script gj

See merge request !259
This commit is contained in:
2022-02-22 09:25:22 +08:00
6 changed files with 820 additions and 6 deletions
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211218, 20220119);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220118, 20220219);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
@@ -0,0 +1,340 @@
package com.fangjia.script.gj.district
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import com.fangjia.core.util.BaiDuApiUtils
import java.sql.ResultSet
class ZsNoExistsToDB {
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
static main(args) {
def task = new ZsNoExistsToDB()
task.run()
// task.addDistrictToDB()
}
def addDistrictToDB(){
def datas = ExcelUtils.read("D:\\数据\\浙商\\不存在小区(结果V1).xlsx",0,1)
def keys = []
def addIndex = 0
for(def d : datas){
if(d['J'] && d['J']=="新增"){
def city = d['A']
def region = d['B']
def name = d['F'].toString().replace("·","")
def address = d['G']
def dist = CommonDatautils.findDistrict(city,region,name,"district")
if(!dist){
dist = CommonDatautils.findDistrict(city,region,d['C'],"district")
}
if(!dist){
keys << city+"##"+region+"##"+name
def alias = []
def str = d['H']
if(str){
def arrs = str.split(",")
for(def s : arrs){
if(!alias.contains(s)){
alias << s
}
}
}
if(address){
address = AddressUtil.formatAddress(city, address)
}
def lng = d['M']
def lat = d['L']
def lng2 = null
def lat2 = null
long lng_new = 0
long lat_new = 0
long lng2_new = 0
long lat2_new = 0
if(lng){
lng = lng.toString().toDouble()
lat = lat.toString().toDouble()
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
lng2 = baidu.x
lat2 = baidu.y
lng_new = lng *10000000000L
lat_new = lat *10000000000L
if(lng2){
lng2_new = lng2 *10000000000L
lat2_new = lat2 *10000000000L
}
}
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",region)
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("address", address)
if(lng_new){
data.put("lat",lat_new)
data.put("lng",lng_new)
data.put("lat2",lat2_new)
data.put("lng2",lng2_new)
}
if(alias){
data.put("alias",alias)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
DbUtils.seaweed.save(data)
FangjiaClientUtils.sendDistrictPriceCalculate(city,region, name, 12);
addIndex++
println "[新增] "+addIndex +" "+ data
}else{
println dist.city +" "+dist.region+" "+dist.name+" 小区已存在"
}
}
}
}
def run(){
def rets = findOpenLog2()
ExcelUtils.write("D:\\数据\\浙商\\不存在小区2.xlsx", rets, "数据")
println rets.size()
// matchMap2()
}
def matchMap2() {
ExcelUtils.multiThreadingModify("D:\\数据\\浙商\\不存在小区.xlsx", { d ->
if(!d['E']){
def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C'])
def map = findStandardObject(d['A'],d['B'],d['C'], rets);
def locationMap = [:]
if (!map || !map.lat) {
rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
map = findStandardObject(d['A'],d['B'],d['C'], rets);
}
if(map){
println map
if(map.adname){
d['E'] = map.adname
}else if(map.region){
d['E'] = map.region
}
d['F'] = map.name
d['G'] = map.address
d['K'] = map.estateType
d['L'] = map.lat
d['M'] = map.lng
d['N'] = map.lat2
d['O'] = map.lng2
d['P'] = map.source
d['Q'] = RegionData.formatRegion(d['A'], d['E'])
}
}
})
}
def findStandardObject(def city, def region, def name, def rets) {
def map;
if (rets) {
if (rets.size() >= 1) {
for (def mapObj : rets) {
// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){
mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
map = mapObj;
break;
}else if(mapObj.address && mapObj.address.contains(name)){
map = mapObj;
break;
}else if(mapObj.name && name){
double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name);
if(score1){
map = mapObj;
break;
}
}
// }
}
}
}
return map;
}
def matchMap(){
ExcelUtils.modify("D:\\数据\\浙商\\不存在小区.xlsx",{data->
def gdData = FangjiaClientUtils.getMapData(data['A'], data['B'], data['C'], data['D'],"", "高德")
if(!gdData){
gdData = FangjiaClientUtils.getMapData(data['A'], "", data['C'], data['D'],"", "高德")
}
if(gdData){
data['E'] = gdData.district
data['F'] = gdData.name
data['G'] = gdData.address
data['H'] = gdData.type
data['I'] = gdData.lng
data['J'] = gdData.lat
data['K'] = gdData.score
}
def bdData = FangjiaClientUtils.getMapData(data['A'], data['B'], data['C'], data['D'],"", "百度")
if(!bdData){
bdData = FangjiaClientUtils.getMapData(data['A'], "", data['C'], data['D'],"", "百度")
}
if(bdData){
data['L'] = bdData.district
data['M'] = bdData.name
data['N'] = bdData.address
data['O'] = bdData.type
data['P'] = bdData.lng
data['Q'] = bdData.lat
data['R'] = bdData.score
}
},0)
}
def findOpenLog2(){
def time = DateUtils.parseFromTightYYYYMMDD("2022-01-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-02-18")
QueryBuilder date_range_cdate = QueryBuilders.rangeQuery("c_date").from(time.getTime()).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007569')).must(QueryBuilders.termQuery("request_uri", '/czbank/evaluation')).must(date_range_cdate)
def rets = []
rets << ["城市", "区域", "名称","地址", "高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","DB存在"]
def datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
def index = 0
def count = 0
def keys = []
for(def d : datas){
if(d.code == 400){
index++
def msg = d.msg
if(msg.contains("该楼盘不存在")){
def arrs = msg.split(" ")
if(arrs){
def rMap = [:]
for(def s : arrs){
if(s.contains(":")){
def ars = s.split(":")
if(ars.size()==2){
rMap.put(ars[0], ars[1])
}else{
rMap.put(ars[0], null)
}
}
}
if(rMap){
count++
def key = rMap.city+"#"+rMap.region+"#"+rMap.name
if(!keys.contains(key)){
keys << key
def params = d.parameter_map
def address = null
if(params && params.address){
address = params.address
}
def r = [rMap.city, rMap.region, rMap.name, address]
if(!rets.contains(r)){
rets << r
}
}
}
}
}else{
println msg
}
}
}
println "400: "+index+" 该楼盘不存在:"+count+" 去重后:"+rets.size()
return rets;
}
def findOpenLog(){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT * FROM open_uri_log WHERE 1=1")
sqlBuffer.append(" and user_id = '1007569'")
sqlBuffer.append(" and request_uri = '/czbank/evaluation'")
sqlBuffer.append(" and c_date >= '2022-01-01'")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def rets = []
rets << ["城市", "区域", "名称","地址", "高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","DB存在"]
def index = 0
def count = 0
while (rs.next()){
def code = rs.getInt("code")
if(code == 400){
index++
def msg = rs.getString("msg")
if(msg && msg.contains("该楼盘不存在")){
def arrs = msg.split(" ")
if(arrs){
def rMap = [:]
for(def s : arrs){
if(s.contains(":")){
def ars = s.split(":")
if(ars.size()==2){
rMap.put(ars[0], ars[1])
}else{
rMap.put(ars[0], null)
}
}
}
if(rMap){
count++
def params = rs.getObject("parameter_map")
println params
def r = [rMap.city, rMap.region, rMap.name]
if(!rets.contains(r)){
rets << r
}
}
}
}else{
println msg
}
}
}
println "400: "+index+" 该楼盘不存在:"+count+" 去重后:"+rets.size()
return rets;
}
}
@@ -0,0 +1,383 @@
package com.fangjia.script.gj.paimai
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.gj.paimai.ChromeDriverPool
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import com.rabbitmq.client.impl.AMQImpl
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.WebElement
import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.support.ui.ExpectedConditions
import org.openqa.selenium.support.ui.WebDriverWait
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
public class BeiKeCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
def task = new BeiKeCrawlByChrome();
task.run()
// task.run1()
// task.addDealRegion();
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def cityMap = getAllCityUrls(driver)
getLogin(driver)
crawlData(driver, cityMap)
}
def addDealRegion(){
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
lastMonth = lastMonth +"-01"
def q = new BasicDBObject()
q.put("region", null)
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
// q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
def dbcursor = DEAL_PRICE_COLL.find(q)
def keys = []
for(def d : dbcursor){
keys << ["city":d.city, "name":d.district_name]
}
for(def m : keys){
//def list = CommonDatautils.findDistrictList(m.city, null, m.name, "district")
def list = CommonDatautils.findDistrictByAlias(m.city, null, m.name, "district")
if(list && list.size()==1){
def dist = list[0]
def query = new BasicDBObject()
query.put("city", m.city)
query.put("region", null)
query.put("district_name", m.name)
// query.put("source", "链家在线")
query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern(lastMonth,"yyyy-MM-dd")))
def data = new BasicDBObject()
data.put("region", dist.region)
def ds = DEAL_PRICE_COLL.count(query)
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
}
}
}
def crawlData(driver, cityMap){
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
cityMap.each{city, cityUrl->
//def detailUrls = getPageUrls(driver, cityUrl)
def detailUrls = getPageDeail(driver, city, cityUrl)
println cityUrl +" 详情链接:"+detailUrls.size()
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
}
}
def getPageUrls(driver, cityUrl){
def stop = false
def page = 1
def detailUrls = []
while(!stop){
def url = cityUrl + "chengjiao/pg${page}/"
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def elements = JQueryUtils.select(".listContent li", doc)
if(elements){
for(def e : elements){
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
}else{
stop = true;
break;
}
}
page++;
}else{
stop = true;
}
}
return detailUrls
}
def getPageDeail(driver, city, cityUrl){
def stop = false
def page = 1
def detailUrls = []
while(!stop){
def url = cityUrl + "/chengjiao/pg${page}/"
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def elements = JQueryUtils.select(".listContent li", doc)
if(elements){
for(def e : elements){
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2021-12-31", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
}else{
stop = true;
break;
}
}
page++;
}else{
stop = true;
}
}
return detailUrls
}
def parse(city,e){
try {
def title = e.select(".title a").text()
if (title) {
def detail = new BasicDBObject()
def arrs = title.split(" ")
detail.city = city
detail.district_name = arrs[0]
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
if(detail.region){
detail.region = RegionData.formatRegion(city, detail.region)
}
def roomStr = arrs[1]
if (roomStr) {
if (roomStr.indexOf("室") > 0) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
}
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
}
}
def areaStr = arrs[2].replace("平米", "")
if (areaStr) {
detail.area = areaStr.toDouble()
}
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = price.toInteger() * 10000
}
if (detail.total_price && detail.area) {
detail.avg_price = (int) (detail.total_price / detail.area)
}
def tradeStr = e.select(".dealDate").text()
if (tradeStr) {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "贝壳找房"
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.url = e.select(".title a").attr("href")
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
}
}
}catch(Exception ex){
ex.printStackTrace()
println e.select(".title a").attr("href")
}
}
def findRegionByThirdPary(city, district_name){
def q = new BasicDBObject()
q.put("city", city)
q.put("name", district_name)
q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
def dbcursor = thirdPartyColl.find(q).sort(new BasicDBObject('s_date', -1)).limit(1)
def dbo = null;
for(def d : dbcursor){
dbo = d
}
if(dbo){
return dbo.region
}
return null;
}
def getContent(driver, detailUrl){
driver.get(detailUrl)
def str = driver.getPageSource()
if(str && str.contains("<title>登录</title>")){
getLogin(driver)
return getContent(driver, detailUrl)
}
return str;
}
def crawlDetail(driver,DEAL_PRICE_COLL, detailUrls){
if(detailUrls){
for(def i=0;i<detailUrls.size();i++){
def detailUrl = detailUrls[i]
try{
def detail = crawling(driver, DEAL_PRICE_COLL, detailUrl)
println ${i}+" "+detail.city + " " + detail.region + " " + detail.district_name
} catch (Exception e) {
e.printStackTrace()
}
}
}
}
def crawling(driver, DEAL_PRICE_COLL, detailUrl){
def content = getContent(driver, detailUrl)
def doc = Jsoup.parse(content);
def eles = JQueryUtils.select(".deal-bread a", doc)
def detail = new BasicDBObject()
if(eles && eles.size()>=4){
detail.city = eles[0].text().replace("房产网","")
detail.region = eles[2].text().replace("二手房成交","")
detail.district_name = eles[eles.size()-1].text().replace("二手房成交","")
}
if(detail.district_name) {
def price = JQueryUtils.selectText(".dealTotalPrice i", doc)
if (price && price.isNumber()) {
detail.total_price = price.toInteger() * 10000
}
def areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("建筑面积", "").replace("㎡", "")
if (areaStr) {
detail.area = areaStr.toDouble()
}
if (detail.total_price && detail.area) {
detail.avg_price = (int) (detail.total_price / detail.area)
}
def tradeStr = JQueryUtils.selectText(".house-title span", doc).replace("成交", "").trim()
if (tradeStr) {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "链家在线"
def dbo = findTrade(DEAL_PRICE_COLL, detail)
if (!dbo) {
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
if (roomStr) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫"))
}
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
def floorStr = JQueryUtils.selectText("li:contains(所在楼层)", doc).replace("所在楼层", "")
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.url = detailUrl
detail.c_date = new Date()
detail.m_date = new Date()
DEAL_PRICE_COLL.save(detail)
}
}
return detail;
}
def findTrade(def detail){
def q = new BasicDBObject()
q.put("city", detail.city)
q.put("region", detail.region)
q.put("district_name", detail.district_name)
q.put("area", detail.area)
q.put("total_price", detail.total_price)
q.put("trade_date", detail.trade_date)
q.put("source", detail.source)
def dbo = DEAL_PRICE_COLL.findOne(q)
return dbo
}
def getAllCityUrls(driver){
driver.get("https://www.ke.com/city/")
Thread.sleep(8000)
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
// if(e.text().contains("天津")){
cityMap.put(e.text(), "https:"+href)
// }
}
}
return cityMap
}
def getLogin(driver){
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F");
WebDriverWait wait = new WebDriverWait(driver,20)
Thread.sleep(10000)
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
elemUsername.sendKeys("15021384995");
elemPassword.sendKeys("12qwaszx");
btn.submit(); // 点击行为;
}
}
@@ -19,6 +19,8 @@ class SnapshotMonitor extends BaseBatch {
// 运维组 运维警报
static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
// 房金部自动检测机器人
// https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736
static void main(String[] args) {
runBatch(new SnapshotMonitor(), args)
@@ -19,18 +19,18 @@ class GetPoiFromExcel {
static void main(String[] args) {
AtomicLong atomicLong = new AtomicLong(0)
List<File> fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\POI"), null, false);
//List<File> fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\POI"), null, false);
List<File> fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\高德反查无ID"), null, false);
for (File file : fileList) {
List<List<String>> seaweeds = ExcelUtils.read(file.getPath())
final cityTemp = file.getName().substring(0, file.getName().indexOf("高德"))
for (List<String> infoTemp : seaweeds) {
final List<String> info = infoTemp
executorService.submit(new Runnable() {
@Override
void run() {
String city = cityTemp
String region = info.get(2)
String seaweedName = info.get(3)
String city = info.get(0)
String region = info.get(1)
String seaweedName = info.get(2)
List<AmapPoi> pois = GaoDeUtils.searchPoi(city, seaweedName, region, "120300|120301|120302");
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
@@ -0,0 +1,89 @@
package com.fangjia.script.temp.shenjian.qianhai
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.xingyebank.CityUtil
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
ExcelUtils.multiThreadingModify("D:\\数据\\前海\\前海征信测试数据test.xlsx",{List item->
if(StringUtils.isBlank(item['B'])){
return;
}
HttpRequest request = new HttpRequest()
def params = [:]
params["address"] = item['B']
params["city"] = getCityFormAddress(item['B'])
params["token"] = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
JSONObject json = request.getJsonContent("http://open.fangjia.com/qhzx/propertyDetail", params, "UTF-8")
if(json.code == 200 && json.result){
JSONObject result = json.result
item['C'] = CityUtil.getProvinceByCity(result.city)
item['D'] = result.city
item['E'] = result.district
item['F'] = result.name
if (result.name) {
item['O'] = '住宅地址'
} else {
item['O'] = '非住宅地址'
}
item['G'] = result.address
item['H'] = result.lng
item['I'] = result.lat
// 小区均价查询
int avgPrice = OpenApiClient.queryDistrictAvgPrice(result.city,result.district,result.name)
if(avgPrice > 0){
item['M'] = avgPrice
}
params.clear()
params["city"] = result.city
params["district"] = result.district
params["name"] = result.name
params["token"] = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
// 小区周边一公里均价
json = request.getJsonContent("http://open.fangjia.com/property/aroundAverage", params, "UTF-8")
if(json.result){
item['J'] = json.getJSONObject("result").averagePrice
}
}
params.clear()
params["city"] = getCityFormAddress(item['B'])
params["address"] = item['B']
params["token"] = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
// 地址认证接口
json = request.getJsonContent("http://open.fangjia.com/address/auth", params, "UTF-8")
if(json && json.result){
JSONObject result = json.result
item['K'] = result.formattedAddress
item['L'] = result.authLevel
}
// 小区详情接口
params.clear()
params["token"] = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
params["city"] = getCityFormAddress(item['B'])
params["name"] = item['F']
params["region"] = item['E']
json = request.getJsonContent("http://open.fangjia.com/district/show", params, "UTF-8")
if(json){
item['N'] = json.buildType
}
println(item)
})
def getCityFormAddress(String addr){
List<String> cities = CommonUtils.findAllCity()
for(String city : cities){
if(addr.contains(city)){
return city
}
}
return null
}