This commit is contained in:
2023-06-21 15:10:18 +08:00
parent 3409e42481
commit 9ed8f6dc21
8 changed files with 115 additions and 17 deletions
@@ -87,7 +87,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
}
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true);
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
saveToPgSql(rows)
@@ -45,6 +45,7 @@ class TradeStatLog {
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
// task.deleteDistrict()
// task.deleteDealData()
// task.deleteDealData2()
@@ -92,13 +93,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys_all;
def rets = []
rets << ["城市","23年2月总量","23年3月总量","23年4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","23年3月总量","23年4月总量","23年5月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2023-02-01"
def date = "2023-03-01"
def endDate = "2023-04-01"
def newDate = "2023-05-01"
def startDate = "2023-03-01"
def date = "2023-04-01"
def endDate = "2023-05-01"
def newDate = "2023-06-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -48,7 +48,7 @@ class HailanTest {
}
def run(){
def inputfile = "D:\\数据\\微元素\\test.xlsx"
def inputfile = "D:\\数据\\微元素\\排查表.xlsx"
// addThirdPrice(inputfile)
// match(inputfile)
// tradeAvgPrice(inputfile)
@@ -1,7 +1,9 @@
package com.fangjia.script.commData
import com.fangjia.core.RegionData
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import groovy.json.JsonSlurper
import net.sf.json.JSONArray
@@ -3871,4 +3873,29 @@ class CommonDatautils {
}
/**
* 根据经纬度获取区域
* @param city
* @param location
* @return
*/
static def getRegionByLocation(city, def location){
def json = GaoDeApiUtils.geocode(location);
def region = null;
if (json && json.info=="OK") {
def regeocode = json.regeocode
if(regeocode && regeocode.addressComponent){
region = regeocode.addressComponent.district
if(!region){
region = regeocode.addressComponent.township
}
}
}
if(region){
region = RegionData.formatRegion(city, region)
}
return region;
}
}
@@ -0,0 +1,59 @@
package com.fangjia.script.district
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoUtils
import com.mongodb.BasicDBObject
class MasterNameInAlias {
static main(args) {
def task = new MasterNameInAlias()
task.run()
}
def run(){
def citys = CommonDatautils.findAllCity()
def allList = []
for(def city : citys){
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name","address","alias","estate_type2", "lat", "lng"])
def cityList = []
cityList << ["城市","区域","名称","物业类型","地址","别名", "距离", "地址状态"]
for(def dist1 : disticts){
def distList = []
for(def dist2 : disticts){
if(dist1._id && dist2._id && dist1._id.toString()!=dist2._id.toString()){
if(dist2.alias && dist2.alias.contains(dist1.name)){
def distance = ''
if(dist1.lng && dist2.lng){
distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, dist2.lng/10000000000d, dist2.lat/10000000000d);
}
distList << [dist2.city, dist2.region, dist2.name, dist2.estate_type2, dist2.address, dist2.alias?dist2.alias.join(' '):'', distance]
// def newAlias = dist2.alias.remove(dist1.name)
// def d = new BasicDBObject()
// d.put("_id", dist2._id)
//
// def udata = new BasicDBObject()
// udata.put("alias", newAlias)
// udata.put("m_date", new Date())
// DbUtils.seaweed.update(d, new BasicDBObject('$set', udata))
}
}
}
if(distList){
cityList << [dist1.city, dist1.region, dist1.name, dist1.estate_type2, dist1.address, dist1.alias?dist1.alias.join(' '):'', '']
cityList.addAll(distList)
// cityList << []
}
}
if(cityList){
allList.addAll(cityList)
println city +" "+cityList.size()
}
}
ExcelUtils.write("d:\\数据\\小区\\主名在别名中.xlsx", allList, "数据")
}
}
@@ -10,6 +10,7 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.lang.StringUtils
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -321,6 +322,7 @@ class DistrictInfoExportBatch extends BaseBatch{
def s = str.toString();
index++;
if(s && s!="null"){
s = s.replace('"','\"')
// if(s.toString().contains(",")){
// s = csvHandlerStr(s.toString())
//
@@ -114,7 +114,10 @@ class HouseExportBatch extends BaseBatch{
// sb.append(s+ "@|@")
// }
// }
sb.append(StringUtils.join(row.toArray(), "@|@"))
def str = StringUtils.join(row.toArray(), "@|@")
str = str.replace('"','\"')
sb.append(str)
sb.append(System.getProperty("line.separator"));
println index
@@ -1,6 +1,7 @@
package com.fangjia.script.gj.paimai
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.gj.ccb.FileUtils
@@ -32,7 +33,7 @@ public class LianjiaCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -5, "yyyy-MM")
static main(args) {
def task = new LianjiaCrawlByChrome();
@@ -300,9 +301,7 @@ public class LianjiaCrawlByChrome {
if(dealDate){
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
println "1..."+new Date()
def dbo = findTrade(durl , "链家在线")
println "2..."+new Date()
if (!dbo) {
parse(city, e, regionText, driver)
}
@@ -333,9 +332,7 @@ public class LianjiaCrawlByChrome {
if(price){
def detailUrl = e.select(".title a").attr("href")
driver.get(detailUrl)
println "3..."+new Date()
def detailDoc = checkDocument(driver, detailUrl);
println "4..."+new Date()
def title = e.select(".title a").text()
if (title) {
def detail = new BasicDBObject()
@@ -353,6 +350,15 @@ public class LianjiaCrawlByChrome {
if(detail.region){
detail.region = RegionData.formatRegion(city, detail.region)
}
if(!detail.region){
def str = detailDoc.toString().substring(detailDoc.toString().indexOf("resblockPosition")+1, detailDoc.toString().size()-1)
str = str.substring(0, str.indexOf("',"))
def locArrs = str.split(",")
def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0].toString().toDouble(), locArrs[1].toString().toDouble())
if(gmap){
detail.region = CommonDatautils.getRegionByLocation(city, gmap.lng+","+gmap.lat)
}
}
def roomStr = arrs[1]
if (roomStr) {
@@ -427,9 +433,7 @@ public class LianjiaCrawlByChrome {
detail.create_date = new Date()
detail.m_date = new Date()
println detail
println "5..."+new Date()
DEAL_PRICE_COLL.save(detail)
println "6..."+new Date()
}
}
}
@@ -581,9 +585,11 @@ public class LianjiaCrawlByChrome {
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
elemUsername.sendKeys("15021384995");
// elemUsername.sendKeys("15021384995");
// elemPassword.sendKeys("12qwaszx");
elemPassword.sendKeys("12qwaszx");
elemUsername.sendKeys("16283166371");
elemPassword.sendKeys("Goojia123456");
btn.submit(); // 点击行为;
Thread.sleep(15000)