update
This commit is contained in:
+1
-1
@@ -87,7 +87,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true);
|
||||
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
|
||||
|
||||
saveToPgSql(rows)
|
||||
|
||||
|
||||
+6
-5
@@ -45,6 +45,7 @@ class TradeStatLog {
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
// task.averageStat()
|
||||
|
||||
// task.deleteDistrict()
|
||||
// task.deleteDealData()
|
||||
// task.deleteDealData2()
|
||||
@@ -92,13 +93,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys_all;
|
||||
def rets = []
|
||||
rets << ["城市","23年2月总量","23年3月总量","23年4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","23年3月总量","23年4月总量","23年5月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2023-02-01"
|
||||
def date = "2023-03-01"
|
||||
def endDate = "2023-04-01"
|
||||
def newDate = "2023-05-01"
|
||||
def startDate = "2023-03-01"
|
||||
def date = "2023-04-01"
|
||||
def endDate = "2023-05-01"
|
||||
def newDate = "2023-06-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
|
||||
+1
-1
@@ -48,7 +48,7 @@ class HailanTest {
|
||||
}
|
||||
|
||||
def run(){
|
||||
def inputfile = "D:\\数据\\微元素\\test.xlsx"
|
||||
def inputfile = "D:\\数据\\微元素\\排查表.xlsx"
|
||||
// addThirdPrice(inputfile)
|
||||
// match(inputfile)
|
||||
// tradeAvgPrice(inputfile)
|
||||
|
||||
+27
@@ -1,7 +1,9 @@
|
||||
package com.fangjia.script.commData
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import groovy.json.JsonSlurper
|
||||
|
||||
import net.sf.json.JSONArray
|
||||
@@ -3871,4 +3873,29 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 根据经纬度获取区域
|
||||
* @param city
|
||||
* @param location
|
||||
* @return
|
||||
*/
|
||||
static def getRegionByLocation(city, def location){
|
||||
def json = GaoDeApiUtils.geocode(location);
|
||||
def region = null;
|
||||
if (json && json.info=="OK") {
|
||||
def regeocode = json.regeocode
|
||||
if(regeocode && regeocode.addressComponent){
|
||||
region = regeocode.addressComponent.district
|
||||
if(!region){
|
||||
region = regeocode.addressComponent.township
|
||||
}
|
||||
}
|
||||
}
|
||||
if(region){
|
||||
region = RegionData.formatRegion(city, region)
|
||||
}
|
||||
return region;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
package com.fangjia.script.district
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
class MasterNameInAlias {
|
||||
|
||||
static main(args) {
|
||||
def task = new MasterNameInAlias()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def allList = []
|
||||
for(def city : citys){
|
||||
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name","address","alias","estate_type2", "lat", "lng"])
|
||||
|
||||
def cityList = []
|
||||
cityList << ["城市","区域","名称","物业类型","地址","别名", "距离", "地址状态"]
|
||||
for(def dist1 : disticts){
|
||||
def distList = []
|
||||
for(def dist2 : disticts){
|
||||
if(dist1._id && dist2._id && dist1._id.toString()!=dist2._id.toString()){
|
||||
if(dist2.alias && dist2.alias.contains(dist1.name)){
|
||||
def distance = ''
|
||||
if(dist1.lng && dist2.lng){
|
||||
distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, dist2.lng/10000000000d, dist2.lat/10000000000d);
|
||||
}
|
||||
distList << [dist2.city, dist2.region, dist2.name, dist2.estate_type2, dist2.address, dist2.alias?dist2.alias.join(' '):'', distance]
|
||||
// def newAlias = dist2.alias.remove(dist1.name)
|
||||
// def d = new BasicDBObject()
|
||||
// d.put("_id", dist2._id)
|
||||
//
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("alias", newAlias)
|
||||
// udata.put("m_date", new Date())
|
||||
// DbUtils.seaweed.update(d, new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
if(distList){
|
||||
cityList << [dist1.city, dist1.region, dist1.name, dist1.estate_type2, dist1.address, dist1.alias?dist1.alias.join(' '):'', '']
|
||||
cityList.addAll(distList)
|
||||
// cityList << []
|
||||
}
|
||||
}
|
||||
|
||||
if(cityList){
|
||||
allList.addAll(cityList)
|
||||
println city +" "+cityList.size()
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\小区\\主名在别名中.xlsx", allList, "数据")
|
||||
}
|
||||
}
|
||||
+2
@@ -10,6 +10,7 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
@@ -321,6 +322,7 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
def s = str.toString();
|
||||
index++;
|
||||
if(s && s!="null"){
|
||||
s = s.replace('"','\"')
|
||||
// if(s.toString().contains(",")){
|
||||
// s = csvHandlerStr(s.toString())
|
||||
//
|
||||
|
||||
+4
-1
@@ -114,7 +114,10 @@ class HouseExportBatch extends BaseBatch{
|
||||
// sb.append(s+ "@|@")
|
||||
// }
|
||||
// }
|
||||
sb.append(StringUtils.join(row.toArray(), "@|@"))
|
||||
def str = StringUtils.join(row.toArray(), "@|@")
|
||||
str = str.replace('"','\"')
|
||||
sb.append(str)
|
||||
|
||||
sb.append(System.getProperty("line.separator"));
|
||||
|
||||
println index
|
||||
|
||||
+15
-9
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
@@ -32,7 +33,7 @@ public class LianjiaCrawlByChrome {
|
||||
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -5, "yyyy-MM")
|
||||
|
||||
static main(args) {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
@@ -300,9 +301,7 @@ public class LianjiaCrawlByChrome {
|
||||
if(dealDate){
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
println "1..."+new Date()
|
||||
def dbo = findTrade(durl , "链家在线")
|
||||
println "2..."+new Date()
|
||||
if (!dbo) {
|
||||
parse(city, e, regionText, driver)
|
||||
}
|
||||
@@ -333,9 +332,7 @@ public class LianjiaCrawlByChrome {
|
||||
if(price){
|
||||
def detailUrl = e.select(".title a").attr("href")
|
||||
driver.get(detailUrl)
|
||||
println "3..."+new Date()
|
||||
def detailDoc = checkDocument(driver, detailUrl);
|
||||
println "4..."+new Date()
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
@@ -353,6 +350,15 @@ public class LianjiaCrawlByChrome {
|
||||
if(detail.region){
|
||||
detail.region = RegionData.formatRegion(city, detail.region)
|
||||
}
|
||||
if(!detail.region){
|
||||
def str = detailDoc.toString().substring(detailDoc.toString().indexOf("resblockPosition")+1, detailDoc.toString().size()-1)
|
||||
str = str.substring(0, str.indexOf("',"))
|
||||
def locArrs = str.split(",")
|
||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0].toString().toDouble(), locArrs[1].toString().toDouble())
|
||||
if(gmap){
|
||||
detail.region = CommonDatautils.getRegionByLocation(city, gmap.lng+","+gmap.lat)
|
||||
}
|
||||
}
|
||||
|
||||
def roomStr = arrs[1]
|
||||
if (roomStr) {
|
||||
@@ -427,9 +433,7 @@ public class LianjiaCrawlByChrome {
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
println "5..."+new Date()
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
println "6..."+new Date()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -581,9 +585,11 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
elemUsername.sendKeys("15021384995");
|
||||
// elemUsername.sendKeys("15021384995");
|
||||
// elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
elemUsername.sendKeys("16283166371");
|
||||
elemPassword.sendKeys("Goojia123456");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(15000)
|
||||
|
||||
Reference in New Issue
Block a user