update
This commit is contained in:
+1
-1
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220618, 20220719);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220718, 20220819);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||
//测试通过匹配接口配房源效果
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||
|
||||
+35
@@ -42,6 +42,10 @@ class ExportOracleData {
|
||||
exportBigdataDeal("2022-06-01", new_dists)
|
||||
|
||||
|
||||
//第三方网站权重表
|
||||
exportWebsiteRate();
|
||||
|
||||
|
||||
//计算结果相关
|
||||
// exportAvmEvaluationGJNB(curMonth)
|
||||
// exportAVMEvaluationItemGJNB(curMonth)
|
||||
@@ -93,6 +97,37 @@ class ExportOracleData {
|
||||
}
|
||||
|
||||
|
||||
def exportWebsiteRate(){
|
||||
def sql = "select DATA_SOURCE_ID,DATA_SOURCE,DATA_SOURCE_MAP,WEIGHT,TYPE,SOURCECODE,CITYID,CITYNAME,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from WEBSITE_CONFIG_GJ";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.DATA_SOURCE_ID
|
||||
data << r.DATA_SOURCE
|
||||
data << r.DATA_SOURCE_MAP
|
||||
data << r.WEIGHT
|
||||
data << r.TYPE
|
||||
data << r.SOURCECODE
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "上传Ftp完成......"
|
||||
}
|
||||
|
||||
|
||||
def exportBigdataPoiData() {
|
||||
def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj";
|
||||
def allRows = []
|
||||
|
||||
+6
-6
@@ -75,13 +75,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
rets << ["城市","4月总量","5月总量","6月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","5月总量","6月总量","7月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2022-04-01"
|
||||
def date = "2022-05-01"
|
||||
def endDate = "2022-06-01"
|
||||
def newDate = "2022-07-01"
|
||||
def startDate = "2022-05-01"
|
||||
def date = "2022-06-01"
|
||||
def endDate = "2022-07-01"
|
||||
def newDate = "2022-08-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
@@ -131,7 +131,7 @@ class TradeStatLog {
|
||||
}
|
||||
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-06成交统计.xlsx", rets, "数据")
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\${newDate}成交统计.xlsx", rets, "数据")
|
||||
println "表格生成完成"
|
||||
}
|
||||
|
||||
|
||||
+19
-1
@@ -70,7 +70,7 @@ class CommonDatautils {
|
||||
|
||||
public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", mongoDbUrl)
|
||||
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", mongoDbUrl)
|
||||
|
||||
public static final DBCollection CITY_INFO_COLL = MongoUtils.getCollectionByName2("fangjia_data_ana", "city_info", mongoDbUrl)
|
||||
|
||||
public static DSCollection getHouseDataStoreConnection(){
|
||||
if(house_dataStoreConnection != null) {
|
||||
@@ -136,6 +136,13 @@ class CommonDatautils {
|
||||
return cityInfos;
|
||||
|
||||
}
|
||||
|
||||
|
||||
public static def findAlCityDetail(city){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("display", city)
|
||||
return SERVER_AREA_COLL.findOne(q);
|
||||
}
|
||||
|
||||
public static def findAllCityByGrade(def city_grade){
|
||||
DBObject q = new BasicDBObject();
|
||||
@@ -3313,5 +3320,16 @@ class CommonDatautils {
|
||||
def count = HOUSE_LOUDONG_COLL.count(q)
|
||||
return count;
|
||||
}
|
||||
|
||||
static def findCityInfo(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
def cur = CITY_INFO_COLL.find(q).sort(new BasicDBObject("weekend",-1)).limit(1)
|
||||
while(cur.hasNext()) {
|
||||
DBObject bsobj = cur.next()
|
||||
return bsobj;
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+8
-3
@@ -24,7 +24,7 @@ class FzDataResultStat {
|
||||
|
||||
static main(args) {
|
||||
def task = new FzDataResultStat()
|
||||
task.priceMarkupRun();
|
||||
// task.priceMarkupRun();
|
||||
task.regionPriceTrendRun()
|
||||
//// task.margeExcel()
|
||||
// task.saveCsvData()
|
||||
@@ -88,9 +88,14 @@ class FzDataResultStat {
|
||||
lastPrice = lastObj[5]
|
||||
}
|
||||
def markup = null
|
||||
if(list && lastPrice){
|
||||
markup = (list[5].toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger()
|
||||
try{
|
||||
if(list && lastPrice && lastPrice.toInteger()>0){
|
||||
markup = (list[5].toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger()
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
def dist = CommonDatautils.findDistrictById(list[1])
|
||||
def flag = "有效"
|
||||
if(dist.status!=0){
|
||||
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
package com.fangjia.script.fuzhou
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.io.FileUtils
|
||||
|
||||
class FzDistrictMatch {
|
||||
|
||||
|
||||
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new FzDistrictMatch()
|
||||
// task.run();
|
||||
// task.addPrice()
|
||||
task.addThirdPrice()
|
||||
}
|
||||
|
||||
def addThirdPrice(){
|
||||
def map = read("D:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202206.csv");
|
||||
ExcelUtils.modify("D:\\数据\\福州\\福州\\福州高端小区匹配.xlsx", { d ->
|
||||
if(d['G']){
|
||||
def priceMap = map.get(d['G'])
|
||||
def prices = []
|
||||
if(priceMap && priceMap.zhugezhaofang && priceMap.zhugezhaofang.isNumber()){
|
||||
d['J'] = priceMap.zhugezhaofang
|
||||
prices << priceMap.zhugezhaofang.toInteger()
|
||||
}
|
||||
|
||||
if(priceMap && priceMap.anjuke && priceMap.anjuke.isNumber()){
|
||||
d['K'] = priceMap.anjuke
|
||||
prices << priceMap.anjuke.toInteger()
|
||||
}
|
||||
|
||||
if(priceMap && priceMap.soufang && priceMap.soufang.isNumber()) {
|
||||
d['L'] = priceMap.soufang
|
||||
prices << priceMap.soufang.toInteger()
|
||||
}
|
||||
|
||||
if(priceMap && priceMap.lianjia && priceMap.lianjia.isNumber()){
|
||||
d['M'] = priceMap.lianjia
|
||||
prices << priceMap.lianjia.toInteger()
|
||||
}
|
||||
|
||||
if(priceMap && priceMap.cityhouse && priceMap.cityhouse.isNumber()){
|
||||
d['N'] = priceMap.cityhouse
|
||||
prices << priceMap.cityhouse.toInteger()
|
||||
}
|
||||
|
||||
if(priceMap && priceMap.fangtan && priceMap.fangtan.isNumber()){
|
||||
d['O'] = priceMap.fangtan
|
||||
prices << priceMap.fangtan.toInteger()
|
||||
}
|
||||
|
||||
if(priceMap && priceMap.fangdoudou && priceMap.fangdoudou.isNumber()){
|
||||
d['P'] = priceMap.fangdoudou
|
||||
prices << priceMap.fangdoudou.toInteger()
|
||||
}
|
||||
if(priceMap && priceMap.maitian && priceMap.maitian.isNumber()){
|
||||
d['Q'] = priceMap.maitian
|
||||
prices << priceMap.maitian.toInteger()
|
||||
}
|
||||
|
||||
if(priceMap && priceMap.get("58tc") && priceMap.get("58tc").isNumber()){
|
||||
d['R'] = priceMap.get("58tc")
|
||||
prices << priceMap.get("58tc").toInteger()
|
||||
}
|
||||
if(prices){
|
||||
d['S'] = (int)(prices.sum()/prices.size())
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def read(fileName){
|
||||
List<String> list1= FileUtils.readLines(new File(fileName))
|
||||
def map1 = [:]
|
||||
for(def str : list1){
|
||||
def s = str.split("@\\|@")
|
||||
def key = s[2]+"##"+s[3]+"##"+s[4]
|
||||
def priceMap = [:]
|
||||
map1.put(s[1], priceMap)
|
||||
|
||||
if(s.size()>=7 && s[6]){
|
||||
priceMap.put("zhugezhaofang", s[6])
|
||||
}
|
||||
|
||||
if(s.size()>=8 && s[7]){
|
||||
priceMap.put("anjuke", s[7])
|
||||
}
|
||||
|
||||
if(s.size()>=9 && s[8]){
|
||||
priceMap.put("soufang", s[8])
|
||||
}
|
||||
|
||||
if(s.size()>=10 && s[9]){
|
||||
priceMap.put("lianjia", s[9])
|
||||
}
|
||||
|
||||
if(s.size()>=11 && s[10]){
|
||||
priceMap.put("cityhouse", s[10])
|
||||
}
|
||||
|
||||
if(s.size()>=12 && s[11]){
|
||||
priceMap.put("fangtan", s[11])
|
||||
}
|
||||
|
||||
if(s.size()>=13 && s[12]){
|
||||
priceMap.put("fangdoudou", s[12])
|
||||
}
|
||||
if(s.size()>=14 && s[13]){
|
||||
priceMap.put("maitian", s[13])
|
||||
}
|
||||
|
||||
try{
|
||||
if(s.size()>=15 && s[14]){
|
||||
priceMap.put("58tc", s[14])
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.getMessage()
|
||||
}
|
||||
|
||||
}
|
||||
return map1
|
||||
}
|
||||
|
||||
def addPrice(){
|
||||
ExcelUtils.modify("D:\\数据\\福州\\福州\\福州高端小区匹配.xlsx", { d ->
|
||||
if(d['G']){
|
||||
def avg_price = findFzDistrictStat( d['G'],202207)
|
||||
d['F'] = avg_price
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
|
||||
def findFzDistrictStat(id,lastMonthDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("comm_id", id)
|
||||
q.put("month", lastMonthDate)
|
||||
def dbcursor = FZ_DISTRICT_STAT_COLL.find(q)
|
||||
for(def d : dbcursor){
|
||||
return d.avg_price
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\福州高端小区匹配.xlsx",{d->
|
||||
def region = d['C']
|
||||
if(!d['G'] && region){
|
||||
def name = d['B'].toString().replace("(","").replace(")","")
|
||||
// name = d['B'].toString().replace("(非钢混)","").replace("(钢混)","")
|
||||
region = RegionData.formatRegion("福州", region)
|
||||
// def dist = CommonDatautils.findDistrict("福州", region, name, "district")
|
||||
// if(dist){
|
||||
// d['G'] = dist._id.toString()
|
||||
// d['H'] = dist.name
|
||||
// }
|
||||
def jsonObject = OpenApiClient.addressMatch("福州", name, "property", false, region, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
d['H'] = jsonObject.get("name")
|
||||
d['G'] = jsonObject.get("id")
|
||||
d['J'] = jsonObject.get("district")
|
||||
}
|
||||
|
||||
}
|
||||
},0)
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202206.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202207.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+41
-24
@@ -75,81 +75,98 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
row << dist.region
|
||||
row << dist.name
|
||||
row << o.s_date
|
||||
|
||||
def zgf_price = getLastThirdPrice(o.zhugezhaofang_esf_price, curPrice)
|
||||
row << zgf_price
|
||||
if(zgf_price){
|
||||
add = true;
|
||||
}
|
||||
// if(o.zhugezhaofang_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
|
||||
def ajk_price = getLastThirdPrice(o.anjuke_esf_price, curPrice)
|
||||
row << ajk_price
|
||||
if(ajk_price){
|
||||
add = true;
|
||||
}
|
||||
// if(o.anjuke_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
|
||||
def sf_price = getLastThirdPrice(o.soufang_esf_price, curPrice)
|
||||
row << sf_price
|
||||
if(sf_price){
|
||||
add = true;
|
||||
}
|
||||
// if(o.soufang_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
|
||||
def lj_price = getLastThirdPrice(o.lianjia_esf_price, curPrice)
|
||||
row << lj_price
|
||||
if(lj_price){
|
||||
add = true;
|
||||
}
|
||||
// if(o.lianjia_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
|
||||
def city_price = getLastThirdPrice(o.cityhouse_esf_price, curPrice)
|
||||
row << city_price
|
||||
if(city_price){
|
||||
add = true;
|
||||
}
|
||||
// if(o.cityhouse_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
|
||||
def ft_price = getLastThirdPrice(o.fangtan_esf_price, curPrice)
|
||||
row << ft_price
|
||||
if(ft_price){
|
||||
add = true;
|
||||
}
|
||||
// if(o.fangtan_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
|
||||
def fdd_price = getLastThirdPrice(o.fangdoudou_esf_price, curPrice)
|
||||
row <<fdd_price
|
||||
if(fdd_price){
|
||||
add = true;
|
||||
}
|
||||
// if(o.fangdoudou_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
|
||||
def mt_price = getLastThirdPrice(o.maitian_esf_price, curPrice)
|
||||
row << mt_price
|
||||
if(mt_price){
|
||||
add = true;
|
||||
}
|
||||
// if(o.maitian_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
|
||||
def tc_price = getLastThirdPrice(o.get("58tc_esf_price"), curPrice)
|
||||
row << tc_price
|
||||
if(tc_price){
|
||||
add = true;
|
||||
}
|
||||
|
||||
// row << o.zhugezhaofang_esf_price
|
||||
// if(o.zhugezhaofang_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
// row << o.anjuke_esf_price
|
||||
// if(o.anjuke_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
// row << o.soufang_esf_price
|
||||
// if(o.soufang_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
//
|
||||
// row << o.lianjia_esf_price
|
||||
// if(o.lianjia_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
//
|
||||
// row << o.cityhouse_esf_price
|
||||
// if(o.cityhouse_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
//
|
||||
// row << o.fangtan_esf_price
|
||||
// if(o.fangtan_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
// row << o.fangdoudou_esf_price
|
||||
// if(o.fangdoudou_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
// row << o.maitian_esf_price
|
||||
// if(o.maitian_esf_price){
|
||||
// add = true;
|
||||
// }
|
||||
//
|
||||
// row << o.get("58tc_esf_price")
|
||||
// if(o.get("58tc_esf_price")){
|
||||
// add = true;
|
||||
// }
|
||||
|
||||
+3
-3
@@ -48,11 +48,11 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2022-06-01"
|
||||
static def statDate = "2022-07-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
task.esfParser();//友达二手房存入mongo
|
||||
// task.esfParser();//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2022年7月\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2022年8月\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
+3
-3
@@ -44,7 +44,7 @@ public class BeiKeCrawlByChrome {
|
||||
|
||||
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
|
||||
WebDriver driver = new ChromeDriver()
|
||||
@@ -376,13 +376,13 @@ public class BeiKeCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
if(e.text().contains("福州")){
|
||||
// if(e.text().contains("福州")){
|
||||
def cityMap = [:]
|
||||
cityMap.city = e.text()
|
||||
cityMap.url = "https:"+href
|
||||
list << cityMap
|
||||
//cityMap.put(e.text(), "https:"+href)
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
return list
|
||||
|
||||
+6
-5
@@ -175,8 +175,8 @@ class ExportPaimai {
|
||||
|
||||
|
||||
def run(){
|
||||
def city = "宿迁"
|
||||
def region = null
|
||||
def city = "苏州"
|
||||
def region = "姑苏"
|
||||
def datas = findPaimaiData(city,region)
|
||||
println "总量:"+datas.size()
|
||||
def rows = []
|
||||
@@ -233,7 +233,7 @@ class ExportPaimai {
|
||||
// println d.title
|
||||
// }
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\京东拍卖(${city}).xlsx",rows,"数据")
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\京东拍卖(${city}-姑苏-撤回中止).xlsx",rows,"数据")
|
||||
|
||||
// batchCreateFile(rows, "D:\\数据\\重庆农商行\\法拍\\淘宝拍卖(新).csv",null,"GBK")
|
||||
println "生成完成........"
|
||||
@@ -452,13 +452,14 @@ class ExportPaimai {
|
||||
if(region){
|
||||
pmquery.put("region",region)
|
||||
}
|
||||
pmquery.put("status", new BasicDBObject('$in', ["已撤回","已暂缓"]))
|
||||
// pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
|
||||
pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
|
||||
pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-07-01")))
|
||||
|
||||
// Pattern pattern = Pattern.compile("rmfysszc.gov.cn", Pattern.MULTILINE);
|
||||
Pattern pattern = Pattern.compile("jd.com", Pattern.MULTILINE);
|
||||
// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
|
||||
pmquery.put("url", new BasicDBObject('$regex', pattern))
|
||||
// pmquery.put("url", new BasicDBObject('$regex', pattern))
|
||||
def dbcursor = paimai43_COLL.find(pmquery).batchSize(2000)
|
||||
def rets = []
|
||||
for(def db : dbcursor){
|
||||
|
||||
+3
-3
@@ -116,7 +116,7 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def cityMap = getAllCityUrls(driver)
|
||||
@@ -446,9 +446,9 @@ public class LianjiaCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
// if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
cityMap.put(e.text(), href)
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
|
||||
+9
-12
@@ -29,21 +29,21 @@ DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "ave
|
||||
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43")
|
||||
|
||||
|
||||
def citys = ["淮安","扬州","南京","苏州","无锡","常州","徐州","镇江","南通","泰州","宿迁","连云港","盐城"]
|
||||
for(def city : citys){
|
||||
def citys = ["乌鲁木齐","昌吉","阿克苏","伊犁","喀什"]
|
||||
//for(def city : citys){
|
||||
//
|
||||
//}
|
||||
|
||||
}
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\江苏\\江苏数据统计.xlsx",{d->
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\江苏\\新疆数据统计.xlsx",{d->
|
||||
// def keys = getDealPrices(d['A'], DEAL_PRICE_COLL)
|
||||
// if(keys){
|
||||
// d['E'] = keys.size()
|
||||
// d['G'] = keys.size()
|
||||
// }
|
||||
|
||||
// def rets = findHousesByCity(d['A'], "sell", 20210701)
|
||||
if(d['A']){
|
||||
def count = findHouseCount(d['A'])
|
||||
d['F'] = count
|
||||
d['H'] = count
|
||||
}
|
||||
|
||||
})
|
||||
@@ -288,7 +288,7 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){
|
||||
//}
|
||||
//
|
||||
|
||||
//def citys = ["淮安","扬州","南京","苏州","无锡","常州","徐州","镇江","南通","泰州","宿迁","连云港","盐城"]
|
||||
|
||||
//def rows = []
|
||||
//rows << ["城市","小区量", "别名量", "地址别名量"]
|
||||
//for(def c : citys){
|
||||
@@ -305,7 +305,4 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){
|
||||
// }
|
||||
// rows << [c, districts.size(), alias_count, addr_alias_count]
|
||||
//}
|
||||
//
|
||||
//
|
||||
//
|
||||
//ExcelUtils.write("D:\\数据\\江苏\\江苏数据统计.xlsx", rows, "结果")
|
||||
//ExcelUtils.write("D:\\数据\\江苏\\新疆数据统计.xlsx", rows, "结果")
|
||||
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class MingShengStat {
|
||||
|
||||
def cityLevel0 = ["北京","上海","广州","深圳"]
|
||||
def cityLevel1 = ["成都","重庆","杭州","武汉","西安","郑州","青岛","长沙","天津","苏州","南京","东莞","沈阳","合肥","佛山"]
|
||||
def cityLevel2 = ["昆明","福州","无锡","厦门","哈尔滨","长春","南昌","济南","宁波","大连","贵阳","温州","石家庄","泉州","南宁","金华","常州","珠海","惠州","嘉兴","南通","中山","保定","兰州","台州","徐州","太原","绍兴","烟台","廊坊"]
|
||||
def cityLevel3 = ["海口","汕头","潍坊","扬州","洛阳","乌鲁木齐","临沂","唐山","镇江","盐城","湖州","赣州","漳州","揭阳","江门","桂林","邯郸","泰州","济宁","呼和浩特","咸阳","芜湖","三亚","阜阳","淮安遵义","银川","衡阳","上饶","柳州","淄博","莆田","绵阳","湛江","商丘","宜昌",
|
||||
"沧州","连云港","南阳","蚌埠","驻马店","滁州","邢台","潮州","秦皇岛","肇庆","荆州","周口","马鞍山","清远","宿州","威海","九江","新乡","信阳","襄阳","岳阳","安庆","菏泽","宜春","黄冈","泰安","宿迁","株洲","宁德","鞍山","南充","六安","大庆","舟山"]
|
||||
def cityLevel4 = ["常德","渭南湖","孝感","丽水","运城","德州","张家口","鄂尔多斯","阳江","泸州","丹东","曲靖","乐山","许昌","湘潭","晋中","安阳","齐齐哈尔","北海","宝鸡","抚州","景德镇","延安","三明","抚顺亳州","日照","西宁","衢州","拉萨","淮北","焦作","平顶山","滨州","吉安","濮阳","眉山","池州","荆门","铜仁","长治",
|
||||
"衡水","铜陵","承德","达州","邵阳","德阳","龙岩","南平","淮南","黄石","营口","东营","吉林","韶关","枣庄","包头","怀化","宣城","临汾","聊城","梅州","盘锦","锦州","榆林","玉林","十堰","汕尾","咸宁","宜宾","永州","益阳","黔南州","黔东南","恩施","红河","大理","大同","鄂州","忻州","吕梁","黄山","开封","郴州","茂名","漯河","葫芦岛","河源","娄底","延边"]
|
||||
def cityLevel5 = ["汉中","辽阳","四平","内江","六盘水","安顺","新余","牡丹江","晋城","自贡","三门峡","赤峰","本溪","防城港","铁岭","随州","广安","广元","天水","遂宁","萍乡","西双版纳","绥化","鹤壁","湘西","松原","阜新","酒泉","张家界","黔西南","保山","昭通","河池","来宾","玉溪","梧州","鹰潭","钦州","云浮","佳木斯","克拉玛依","呼伦贝尔","贺州",
|
||||
"通化","阳泉","朝阳","百色","毕节","贵港","丽江","安康","通辽","德宏","朔州","伊犁","文山","楚雄","嘉峪关","凉山","资阳","锡林郭勒盟","雅安","普洱","崇左","庆阳","巴音郭楞(巴州)","乌兰察布","白山","昌吉","白城","兴安盟","定西","喀什","白银","陇南","巴彦淖尔","巴中","鸡西","乌海","临沧","海东","张掖","商洛","黑河","哈密","吴忠","阿克苏"]
|
||||
|
||||
def ajkNewDistsMap = ["北京": 538, "上海": 1928, "广州": 1834, "深圳": 1149, "武汉": 2181, "大连": 866, "南京": 1442, "杭州": 2245, "太原": 497, "石家庄": 723, "重庆": 2700, "西安": 1693, "福州": 305, "济南": 1003, "宁波": 1304, "成都": 3004, "天津": 1614, "昆明": 755, "南昌": 697, "青岛": 1843, "厦门": 253, "郑州": 1385, "长沙": 1811, "长春": 741, "合肥": 1265, "呼和浩特": 462, "沈阳": 456, "贵阳": 499, "拉萨": 82, "宝鸡": 249, "汕头": 411, "苏州": 1531, "温州": 566, "泉州": 565, "邯郸": 314, "唐山": 442, "沧州": 248, "洛阳": 560, "南阳": 422, "襄阳": 302, "吕梁": 79, "大同": 171, "潍坊": 739, "南通": 735, "衡阳": 425, "中山": 590, "宜昌": 360, "许昌": 384, "盐城": 459, "济宁": 510, "龙岩": 227, "上饶": 288, "马鞍山": 199, "赣州": 385, "金华": 479, "常德": 394, "延边": 71, "泰州": 304, "三亚": 456, "淮安": 488, "咸宁": 307, "无锡": 703, "常州": 470, "镇江": 382, "烟台": 917, "绍兴": 714, "莆田": 261, "东莞": 729, "佛山": 1161, "株洲": 285, "黄石": 151, "孝感": 379]
|
||||
def bkNewDistsMap = ["北京": 316, "上海": 2002, "广州": 1350, "深圳": 1261, "武汉": 2308, "大连": 438, "南京": 579, "杭州": 1892, "太原": 418, "石家庄": 582, "重庆": 2122, "西安": 1608, "福州": 512, "济南": 779, "宁波": 592, "成都": 2849, "天津": 1039, "昆明": 1262, "南昌": 429, "青岛": 1275, "厦门": 387, "郑州": 1146, "长沙": 1434, "长春": 433, "合肥": 807, "呼和浩特": 294, "沈阳": 1137, "贵阳": 242, "拉萨": 3, "宝鸡": 50, "苏州": 1129, "温州": 126, "泉州": 410, "邯郸": 114, "唐山": 181, "沧州": 50, "洛阳": 192, "南阳": 290, "襄阳": 55, "大同": 62, "潍坊": 332, "南通": 509, "衡阳": 83, "中山": 516, "宜昌": 147, "许昌": 156, "盐城": 211, "济宁": 205, "龙岩": 5, "上饶": 160, "马鞍山": 120, "赣州": 236, "金华": 124, "常德": 116, "三亚": 164, "淮安": 278, "咸宁": 75, "无锡": 575, "常州": 221, "镇江": 310, "烟台": 548, "绍兴": 175, "莆田": 13, "东莞": 622, "佛山": 971, "株洲": 134, "黄石": 88, "孝感": 94]
|
||||
def ljNewDistMap = ["北京": 222, "上海": 1758, "广州": 1342, "深圳": 1240, "武汉": 2034, "大连": 433, "南京": 531, "杭州": 1845, "太原": 400, "石家庄": 566, "重庆": 2057, "西安": 1513, "济南": 685, "宁波": 553, "成都": 2683, "天津": 1015, "昆明": 1255, "南昌": 404, "青岛": 1228, "厦门": 361, "郑州": 986, "长沙": 1393, "长春": 367, "合肥": 786, "呼和浩特": 176, "沈阳": 1102, "贵阳": 187, "宝鸡": 0, "苏州": 1018, "温州": 222, "泉州": 329, "邯郸": 222, "唐山": 222, "洛阳": 0, "襄阳": 54, "潍坊": 222, "南通": 413, "衡阳": 0, "中山": 486, "宜昌": 7, "许昌": 0, "盐城": 0, "济宁": 0, "上饶": 0, "马鞍山": 0, "赣州": 0, "金华": 0, "常德": 0, "三亚": 156, "淮安": 0, "无锡": 550, "常州": 0, "镇江": 255, "烟台": 479, "绍兴": 150, "东莞": 589, "佛山": 948, "株洲": 0, "黄石": 40]
|
||||
def sfNewDistMap =["北京": 222, "上海": 1758, "广州": 1342, "深圳": 1240, "武汉": 2034, "大连": 433, "南京": 531, "杭州": 1845, "太原": 400, "石家庄": 566, "重庆": 2057, "西安": 1513, "济南": 685, "宁波": 553, "成都": 2683, "天津": 1015, "昆明": 1255, "南昌": 404, "青岛": 1228, "厦门": 361, "郑州": 986, "长沙": 1393, "长春": 367, "合肥": 786, "呼和浩特": 176, "沈阳": 1102, "贵阳": 187, "宝鸡": 0, "苏州": 1018, "温州": 222, "泉州": 329, "邯郸": 222, "唐山": 222, "洛阳": 0, "襄阳": 54, "潍坊": 222, "南通": 413, "衡阳": 0, "中山": 486, "宜昌": 7, "许昌": 0, "盐城": 0, "济宁": 0, "上饶": 0, "马鞍山": 0, "赣州": 0, "金华": 0, "常德": 0, "三亚": 156, "淮安": 0, "无锡": 550, "常州": 0, "镇江": 255, "烟台": 479, "绍兴": 150, "东莞": 589, "佛山": 948, "株洲": 0, "黄石": 40]
|
||||
def chNewDistMap = ["北京": 408, "上海": 532, "广州": 559, "深圳": 763, "武汉": 775, "大连": 7, "南京": 1137, "杭州": 517, "太原": 293, "石家庄": 567, "重庆": 817, "西安": 439, "济南": 544, "宁波": 555, "成都": 1094, "天津": 754, "昆明": 446, "南昌": 379, "青岛": 840, "厦门": 267, "郑州": 960, "长沙": 767, "长春": 299, "合肥": 404, "呼和浩特": 136, "沈阳": 385, "贵阳": 330, "拉萨": 41, "宝鸡": 145, "汕头": 220, "苏州": 715, "温州": 422, "泉州": 312, "邯郸": 425, "唐山": 317, "沧州": 298, "洛阳": 268, "南阳": 233, "襄阳": 84, "吕梁": 38, "大同": 121, "潍坊": 421, "南通": 328, "衡阳": 329, "中山": 279, "宜昌": 158, "许昌": 161, "盐城": 165, "济宁": 251, "龙岩": 90, "上饶": 136, "马鞍山": 69, "赣州": 265, "金华": 252, "常德": 192, "延边": 59, "泰州": 122, "三亚": 120, "淮安": 150, "咸宁": 175, "无锡": 468, "常州": 259, "镇江": 223, "烟台": 420, "绍兴": 368, "莆田": 106, "东莞": 249, "佛山": 438, "株洲": 216, "黄石": 96, "孝感": 160]
|
||||
def jwNewDistMap = ["北京": 510, "上海": 1353, "广州": 1193, "深圳": 697, "武汉": 1746, "大连": 539, "南京": 1219, "杭州": 2030, "太原": 631, "石家庄": 521, "重庆": 2165, "西安": 1518, "济南": 811, "宁波": 1023, "成都": 3192, "天津": 91, "昆明": 764, "南昌": 632, "青岛": 1420, "厦门": 274, "郑州": 1509, "长沙": 1474, "长春": 400, "合肥": 1112, "呼和浩特": 320, "沈阳": 648, "贵阳": 501, "拉萨": 75, "宝鸡": 217, "汕头": 350, "苏州": 1210, "温州": 628, "泉州": 407, "邯郸": 203, "唐山": 638, "沧州": 367, "洛阳": 509, "南阳": 311, "襄阳": 206, "吕梁": 48, "大同": 159, "潍坊": 1041, "南通": 883, "衡阳": 420, "中山": 457, "宜昌": 319, "许昌": 151, "盐城": 488, "济宁": 534, "龙岩": 161, "上饶": 205, "马鞍山": 181, "赣州": 380, "金华": 424, "常德": 305, "延边": 102, "泰州": 172, "三亚": 460, "淮安": 368, "咸宁": 223, "无锡": 509, "常州": 420, "镇江": 506, "烟台": 976, "绍兴": 608, "莆田": 226, "东莞": 1270, "佛山": 894, "株洲": 274, "黄石": 170, "孝感": 233]
|
||||
def zgNewDistMap = ["南京": 711, "大连": 143, "青岛": 1117, "济南": 577, "昆明": 726, "中山": 466, "佛山": 852, "盐城": 384, "北京": 563, "无锡": 342, "东莞": 428, "烟台": 853, "厦门": 289, "南通": 325, "太原": 310, "泉州": 354, "贵阳": 1324, "马鞍山": 141, "上海": 1776, "苏州": 385, "郑州": 610, "常州": 230, "洛阳": 228, "潍坊": 795, "赣州": 238, "淮安": 268, "深圳": 1034, "长沙": 1252, "石家庄": 296, "金华": 156, "南昌": 383, "泰州": 255, "广州": 1193, "三亚": 341, "宁波": 475, "唐山": 382, "镇江": 495, "济宁": 330, "莆田": 213, "沧州": 155, "宝鸡": 173, "重庆": 634, "沈阳": 494, "长春": 182, "襄阳": 164, "天津": 556, "合肥": 366, "宜昌": 252, "温州": 428, "南阳": 379, "成都": 2452, "绍兴": 147, "呼和浩特": 337, "武汉": 845, "西安": 1407, "邯郸": 220, "常德": 259, "杭州": 568, "黄石": 143, "大同": 151, "许昌": 289, "延边": 72, "咸宁": 230, "拉萨": 60, "株洲": 225, "上饶": 210, "孝感": 229, "吕梁": 78, "汕头": 329, "龙岩": 150, "衡阳": 328]
|
||||
|
||||
def fields = ["city","region","district_name","estate_type","area","avg_price","total_price","direction","fitment","room","hall","toilet","floor","height","build_structure","comm_id","dataType1","purpose","root","url", "p_date"]
|
||||
def no_fields = ["build_date","build_type","floor_info"]
|
||||
static main(args) {
|
||||
def task = new MingShengStat();
|
||||
// task.run()
|
||||
// task.addNewLevel
|
||||
task.addHouseRate()
|
||||
// task.houseFieldStat()
|
||||
}
|
||||
|
||||
def houseFieldStat(){
|
||||
def citys = ["北京","上海","广州","深圳","武汉","大连","南京","杭州","太原","石家庄","重庆","西安","福州","济南","宁波","成都","天津","昆明","南昌","青岛","厦门","郑州","长沙","长春","合肥","呼和浩特","沈阳","贵阳","拉萨","宝鸡","汕头","苏州","温州","泉州","邯郸","唐山","沧州","洛阳","南阳","襄阳","吕梁","大同","潍坊","南通","衡阳","中山","宜昌","许昌","盐城","济宁","龙岩","上饶","马鞍山","赣州","金华","常德","延边","泰州","三亚","淮安","咸宁","无锡","常州","镇江","烟台","绍兴","莆田","东莞","佛山","株洲","黄石","孝感","香港"]
|
||||
def rows = []
|
||||
for(def c : citys){
|
||||
def row = []
|
||||
row << c
|
||||
def total = findHouseCount(c)
|
||||
for(def f : no_fields){
|
||||
def num = findValueByCity(c, f);
|
||||
if(num){
|
||||
def rate = com.fangjia.script.common.DataHandleUtils.getDoubleRound(4, num*100/total)
|
||||
row << rate
|
||||
}else{
|
||||
row << 0
|
||||
}
|
||||
}
|
||||
rows << row
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\民生\\字段覆盖率统计2.xlsx", rows, "字段统计")
|
||||
}
|
||||
|
||||
def addHouseRate(){
|
||||
// def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Downloads\\估值数据统计结果20220816.xlsx",0,1)
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\民生\\城市统计.xlsx",{d->
|
||||
if(d['B']){
|
||||
def cityInfoDto = CommonDatautils.findCityInfo(d['B'])
|
||||
d['G'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, cityInfoDto.price_coverage*100)
|
||||
def dists = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["city","region","name"])
|
||||
d['F'] = dists.size()
|
||||
d['D'] = statNewDistCount(d['B'])
|
||||
d['E'] = findHouseCount(d['B'])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def findValueByCity(city, field){
|
||||
def count = 0
|
||||
def sql = "select count(*) as count from seawater where p_date >= 20200801 and city='${city}' and cat='sell' and ${field} is not null"
|
||||
ElasticUtils.executeSqlQuery(sql, {
|
||||
count = it.count
|
||||
println("$city\t${field} ${it.count}")
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
def findHouseCount(city){
|
||||
def count = 0
|
||||
ElasticUtils.executeSqlQuery("select count(*) as count from seawater where p_date >= 20200801 and city='${city}' and cat='sell'", {
|
||||
count = it.count
|
||||
println("$city\t${it.count}")
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
def statNewDistCount(def city){
|
||||
def sum = 0
|
||||
if(ajkNewDistsMap.get(city)){
|
||||
sum = sum + ajkNewDistsMap.get(city)
|
||||
}
|
||||
if(bkNewDistsMap.get(city)){
|
||||
sum = sum + bkNewDistsMap.get(city)
|
||||
}
|
||||
if(ljNewDistMap.get(city)){
|
||||
sum = sum + ljNewDistMap.get(city)
|
||||
}
|
||||
if(sfNewDistMap.get(city)){
|
||||
sum = sum + sfNewDistMap.get(city)
|
||||
}
|
||||
if(chNewDistMap.get(city)){
|
||||
sum = sum + chNewDistMap.get(city)
|
||||
}
|
||||
if(jwNewDistMap.get(city)){
|
||||
sum = sum + jwNewDistMap.get(city)
|
||||
}
|
||||
if(zgNewDistMap.get(city)){
|
||||
sum = sum + zgNewDistMap.get(city)
|
||||
}
|
||||
return sum
|
||||
}
|
||||
|
||||
def addNewLevel(){
|
||||
ExcelUtils.modify("D:\\数据\\民生\\城市统计.xlsx",{d->
|
||||
if(d['B']){
|
||||
if(cityLevel0.contains(d['B'])){
|
||||
d['D'] = "一线"
|
||||
}else if(cityLevel1.contains(d['B'])){
|
||||
d['D'] = "新一线"
|
||||
}else if(cityLevel2.contains(d['B'])){
|
||||
d['D'] = "二线"
|
||||
}else if(cityLevel3.contains(d['B'])){
|
||||
d['D'] = "三线"
|
||||
}else if(cityLevel4.contains(d['B'])){
|
||||
d['D'] = "四线"
|
||||
}else if(cityLevel5.contains(d['B'])){
|
||||
d['D'] = "五线"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = ["北京","上海","广州","深圳","武汉","大连","南京","杭州","太原","石家庄","重庆","西安","福州","济南","宁波","成都","天津","昆明","南昌","青岛","厦门","郑州","长沙","长春","合肥","呼和浩特","沈阳","贵阳","拉萨","宝鸡","汕头","苏州","温州","泉州","邯郸","唐山","沧州","洛阳","南阳","襄阳","吕梁","大同","潍坊","南通","衡阳","中山","宜昌","许昌","盐城","济宁","龙岩","上饶","马鞍山","赣州","金华","常德","延边","泰州","三亚","淮安","咸宁","无锡","常州","镇江","烟台","绍兴","莆田","东莞","佛山","株洲","黄石","孝感","香港"]
|
||||
def rows = []
|
||||
for(def c : citys){
|
||||
def dbo = CommonDatautils.findAlCityDetail(c)
|
||||
def row = []
|
||||
if(dbo){
|
||||
row << dbo.province
|
||||
row << c
|
||||
row << dbo.new_city_grade/1000
|
||||
}else{
|
||||
row << ""
|
||||
row << c
|
||||
row << ""
|
||||
}
|
||||
rows << row
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\民生\\城市统计.xlsx", rows, "数据")
|
||||
}
|
||||
|
||||
}
|
||||
+4
-4
@@ -40,11 +40,11 @@ class DataStat {
|
||||
|
||||
|
||||
def run(){
|
||||
def source = "城市房产"//
|
||||
def citys = findCrawlCity(source, 202205);
|
||||
def source = "365淘房"//
|
||||
def citys = findCrawlCity(source, 202206);
|
||||
for(def city : citys){
|
||||
def count1 = findCount(source, city, 202205);
|
||||
def count2 = findCount(source, city, 202206);
|
||||
def count1 = findCount(source, city, 202206);
|
||||
def count2 = findCount(source, city, 202207);
|
||||
println city +" "+count1+" "+count2
|
||||
// println count1
|
||||
// println "--- ----"
|
||||
|
||||
+3
-2
@@ -23,7 +23,8 @@ class SynchDistrictPrice {
|
||||
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
|
||||
|
||||
//,"城市房产":"城市房产"
|
||||
def esSources = ["365淘房":"365淘房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","吉屋":"吉屋"]//
|
||||
//"365淘房":"365淘房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房",
|
||||
def esSources = ["我爱我家":"我爱我家","安居客小程序":"安居客","搜房":"搜房","贝壳找房":"贝壳找房",]//"吉屋":"吉屋","365淘房":"365淘房"
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
// task.run()
|
||||
@@ -46,7 +47,7 @@ class SynchDistrictPrice {
|
||||
def findAllEsData(city,esSource, mgSource){
|
||||
def names = new HashSet()
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>'2022-08-01'", {d->
|
||||
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='2022-07-01' order by m_date", {d->
|
||||
def s = d.city +"#"+d.region+"#"+d.name
|
||||
if(!names.contains(s)){
|
||||
names << s
|
||||
|
||||
Reference in New Issue
Block a user