福州同步数据修改,增加人工删除数据
This commit is contained in:
+3
-3
@@ -373,10 +373,10 @@ class DealPriceDataProcess {
|
||||
def citys = CcbCommonUtils.citys
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def publishTime = '2021-10-01'
|
||||
def month = 202204
|
||||
def month = 202205
|
||||
def rows = []
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty(city, 202205)
|
||||
def rets = findPeGteThirdParty(city, 202206)
|
||||
def total = 0
|
||||
println city +" "+rets.size()
|
||||
for(def m : rets){
|
||||
@@ -388,7 +388,7 @@ class DealPriceDataProcess {
|
||||
}
|
||||
|
||||
def findPeGteThirdParty(cityName, yearmonth){
|
||||
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>50"
|
||||
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>30"
|
||||
def rets = []
|
||||
sqlInstance236.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
|
||||
+16
-1
@@ -23,7 +23,8 @@ class LoadGuide {
|
||||
}
|
||||
|
||||
def run(){
|
||||
addPersonGuided("D:\\数据\\建行\\人工指导价更新.xlsx")
|
||||
// addName("D:\\数据\\建行\\房产估值\\人工指导价更新.xlsx")
|
||||
addPersonGuided("D:\\数据\\建行\\房产估值\\人工指导价更新.xlsx")
|
||||
// match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
|
||||
|
||||
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx")
|
||||
@@ -38,6 +39,20 @@ class LoadGuide {
|
||||
|
||||
}
|
||||
|
||||
def addName(filename){
|
||||
ExcelUtils.multiThreadingModify(filename, { d ->
|
||||
if(d['A']){
|
||||
def dist = CommonDatautils.findDistrictById(d['A'].toString())
|
||||
if(dist){
|
||||
d['B'] = dist.city
|
||||
d['C'] = dist.region
|
||||
d['D'] = dist.name
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def addPersonGuided(filename){
|
||||
ExcelUtils.modify(filename, { d ->
|
||||
def city = d['B']
|
||||
|
||||
+25
-23
@@ -45,7 +45,9 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
|
||||
def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
|
||||
|
||||
def url = "https://sh.anjuke.com"
|
||||
def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com")
|
||||
println s
|
||||
|
||||
//def districts = CommonDatautils.findAllFiledDistrict("district", "南宁")
|
||||
//for(def d : districts){
|
||||
@@ -324,28 +326,28 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
|
||||
//
|
||||
//},0)
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\删除重复小区.xlsx",{d->
|
||||
if(d['A']){
|
||||
// def region = d['B']
|
||||
// def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
// if(newRegion) {
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_cat", 11)
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", dist._id)
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println d
|
||||
}
|
||||
// }
|
||||
}
|
||||
},0)
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\删除重复小区.xlsx",{d->
|
||||
// if(d['A']){
|
||||
//// def region = d['B']
|
||||
//// def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
//// if(newRegion) {
|
||||
//// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("name_cat", 11)
|
||||
// udata.put("status", 1)
|
||||
// udata.put("m_date",new Date())
|
||||
//
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("_id", dist._id)
|
||||
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
// }else{
|
||||
// println d
|
||||
// }
|
||||
//// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
//def citys = CcbCommonUtils.citys
|
||||
//def rets = []
|
||||
|
||||
+21
@@ -239,6 +239,27 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
public static def findRemoveDistricts(def city, def cat, def fields, def m_date){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("status", 1);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
q.put("m_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(m_date)))
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
for(def field : fields){
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
List<DBObject> rets = []
|
||||
for(def obj : cursor){
|
||||
rets << obj
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取城市中小区的经纬度
|
||||
* @param city
|
||||
|
||||
+6
-3
@@ -68,14 +68,17 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
def createDatas(def city){
|
||||
def result = []
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", null);
|
||||
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
|
||||
districts.addAll(removeDists)
|
||||
|
||||
def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1)
|
||||
def startDate = DateUtils.getFirstDayOfMonth(lastDate);
|
||||
|
||||
def fzAddrMap = getFzAddress();
|
||||
for(def rMap : districts){
|
||||
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
|
||||
if(dbo){
|
||||
//价格没有也提供
|
||||
// def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
|
||||
// if(dbo){
|
||||
def row = []
|
||||
row << rMap.get("_id").toString()
|
||||
row << rMap.city
|
||||
@@ -112,7 +115,7 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
}
|
||||
//row << processAlias(rMap.addr_alias)
|
||||
result << row
|
||||
}
|
||||
// }
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
+18
-30
@@ -88,6 +88,9 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
|
||||
def createDatas(def city){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", null);
|
||||
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
|
||||
districts.addAll(removeDists)
|
||||
|
||||
def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
|
||||
def dataStr = DateUtils.add(endDateStr, -30)
|
||||
def startDate = DateUtils.parseFromTightYYYYMMDD(dataStr)
|
||||
@@ -105,9 +108,10 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
for(def rMap : districts){
|
||||
|
||||
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
|
||||
def curPrice = 0
|
||||
if(dbo){
|
||||
def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString())
|
||||
def curPrice = dbo.get("avg_price")
|
||||
curPrice = dbo.get("avg_price")
|
||||
if(lastMonthPrice && curPrice){
|
||||
def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice
|
||||
if(Math.abs(markup)>3){
|
||||
@@ -120,36 +124,20 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
|
||||
}
|
||||
}
|
||||
|
||||
def m = []
|
||||
m << dbo.get("_id").toString()//均价ID
|
||||
m << rMap.get("_id").toString()//楼盘ID
|
||||
m << rMap.city
|
||||
m << rMap.region
|
||||
m << rMap.name
|
||||
m << curPrice
|
||||
// def lastMonthPrice = map2.get(rMap.city + "##" + rMap.region+"##"+rMap.name)
|
||||
// if(lastMonthPrice && dbo.get("avg_price")){
|
||||
// lastMonthPrice = lastMonthPrice.toInteger()
|
||||
// def markup = (dbo.get("avg_price")-lastMonthPrice)*100.0/lastMonthPrice
|
||||
// if(Math.abs(markup)>10){
|
||||
// def random = 0
|
||||
// if(dbo.get("avg_price")-lastMonthPrice>=0){
|
||||
// random = com.fangjia.core.util.RandomUtil.random(1, 9)
|
||||
// }else{
|
||||
// random = com.fangjia.core.util.RandomUtil.random(-9, 0)
|
||||
// }
|
||||
// def newPrice = lastMonthPrice*random/100 + lastMonthPrice
|
||||
// m << newPrice.toInteger()
|
||||
// }else{
|
||||
// m << dbo.get("avg_price")
|
||||
// }
|
||||
// }else{
|
||||
// m << dbo.get("avg_price")
|
||||
// }
|
||||
m << curDate.toInteger();
|
||||
result << m
|
||||
}
|
||||
def m = []
|
||||
if(curPrice){
|
||||
m << dbo.get("_id").toString()//均价ID
|
||||
}else{
|
||||
m << ""
|
||||
}
|
||||
m << rMap.get("_id").toString()//楼盘ID
|
||||
m << rMap.city
|
||||
m << rMap.region
|
||||
m << rMap.name
|
||||
m << curPrice
|
||||
m << curDate.toInteger();
|
||||
result << m
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -10,11 +10,11 @@ import java.text.SimpleDateFormat
|
||||
|
||||
class TestFjtLog {
|
||||
|
||||
public static final DBCollection COLL = MongoUtils.getCollectionByName("fangjiatong_log", "fjt_select_log", "192.168.0.42", 52002)
|
||||
public static final DBCollection COLL = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002)
|
||||
|
||||
static main(args) {
|
||||
def q = new BasicDBObject()
|
||||
q.put("cat","土地")
|
||||
// q.put("cat","土地")
|
||||
def dbcursor = COLL.find(q).sort(new BasicDBObject("_id",-1))
|
||||
|
||||
def map = [:]
|
||||
|
||||
+9
-5
@@ -25,11 +25,11 @@ class AnjukeHouseCrawl {
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
|
||||
def href = "https://nanjing.anjuke.com/community/view/634969"
|
||||
// def doc = getDocument(href, driver)
|
||||
def doc = ClaABuYun.getResponse(href)
|
||||
// def newUrl = href.substring(0, href.indexOf("?"))
|
||||
sendMq(href, doc, "district_detail_anjuke")
|
||||
def href = "https://sh.esf.fang.com/loupan/1210043376.htm"
|
||||
def doc = getDocument(href, driver)
|
||||
// def doc = ClaABuYun.getResponse(href)
|
||||
// def newUrl = href.substring(0, hrefs.indexOf("?"))
|
||||
sendMq(href, doc, "district_detail_fang")
|
||||
|
||||
// for(def i=1;i<50;i++){
|
||||
// def url = "https://fz.anjuke.com/sale/guloue/p${i}/"
|
||||
@@ -201,6 +201,10 @@ class AnjukeHouseCrawl {
|
||||
if(!els){
|
||||
els = JQueryUtils.select(".tab-cont-right", doc)
|
||||
}
|
||||
|
||||
if(!els){
|
||||
els = JQueryUtils.select(".village_info", doc)
|
||||
}
|
||||
if(els && els.size()>0){
|
||||
stop = true;
|
||||
}else{
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ class AnjukeSendMq {
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = collection.find(q).limit(30)
|
||||
for(def d : dbcursor){
|
||||
sendMq(d.url, d.content, "district_detail_anjuke")
|
||||
sendMq(d.url, d.content, "community_anjuke")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+47
-1
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.location
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.gj.paimai.DataUtil
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
@@ -13,7 +14,9 @@ class DistrictMapMatch {
|
||||
def task = new DistrictMapMatch()
|
||||
// task.mapMatch()
|
||||
// task.mapMatch1()
|
||||
task.distince()
|
||||
// task.distince()
|
||||
|
||||
task.mapMatch2()
|
||||
}
|
||||
|
||||
def distince(){
|
||||
@@ -93,6 +96,49 @@ class DistrictMapMatch {
|
||||
}
|
||||
|
||||
|
||||
def mapMatch2(){
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\待入库小区(1).xlsx",{ d->
|
||||
if(!d['M']){
|
||||
index++
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(d['C'],d['J'],d['E'])
|
||||
def map = findStandardObject2(d['C'],d['J'],d['E'], d['A'], rets);
|
||||
// if (!map || !map.lat) {
|
||||
// rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
|
||||
// map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
|
||||
// }
|
||||
if(map){
|
||||
d['K'] = map.id
|
||||
if(map.adname){
|
||||
d['L'] = map.adname
|
||||
}else if(map.region){
|
||||
d['L'] = map.region
|
||||
}
|
||||
d['M'] = map.name
|
||||
d['N'] = map.address
|
||||
|
||||
d['O'] = map.lng
|
||||
d['P'] = map.lat
|
||||
if(map.type){
|
||||
d['Q'] = map.type
|
||||
}else if(map.estateType){
|
||||
d['Q'] = map.estateType
|
||||
}
|
||||
d['R'] = map.score
|
||||
|
||||
if(d['O'] && d['AM']){
|
||||
def location = d['AM']
|
||||
def lng = location.toString().substring(location.toString().indexOf(": ")+2, location.toString().indexOf(","))
|
||||
def lat = location.toString().substring(location.toString().lastIndexOf(": ")+2, location.toString().indexOf("}"))
|
||||
def distance = GeoUtils.calculateDistance(d['O'].toDouble(), d['P'].toDouble(), lng.toDouble(), lat.toDouble())
|
||||
d['S'] = distance
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findStandardObject2(def city, def region, def name, def address, def rets) {
|
||||
if (rets) {
|
||||
def bestObj = [:]
|
||||
|
||||
@@ -308,4 +308,5 @@ class DataUtil {
|
||||
return s
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+29
-5
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.alibaba.fastjson.JSON
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
@@ -32,7 +33,20 @@ class PaimaiTest {
|
||||
// task.stat()
|
||||
// task.updatePaimai()
|
||||
// task.updateFieldPaimai()
|
||||
task.statCity()
|
||||
// task.statCity()
|
||||
task.jsStat()
|
||||
}
|
||||
|
||||
def jsStat(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","苏州")
|
||||
def dbcursor = paimai43_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def rets = []
|
||||
for(def db : dbcursor){
|
||||
def m = [:]
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def statCity(){
|
||||
@@ -60,16 +74,26 @@ class PaimaiTest {
|
||||
|
||||
def updateFieldPaimai(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", new BasicDBObject('$in',["重庆","重庆市"]))
|
||||
// q.put("city", new BasicDBObject('$in',["重庆","重庆市"]))
|
||||
def index = 0
|
||||
def dbcursor = paimai43_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
for(def obj : dbcursor) {
|
||||
index++
|
||||
println index
|
||||
if(obj.address && obj.address.contains("可贷款")){
|
||||
obj.put("address", obj.address.substring(0,obj.address.indexOf("可贷款")))
|
||||
paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
|
||||
def oldCity = obj.city
|
||||
if(oldCity){
|
||||
def newCity = RegionData.formatCity(oldCity)
|
||||
if(newCity && newCity!=oldCity){
|
||||
obj.put("city", newCity)
|
||||
println index +" "+ oldCity +" "+newCity
|
||||
paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
|
||||
|
||||
}
|
||||
}
|
||||
// if(obj.address && obj.address.contains("可贷款")){
|
||||
// obj.put("address", obj.address.substring(0,obj.address.indexOf("可贷款")))
|
||||
// paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+3
-3
@@ -40,11 +40,11 @@ class DataStat {
|
||||
|
||||
|
||||
def run(){
|
||||
def source = "贝壳找房"//
|
||||
def citys = findCrawlCity(source, 202202);
|
||||
def source = "Q房"//
|
||||
def citys = findCrawlCity(source, 202204);
|
||||
for(def city : citys){
|
||||
def count1 = findCount(source, city, 202204);
|
||||
def count2 = findCount(source, city, 202205);
|
||||
def count2 = findCount(source, city, 202203);
|
||||
println city +" "+count1+" "+count2
|
||||
// println count1
|
||||
// println "--- ----"
|
||||
|
||||
+13
-7
@@ -33,18 +33,18 @@ class SynchDistrictPrice {
|
||||
|
||||
esSourceMap.each { k, v ->
|
||||
for(def city : citys){
|
||||
def rets = findAllEsData(city, k, v)
|
||||
def rets = findAllEsData(city, v)
|
||||
println city +" "+rets.size()
|
||||
synchData(city, rets, v)
|
||||
synchData(city, rets, "贝壳找房")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findAllEsData(source){
|
||||
def findAllEsData(city,source){
|
||||
def names = new HashSet()
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery("select * from community where source='贝壳'", {d->
|
||||
ElasticUtils.executeSqlQuery("select * from community where source='贝壳' and city='${city}'", {d->
|
||||
def s = d.city +"#"+d.region+"#"+d.name
|
||||
if(!names.contains(s)){
|
||||
names << s
|
||||
@@ -56,7 +56,7 @@ class SynchDistrictPrice {
|
||||
data.put("avg_price", d.avg_price)
|
||||
data.put("source", source)
|
||||
data.put("s_date", SHORT_DATE)
|
||||
rets << d
|
||||
rets << data
|
||||
}
|
||||
})
|
||||
return rets;
|
||||
@@ -132,8 +132,14 @@ class SynchDistrictPrice {
|
||||
def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
|
||||
if(obj){
|
||||
updateIndex++
|
||||
thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
|
||||
thirdPartyColl.save(r)
|
||||
// thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
|
||||
// thirdPartyColl.save(r)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("avg_price", r.avg_price)
|
||||
udata.put("m_date",new Date())
|
||||
thirdPartyColl.update(new BasicDBObject('_id',obj._id), udata)
|
||||
|
||||
}else{
|
||||
try{
|
||||
addIndex++
|
||||
|
||||
Reference in New Issue
Block a user