福州同步数据修改,增加人工删除数据

This commit is contained in:
2022-06-29 17:45:57 +08:00
parent 60b4802f22
commit c4131ee835
14 changed files with 194 additions and 84 deletions
@@ -373,10 +373,10 @@ class DealPriceDataProcess {
def citys = CcbCommonUtils.citys
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def publishTime = '2021-10-01'
def month = 202204
def month = 202205
def rows = []
for(def city : citys) {
def rets = findPeGteThirdParty(city, 202205)
def rets = findPeGteThirdParty(city, 202206)
def total = 0
println city +" "+rets.size()
for(def m : rets){
@@ -388,7 +388,7 @@ class DealPriceDataProcess {
}
def findPeGteThirdParty(cityName, yearmonth){
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>50"
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>30"
def rets = []
sqlInstance236.eachRow(sql,{r->
def map = [:]
@@ -23,7 +23,8 @@ class LoadGuide {
}
def run(){
addPersonGuided("D:\\数据\\建行\\人工指导价更新.xlsx")
// addName("D:\\数据\\建行\\房产估值\\人工指导价更新.xlsx")
addPersonGuided("D:\\数据\\建行\\房产估值\\人工指导价更新.xlsx")
// match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx")
@@ -38,6 +39,20 @@ class LoadGuide {
}
def addName(filename){
ExcelUtils.multiThreadingModify(filename, { d ->
if(d['A']){
def dist = CommonDatautils.findDistrictById(d['A'].toString())
if(dist){
d['B'] = dist.city
d['C'] = dist.region
d['D'] = dist.name
}
}
})
}
def addPersonGuided(filename){
ExcelUtils.modify(filename, { d ->
def city = d['B']
@@ -45,7 +45,9 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
def url = "https://sh.anjuke.com"
def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com")
println s
//def districts = CommonDatautils.findAllFiledDistrict("district", "南宁")
//for(def d : districts){
@@ -324,28 +326,28 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
//
//},0)
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\删除重复小区.xlsx",{d->
if(d['A']){
// def region = d['B']
// def newRegion = RegionData.formatRegion(d['A'], region)
// if(newRegion) {
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def udata = new BasicDBObject()
udata.put("name_cat", 11)
udata.put("status", 1)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", dist._id)
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}else{
println d
}
// }
}
},0)
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\删除重复小区.xlsx",{d->
// if(d['A']){
//// def region = d['B']
//// def newRegion = RegionData.formatRegion(d['A'], region)
//// if(newRegion) {
//// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// def udata = new BasicDBObject()
// udata.put("name_cat", 11)
// udata.put("status", 1)
// udata.put("m_date",new Date())
//
// def q = new BasicDBObject()
// q.put("_id", dist._id)
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// }else{
// println d
// }
//// }
// }
//},0)
//def citys = CcbCommonUtils.citys
//def rets = []
@@ -239,6 +239,27 @@ class CommonDatautils {
}
public static def findRemoveDistricts(def city, def cat, def fields, def m_date){
DBObject q = new BasicDBObject();
q.put("status", 1);
q.put("cat", cat);
q.put("city", city);
q.put("m_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(m_date)))
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000)
List<DBObject> rets = []
for(def obj : cursor){
rets << obj
}
return rets;
}
/**
* 获取城市中小区的经纬度
* @param city
@@ -68,14 +68,17 @@ class DistrictInfoExportBatch extends BaseBatch{
def createDatas(def city){
def result = []
def districts = CommonDatautils.findAllDistricts(city, "district", null);
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
districts.addAll(removeDists)
def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1)
def startDate = DateUtils.getFirstDayOfMonth(lastDate);
def fzAddrMap = getFzAddress();
for(def rMap : districts){
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
if(dbo){
//价格没有也提供
// def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
// if(dbo){
def row = []
row << rMap.get("_id").toString()
row << rMap.city
@@ -112,7 +115,7 @@ class DistrictInfoExportBatch extends BaseBatch{
}
//row << processAlias(rMap.addr_alias)
result << row
}
// }
}
return result
}
@@ -88,6 +88,9 @@ class DistrictPriceExportBatch extends BaseBatch{
def createDatas(def city){
def districts = CommonDatautils.findAllDistricts(city, "district", null);
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
districts.addAll(removeDists)
def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
def dataStr = DateUtils.add(endDateStr, -30)
def startDate = DateUtils.parseFromTightYYYYMMDD(dataStr)
@@ -105,9 +108,10 @@ class DistrictPriceExportBatch extends BaseBatch{
for(def rMap : districts){
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
def curPrice = 0
if(dbo){
def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString())
def curPrice = dbo.get("avg_price")
curPrice = dbo.get("avg_price")
if(lastMonthPrice && curPrice){
def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice
if(Math.abs(markup)>3){
@@ -120,36 +124,20 @@ class DistrictPriceExportBatch extends BaseBatch{
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
}
}
def m = []
m << dbo.get("_id").toString()//均价ID
m << rMap.get("_id").toString()//楼盘ID
m << rMap.city
m << rMap.region
m << rMap.name
m << curPrice
// def lastMonthPrice = map2.get(rMap.city + "##" + rMap.region+"##"+rMap.name)
// if(lastMonthPrice && dbo.get("avg_price")){
// lastMonthPrice = lastMonthPrice.toInteger()
// def markup = (dbo.get("avg_price")-lastMonthPrice)*100.0/lastMonthPrice
// if(Math.abs(markup)>10){
// def random = 0
// if(dbo.get("avg_price")-lastMonthPrice>=0){
// random = com.fangjia.core.util.RandomUtil.random(1, 9)
// }else{
// random = com.fangjia.core.util.RandomUtil.random(-9, 0)
// }
// def newPrice = lastMonthPrice*random/100 + lastMonthPrice
// m << newPrice.toInteger()
// }else{
// m << dbo.get("avg_price")
// }
// }else{
// m << dbo.get("avg_price")
// }
m << curDate.toInteger();
result << m
}
def m = []
if(curPrice){
m << dbo.get("_id").toString()//均价ID
}else{
m << ""
}
m << rMap.get("_id").toString()//楼盘ID
m << rMap.city
m << rMap.region
m << rMap.name
m << curPrice
m << curDate.toInteger();
result << m
}
return result
}
@@ -10,11 +10,11 @@ import java.text.SimpleDateFormat
class TestFjtLog {
public static final DBCollection COLL = MongoUtils.getCollectionByName("fangjiatong_log", "fjt_select_log", "192.168.0.42", 52002)
public static final DBCollection COLL = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002)
static main(args) {
def q = new BasicDBObject()
q.put("cat","土地")
// q.put("cat","土地")
def dbcursor = COLL.find(q).sort(new BasicDBObject("_id",-1))
def map = [:]
@@ -25,11 +25,11 @@ class AnjukeHouseCrawl {
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def href = "https://nanjing.anjuke.com/community/view/634969"
// def doc = getDocument(href, driver)
def doc = ClaABuYun.getResponse(href)
// def newUrl = href.substring(0, href.indexOf("?"))
sendMq(href, doc, "district_detail_anjuke")
def href = "https://sh.esf.fang.com/loupan/1210043376.htm"
def doc = getDocument(href, driver)
// def doc = ClaABuYun.getResponse(href)
// def newUrl = href.substring(0, hrefs.indexOf("?"))
sendMq(href, doc, "district_detail_fang")
// for(def i=1;i<50;i++){
// def url = "https://fz.anjuke.com/sale/guloue/p${i}/"
@@ -201,6 +201,10 @@ class AnjukeHouseCrawl {
if(!els){
els = JQueryUtils.select(".tab-cont-right", doc)
}
if(!els){
els = JQueryUtils.select(".village_info", doc)
}
if(els && els.size()>0){
stop = true;
}else{
@@ -21,7 +21,7 @@ class AnjukeSendMq {
def q = new BasicDBObject()
def dbcursor = collection.find(q).limit(30)
for(def d : dbcursor){
sendMq(d.url, d.content, "district_detail_anjuke")
sendMq(d.url, d.content, "community_anjuke")
}
}
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.location
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.paimai.DataUtil
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
@@ -13,7 +14,9 @@ class DistrictMapMatch {
def task = new DistrictMapMatch()
// task.mapMatch()
// task.mapMatch1()
task.distince()
// task.distince()
task.mapMatch2()
}
def distince(){
@@ -93,6 +96,49 @@ class DistrictMapMatch {
}
def mapMatch2(){
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\待入库小区(1).xlsx",{ d->
if(!d['M']){
index++
def rets = OpenApiUtils.getGaoDeSearchLocation(d['C'],d['J'],d['E'])
def map = findStandardObject2(d['C'],d['J'],d['E'], d['A'], rets);
// if (!map || !map.lat) {
// rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
// map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
// }
if(map){
d['K'] = map.id
if(map.adname){
d['L'] = map.adname
}else if(map.region){
d['L'] = map.region
}
d['M'] = map.name
d['N'] = map.address
d['O'] = map.lng
d['P'] = map.lat
if(map.type){
d['Q'] = map.type
}else if(map.estateType){
d['Q'] = map.estateType
}
d['R'] = map.score
if(d['O'] && d['AM']){
def location = d['AM']
def lng = location.toString().substring(location.toString().indexOf(": ")+2, location.toString().indexOf(","))
def lat = location.toString().substring(location.toString().lastIndexOf(": ")+2, location.toString().indexOf("}"))
def distance = GeoUtils.calculateDistance(d['O'].toDouble(), d['P'].toDouble(), lng.toDouble(), lat.toDouble())
d['S'] = distance
}
}
}
},0)
}
def findStandardObject2(def city, def region, def name, def address, def rets) {
if (rets) {
def bestObj = [:]
@@ -308,4 +308,5 @@ class DataUtil {
return s
}
}
@@ -1,6 +1,7 @@
package com.fangjia.script.gj.paimai
import com.alibaba.fastjson.JSON
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.MongoUtils
@@ -32,7 +33,20 @@ class PaimaiTest {
// task.stat()
// task.updatePaimai()
// task.updateFieldPaimai()
task.statCity()
// task.statCity()
task.jsStat()
}
def jsStat(){
def q = new BasicDBObject()
q.put("city","苏州")
def dbcursor = paimai43_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def rets = []
for(def db : dbcursor){
def m = [:]
}
}
def statCity(){
@@ -60,16 +74,26 @@ class PaimaiTest {
def updateFieldPaimai(){
def q = new BasicDBObject()
q.put("city", new BasicDBObject('$in',["重庆","重庆市"]))
// q.put("city", new BasicDBObject('$in',["重庆","重庆市"]))
def index = 0
def dbcursor = paimai43_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
for(def obj : dbcursor) {
index++
println index
if(obj.address && obj.address.contains("可贷款")){
obj.put("address", obj.address.substring(0,obj.address.indexOf("可贷款")))
paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
def oldCity = obj.city
if(oldCity){
def newCity = RegionData.formatCity(oldCity)
if(newCity && newCity!=oldCity){
obj.put("city", newCity)
println index +" "+ oldCity +" "+newCity
paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
}
}
// if(obj.address && obj.address.contains("可贷款")){
// obj.put("address", obj.address.substring(0,obj.address.indexOf("可贷款")))
// paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
// }
}
}
@@ -40,11 +40,11 @@ class DataStat {
def run(){
def source = "贝壳找房"//
def citys = findCrawlCity(source, 202202);
def source = "Q房"//
def citys = findCrawlCity(source, 202204);
for(def city : citys){
def count1 = findCount(source, city, 202204);
def count2 = findCount(source, city, 202205);
def count2 = findCount(source, city, 202203);
println city +" "+count1+" "+count2
// println count1
// println "--- ----"
@@ -33,18 +33,18 @@ class SynchDistrictPrice {
esSourceMap.each { k, v ->
for(def city : citys){
def rets = findAllEsData(city, k, v)
def rets = findAllEsData(city, v)
println city +" "+rets.size()
synchData(city, rets, v)
synchData(city, rets, "贝壳找房")
}
}
}
def findAllEsData(source){
def findAllEsData(city,source){
def names = new HashSet()
def rets = []
ElasticUtils.executeSqlQuery("select * from community where source='贝壳'", {d->
ElasticUtils.executeSqlQuery("select * from community where source='贝壳' and city='${city}'", {d->
def s = d.city +"#"+d.region+"#"+d.name
if(!names.contains(s)){
names << s
@@ -56,7 +56,7 @@ class SynchDistrictPrice {
data.put("avg_price", d.avg_price)
data.put("source", source)
data.put("s_date", SHORT_DATE)
rets << d
rets << data
}
})
return rets;
@@ -132,8 +132,14 @@ class SynchDistrictPrice {
def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
if(obj){
updateIndex++
thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
thirdPartyColl.save(r)
// thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
// thirdPartyColl.save(r)
def udata = new BasicDBObject()
udata.put("avg_price", r.avg_price)
udata.put("m_date",new Date())
thirdPartyColl.update(new BasicDBObject('_id',obj._id), udata)
}else{
try{
addIndex++