Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
+10
-8
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳2024年3月份二手房成交数据汇总.xls", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\20240619--深圳成交数据.xlsx", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
@@ -72,19 +72,21 @@ for(def d : datas){
|
||||
|
||||
}
|
||||
|
||||
map.area = d['F'].toString().toDouble()
|
||||
map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
|
||||
map.avg_price = (int)(d['H'].toString().toDouble())
|
||||
map.area = d['F'].replace("平","").toString().toDouble()
|
||||
map.total_price = (int)(d['G'].replace("万","").toString().toDouble().toInteger()*10000)
|
||||
map.avg_price = (int)(d['H'].replace("元/平","").toString().toDouble())
|
||||
if(d['I']){
|
||||
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
|
||||
map.sell_price = (int)(d['I'].replace("万","").toString().toDouble()*10000)
|
||||
}
|
||||
|
||||
map.trade_date = d['J']
|
||||
// def trade_date = d['J']
|
||||
// if(trade_date){
|
||||
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd')
|
||||
// }else{
|
||||
def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31")
|
||||
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
||||
// }
|
||||
// else{
|
||||
// def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31")
|
||||
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
||||
// }
|
||||
|
||||
map.source = "深房家族"
|
||||
|
||||
+25
-16
@@ -35,7 +35,7 @@ import java.util.regex.Pattern
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM"));
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
@@ -62,20 +62,20 @@ class BigdataDealBatch extends BaseBatch{
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def date = DateUtils.parseFromTightByPattern("2024-5-21", "yyyy-MM-dd")
|
||||
def objectId = dateToObjectId(date)
|
||||
println objectId
|
||||
// def date = DateUtils.parseFromTightByPattern("2024-5-21", "yyyy-MM-dd")
|
||||
// def objectId = dateToObjectId(date)
|
||||
// println objectId
|
||||
|
||||
//664a22000000000000000000 5-20
|
||||
//664b73800000000000000000 5-21
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
//
|
||||
// println "curDate:"+curDate
|
||||
// def allCitys = CommonDatautils.findAllCity()
|
||||
// for(def city : allCitys){
|
||||
// addDealPriceMatch(city);
|
||||
// }
|
||||
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
|
||||
println "curDate:"+curDate
|
||||
def allCitys = CommonDatautils.findAllCity()
|
||||
for(def city : allCitys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
|
||||
|
||||
def rows = []
|
||||
@@ -726,15 +726,24 @@ class BigdataDealBatch extends BaseBatch{
|
||||
t.name = t.name +"商住楼"
|
||||
}
|
||||
}
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, null);
|
||||
if(!rMap && t.address){
|
||||
def flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
|
||||
def flag = false
|
||||
def address = null
|
||||
if(t.address){
|
||||
flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
|
||||
if(flag){
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address);
|
||||
address = t.address
|
||||
}
|
||||
}
|
||||
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, address);
|
||||
if(!rMap && !t.name.toString().endsWith("小区")){
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name+"小区", address);
|
||||
}
|
||||
|
||||
if(rMap && rMap.region==region){
|
||||
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
|
||||
}else if(rMap && rMap.address && address && rMap.region!=region && address.contains(rMap.address)){
|
||||
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
|
||||
}else {
|
||||
if (rMap) {
|
||||
println "区域不同"
|
||||
@@ -752,7 +761,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
}
|
||||
}
|
||||
);
|
||||
)
|
||||
}
|
||||
latch.await()
|
||||
println city +" "+trades.size()+" "+new Date().toString()
|
||||
|
||||
+2
-1
@@ -634,7 +634,8 @@ class CcbCommonUtils {
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate))
|
||||
q.put("match_id", new BasicDBObject('$exists', false))
|
||||
// q.put("district_name","中海寰宇天下")
|
||||
// q.put("district_name","新明胡同")
|
||||
// q.put("address","桂苑大道135号")
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def total = dbcursor.count()
|
||||
def rets = new HashSet()
|
||||
|
||||
+1
-1
@@ -389,7 +389,7 @@ class DealPriceDataProcess {
|
||||
def deleteThirdPartyData(){
|
||||
// ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水"
|
||||
// def citys = ["310000","440100","500000","410100","430100","330100","340100","320100","370200","320500","120000","442000","320300","350200","360100","320400","210200","530100","469005","469028","469029","230700","610700","533400","650400","511100","131100"]//CcbCommonUtils.citys_all
|
||||
def citys = ["上海","广州"]//CcbCommonUtils.citys_all
|
||||
def citys = CcbCommonUtils.citys_all
|
||||
def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
+1
-1
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-08")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-22")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
|
||||
+15
-2
@@ -1289,7 +1289,7 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240607人工指导价.xlsx"
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\国信达指导价6-21.xlsx"
|
||||
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
if(d['J']){
|
||||
@@ -1301,7 +1301,7 @@ ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// d['H'] = dist._id.toString()
|
||||
// }
|
||||
// }
|
||||
if(!d['A'] && d['J'] instanceof String){
|
||||
if(d['J'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['J'])
|
||||
if(dist){
|
||||
d['A'] = dist.city
|
||||
@@ -1318,6 +1318,19 @@ ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
},0)
|
||||
|
||||
|
||||
//def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(file, { d ->
|
||||
// if(d['A']){
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
// d['C'] = dist.city
|
||||
// d['D'] = dist.region
|
||||
// d['E'] = dist.name
|
||||
// def random = RandomUtil.random(-300,50)
|
||||
// d['F'] = random + d['B']
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
//def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(filePath, { d ->
|
||||
// if(d['B']){
|
||||
|
||||
+4
-4
@@ -40,8 +40,8 @@ class TradeStatLog {
|
||||
|
||||
static main(args) {
|
||||
def task = new TradeStatLog()
|
||||
task.run()//统计最近3个月抓取的成交数据量
|
||||
// task.exportTrade()//导出未入库成交
|
||||
// task.run()//统计最近3个月抓取的成交数据量
|
||||
task.exportTrade()//导出未入库成交
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
// task.averageStat()
|
||||
@@ -499,8 +499,8 @@ class TradeStatLog {
|
||||
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
|
||||
def rets = []
|
||||
rets << headers
|
||||
def startDate = "2023-08-01"
|
||||
def endDate = "2023-10-01"
|
||||
def startDate = "2024-04-01"
|
||||
def endDate = "2024-06-01"
|
||||
for(def city : citys){
|
||||
|
||||
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
|
||||
|
||||
+34
-34
@@ -1846,39 +1846,6 @@ class CommonDatautils {
|
||||
}
|
||||
return priceMap;
|
||||
}
|
||||
|
||||
|
||||
public static def queryDistrictPrice(city, region, name, weekend, cat){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
queryCondition.put("region", region);
|
||||
queryCondition.put("district_name", name);
|
||||
queryCondition.put("weekend", weekend);
|
||||
|
||||
if(cat == "district"){
|
||||
BasicDBObject queryObj = new BasicDBObject();
|
||||
queryObj.putAt(QueryOperators.IN, ["小区","别墅"]);
|
||||
queryCondition.put("estate_type", queryObj);
|
||||
}else if(cat == "office"){
|
||||
queryCondition.put("estate_type", "写字楼");
|
||||
}else if(cat == "shop"){
|
||||
queryCondition.put("estate_type", "商铺");
|
||||
}
|
||||
|
||||
|
||||
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
|
||||
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def price = 0;
|
||||
while(cur.hasNext()) {
|
||||
DBObject dbo = cur.next()
|
||||
price = dbo.get("avg_price");
|
||||
if(price){
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return price;
|
||||
}
|
||||
|
||||
|
||||
public static def queryFirstDistrictPrice(city, region, name){
|
||||
@@ -1917,6 +1884,39 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
public static def queryDistrictPrice(city, region, name, weekend, cat){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
queryCondition.put("region", region);
|
||||
queryCondition.put("district_name", name);
|
||||
queryCondition.put("weekend", weekend);
|
||||
|
||||
if(cat == "district"){
|
||||
BasicDBObject queryObj = new BasicDBObject();
|
||||
queryObj.putAt(QueryOperators.IN, ["小区","别墅"]);
|
||||
queryCondition.put("estate_type", queryObj);
|
||||
}else if(cat == "office"){
|
||||
queryCondition.put("estate_type", "写字楼");
|
||||
}else if(cat == "shop"){
|
||||
queryCondition.put("estate_type", "商铺");
|
||||
}
|
||||
|
||||
|
||||
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
|
||||
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def price = 0;
|
||||
while(cur.hasNext()) {
|
||||
DBObject dbo = cur.next()
|
||||
price = dbo.get("avg_price");
|
||||
if(price){
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return price;
|
||||
}
|
||||
|
||||
|
||||
public static def queryDistrictPrice(city, region, name){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
@@ -4679,7 +4679,7 @@ class CommonDatautils {
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
q.put("province",new BasicDBObject('$in', ['福建省','重庆市']))
|
||||
// q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省']))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
|
||||
@@ -13,12 +13,15 @@ import net.sf.json.groovy.JsonSlurper
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
class OpenApiUtils {
|
||||
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){
|
||||
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag, def suggestName){
|
||||
address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","").replace("%","").replace("\"","")
|
||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
|
||||
if(preciseLevelflag){
|
||||
url = url +"&preciseLevel=strict"
|
||||
}
|
||||
if(suggestName){
|
||||
url = url +"&suggestName="+suggestName
|
||||
}
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
def rMap = [:]
|
||||
if(content){
|
||||
|
||||
+17
-17
@@ -39,7 +39,7 @@ class DevProcess {
|
||||
|
||||
// task.calAddrDistince()
|
||||
// task.updateCityAndDist()
|
||||
task.addDevAddres1() //按添加地址中包含开发区名称的地址
|
||||
// task.addDevAddres1() //按添加地址中包含开发区名称的地址
|
||||
// task.addDevAddres2()//添加相差2公里范围内的地址
|
||||
// task.matchDevAndAddAddress()
|
||||
|
||||
@@ -48,7 +48,7 @@ class DevProcess {
|
||||
// task.createDevMapping()
|
||||
// task.deleteAddr()
|
||||
// task.deleteAddr2()
|
||||
// task.synchDevTo43()
|
||||
task.synchDevTo43()
|
||||
// task.addNewDev2()
|
||||
|
||||
// task.addAlias()
|
||||
@@ -644,16 +644,16 @@ class DevProcess {
|
||||
|
||||
|
||||
def matchDevAndAddAddress(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建去重地址入库.xlsx",{d->
|
||||
if(d['J']){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建重庆去重后地址.xlsx",{d->
|
||||
if(d['E']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['J'])
|
||||
q.put("name", d['E'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
|
||||
d['P'] = obj._id.toString()
|
||||
d['Q'] = obj.name
|
||||
def address = d['L']+d['M']
|
||||
d['F'] = obj._id.toString()
|
||||
d['G'] = obj.name
|
||||
def address = d['D']
|
||||
q.put("address", address)
|
||||
// def street = d['G']
|
||||
// q.put("township", street)
|
||||
@@ -685,14 +685,14 @@ class DevProcess {
|
||||
|
||||
|
||||
def matchDevTownship(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\区域匹配开发区.xlsx",{d->
|
||||
if(d['C']){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建重庆去重后地址.xlsx",{d->
|
||||
if(d['D']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['C'])
|
||||
q.put("name", d['D'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['E'] = obj._id.toString()
|
||||
def street = d['D']
|
||||
d['F'] = obj._id.toString()
|
||||
def street = d['E']
|
||||
q.put("township", street)
|
||||
def sameObj = DEV_MAPPING_CONN.findOne(q)
|
||||
if(!sameObj){
|
||||
@@ -706,11 +706,11 @@ class DevProcess {
|
||||
}
|
||||
|
||||
udata.put("name",obj.name)
|
||||
// udata.put("type", "乡镇街道")
|
||||
// udata.put("township", street)
|
||||
udata.put("type", "乡镇街道")
|
||||
udata.put("township", street)
|
||||
|
||||
udata.put("type", "区域")
|
||||
udata.put("dev_district", street)
|
||||
// udata.put("type", "区域")
|
||||
// udata.put("dev_district", street)
|
||||
udata.put("c_date",new Date())
|
||||
DEV_MAPPING_CONN.save(udata)
|
||||
println obj.name +" "+street
|
||||
|
||||
+25
-14
@@ -38,17 +38,17 @@ class PoiLocation {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def citys = ["莆田市","宁德市","三明市","龙岩市","南平市","漳州市","福州市","厦门市","泉州市","重庆市"]
|
||||
def citys = ["重庆市","杭州市","泉州市","东莞市","福州市","厦门市","佛山市","天津市","武汉市","宁波市","郑州市","长沙市","青岛市","上海市","西安市","温州市","无锡市","南京市","湖州市","合肥市","临沂市","石家庄市","昆明市","金华市","保定市","成都市","济南市","北京市","沈阳市","潍坊市","广州市","惠州市","嘉兴市","赣州市","苏州市","南通市","南宁市","台州市","南阳市","徐州市","长春市","深圳市","哈尔滨市","漳州市","中山市","常州市","烟台市","大连市","邯郸市","济宁市","盐城市","绍兴市","唐山市","洛阳市","阜阳市","菏泽市","贵阳市","沧州市","遵义市","莆田市","南昌市","商丘市","周口市","邢台市","江门市","衡阳市","邵阳市","信阳市","廊坊市","扬州市","泰州市","新乡市","太原市","德州市","淄博市","宁德市","绵阳市","汕头市","聊城市","淮安市","驻马店市","黄冈市","宿迁市","三明市","六安市","龙岩市","桂林市","岳阳市","南平市","揭阳市","上饶市","九江市","南充市","连云港市","襄阳市","泰安市","常德市","郴州市","安阳市","安庆市","湛江市","永州市","茂名市","曲靖市","宜宾市","肇庆市","宿州市","兰州市","运城市","镇江市","滁州市","威海市","梅州市","宜春市","清远市","亳州市","怀化市","荆州市","平顶山市","开封市","玉林市","咸阳市","宜昌市","株洲市","乌鲁木齐市","滨州市","渭南市","临汾市","榆林市","芜湖市","银川市","衡水市","毕节市","枣庄市","吉安市","张家口市","柳州市","许昌市","泸州市","呼和浩特市","达州市","德阳市","赤峰市","珠海市","孝感市","益阳市","娄底市","乐山市","恩施土家族苗族自治州","焦作市","湘潭市","海口市","红河哈尼族彝族自治州","黔南布依族苗族自治州","河源市","汉中市","日照市","大理白族自治州","濮阳市","黔东南苗族侗族自治州","秦皇岛市","晋中市","宝鸡市","吉林市","包头市","蚌埠市","东营市","宣城市","韶关市","衢州市","十堰市","承德市","丽水市","眉山市","鞍山市","昭通市","凉山彝族自治州","长治市","抚州市","鄂尔多斯市","吕梁市","百色市","齐齐哈尔市","贵港市","潮州市","黔西南布依族苗族自治州","广安市","淮南市","河池市","大同市","铜仁市","阳江市","咸宁市","营口市","绥化市","马鞍山市","安康市","荆门市","忻州市","广元市","内江市","遂宁市","西宁市","大庆市","汕尾市","文山壮族苗族自治州","锦州市","延安市","巴中市","呼伦贝尔市","楚雄彝族自治州","黄石市","玉溪市","六盘水市","通辽市","钦州市","云浮市","梧州市","漯河市","晋城市","庆阳市","安顺市","丹东市","自贡市","朝阳市","葫芦岛市","天水市","三门峡市","湘西土家族苗族自治州","北海市","陇南市","普洱市","延边朝鲜族自治州","保山市","定西市","随州市","喀什地区","黄山市","佳木斯市","巴彦淖尔市","牡丹江市","舟山市","资阳市","商洛市","松原市","锡林郭勒盟","伊犁哈萨克自治州","萍乡市","铁岭市","乌兰察布市","丽江市","景德镇市","张家界市","抚顺市","通化市","淮北市","来宾市","临沧市","阿克苏地区","贺州市","盘锦市","昌吉回族自治州","池州市","雅安市","三亚市","平凉市","四平市","酒泉市","西双版纳傣族自治州","鹤壁市","辽阳市","白城市","吴忠市","崇左市","阜新市","朔州市","白银市","拉萨市","巴音郭楞蒙古自治州","张掖市","铜陵市","武威市","临夏回族自治州","德宏傣族景颇族自治州","新余市","甘孜藏族自治州","兴安盟","本溪市","鄂州市","黑河市","阿坝藏族羌族自治州","阳泉市","防城港市","鸡西市","仙桃市","鹰潭市","固原市","白山市","中卫市","双鸭山市","攀枝花市","石嘴山市","和田地区","海东市","塔城地区","日喀则市","济源市","伊春市","天门市","辽源市","潜江市","那曲市","铜川市","海西蒙古族藏族自治州","鹤岗市","昌都市","阿拉善盟","乌海市","儋州市","哈密市","克拉玛依市","七台河市","金昌市","甘南藏族自治州","琼海市","迪庆藏族自治州","石河子市","阿勒泰地区","文昌市","玉树藏族自治州","嘉峪关市","万宁市","怒江傈僳族自治州","林芝市","海南藏族自治州","吐鲁番市","澄迈县","陵水黎族自治县","博尔塔拉蒙古自治州","山南市","阿里地区","大兴安岭地区","海北藏族自治州","东方市","乐东黎族自治县","克孜勒苏柯尔克孜自治州","果洛藏族自治州","临高县","阿拉尔市","黄南藏族自治州","神农架林区","屯昌县","定安县","昌江黎族自治县","五家渠市","保亭黎族苗族自治县","琼中黎族苗族自治县","五指山市","白沙黎族自治县","北屯市","图木舒克市","可克达拉市","双河市","铁门关市","毕节地区","昆玉市","铜仁地区","海东地区","三沙市","吐鲁番地区","哈密地区","日喀则地区","林芝地区","昌都地区","山南地区","那曲地区"]
|
||||
for(def city : citys){
|
||||
findPois(city)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findPois(def city){
|
||||
def findPois(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("run_status", new BasicDBObject('$exists', false))
|
||||
q.put("run_status", new BasicDBObject('$ne', 1))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("_id", 1)
|
||||
@@ -67,8 +67,8 @@ class PoiLocation {
|
||||
index++
|
||||
def datas = checkData(index, rets)
|
||||
// updateToEs(datas)
|
||||
updateToEs2(datas)
|
||||
updateToMG(datas)
|
||||
// updateToEs2(datas)
|
||||
// updateToMG(datas)
|
||||
rets = []
|
||||
print "."
|
||||
}
|
||||
@@ -92,6 +92,10 @@ class PoiLocation {
|
||||
if(map.update_status && map.update_status!=100){
|
||||
println "[ ${index} ]"+d.city +" "+totalCount.get() +"/"+rets.size()+" "+map.id+" "+d.district+" Ô£º"+d.township+" Уº"+map.township
|
||||
}
|
||||
if (map.update_status && map.update_status !=100) {
|
||||
updateMapPoi(map.id, map.township)
|
||||
}
|
||||
updateToMG(map)
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println d.city +" "+d.name
|
||||
@@ -115,7 +119,9 @@ class PoiLocation {
|
||||
def regionFlag = null
|
||||
if(d.district){
|
||||
for(def rs : regions){
|
||||
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
|
||||
if(rs.shape && rs.shape.coordinates){
|
||||
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -229,9 +235,14 @@ class PoiLocation {
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = JSONObject.fromObject(res)
|
||||
println(response)
|
||||
try{
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = JSONObject.fromObject(res)
|
||||
println(response)
|
||||
}catch(Exception e){
|
||||
println "error : "+e.getMessage()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -293,10 +304,10 @@ class PoiLocation {
|
||||
}
|
||||
|
||||
|
||||
def updateToMG(rows){
|
||||
def rets = []
|
||||
for(def i=0;i<rows.size();i++){
|
||||
def m = rows[i]
|
||||
def updateToMG(m){
|
||||
// def rets = []
|
||||
// for(def i=0;i<rows.size();i++){
|
||||
// def m = rows[i]
|
||||
def udata = new BasicDBObject()
|
||||
m.each{k,v->
|
||||
if(k!="id"){
|
||||
@@ -311,7 +322,7 @@ class PoiLocation {
|
||||
println m.toString()+" ÎÞ·¨ÐÞ¸Ä-ÎÞÖµ"
|
||||
}
|
||||
|
||||
}
|
||||
// }
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -50,7 +50,7 @@ class TownMiddleProcess {
|
||||
//修改乡镇街道别名
|
||||
// updateTownAlias()
|
||||
//中间表同步到100
|
||||
// synchTown()
|
||||
synchTown()
|
||||
//添加地级市标志
|
||||
// addCityFlag()
|
||||
// addPinyin()
|
||||
|
||||
+80
-4
@@ -3,9 +3,11 @@ package com.fangjia.script.gj.data_handle
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
@@ -27,7 +29,9 @@ class TownshipStrandUpdate {
|
||||
|
||||
static main(args) {
|
||||
def task = new TownshipStrandUpdate()
|
||||
task.process()
|
||||
// task.process()
|
||||
task.updateTown()
|
||||
// task.stat()
|
||||
|
||||
}
|
||||
|
||||
@@ -42,13 +46,14 @@ class TownshipStrandUpdate {
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
citys = citys.reverse()
|
||||
for(def city : citys){
|
||||
if(["中山"].contains(city)){
|
||||
//
|
||||
if(["中山","东莞"].contains(city)){
|
||||
continue
|
||||
}
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\小区\\小区乡镇街道格式化-${city}.xlsx"
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
|
||||
File file=new File(fileName);
|
||||
if(!file.exists()){
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, "district", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, "shop", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
|
||||
dists = checkData(dists)
|
||||
def datas = createDatas(dists)
|
||||
ExcelUtils.write(fileName, datas, "${city}")
|
||||
@@ -56,6 +61,77 @@ class TownshipStrandUpdate {
|
||||
}
|
||||
}
|
||||
|
||||
// 相同且为统计局乡镇
|
||||
// 更正
|
||||
// 不同且不为统计局乡镇
|
||||
// 区域不同
|
||||
// 地图无乡镇街道
|
||||
// 城市不同
|
||||
|
||||
def stat(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
citys = citys.reverse()
|
||||
def rets = []
|
||||
rets << ["城市", "小区量", "更正量","相同且为统计局乡镇","不同且不为统计局乡镇","区域不同","地图无乡镇街道","城市不同","无值"]
|
||||
for(def city : citys){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
|
||||
File file=new File(fileName);
|
||||
def map = [:]
|
||||
def datas = null
|
||||
if(file.exists()){
|
||||
datas = ExcelUtils.read(fileName,0, 1)
|
||||
for(def d : datas){
|
||||
def key = "空值"
|
||||
if(d['S']){
|
||||
key = d['S']
|
||||
}
|
||||
def count = map.get(key)
|
||||
if(!count){
|
||||
count = 0
|
||||
}
|
||||
map.put(key, count+1)
|
||||
}
|
||||
}
|
||||
|
||||
println city
|
||||
map.each {k,v->
|
||||
println k+" "+v
|
||||
}
|
||||
println "--------------------------------------------------------------------------------------------"
|
||||
if(map){
|
||||
rets << [city, datas.size(), map.get("更正"),map.get("相同且为统计局乡镇"),map.get("不同且不为统计局乡镇"),map.get("区域不同"),map.get("地图无乡镇街道"),map.get("城市不同"),map.get("空值")]
|
||||
}
|
||||
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\乡镇街道\\乡镇街道更新结果统计.xlsx", rets, "商铺")
|
||||
}
|
||||
|
||||
def updateTown(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
citys = citys.reverse()
|
||||
for(def city : citys){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
println fileName
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
if(d['S']!='相同且为统计局乡镇' && d['S']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['A'])
|
||||
q.put("region", d['B'])
|
||||
q.put("name", d['C'])
|
||||
q.put("status", 0)
|
||||
q.put("cat","shop")
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("township", d['R'])
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def checkData(def dists){
|
||||
def totalCount = new AtomicLong()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
|
||||
+5
-7
@@ -36,18 +36,16 @@ class SeaweedDataMatchProcess {
|
||||
}
|
||||
}
|
||||
|
||||
if(!rMap && name){
|
||||
rMap = OpenApiUtils.matchDistrict(city, name, region, false)
|
||||
}
|
||||
|
||||
|
||||
if(!rMap && address){
|
||||
if(!rMap && address && name){
|
||||
def flag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||
if(flag){
|
||||
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
|
||||
rMap = OpenApiUtils.matchDistrict(city, address, region, true, name)
|
||||
}
|
||||
}
|
||||
|
||||
if(!rMap && name){
|
||||
rMap = OpenApiUtils.matchDistrict(city, name, region, false, null)
|
||||
}
|
||||
// if(rMap && rMap.id && region.contains(rMap.region)) {
|
||||
// def obj = CommonDatautils.findDistrictById(rMap.id.toString());
|
||||
// if (obj) {
|
||||
|
||||
+12
-9
@@ -33,11 +33,11 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\神经网络计算.xlsx"
|
||||
addGXDPrice(filePath)
|
||||
def filePath = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\0621.xlsx"
|
||||
// addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
// addFjwPrice(filePath)
|
||||
addFjwPrice(filePath)
|
||||
// exportPkPrice()
|
||||
// addRegion()
|
||||
// addTradePrice(filePath)
|
||||
@@ -205,7 +205,7 @@ class AddPriceTest {
|
||||
def addThirdPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (!d['X']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['Y'])
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name)
|
||||
d['X'] = thirdPrice
|
||||
@@ -220,8 +220,8 @@ class AddPriceTest {
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
// q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404))
|
||||
q.put("s_date", 202404)
|
||||
q.put("s_date", new BasicDBObject('$gte', 202403))
|
||||
// q.put("s_date", 202404)
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def price = null
|
||||
for(def db : dbcursor){
|
||||
@@ -271,13 +271,16 @@ class AddPriceTest {
|
||||
def addFjwPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
if(!d['L']){
|
||||
if(d['B'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
if(d['A'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
d['A'] = dist.city
|
||||
d['B'] = dist.region
|
||||
d['C'] = dist.name
|
||||
// d['N'] = dist.region
|
||||
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(dbo){
|
||||
// d['W'] = dbo.avg_price
|
||||
d['W'] = dbo.avg_price
|
||||
// d['X'] = dbo.cal_type
|
||||
d['L'] = dbo.avg_price_2b
|
||||
}
|
||||
|
||||
@@ -234,5 +234,10 @@
|
||||
<artifactId>okhttp</artifactId>
|
||||
<version>3.14.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka_2.13</artifactId>
|
||||
<version>3.4.0</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
||||
+7
-1
@@ -4,6 +4,7 @@ import com.fangjia.dal.QueryParam;
|
||||
import com.fangjia.dal.SortParam;
|
||||
import com.fangjia.dal.mongo.open.dao.OpenUriLogDao;
|
||||
import com.fangjia.dal.mongo.tool.model.OpenUriLog;
|
||||
import com.fangjia.service.util.kafka.KafkaProducerUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
@@ -13,9 +14,12 @@ import java.util.List;
|
||||
public class OpenUriLogServiceImpl implements OpenUriLogService {
|
||||
|
||||
private OpenUriLogDao openUriLogDao;
|
||||
private KafkaProducerUtils kafkaProducerUtils;
|
||||
|
||||
public void save(OpenUriLog openUriLog) {
|
||||
this.openUriLogDao.save(openUriLog);
|
||||
// 改造为向Kafka中发送消息
|
||||
kafkaProducerUtils.sendLog(openUriLog);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -95,5 +99,7 @@ public class OpenUriLogServiceImpl implements OpenUriLogService {
|
||||
this.openUriLogDao = openUriLogDao;
|
||||
}
|
||||
|
||||
|
||||
public void setKafkaProducerUtils(KafkaProducerUtils kafkaProducerUtils) {
|
||||
this.kafkaProducerUtils = kafkaProducerUtils;
|
||||
}
|
||||
}
|
||||
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package com.fangjia.service.util.kafka;
|
||||
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.fangjia.dal.mongo.tool.model.OpenUriLog;
|
||||
import org.apache.kafka.clients.producer.KafkaProducer;
|
||||
import org.apache.kafka.clients.producer.ProducerConfig;
|
||||
import org.apache.kafka.clients.producer.ProducerRecord;
|
||||
import org.apache.kafka.common.serialization.StringSerializer;
|
||||
import org.springframework.beans.factory.DisposableBean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
@Configuration
|
||||
@Service
|
||||
public class KafkaProducerUtils implements DisposableBean {
|
||||
public final KafkaProducer<String, String> producer;
|
||||
|
||||
public KafkaProducerUtils() {
|
||||
Properties properties = new Properties();
|
||||
properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.0.100:30092");
|
||||
properties.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
|
||||
properties.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
|
||||
|
||||
this.producer = new KafkaProducer<String, String>(properties);
|
||||
}
|
||||
|
||||
public void sendLog(OpenUriLog openUriLog) {
|
||||
try {
|
||||
String msg = JSONObject.toJSONString(openUriLog);
|
||||
ProducerRecord<String, String> record = new ProducerRecord<String, String>("log-data-topic", msg);
|
||||
this.producer.send(record);
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() throws Exception {
|
||||
producer.close();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user