update
This commit is contained in:
@@ -7,7 +7,7 @@ import java.sql.ResultSet
|
||||
class EsSearchUtils {
|
||||
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
@@ -394,6 +394,7 @@ class FiledStrand {
|
||||
}
|
||||
return estateCharege;
|
||||
}
|
||||
|
||||
static def processHouseholdCount(def household_count, def estateType2){
|
||||
if(household_count){
|
||||
household_count = household_count.toInteger()
|
||||
|
||||
@@ -29,6 +29,8 @@ class OpenApiUtils {
|
||||
rMap.region = dto.district
|
||||
rMap.name = dto.name
|
||||
rMap.id = dto.id
|
||||
rMap.lng = dto.lng
|
||||
rMap.lat = dto.lat
|
||||
if(dto.address && dto.address!="null"){
|
||||
rMap.address = dto.address
|
||||
}
|
||||
|
||||
+36
-1
@@ -6,7 +6,40 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
class WuXiAddressTest {
|
||||
static main(args) {
|
||||
def task = new WuXiAddressTest()
|
||||
task.run()
|
||||
//task.run()
|
||||
task.addAddress()
|
||||
}
|
||||
|
||||
def addAddress(){
|
||||
ExcelUtils.modify("D:\\数据\\无锡\\教育机构-标准化(已处理)(3)(1).xlsx",{d->
|
||||
def prov = d['C']
|
||||
def city = d['D']
|
||||
def region = d['E']
|
||||
def address = d['B']
|
||||
def road = d['J']
|
||||
def street = d['R']
|
||||
|
||||
def newAddress = prov+"省"+city +"市"
|
||||
if(region == "惠山"){
|
||||
newAddress = newAddress + region +"区"
|
||||
}else{
|
||||
newAddress = newAddress + region +"市"
|
||||
}
|
||||
|
||||
d['U'] = d['S'].replace(region+"市", "").replace(d['R'], "")
|
||||
def dist = d['U']
|
||||
|
||||
def s1 = null
|
||||
if(road){
|
||||
s1 = address.toString().substring(address.toString().indexOf(road)+road.size()).trim().replace(" ","")
|
||||
newAddress = newAddress + street + dist + road + s1
|
||||
}else if(d['L']){
|
||||
s1 = address.toString().substring(address.toString().indexOf(d['L'])+d['L'].size()).trim().replace(" ","")
|
||||
newAddress = newAddress + street + dist + d['L'] + s1
|
||||
}
|
||||
|
||||
d['V'] = newAddress
|
||||
},0)
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -18,6 +51,8 @@ class WuXiAddressTest {
|
||||
//
|
||||
// }
|
||||
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\无锡\\教育机构-标准化(已处理).xlsx",{d->
|
||||
if(!d['S']){
|
||||
def locationStr = getLocationInfo(d['D'], d['Q'])
|
||||
|
||||
+3
-3
@@ -47,11 +47,11 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2021-05-01"
|
||||
static def statDate = "2021-06-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser();//友达二手房存入mongo
|
||||
task.esfParser();//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2021年6月\\二手房\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2021年7月\\二手房\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
@@ -7,6 +7,13 @@ import java.util.regex.Pattern
|
||||
class CcbCodeUtils {
|
||||
|
||||
|
||||
//否:0 是:1
|
||||
static def parseFlag(String flag){
|
||||
if(flag && flag == "是"){
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static def parseBuildYear(String buildDate) {
|
||||
if(buildDate){
|
||||
@@ -53,6 +60,44 @@ class CcbCodeUtils {
|
||||
return null
|
||||
}
|
||||
|
||||
// 高层塔楼 0101 增加“超高层” 0108
|
||||
// 高层板楼 0102 增加“老洋房” 0109
|
||||
// 小高层板楼 0103 增加“花园洋房” 0110
|
||||
// 多层板楼 0104 增加“四合院” 0111
|
||||
// 独栋别墅 0105 增加“老式别墅洋房” 0112
|
||||
// 联排别墅 0106
|
||||
// 叠拼别墅 0107
|
||||
// 其他建筑形式 0199
|
||||
|
||||
static def parseBuildType(def buildType){
|
||||
if(buildType){
|
||||
if(buildType.contains("老式别墅洋房")){
|
||||
return "0112"
|
||||
}else if(buildType.contains("四合院")){
|
||||
return "0111"
|
||||
}else if(buildType.contains("花园洋房")){
|
||||
return "0110"
|
||||
}else if(buildType.contains("老洋房")){
|
||||
return "0109"
|
||||
}else if(buildType.contains("叠拼")){
|
||||
return "0107"
|
||||
}else if(buildType.contains("联排")){
|
||||
return "0106"
|
||||
}else if(buildType.contains("独栋")){
|
||||
return "0105"
|
||||
}else if(buildType.contains("多层")){
|
||||
return "0104"
|
||||
}else if(buildType.contains("超高层")){
|
||||
return "0108"
|
||||
}else if(buildType.contains("小高层")){
|
||||
return "0103"
|
||||
}else if(buildType.contains("高层")){
|
||||
return "0102"
|
||||
}
|
||||
}
|
||||
return "0199"
|
||||
}
|
||||
|
||||
|
||||
//朝向 南北通透:0301 东南:0302 南:0303 东西通透:0304 东:0305 西南:0306 西:0307 北:0308 东北:0309 西北:0310 其他朝向:0399
|
||||
static def getDirectionCode(def direction){
|
||||
@@ -161,4 +206,19 @@ class CcbCodeUtils {
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
//房屋用途
|
||||
static def parseHouseUse(String houseUse){
|
||||
if(houseUse){
|
||||
if(houseUse.contains("住宅")){
|
||||
return 11;
|
||||
}else if(houseUse.contains("商住")){
|
||||
return 13;
|
||||
}else{
|
||||
return 99
|
||||
}
|
||||
}
|
||||
return 99
|
||||
}
|
||||
}
|
||||
|
||||
+15
-15
@@ -3,13 +3,13 @@ package com.fangjia.script.gj.ccb
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
class FileUtils {
|
||||
static def batchCreateFile(ret, fileName, headers,charset){
|
||||
static def batchCreateFile(ret, fileName, headers,charset, isAppend){
|
||||
File file = new File(fileName);
|
||||
if (file.exists()) {
|
||||
file.delete();
|
||||
}
|
||||
// if (file.exists()) {
|
||||
// file.delete();
|
||||
// }
|
||||
|
||||
OutputStream out = new FileOutputStream(fileName);
|
||||
OutputStream out = new FileOutputStream(fileName, isAppend);
|
||||
def sb = new StringBuffer();
|
||||
|
||||
sb.append(headers.join("|@|"))
|
||||
@@ -20,18 +20,18 @@ class FileUtils {
|
||||
|
||||
def index = 0
|
||||
for(def row : ret){
|
||||
index++
|
||||
sb.append(StringUtils.join(row.toArray(), "|@|"))
|
||||
sb.append(System.getProperty("line.separator"));
|
||||
println index
|
||||
if(row){
|
||||
index++
|
||||
sb.append(StringUtils.join(row.toArray(), "|@|"))
|
||||
sb.append(System.getProperty("line.separator"));
|
||||
|
||||
if(index%10000==0){
|
||||
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
|
||||
if(index%10000==0){
|
||||
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(sb.length()>0){
|
||||
|
||||
+18
-6
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -10,22 +11,23 @@ import com.mongodb.DBCollection
|
||||
class RegionPriceExport {
|
||||
|
||||
DBCollection REGION_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "region_sell_stat", "mongo.db.fangjia.com")
|
||||
DBCollection REGION_STAT = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
|
||||
static main(args) {
|
||||
def task = new RegionPriceExport();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def months = [202010,202011,202012,202101,202102,202103]
|
||||
def regions = CommonUtils.getAllRegions("福州","region")
|
||||
def months = ["20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"]
|
||||
def regions = CommonUtils.getAllRegions("佛山","region")
|
||||
def rets = []
|
||||
rets << ["城市","区域","202010","202011","202012","202101","202102","202103"]
|
||||
rets << ["城市","区域","20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"]
|
||||
regions.each{region,values->
|
||||
def data = []
|
||||
data << "福州"
|
||||
data << "佛山"
|
||||
data << region
|
||||
for(def month :months){
|
||||
def dbo = findPrice("福州", region, month);
|
||||
def dbo = findWeekendPrice("佛山", region, month);
|
||||
if(dbo){
|
||||
data << dbo.avg_price
|
||||
}else{
|
||||
@@ -34,7 +36,7 @@ class RegionPriceExport {
|
||||
}
|
||||
rets << data
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\福州\\福州区域月均价数据.xlsx",rets, "价格")
|
||||
ExcelUtils.write("d:\\数据\\福州\\佛山区域均价数据.xlsx",rets, "价格")
|
||||
}
|
||||
|
||||
def findPrice(city, region, month){
|
||||
@@ -45,4 +47,14 @@ class RegionPriceExport {
|
||||
def dbo = REGION_SELL_STAT.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
|
||||
def findWeekendPrice(city, region, month){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("cat", "区域")
|
||||
q.put("weekend", DateUtils.parseFromTightByPattern(month, "yyyyMMdd"))
|
||||
def dbo = REGION_STAT.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
}
|
||||
|
||||
+25
-21
@@ -1,6 +1,8 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -13,7 +15,7 @@ class ExportData {
|
||||
|
||||
def static final DBCollection thirdDistrictColl = CrawalDBUtil.getThirdDistrictCol();
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
@@ -24,8 +26,8 @@ class ExportData {
|
||||
|
||||
static main(args) {
|
||||
def task = new ExportData();
|
||||
task.exportThird()
|
||||
// task.exportMap();
|
||||
// task.exportThird()
|
||||
task.exportMap();
|
||||
}
|
||||
|
||||
def exportMap(){
|
||||
@@ -51,19 +53,18 @@ class ExportData {
|
||||
//"内蒙古"---" 包头","赤峰","鄂尔多斯","乌海","巴彦淖尔","呼伦贝尔","乌兰察布","通辽","兴安盟","锡林郭勒盟","阿拉善盟","呼和浩特"
|
||||
//宁夏 ----- 中卫 固原 石嘴山 吴忠 银川
|
||||
//青海 ---- 海东 海北州 黄南州 海南州 果洛州 玉树州 海西州 西宁
|
||||
def citys = ["潜江","来宾","河源","甘南","宣城","朔州","景德镇","呼和浩特","固原","佛山","玉溪","江门","黄山","海口","梧州","辽源","亳州","六安","大理","滁州","韶关","拉萨","北屯","锡林郭勒盟","海北州","肇庆","梅州","重庆","玉树州","石嘴山","清远","钦州",
|
||||
"中卫","兴安盟","天门","贺州","湛江","文山州","图木舒克","通辽","茂名","海南州","贵港","铜陵","铁门关","汕头","临沧","楚雄","昌都","德宏州","北海","保山","阳江","西双版纳州","石河子","琼海","甘孜州","阿坝州","万宁","怒江州","揭阳","潮州","云浮",
|
||||
"防城港","池州","文昌","儋州","阿拉善盟","阿拉尔","五家渠","山南","日喀则","乐东","黄南州"]
|
||||
//"沈阳","大连","抚顺","鞍山"
|
||||
//"贵阳","朝阳","本溪","邯郸","延安","保定","铁岭","阜新","辽阳","张家口","大庆"
|
||||
//"沧州","武汉","渭南","成都","西宁","汉中","丹东","青岛",
|
||||
//"遵义","衡水","锦州","葫芦岛","毕节","宜昌","济宁","南昌","郑州","白银","绥化","兰州","滨州","徐州","宝鸡","济南","邢台","鸡西","新乡","上饶","宜春","营口","烟台","盘锦","安阳","承德","吉安","黑河","德州",
|
||||
//"榆林","洛阳","武威","双鸭山","淄博","铜仁","齐齐哈尔","福州","佳木斯","南宁","铜川","天水","襄阳","张掖","三门峡","酒泉","台州","南阳","吉林","嘉峪关","太原","平顶山","金华","商丘","潍坊","十堰","聊城","临沂","定西","平凉","晋中","南充","荆州","哈密","九江","南京","六盘水","温州","信阳","绵阳","广安","枣庄","开封","东营","常州","邵阳","秦皇岛","萍乡","安顺","伊春","鹤岗","达州","焦作","黄冈","孝感","海西州", "白城",
|
||||
// def citys = ["潜江","来宾","河源","甘南","宣城","朔州","景德镇","呼和浩特","固原","佛山","玉溪","江门","黄山","海口","梧州","辽源","亳州","六安","大理","滁州","韶关","拉萨","北屯","锡林郭勒盟","海北州","肇庆","梅州","重庆","玉树州","石嘴山","清远","钦州",
|
||||
// "中卫","兴安盟","天门","贺州","湛江","文山州","图木舒克","通辽","茂名","海南州","贵港","铜陵","铁门关","汕头","临沧","楚雄","昌都","德宏州","北海","保山","阳江","西双版纳州","石河子","琼海","甘孜州","阿坝州","万宁","怒江州","揭阳","潮州","云浮",
|
||||
// "防城港","池州","文昌","儋州","阿拉善盟","阿拉尔","五家渠","山南","日喀则","乐东","黄南州"]
|
||||
def citys = ["安庆","西宁","日照","长春","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"];
|
||||
|
||||
for(def city : citys){
|
||||
def path = "D:\\数据\\小区\\地图\\${city}.xlsx";
|
||||
def datas = findAllDatas(city, null)
|
||||
println "${city}共计:"+datas.size()
|
||||
ExcelUtils.write(path, datas, "数据")
|
||||
}
|
||||
|
||||
def path = "D:\\数据\\小区\\地图\\建行信用卡城市(潜江--黄南州).xlsx";
|
||||
def datas = findAllDatas(citys, null)
|
||||
println "共计:"+datas.size()
|
||||
ExcelUtils.write(path, datas, "数据")
|
||||
}
|
||||
|
||||
|
||||
@@ -98,9 +99,9 @@ class ExportData {
|
||||
|
||||
|
||||
|
||||
def findAllDatas(def citys, def pname){
|
||||
def findAllDatas(def city, def pname){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT address,pname,cityname,type,adname,name,source,m_date,location.lon,location.lat FROM amap_poi WHERE 1=1")
|
||||
sqlBuffer.append("SELECT address,pname,cityname,type,adname,name,source,m_date,location FROM amap_poi WHERE 1=1 and cityname='${city}' and type like '商务住宅%'")
|
||||
// sqlBuffer.append(" and pname = '"+pname+"'")
|
||||
// sqlBuffer.append(" and cityname = '"+pname+"'")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
@@ -113,16 +114,19 @@ class ExportData {
|
||||
if(source && source=="高德"){
|
||||
println index
|
||||
def type = rs.getObject("type")
|
||||
def city = rs.getString("cityname")
|
||||
if(citys.contains(city) && type.contains("商务住宅")){
|
||||
//def city = rs.getString("cityname")
|
||||
if(type.contains("住宅")){
|
||||
def ret = []
|
||||
ret << city
|
||||
ret << rs.getString("adname")
|
||||
ret << rs.getString("name")
|
||||
ret << rs.getString("address")
|
||||
ret << rs.getString("type")
|
||||
ret << rs.getFloat("location.lon")
|
||||
ret << rs.getFloat("location.lat")
|
||||
def loaction = rs.getObject("location")
|
||||
if(loaction){
|
||||
ret << DataHandleUtils.getDoubleRound(6, loaction.x)
|
||||
ret << DataHandleUtils.getDoubleRound(6, loaction.y)
|
||||
}
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
|
||||
+4
-1
@@ -68,13 +68,16 @@ class FangjiaClientUtils {
|
||||
|
||||
public static def addressStand(def address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
|
||||
|
||||
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
|
||||
def content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
|
||||
+18
-13
@@ -1,9 +1,12 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsSearchUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.yinjihaun.lng.UpdateDistrictLocation
|
||||
|
||||
import java.util.regex.Pattern
|
||||
@@ -20,20 +23,13 @@ class MapDistrictProcess {
|
||||
* @return
|
||||
*/
|
||||
def run(){
|
||||
//"江苏地图.xlsx","浙江地图.xlsx","福建地图.xlsx","江西地图.xlsx","山东地图.xlsx","安徽地图.xlsx",
|
||||
//"上海地图.xlsx","北京地图.xlsx"
|
||||
//"安徽地图.xlsx","江苏地图.xlsx","浙江地图.xlsx","福建地图.xlsx","江西地图.xlsx","山东地图.xlsx",
|
||||
|
||||
//"重庆地图.xlsx","天津地图.xlsx",
|
||||
//"上海地图.xlsx","北京地图.xlsx",
|
||||
//"河北地图.xlsx","河南地图.xlsx","湖北地图.xlsx","湖南地图.xlsx","广西地图.xlsx",
|
||||
//"吉林地图.xlsx","辽宁地图.xlsx""辽宁地图.xlsx","吉林地图.xlsx","黑龙江地图.xlsx","山西地图.xlsx","海南地图.xlsx"]
|
||||
def files = ["青海地图.xlsx","宁夏地图.xlsx","内蒙古地图.xlsx","云南地图.xlsx","陕西地图.xlsx","新疆地图.xlsx","四川地图.xlsx"]
|
||||
def files = ["武汉.xlsx"]
|
||||
for(def fileName : files){
|
||||
def filePath = "D:\\数据\\小区\\地图\\"+fileName;
|
||||
typeFilter(filePath)
|
||||
nameStandard(filePath);
|
||||
dataFilter(filePath)
|
||||
// typeFilter(filePath)
|
||||
// nameStandard(filePath);
|
||||
// dataFilter(filePath)
|
||||
matchDistDatas(filePath)
|
||||
houseCalculate(filePath);
|
||||
////// //统计是否存在多个区域
|
||||
@@ -114,15 +110,24 @@ class MapDistrictProcess {
|
||||
|
||||
def matchDistDatas(def path){
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify(path,{ List row ->
|
||||
ExcelUtils.modify(path,{ List row ->
|
||||
index ++
|
||||
if(!row['I'] && !row['J']){
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['H'], row['D'],);
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['B'], row['H'], row['D']);
|
||||
if(rMap && (rMap.region==row['B'] || row['B'].contains(rMap.region))){
|
||||
row['J'] = rMap.city
|
||||
row['K'] = rMap.region
|
||||
row['L'] = rMap.name
|
||||
row['M'] = rMap.address
|
||||
|
||||
if(rMap.lat){
|
||||
def baidu = BaiDuApiUtils.geoconv(row['F'].toDouble(), row['G'].toDouble(), 3)
|
||||
def maplng = baidu.x
|
||||
def maplat = baidu.y
|
||||
row['X'] = GeoHelper.GetDistance(maplat, maplng, rMap.lat, rMap.lng)
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['L'])
|
||||
row['Y'] = score
|
||||
}
|
||||
}
|
||||
println index +" "+row
|
||||
}
|
||||
|
||||
+8
-5
@@ -26,15 +26,18 @@ class SeaweedDataMatchProcess {
|
||||
}
|
||||
}
|
||||
|
||||
if(!rMap && address){
|
||||
// if (findPattern(address)){
|
||||
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
|
||||
// }
|
||||
}
|
||||
if(!rMap && name){
|
||||
rMap = OpenApiUtils.matchDistrict(city, name, region, false)
|
||||
}
|
||||
|
||||
if(!rMap && address){
|
||||
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
|
||||
if(rMap && rMap.region && !region.contains(rMap.region)){
|
||||
rMap = null
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
if(rMap && rMap.id && region.contains(rMap.region)) {
|
||||
|
||||
+1
-1
@@ -49,7 +49,7 @@ class ZsPersonProcess {
|
||||
// def loactionMap = task.matchMap(dist)
|
||||
// println loactionMap
|
||||
|
||||
// task.matchMap()
|
||||
task.matchMap()
|
||||
|
||||
// task.matchDb()
|
||||
// task.matchDb2()
|
||||
|
||||
+26
-28
@@ -19,9 +19,9 @@ class EsMonthHouseStat {
|
||||
def task = new EsMonthHouseStat()
|
||||
// task.run()
|
||||
// task.tradeRun()
|
||||
// task.houseRun()
|
||||
task.houseRun()
|
||||
|
||||
task.statHouseUpdateCount()
|
||||
// task.statHouseUpdateCount()
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -99,37 +99,35 @@ class EsMonthHouseStat {
|
||||
}
|
||||
|
||||
def houseRun(){
|
||||
ExcelUtils.multiThreadingModify("d:\\数据\\小区案例数据覆盖率.xlsx",{d->
|
||||
def city = d['B']
|
||||
if(city == "湖州"){
|
||||
def count = findHouseCount(city, 20210301, 20210601)
|
||||
d['E'] = count
|
||||
ExcelUtils.multiThreadingModify("d:\\数据\\建行\\城市覆盖率统计.xlsx",{d->
|
||||
def city = d['A']
|
||||
def count = findHouseCount(city, 20190709, 20210709)
|
||||
d['E'] = count
|
||||
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]);
|
||||
def disthouseCount = 0
|
||||
for(def dist : districts){
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]);
|
||||
def disthouseCount = 0
|
||||
for(def dist : districts){
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
|
||||
def distCount = findHouseCount(dist.city, dist.region, alias, 20210301, 20210601)
|
||||
if(distCount){
|
||||
disthouseCount++
|
||||
}
|
||||
println dist.city+" "+dist.region+" "+dist.name+" "+distCount
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
println "disthouseCount:"+disthouseCount
|
||||
println districts.size()
|
||||
def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
|
||||
d['D'] = precent+"%"
|
||||
def distCount = findHouseCount(dist.city, dist.region, alias, 20190709, 20210709)
|
||||
if(distCount){
|
||||
disthouseCount++
|
||||
}
|
||||
println dist.city+" "+dist.region+" "+dist.name+" "+distCount
|
||||
}
|
||||
|
||||
println "disthouseCount:"+disthouseCount
|
||||
println districts.size()
|
||||
def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
|
||||
d['F'] = precent+"%"
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -42,7 +42,7 @@ class ZhugefangCrawl4 extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
citys.addAll(["东方","东莞","临沂","五常","五指山","保亭","修文","合肥","吴江","咸阳","商洛","宁乡","德州","成都","文昌","无锡","普兰店","珠海","琼中","琼海","白沙县","石家庄","肥西","贵阳","郑州","重庆","陵水","","","",""])
|
||||
citys.addAll(["福州"])
|
||||
// getNoDataCitys();
|
||||
def cityurl = "http://www.zhuge.com/areachange/"
|
||||
def cityUrls = getCityUrlCrawal(cityurl);
|
||||
|
||||
@@ -140,6 +140,7 @@ public class OpenApiClient {
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
public static JSONObject addressMatchV2(String city, String input, String cat, Boolean useMap = false, String suggestDistrict = null, String preciseLevel = "fuzzy", boolean verify = false) {
|
||||
|
||||
if (StringUtils.isBlank(input)) {
|
||||
|
||||
Reference in New Issue
Block a user