This commit is contained in:
2021-07-22 10:01:40 +08:00
parent ed0f3a05d4
commit be891b3229
16 changed files with 220 additions and 96 deletions
@@ -7,7 +7,7 @@ import java.sql.ResultSet
class EsSearchUtils {
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -394,6 +394,7 @@ class FiledStrand {
}
return estateCharege;
}
static def processHouseholdCount(def household_count, def estateType2){
if(household_count){
household_count = household_count.toInteger()
@@ -29,6 +29,8 @@ class OpenApiUtils {
rMap.region = dto.district
rMap.name = dto.name
rMap.id = dto.id
rMap.lng = dto.lng
rMap.lat = dto.lat
if(dto.address && dto.address!="null"){
rMap.address = dto.address
}
@@ -6,7 +6,40 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
class WuXiAddressTest {
static main(args) {
def task = new WuXiAddressTest()
task.run()
//task.run()
task.addAddress()
}
def addAddress(){
ExcelUtils.modify("D:\\数据\\无锡\\教育机构-标准化(已处理)(3)(1).xlsx",{d->
def prov = d['C']
def city = d['D']
def region = d['E']
def address = d['B']
def road = d['J']
def street = d['R']
def newAddress = prov+"省"+city +"市"
if(region == "惠山"){
newAddress = newAddress + region +"区"
}else{
newAddress = newAddress + region +"市"
}
d['U'] = d['S'].replace(region+"市", "").replace(d['R'], "")
def dist = d['U']
def s1 = null
if(road){
s1 = address.toString().substring(address.toString().indexOf(road)+road.size()).trim().replace(" ","")
newAddress = newAddress + street + dist + road + s1
}else if(d['L']){
s1 = address.toString().substring(address.toString().indexOf(d['L'])+d['L'].size()).trim().replace(" ","")
newAddress = newAddress + street + dist + d['L'] + s1
}
d['V'] = newAddress
},0)
}
def run(){
@@ -18,6 +51,8 @@ class WuXiAddressTest {
//
// }
ExcelUtils.multiThreadingModify("D:\\数据\\无锡\\教育机构-标准化(已处理).xlsx",{d->
if(!d['S']){
def locationStr = getLocationInfo(d['D'], d['Q'])
@@ -47,11 +47,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-05-01"
static def statDate = "2021-06-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser();//友达二手房存入mongo
task.esfParser();//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2021年6月\\二手房\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2021年7月\\二手房\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -7,6 +7,13 @@ import java.util.regex.Pattern
class CcbCodeUtils {
//否:0 是:1
static def parseFlag(String flag){
if(flag && flag == "是"){
return 1;
}
return 0;
}
static def parseBuildYear(String buildDate) {
if(buildDate){
@@ -53,6 +60,44 @@ class CcbCodeUtils {
return null
}
// 高层塔楼 0101 增加“超高层” 0108
// 高层板楼 0102 增加“老洋房” 0109
// 小高层板楼 0103 增加“花园洋房” 0110
// 多层板楼 0104 增加“四合院” 0111
// 独栋别墅 0105 增加“老式别墅洋房” 0112
// 联排别墅 0106
// 叠拼别墅 0107
// 其他建筑形式 0199
static def parseBuildType(def buildType){
if(buildType){
if(buildType.contains("老式别墅洋房")){
return "0112"
}else if(buildType.contains("四合院")){
return "0111"
}else if(buildType.contains("花园洋房")){
return "0110"
}else if(buildType.contains("老洋房")){
return "0109"
}else if(buildType.contains("叠拼")){
return "0107"
}else if(buildType.contains("联排")){
return "0106"
}else if(buildType.contains("独栋")){
return "0105"
}else if(buildType.contains("多层")){
return "0104"
}else if(buildType.contains("超高层")){
return "0108"
}else if(buildType.contains("小高层")){
return "0103"
}else if(buildType.contains("高层")){
return "0102"
}
}
return "0199"
}
//朝向 南北通透:0301 东南:0302 南:0303 东西通透:0304 东:0305 西南:0306 西:0307 北:0308 东北:0309 西北:0310 其他朝向:0399
static def getDirectionCode(def direction){
@@ -161,4 +206,19 @@ class CcbCodeUtils {
}
return 0;
}
//房屋用途
static def parseHouseUse(String houseUse){
if(houseUse){
if(houseUse.contains("住宅")){
return 11;
}else if(houseUse.contains("商住")){
return 13;
}else{
return 99
}
}
return 99
}
}
@@ -3,13 +3,13 @@ package com.fangjia.script.gj.ccb
import org.apache.commons.lang.StringUtils
class FileUtils {
static def batchCreateFile(ret, fileName, headers,charset){
static def batchCreateFile(ret, fileName, headers,charset, isAppend){
File file = new File(fileName);
if (file.exists()) {
file.delete();
}
// if (file.exists()) {
// file.delete();
// }
OutputStream out = new FileOutputStream(fileName);
OutputStream out = new FileOutputStream(fileName, isAppend);
def sb = new StringBuffer();
sb.append(headers.join("|@|"))
@@ -20,18 +20,18 @@ class FileUtils {
def index = 0
for(def row : ret){
index++
sb.append(StringUtils.join(row.toArray(), "|@|"))
sb.append(System.getProperty("line.separator"));
println index
if(row){
index++
sb.append(StringUtils.join(row.toArray(), "|@|"))
sb.append(System.getProperty("line.separator"));
if(index%10000==0){
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
if(index%10000==0){
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
}
}
}
if(sb.length()>0){
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.data
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
@@ -10,22 +11,23 @@ import com.mongodb.DBCollection
class RegionPriceExport {
DBCollection REGION_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "region_sell_stat", "mongo.db.fangjia.com")
DBCollection REGION_STAT = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new RegionPriceExport();
task.run()
}
def run(){
def months = [202010,202011,202012,202101,202102,202103]
def regions = CommonUtils.getAllRegions("福州","region")
def months = ["20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"]
def regions = CommonUtils.getAllRegions("佛山","region")
def rets = []
rets << ["城市","区域","202010","202011","202012","202101","202102","202103"]
rets << ["城市","区域","20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"]
regions.each{region,values->
def data = []
data << "福州"
data << "佛山"
data << region
for(def month :months){
def dbo = findPrice("福州", region, month);
def dbo = findWeekendPrice("佛山", region, month);
if(dbo){
data << dbo.avg_price
}else{
@@ -34,7 +36,7 @@ class RegionPriceExport {
}
rets << data
}
ExcelUtils.write("d:\\数据\\福州\\福州区域月均价数据.xlsx",rets, "价格")
ExcelUtils.write("d:\\数据\\福州\\佛山区域均价数据.xlsx",rets, "价格")
}
def findPrice(city, region, month){
@@ -45,4 +47,14 @@ class RegionPriceExport {
def dbo = REGION_SELL_STAT.findOne(q)
return dbo
}
def findWeekendPrice(city, region, month){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("cat", "区域")
q.put("weekend", DateUtils.parseFromTightByPattern(month, "yyyyMMdd"))
def dbo = REGION_STAT.findOne(q)
return dbo
}
}
@@ -1,6 +1,8 @@
package com.fangjia.script.gj.district
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
@@ -13,7 +15,7 @@ class ExportData {
def static final DBCollection thirdDistrictColl = CrawalDBUtil.getThirdDistrictCol();
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -24,8 +26,8 @@ class ExportData {
static main(args) {
def task = new ExportData();
task.exportThird()
// task.exportMap();
// task.exportThird()
task.exportMap();
}
def exportMap(){
@@ -51,19 +53,18 @@ class ExportData {
//"内蒙古"---" 包头","赤峰","鄂尔多斯","乌海","巴彦淖尔","呼伦贝尔","乌兰察布","通辽","兴安盟","锡林郭勒盟","阿拉善盟","呼和浩特"
//宁夏 ----- 中卫 固原 石嘴山 吴忠 银川
//青海 ---- 海东 海北州 黄南州 海南州 果洛州 玉树州 海西州 西宁
def citys = ["潜江","来宾","河源","甘南","宣城","朔州","景德镇","呼和浩特","固原","佛山","玉溪","江门","黄山","海口","梧州","辽源","亳州","六安","大理","滁州","韶关","拉萨","北屯","锡林郭勒盟","海北州","肇庆","梅州","重庆","玉树州","石嘴山","清远","钦州",
"中卫","兴安盟","天门","贺州","湛江","文山州","图木舒克","通辽","茂名","海南州","贵港","铜陵","铁门关","汕头","临沧","楚雄","昌都","德宏州","北海","保山","阳江","西双版纳州","石河子","琼海","甘孜州","阿坝州","万宁","怒江州","揭阳","潮州","云浮",
"防城港","池州","文昌","儋州","阿拉善盟","阿拉尔","五家渠","山南","日喀则","乐东","黄南州"]
//"沈阳","大连","抚顺","鞍山"
//"贵阳","朝阳","本溪","邯郸","延安","保定","铁岭","阜新","辽阳","张家口","大庆"
//"沧州","武汉","渭南","成都","西宁","汉中","丹东","青岛",
//"遵义","衡水","锦州","葫芦岛","毕节","宜昌","济宁","南昌","郑州","白银","绥化","兰州","滨州","徐州","宝鸡","济南","邢台","鸡西","新乡","上饶","宜春","营口","烟台","盘锦","安阳","承德","吉安","黑河","德州",
//"榆林","洛阳","武威","双鸭山","淄博","铜仁","齐齐哈尔","福州","佳木斯","南宁","铜川","天水","襄阳","张掖","三门峡","酒泉","台州","南阳","吉林","嘉峪关","太原","平顶山","金华","商丘","潍坊","十堰","聊城","临沂","定西","平凉","晋中","南充","荆州","哈密","九江","南京","六盘水","温州","信阳","绵阳","广安","枣庄","开封","东营","常州","邵阳","秦皇岛","萍乡","安顺","伊春","鹤岗","达州","焦作","黄冈","孝感","海西州", "白城",
// def citys = ["潜江","来宾","河源","甘南","宣城","朔州","景德镇","呼和浩特","固原","佛山","玉溪","江门","黄山","海口","梧州","辽源","亳州","六安","大理","滁州","韶关","拉萨","北屯","锡林郭勒盟","海北州","肇庆","梅州","重庆","玉树州","石嘴山","清远","钦州",
// "中卫","兴安盟","天门","贺州","湛江","文山州","图木舒克","通辽","茂名","海南州","贵港","铜陵","铁门关","汕头","临沧","楚雄","昌都","德宏州","北海","保山","阳江","西双版纳州","石河子","琼海","甘孜州","阿坝州","万宁","怒江州","揭阳","潮州","云浮",
// "防城港","池州","文昌","儋州","阿拉善盟","阿拉尔","五家渠","山南","日喀则","乐东","黄南州"]
def citys = ["安庆","西宁","日照","长春","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"];
for(def city : citys){
def path = "D:\\数据\\小区\\地图\\${city}.xlsx";
def datas = findAllDatas(city, null)
println "${city}共计:"+datas.size()
ExcelUtils.write(path, datas, "数据")
}
def path = "D:\\数据\\小区\\地图\\建行信用卡城市(潜江--黄南州).xlsx";
def datas = findAllDatas(citys, null)
println "共计:"+datas.size()
ExcelUtils.write(path, datas, "数据")
}
@@ -98,9 +99,9 @@ class ExportData {
def findAllDatas(def citys, def pname){
def findAllDatas(def city, def pname){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT address,pname,cityname,type,adname,name,source,m_date,location.lon,location.lat FROM amap_poi WHERE 1=1")
sqlBuffer.append("SELECT address,pname,cityname,type,adname,name,source,m_date,location FROM amap_poi WHERE 1=1 and cityname='${city}' and type like '商务住宅%'")
// sqlBuffer.append(" and pname = '"+pname+"'")
// sqlBuffer.append(" and cityname = '"+pname+"'")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
@@ -113,16 +114,19 @@ class ExportData {
if(source && source=="高德"){
println index
def type = rs.getObject("type")
def city = rs.getString("cityname")
if(citys.contains(city) && type.contains("商务住宅")){
//def city = rs.getString("cityname")
if(type.contains("住宅")){
def ret = []
ret << city
ret << rs.getString("adname")
ret << rs.getString("name")
ret << rs.getString("address")
ret << rs.getString("type")
ret << rs.getFloat("location.lon")
ret << rs.getFloat("location.lat")
def loaction = rs.getObject("location")
if(loaction){
ret << DataHandleUtils.getDoubleRound(6, loaction.x)
ret << DataHandleUtils.getDoubleRound(6, loaction.y)
}
rets << ret
}
}
@@ -68,13 +68,16 @@ class FangjiaClientUtils {
public static def addressStand(def address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
def content = new HttpRequest().newGet(url, null, "utf-8")
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
@@ -1,9 +1,12 @@
package com.fangjia.script.gj.district
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DataUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.yinjihaun.lng.UpdateDistrictLocation
import java.util.regex.Pattern
@@ -20,20 +23,13 @@ class MapDistrictProcess {
* @return
*/
def run(){
//"江苏地图.xlsx","浙江地图.xlsx","福建地图.xlsx","江西地图.xlsx","山东地图.xlsx","安徽地图.xlsx",
//"上海地图.xlsx","北京地图.xlsx"
//"安徽地图.xlsx","江苏地图.xlsx","浙江地图.xlsx","福建地图.xlsx","江西地图.xlsx","山东地图.xlsx",
//"重庆地图.xlsx","天津地图.xlsx",
//"上海地图.xlsx","北京地图.xlsx",
//"河北地图.xlsx","河南地图.xlsx","湖北地图.xlsx","湖南地图.xlsx","广西地图.xlsx",
//"吉林地图.xlsx","辽宁地图.xlsx""辽宁地图.xlsx","吉林地图.xlsx","黑龙江地图.xlsx","山西地图.xlsx","海南地图.xlsx"]
def files = ["青海地图.xlsx","宁夏地图.xlsx","内蒙古地图.xlsx","云南地图.xlsx","陕西地图.xlsx","新疆地图.xlsx","四川地图.xlsx"]
def files = ["武汉.xlsx"]
for(def fileName : files){
def filePath = "D:\\数据\\小区\\地图\\"+fileName;
typeFilter(filePath)
nameStandard(filePath);
dataFilter(filePath)
// typeFilter(filePath)
// nameStandard(filePath);
// dataFilter(filePath)
matchDistDatas(filePath)
houseCalculate(filePath);
////// //统计是否存在多个区域
@@ -114,15 +110,24 @@ class MapDistrictProcess {
def matchDistDatas(def path){
def index = 0
ExcelUtils.multiThreadingModify(path,{ List row ->
ExcelUtils.modify(path,{ List row ->
index ++
if(!row['I'] && !row['J']){
def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['H'], row['D'],);
def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['B'], row['H'], row['D']);
if(rMap && (rMap.region==row['B'] || row['B'].contains(rMap.region))){
row['J'] = rMap.city
row['K'] = rMap.region
row['L'] = rMap.name
row['M'] = rMap.address
if(rMap.lat){
def baidu = BaiDuApiUtils.geoconv(row['F'].toDouble(), row['G'].toDouble(), 3)
def maplng = baidu.x
def maplat = baidu.y
row['X'] = GeoHelper.GetDistance(maplat, maplng, rMap.lat, rMap.lng)
def score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['L'])
row['Y'] = score
}
}
println index +" "+row
}
@@ -26,15 +26,18 @@ class SeaweedDataMatchProcess {
}
}
if(!rMap && address){
// if (findPattern(address)){
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
// }
}
if(!rMap && name){
rMap = OpenApiUtils.matchDistrict(city, name, region, false)
}
if(!rMap && address){
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
if(rMap && rMap.region && !region.contains(rMap.region)){
rMap = null
}
}
if(rMap && rMap.id && region.contains(rMap.region)) {
@@ -49,7 +49,7 @@ class ZsPersonProcess {
// def loactionMap = task.matchMap(dist)
// println loactionMap
// task.matchMap()
task.matchMap()
// task.matchDb()
// task.matchDb2()
@@ -19,9 +19,9 @@ class EsMonthHouseStat {
def task = new EsMonthHouseStat()
// task.run()
// task.tradeRun()
// task.houseRun()
task.houseRun()
task.statHouseUpdateCount()
// task.statHouseUpdateCount()
}
def run(){
@@ -99,37 +99,35 @@ class EsMonthHouseStat {
}
def houseRun(){
ExcelUtils.multiThreadingModify("d:\\数据\\小区案例数据覆盖率.xlsx",{d->
def city = d['B']
if(city == "湖州"){
def count = findHouseCount(city, 20210301, 20210601)
d['E'] = count
ExcelUtils.multiThreadingModify("d:\\数据\\建行\\城市覆盖率统计.xlsx",{d->
def city = d['A']
def count = findHouseCount(city, 20190709, 20210709)
d['E'] = count
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]);
def disthouseCount = 0
for(def dist : districts){
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]);
def disthouseCount = 0
for(def dist : districts){
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
def distCount = findHouseCount(dist.city, dist.region, alias, 20210301, 20210601)
if(distCount){
disthouseCount++
}
println dist.city+" "+dist.region+" "+dist.name+" "+distCount
alias = dist.alias
}else{
alias << dist.name
}
println "disthouseCount:"+disthouseCount
println districts.size()
def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
d['D'] = precent+"%"
def distCount = findHouseCount(dist.city, dist.region, alias, 20190709, 20210709)
if(distCount){
disthouseCount++
}
println dist.city+" "+dist.region+" "+dist.name+" "+distCount
}
println "disthouseCount:"+disthouseCount
println districts.size()
def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
d['F'] = precent+"%"
},0)
}
@@ -42,7 +42,7 @@ class ZhugefangCrawl4 extends BaseBatch{
@Override
void process(Object args) {
citys.addAll(["东方","东莞","临沂","五常","五指山","保亭","修文","合肥","吴江","咸阳","商洛","宁乡","德州","成都","文昌","无锡","普兰店","珠海","琼中","琼海","白沙县","石家庄","肥西","贵阳","郑州","重庆","陵水","","","",""])
citys.addAll(["福州"])
// getNoDataCitys();
def cityurl = "http://www.zhuge.com/areachange/"
def cityUrls = getCityUrlCrawal(cityurl);
@@ -140,6 +140,7 @@ public class OpenApiClient {
return null
}
public static JSONObject addressMatchV2(String city, String input, String cat, Boolean useMap = false, String suggestDistrict = null, String preciseLevel = "fuzzy", boolean verify = false) {
if (StringUtils.isBlank(input)) {