Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj

# Conflicts:
#	projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy
This commit is contained in:
2022-11-05 11:34:07 +08:00
22 changed files with 1685 additions and 121 deletions
@@ -25,8 +25,6 @@ import java.util.zip.GZIPOutputStream;
import com.hankcs.hanlp.HanLP;
import com.hankcs.hanlp.dictionary.py.Pinyin;
import com.sun.javafx.scene.transform.TransformUtils;
import net.sf.json.JSON;
import net.sf.json.JSONArray;
import net.sf.json.JSONNull;
import net.sf.json.JSONObject;
@@ -8,6 +8,7 @@ import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
@@ -35,7 +36,7 @@ class BigdataCommunityBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys_92;
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
@@ -107,7 +108,6 @@ class BigdataCommunityBatch extends BaseBatch{
if(ret){
rets << ret
}
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -175,38 +175,25 @@ class BigdataCommunityBatch extends BaseBatch{
ret << "pa101"
ret << ""
if(dist.address){
if(dist.address.toString().contains("弄")){
String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄"))
if(number){
ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number))
ret << dist.address.toString().substring(dist.address.toString().indexOf(number), dist.address.toString().size())
def jsonObj = null
if(dist.address){
jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address)
}
if(jsonObj && jsonObj.result && jsonObj.result.street){
if(jsonObj.result.street){
ret << jsonObj.result.street
}else{
ret << ""
}
if(jsonObj.result.streetNumber){
ret << jsonObj.result.streetNumber
}else{
ret << ""
}
}else{
def jsonObj = null
if(dist.address){
// jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
jsonObj = OpenApiClient.standard1Action(dist.city, dist.address)
}
if(jsonObj && jsonObj.street){
if(jsonObj.street){
ret << jsonObj.street
}else{
ret << ""
}
if(jsonObj.streetNumber){
ret << jsonObj.streetNumber
}else{
ret << ""
}
}else{
ret << ""
ret << ""
}
ret << ""
ret << ""
}
}else{
ret << ""
@@ -225,7 +212,6 @@ class BigdataCommunityBatch extends BaseBatch{
println dist.city + " "+dist.name+" error"
e.printStackTrace()
}
return ret;
}
}
@@ -58,7 +58,7 @@ class BigdataDealBatch extends BaseBatch{
public void process(Object args) {
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
// def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args)
if(args && args.size()>=2){
curDate = args[1].toString().toInteger()
}
@@ -448,7 +448,6 @@ class BigdataForsaleBatch extends BaseBatch{
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
@@ -11,7 +11,7 @@ class CcbFtpTestUtils {
static def ftpPort = 17495
static def ftpUser = "ccb"
static def ftpPassword = "ccb!@#"
static def ftpPath = "/home/ccb/output/"
static def ftpPath = "/home/ccb_evaluation/output/"
static def ftpPath_rh = "/home/ccb_rh/output/"
@@ -70,6 +70,22 @@ class CcbFtpTestUtils {
}
static def listFileNames(month){
//上传到ftp
sftpChannel = reConnect();
Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel);
return files
}
static def getFileLength(String path, String month){
sftpChannel = reConnect();
def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel)
return size
}
static main(args) {
@@ -3,6 +3,7 @@ package com.fangjia.script.ccbevaluation
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.jiangnanblank.SFTP
import com.jcraft.jsch.ChannelSftp
import com.jcraft.jsch.SftpATTRS
import org.apache.commons.codec.digest.DigestUtils
class CcbFtpUtils {
@@ -64,6 +65,13 @@ class CcbFtpUtils {
return files
}
static def getFileLength(String path, String month){
sftpChannel = reConnect();
def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel)
return size
}
static main(args) {
@@ -29,28 +29,28 @@ class ExportOracleData {
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
def new_dists = []
//不会变
// exportEvaluationRateData()
// exportBigdataPoiData()//
// exportBigdataDistrictData()//
//
// exportGuidedPrice()//随人工改变//
// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策//
//
//// //处理小区情况
// exportBigdataCommunity()//
// exportCommunityStaging()//没cityid
// exportCommunityBuileType()//
//不会变 10.25
exportEvaluationRateData()
exportBigdataPoiData()//
exportBigdataDistrictData()//
// exportThirdPartyPrice2(lastMonth, new_dists)//
// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19
exportGuidedPrice()//随人工改变//
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策//
// //处理小区情况
exportBigdataCommunity()//
exportCommunityStaging()//没cityid
exportCommunityBuileType()//
exportThirdPartyPrice2(lastMonth, new_dists)//
exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19
exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01
//第三方网站权重表
// exportWebsiteRate();
//计算结果相关
exportWebsiteRate();
//
//
// //计算结果相关
// exportAvmEvaluationGJNB(curMonth)
// exportAVMEvaluationItemGJNB(curMonth)
//
@@ -62,12 +62,12 @@ class ExportOracleData {
//
// exportCommunityPdescribeGj(curMonth)
// exportCommCaseDetailGj(curMonth)
def curDate = curMonth.toString()+"28"
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
//ok文件
exportLastFile()
//
// def curDate = curMonth.toString()+"28"
// SSHHelper jschCommand = new SSHHelper();
// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
//// ok文件
// exportLastFile()
}
@@ -12,13 +12,14 @@ class ExportOracleEvaluationData {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def citys = null
static main(args) {
def task = new ExportOracleEvaluationData()
task.run()
task.run(args)
}
def run() {
def run(args) {
citys = CcbCommonUtils.getRunningCitys(args)
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
//计算结果相关
@@ -369,7 +370,7 @@ class ExportOracleEvaluationData {
def exportCommCaseDetailGj(month){
def citys = CcbCommonUtils.citys;
// def citys = CcbCommonUtils.citys;
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
@@ -36,7 +36,7 @@ class GuiDingPriceBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys_252;
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
for (def city : citys) {
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2021-09-01", "2022-10-12")
def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-22", "2022-10-27")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -1001,10 +1001,10 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入人工查价_0922_300.xlsx"
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入_1020.xlsx"
//ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['A']){
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['A']){
// def region = RegionData.formatRegion(d['A'], d['B'])
// if(region){
// def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
@@ -1013,18 +1013,18 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
// d['F'] = dist._id.toString()
// }
// }
//// if(!d['Q']){
//// def dist = CommonDatautils.findDistrictById(d['A'])
//////
//// if(dist){
//// d['H'] = dist.region
////// d['G'] = dist.name
//// d['I'] = dist.status
//// }
//// }
if(!d['F']){
def dist = CommonDatautils.findDistrictById(d['A'])
//
// }
//},0)
if(dist){
// d['H'] = dist.region
// d['G'] = dist.name
d['F'] = dist.status
}
}
}
},0)
@@ -0,0 +1,146 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class AddressCut extends BaseBatch{
static main(args) {
runBatch(new AddressCut(), args);
}
@Override
public void process(Object args) {
sameAddr()
}
def sameAddr() {
def citys = CommonDatautils.findAllCity()
def dataMap = [:]
def index = 0
for (def c : citys) {
index++
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name", "address"])
def addrMap = [:]
for (def dist : dists) {
if (dist.address) {
def rets = addrMap.get(dist.address)
if (!rets) {
rets = []
}
rets << dist
addrMap.put(dist.address, rets)
}
}
dataMap.put(c, addrMap)
println index +" "+c
}
def sameAddrList = []
for(def c : citys) {
def cityMap = dataMap.get(c)
if(cityMap){
cityMap.each{addr,rets->
if(rets && rets.size()>1){
def keys = new HashSet()
for(def d : rets){
if(d.name){
def name = d.name.replace("一期","").replace("二期","").replace("三期","").replace("四期","").replace("五期","").replace("六期","").replace("七期","").replace("八期","").replace("九期","").replace("十期","").replace("一区","").replace("二区","").replace("三区","").replace("四区","").replace("五区","").replace("六区","").replace("七区","").replace("八区","").replace("九区","").replace("十区","").replace("A区","").replace("B区","").replace("C区","").replace("D区","")
.replace("十一街区","").replace("十二街区","").replace("十三街区","").replace("十四街区","").replace("十五街区","").replace("十六街区","").replace("十七街区","").replace("十八街区","").replace("十九街区","").replace("一街区","").replace("二街区","").replace("三街区","").replace("四街区","").replace("五街区","").replace("六街区","").replace("七街区","").replace("八街区","").replace("九街区","").replace("十街区","")
.replace("公寓","").replace("别墅","").replace("商住楼","").replace("花园","").replace("南区","").replace("北区","").replace("东区","").replace("西区","")
keys << d.region +"##"+ name
}
}
if(keys && keys.size()>1){
for(def d : rets) {
if (d.name) {
sameAddrList << [d.city, d.region, d.name, d.address]
}
}
}
}
}
}
}
//ExcelUtils.write("D:\\数据\\小区\\同地址小区.xlsx", sameAddrList, "相同地址小区")
FileUtils.batchCreateFile(sameAddrList, "D:\\数据\\小区\\同地址小区.txt", null, "utf-8", false);
println "完成"
}
def addrCut(){
def citys = CommonDatautils.findAllCity()
def index = 0
def fileName = "/home/goojia/gj/address_split.txt"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\address_split.txt"
}
for(def c : citys) {
index++
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name","address"])
def rets = cut(dists)
FileUtils.batchCreateFile(rets, fileName, null, "utf-8", true);
println index +" "+ c +" "+ rets.size()
}
}
def cut(rets){
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
final CountDownLatch latch = new CountDownLatch(rets.size())
def totalCount = new AtomicLong()
def rows = new CopyOnWriteArrayList()
for(def i=0;i<rets.size();i++){
final def d = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(d.address){
def jsonObj = FangjiaClientUtils.addressStructure(d.city, d.address)
if(jsonObj && jsonObj.result){
def udata = new BasicDBObject()
def street = null
def street_number = null
if(jsonObj.result.street){
street = jsonObj.result.street
}
if(jsonObj.result.streetNumber){
street_number = jsonObj.result.streetNumber
}
if(street || street_number){
println totalCount.get() +" "+d.city +" "+d.address +" "+street+" "+street_number
rows << [d.city,d.region, d.name, d.address, street, street_number]
// udata.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rows;
}
}
@@ -0,0 +1,112 @@
package com.fangjia.script.ccbevaluation.ronghe
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import com.jcraft.jsch.ChannelExec;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.Session
import com.jcraft.jsch.*;
class SSHHelperTest {
private Session session = null;
private Channel channel = null;
private String sftpHost = "192.168.0.86";
private int sftpPort = 17495;
private String sftpUserName = "ccb";
private String sftpPassword = "ccb!@#";
private int timeout = 30000;
/**
* 获取连接
* @return
*/
private ChannelExec getChannelExec() {
try {
if (channel != null && channel.isConnected()) {
return (ChannelExec) channel;
}
JSch jSch = new JSch();
if (session == null || !session.isConnected()) {
session = jSch.getSession(sftpUserName, sftpHost, sftpPort);
session.setPassword(sftpPassword);
Properties config = new Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.setTimeout(timeout);
session.connect();
}
channel = session.openChannel("exec");
} catch (Exception e) {
if (session != null) {
session.disconnect();
session = null;
}
channel = null;
}
return channel == null ? null : (ChannelExec) channel;
}
/**
* 关闭连接
*/
private void closeChannel() {
try {
if (channel != null) {
channel.disconnect();
channel = null;
}
if (session != null) {
session.disconnect();
session = null;
}
} catch (Exception e) {
System.out.println(e);
}
}
/**
* 执行服务器端命令
*/
public boolean executeCommand(String command) {
boolean flag = false;
ChannelExec channelExec = getChannelExec();
if (channelExec == null) {
return false;
}
try {
channelExec.setInputStream(null);
channelExec.setErrStream(System.err);
channelExec.setCommand(command);
InputStream ins = channelExec.getInputStream();
channelExec.connect();
BufferedReader reader = new BufferedReader(new InputStreamReader(ins, "utf-8"));
String line = null;
while ((line = reader.readLine()) != null) {
System.out.println(line);
}
reader.close();
closeChannel();
flag = true;
} catch (Exception e) {
System.out.println(e);
flag = false;
}
return flag;
}
public static void main(String[] args) {
SSHHelperTest jschCommand = new SSHHelperTest();
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/package.sh 20220110"));
}
}
@@ -29,7 +29,30 @@ class DataMatch {
// task.run3()
// println task.findAddress("昌州街道棠香路")
task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
task.fzWeiyuanshuMatch("D:\\数据\\福州\\连江楼盘.xlsx")
}
def fzWeiyuanshuMatch(fileName){
ExcelUtils.multiThreadingModify(fileName,{d->
// if(!d['G']){
// def arr = OpenApiClient.search("福州", "连江", d['C'])
// if(arr){
// d['E'] = arr.city
// d['F'] = arr.district
// d['G'] = arr.name
// }
// }
def results = null
if(d['H'] && d['H']=="F" && d['C']){
results = OpenApiClient.addressMatch("福州",d['C'], "property", false, "连江", null)
}
if(results && results.precise>=0){
d['J'] = results.district
d['K'] = results.name
}
},0)
}
def run1(){
@@ -67,6 +67,7 @@ class FangjiaClientUtils {
//分词 省,市,区
public static def addressStand(def city, def address){
def jsonObj = null
def content = null
@@ -84,6 +85,22 @@ class FangjiaClientUtils {
return jsonObj;
}
//分词基础上的标准化,complete=false 不不全 short=true 省市区返回简拼
public static def addressStructure(def city, def address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def arroundPrice(city,address){
def jsonObj = null
def content = null
@@ -38,8 +38,8 @@ class PaimaiDataSend {
// task.jdRunByMongo()
// task.aliZcRun()
task.aliZcRun2()
// task.aliFpRun()
// task.aliZcRun2()
task.aliFpRun()
// task.rmfysszcRun()
// task.jdRun()
// task.checkUrl()
@@ -334,10 +334,10 @@ class PaimaiDataSend {
def index = 0
def q = new BasicDBObject()
// q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
q.put("province","江苏")
q.put("sub_type","土地")
// q.put("province","江苏")
// q.put("sub_type","土地")
// q.put("parser_status", new BasicDBObject('$ne',2))
q.put("url","https://susong-item.taobao.com/auction/671320162557.htm")
q.put("url","https://sf-item.taobao.com/sf_item/673646300374.htm")
//https://sf-item.taobao.com/sf_item/657131607486.htm
// q.put("parser_status", new BasicDBObject('$exists',false))
// q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01")))
@@ -209,4 +209,20 @@ class SFTP {
return isDirExistFlag;
}
public long getFileSize(String srcSftpFilePath, ChannelSftp sftp) {
long fileSize;//文件大于等于0则存在
try {
SftpATTRS sftpATTRS = sftp.lstat(srcSftpFilePath);
fileSize = sftpATTRS.getSize();
} catch (Exception e) {
e.printStackTrace()
fileSize = -1;//获取文件大小异常
if (e.getMessage().toLowerCase().equals("no such file")) {
fileSize = -2;//文件不存在
}
}
return fileSize;
}
}
@@ -1,6 +1,7 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.util.BaiDuUtils
import com.fangjia.script.temp.shenjian.util.EsUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.util.ElasticUtils
@@ -15,25 +16,26 @@ import java.util.concurrent.Executors
*/
class FillLngLat {
static ExecutorService executorService = Executors.newFixedThreadPool(4)
// static ExecutorService executorService = Executors.newFixedThreadPool(4)
static void main(String[] args) {
ElasticUtils.executeSqlQuery("select * from dws_poi where location is not null", { dataT ->
ElasticUtils.executeSqlQuery("select * from dws_poi where location is null", { dataT ->
final def data = dataT
executorService.submit(new Runnable() {
@Override
void run() {
// executorService.submit(new Runnable() {
// @Override
// void run() {
println(data)
def lngLat = GaoDeUtils.parserByGaode(data.city, data.region + data.name + data.address)
def lngLat = BaiDuUtils.parserByBaidu(data.city, data.region + data.name + data.address)
Thread.sleep(1000)
if (lngLat != null && lngLat.size() == 2 ) {
JSONObject jsonObject = new JSONObject()
jsonObject.put("lon", Double.valueOf(lngLat['lng']))
jsonObject.put("lat", Double.valueOf(lngLat['lat']))
EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id))
}
}
})
// }
// })
})
executorService.shutdown()
//executorService.shutdown()
}
}
@@ -0,0 +1,48 @@
package com.fangjia.script.temp.shenjian.util
import com.alibaba.fastjson.JSONObject
import com.fangjia.core.util.GeoUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.DataUtil
import org.apache.commons.lang.StringUtils
/**
* 百度工具类
*
* @author shenjian
* @since 2022/10/28
*/
class BaiDuUtils {
def static final String KEY = "6w9SEHS3hnAFjmY2oEhqIDdgvLPKWXu5"
/**
* 百度地址逆解析,返回高德坐标!!!!!!!!!!!!!!!
* @param city
* @param address
*/
def static parserByBaidu(def city, def address) {
Map<String, String> params = new HashMap<String, String>()
params.put("city", city)
if (StringUtils.isBlank(address)) {
return []
}
params.put("address", address.replaceAll("null", ""))
params.put("output", "json")
params.put("ak", KEY)
params.put("callback", "showLocation")
String content
content = new HttpRequest().get("https://api.map.baidu.com/geocoding/v3/", params, "utf-8")
//content = "showLocation&&showLocation({\"status\":0,\"result\":{\"location\":{\"lng\":131.04009309945693,\"lat\":45.78426770180344},\"precise\":1,\"confidence\":70,\"comprehension\":100,\"level\":\"地产小区\"}})"
content = content.replaceAll("showLocation&&showLocation\\(", "").replaceAll("\\)", "")
JSONObject json = JSONObject.parseObject(content)
if (json.status == 0 && json.result && json.result.location) {
def bd_lng = json.result.location.lng
def bd_lat = json.result.location.lat
Map<String, Double> locationMap = GeoUtils.bd09ToGcj02(bd_lat, bd_lng)
return ["lng": com.fangjia.core.util.DataUtil.getDoubleRound(6, locationMap['lng']), "lat": com.fangjia.core.util.DataUtil.getDoubleRound(6, locationMap['lat'])]
}
return []
}
}
@@ -91,11 +91,9 @@ class DataStandardBatch extends BaseBatch{
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
cityMap.each{city,alias->
// if(city == "北京"){
println "开始加载【${city}】"
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
districtMap.putAll(detailMap)
// }
println "开始加载【${city}】"
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
districtMap.putAll(detailMap)
}
while(flag){
@@ -122,9 +120,9 @@ class DataStandardBatch extends BaseBatch{
def final AtomicInteger count_ = new AtomicInteger(0)
def final AtomicInteger successIndex = new AtomicInteger(0)
datas.each { dbo ->
fixedThreadPool.execute(
new Runnable() {
void run() {
// fixedThreadPool.execute(
// new Runnable() {
// void run() {
try{
println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】"
if(dbo.name){
@@ -164,12 +162,12 @@ class DataStandardBatch extends BaseBatch{
}finally {
latch.countDown()
}
}
}
);
// }
// }
// );
}
latch.await()
// latch.await()
long end = System.currentTimeMillis()
println "第【${rowIndex}】次中匹配成功的:"+successCount+" 个,耗时:"+ (end - start)+" "+ DateUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss")
@@ -502,6 +500,8 @@ class DataStandardBatch extends BaseBatch{
type = 12;//区域不同
district = null;
}else{
def score = CosineSimilarAlgorithm.getSimilarity(district.name, copyObj.name)
district.score = score //此处用来做测试
type=15
}
}
@@ -922,7 +922,7 @@ class DataStandardBatch extends BaseBatch{
qdata.put("s_date", SHORT_DATE)
// qdata.put("city","北京")
// qdata.put("source","安居客")
// qdata.put("name",Pattern.compile("金隅澜湾.*"))
// qdata.put("name",Pattern.compile("龚家宅.*"))
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
def ret = []
try{
@@ -1065,6 +1065,9 @@ class DataStandardBatch extends BaseBatch{
udata2.put("match_region", district.region);
udata2.put("match_name", district.name);
udata2.put("match_address", district.address);
if(district.score){
udata2.put("score", district.score);
}
udata2.put("avg_price", data.avg_price);
udata2.put("type", type);
udata2.put("tid", data._id.toString());
@@ -81,8 +81,8 @@ class ThirdDataToGatherBatch extends BaseBatch{
println "小区添加价格完成"
districts = addOtherWebsitePrice(districts);
println "小区添加第三方价格完成"
districts = addLevel(districts);
println "Ð¡ÇøÆÀ¼¶Íê³É"
// districts = addLevel(districts);
// println "Ð¡ÇøÆÀ¼¶Íê³É"
saveOrUpdate(districts);
println city + "保存修改完成"
toLineDB(city)
@@ -101,16 +101,16 @@ class ThirdDataToGatherBatch extends BaseBatch{
def tempPrice = d.get(field+"_price")
if(!tempPrice){
d.put(field+"_price", m.avg_price.toInteger())
if(d.fangjia_price && m.avg_price){
if(d.fangjia_price){
try{
d.put(field+"_diff", diffPercent(d.fangjia_price.toInteger(), m.avg_price.toInteger()))
}catch(Exception e){
println "dddd"
}
}
}
// if(d.fangjia_price && m.avg_price){
// if(d.fangjia_price){
// try{
// d.put(field+"_diff", diffPercent(d.fangjia_price.toInteger(), m.avg_price.toInteger()))
// }catch(Exception e){
// println "dddd"
// }
//
// }
// }
}
}else{
@@ -208,8 +208,9 @@ class ThirdDataToGatherBatch extends BaseBatch{
query.put("s_date", SHORT_DATE)
DBObject udata = new BasicDBObject()
udata.put("level",d.level);
udata.put("last_price", d.last_price);
// udata.put("level",d.level);
// udata.put("last_price", d.last_price);
udata.put("seaweed_id", d._id.toString())
udata.put("fangjia_price", d.fangjia_price);
fieldMap.each{k,v->
@@ -52,6 +52,22 @@ public class OpenApiClient {
return false
}
public static def search(def city, def district, def name){
Map<String, String> params = new HashMap<String, String>()
params.put("city", city);
params.put("keyword", name);
params.put("district", district)
params.put("token", TOKEN);
String content = new HttpRequest().get("http://open.fangjia.com/property/search", params, "utf-8")
JSONObject jsonObj = JSONObject.fromObject(content)
JSONArray array = jsonObj.get("result")
def matchObj = null
if(array){
matchObj = array[0]
}
return matchObj
}
public static List searchDistrict(double lng, double lat, int r, String city) {
DecimalFormat format = new DecimalFormat("#.0000000000");
try {
@@ -348,6 +364,7 @@ public class OpenApiClient {
}
//此接口地址标准化,带补全
public static JSONObject standard1Action(String city, String address) {
try {
Map<String, String> params = new HashMap<String, String>();