Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
@@ -58,7 +58,7 @@
|
||||
</ul>
|
||||
</div>
|
||||
</li>
|
||||
<li class="<#if controllerName=='app'>current</#if>"><a href="${wwwUrl}/app/index">APP下载</a></li>
|
||||
<#-- <li class="<#if controllerName=='app'>current</#if>"><a href="${wwwUrl}/app/index">APP下载</a></li>-->
|
||||
|
||||
</ul>
|
||||
|
||||
@@ -148,9 +148,9 @@
|
||||
</ul>
|
||||
</li>
|
||||
|
||||
<li>
|
||||
<a href="${wwwUrl}/app/index" class="<#if controllerName=='app'>current</#if>">APP下载</a>
|
||||
</li>
|
||||
<#-- <li>-->
|
||||
<#-- <a href="${wwwUrl}/app/index" class="<#if controllerName=='app'>current</#if>">APP下载</a>-->
|
||||
<#-- </li>-->
|
||||
<!--<li>
|
||||
<a href="javascript:void(0)" onclick="applyLoan()" class="<#if controllerName=='loan'>current</#if>">我要贷款</a>
|
||||
</li>-->
|
||||
|
||||
+1
-8
@@ -23,14 +23,7 @@ class CityInfoEmailSender extends BaseBatch{
|
||||
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
|
||||
final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files')
|
||||
def emails = [
|
||||
"wangsheng@fangjia.com"
|
||||
,"xuyejun@fangjia.com"
|
||||
,"tiki@fangjia.com"
|
||||
,"zhangzx@fangjia.com"
|
||||
,"panyingjiao@fangjia.com"
|
||||
,"yangqixing@fangjia.com"
|
||||
,"james.lin@fangjia.com"
|
||||
,"jhu@fangjia.com"
|
||||
"gaojuan@fangjia.com"
|
||||
]
|
||||
|
||||
|
||||
|
||||
-1
@@ -383,7 +383,6 @@ class DealPriceDataProcess {
|
||||
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
|
||||
sqlInstance236.executeUpdate(sql)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+20
-1
@@ -48,13 +48,14 @@ class AnjukeAppCrawl {
|
||||
// task.run();
|
||||
// task.run2()
|
||||
|
||||
task.addThirdPrice()
|
||||
// task.addThirdPrice()
|
||||
// task.saveToEs()
|
||||
// task.saveToEs1()
|
||||
|
||||
// task.groupByCity()
|
||||
// task.selectNoSave()
|
||||
// task.statCount()
|
||||
task.exportFzDist()
|
||||
}
|
||||
def saveToEs1(){
|
||||
def lists = getAllAppDatas()
|
||||
@@ -75,6 +76,24 @@ class AnjukeAppCrawl {
|
||||
})
|
||||
}
|
||||
|
||||
def exportFzDist(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","福州")
|
||||
def dbcursor = detail.find(q).batchSize(5000)
|
||||
def index = 0
|
||||
def rets = []
|
||||
rets << ["city","region","name","_id","id","sale_num","city_short","source"]
|
||||
for(def d : dbcursor){
|
||||
def count = 0
|
||||
if(d.propInfo.saleNum){
|
||||
count = d.propInfo.saleNum.toInteger()
|
||||
}
|
||||
rets << [d.base.cityName, d.base.areaName, d.base.name, d._id.toString(),d.base.id, count, "fz","安居客"]
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\福州\\福州小区.xlsx", rets, "数据")
|
||||
}
|
||||
|
||||
def selectNoSave(){
|
||||
def esData = findAllEsData()
|
||||
def q = new BasicDBObject()
|
||||
|
||||
+16
-5
@@ -25,14 +25,14 @@ class AnjukeHouseCrawl {
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
|
||||
def href = "https://sh.esf.fang.com/loupan/1210043376.htm"
|
||||
def href = "https://fz.anjuke.com/prop/view/S2512363126645770"
|
||||
def doc = getDocument(href, driver)
|
||||
// def doc = ClaABuYun.getResponse(href)
|
||||
// def newUrl = href.substring(0, hrefs.indexOf("?"))
|
||||
sendMq(href, doc, "district_detail_fang")
|
||||
sendMq(href, doc, "sell_anjuke")
|
||||
|
||||
// for(def i=1;i<50;i++){
|
||||
// def url = "https://fz.anjuke.com/sale/guloue/p${i}/"
|
||||
// for(def i=25;i<50;i++){
|
||||
// def url = "https://fz.anjuke.com/sale/jinana/p${i}/"
|
||||
// def doc = getDocument(url, driver)
|
||||
// def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc)
|
||||
// println "页数:"+i+" 记录数:"+eles.size()
|
||||
@@ -203,7 +203,18 @@ class AnjukeHouseCrawl {
|
||||
}
|
||||
|
||||
if(!els){
|
||||
els = JQueryUtils.select(".village_info", doc)
|
||||
els = JQueryUtils.select(".detail-content", doc)
|
||||
}
|
||||
if(!els){
|
||||
els = JQueryUtils.select(".content", doc)
|
||||
}
|
||||
|
||||
if(!els){
|
||||
els = JQueryUtils.select(".wrapper", doc)
|
||||
}
|
||||
|
||||
if(!els){
|
||||
els = JQueryUtils.select(".main-wrap", doc)
|
||||
}
|
||||
if(els && els.size()>0){
|
||||
stop = true;
|
||||
|
||||
+2
-2
@@ -23,14 +23,14 @@ class ZhugeFangDistCrawl {
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
def url = "https://fz.esfxiaoqu.zhuge.com/pingtan/"
|
||||
def url = "https://fz.esfxiaoqu.zhuge.com/changle/"
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver.manage().deleteAllCookies()
|
||||
getRegions(driver, url)
|
||||
}
|
||||
|
||||
def getRegions(driver, url) {
|
||||
for(def i=1;i<=4;i++){
|
||||
for(def i=1;i<=7;i++){
|
||||
def newUrl = url+"page${i}/"
|
||||
println newUrl
|
||||
def doc = getDocument(newUrl, driver)
|
||||
|
||||
+147
@@ -0,0 +1,147 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class ExportPaimai {
|
||||
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
def task = new ExportPaimai();
|
||||
task.run()
|
||||
// task.addressStandard()
|
||||
// task.addPrice()
|
||||
// task.stat()
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def file = "D:\\数据\\重庆农商行\\法拍\\宿迁法拍数据.xlsx"
|
||||
def datas = ExcelUtils.read(file, 0, 1)
|
||||
def index1 = 0
|
||||
|
||||
def rates1 = []
|
||||
for(def d : datas){
|
||||
if(d['P'] && d['P']=='已成交'){
|
||||
if(d['M'] && d['M']=="住宅")
|
||||
// if(d['K']>=150){
|
||||
// index1++
|
||||
// def sumTradePrice = d['S']
|
||||
// def marketPrice = d['Y']
|
||||
// rates1 << sumTradePrice/marketPrice
|
||||
// }
|
||||
if(d['S']>2000000){
|
||||
index1++
|
||||
def sumTradePrice = d['S']
|
||||
def marketPrice = d['Y']
|
||||
rates1 << sumTradePrice/marketPrice
|
||||
}
|
||||
}
|
||||
}
|
||||
def rate = rates1.sum()/rates1.size()
|
||||
println index1 +" "+ rate
|
||||
}
|
||||
|
||||
def addPrice(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\宿迁法拍数据.xlsx",{d->
|
||||
if(!d['Y'] && d['Z']){
|
||||
d['Y'] = d['Z']
|
||||
}
|
||||
})
|
||||
}
|
||||
def addressStandard(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\重庆法拍数据.xlsx",{d->
|
||||
def formatAddress = AddressUtil.formatAddress(d['C'], d['A'])
|
||||
if(formatAddress){
|
||||
d['F'] = formatAddress
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findPaimaiData()
|
||||
println "总量:"+datas.size()
|
||||
def rows = []
|
||||
def header = ["标地物地址","省","城市","区域","地址","标题提取地址","楼栋号","单元","室号","楼盘名称","建筑面积","竣工日期","房屋用途","总楼层",
|
||||
"拍卖状态","是否成交","拍卖阶段","拍卖方式","成交价","拍卖开始时间","拍卖结束时间","起拍价","保证金","加价幅度","评估价","市场价",
|
||||
"变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数","URL"]
|
||||
rows << header
|
||||
def set = new HashSet()
|
||||
def index = 0
|
||||
for(def d : datas){
|
||||
if(!set.contains(d.title)){
|
||||
index++
|
||||
println index
|
||||
set << d.title
|
||||
def row = []
|
||||
row << d.title
|
||||
row << d.province
|
||||
row << d.city
|
||||
row << d.region
|
||||
row << d.address
|
||||
row << ""
|
||||
// def formatAddress = AddressUtil.formatAddress(d.city, d.title)
|
||||
// if(formatAddress){
|
||||
// row << formatAddress
|
||||
// }else{
|
||||
// row << ""
|
||||
// }
|
||||
row << d.houseNum
|
||||
row << d.unitNum
|
||||
row << d.roomNum
|
||||
row << d.matchName
|
||||
row << d.area
|
||||
row << d.buildYear
|
||||
row << d.estateType
|
||||
row << ""
|
||||
row << ""
|
||||
row << d.status
|
||||
row << d.stage
|
||||
row << ""
|
||||
row << d.tradePrice
|
||||
row << DateUtils.formatDateByPattern(d.startDate, "yyyy-MM-dd")
|
||||
row << DateUtils.formatDateByPattern(d.finishDate, "yyyy-MM-dd")
|
||||
row << d.startPrice
|
||||
row << d.depositPrice
|
||||
row << d.addPrice
|
||||
row << d.evalatePrice
|
||||
row << d.marketPrice
|
||||
row << d.sellOffAdvancePay
|
||||
row << d.pricePkWeekly
|
||||
row << d.delayWeekly
|
||||
row << d.preemptivePerson
|
||||
row << d.enrollCount
|
||||
row << d.lookCount
|
||||
row << d.callCount
|
||||
row << d.delayCount
|
||||
row << d.url
|
||||
|
||||
rows << row
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\重庆法拍原数据.xlsx",rows,"数据")
|
||||
}
|
||||
|
||||
def findPaimaiData(){
|
||||
def pmquery = new BasicDBObject()
|
||||
pmquery.put("city","重庆")
|
||||
pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
|
||||
pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-06-01")))
|
||||
pmquery.put("status","已成交")
|
||||
def dbcursor = paimai43_COLL.find(pmquery).sort(new BasicDBObject("finishDate",-1))
|
||||
def rets = []
|
||||
for(def db : dbcursor){
|
||||
// if(db.estateType && db.estateType=="车库"){
|
||||
// continue
|
||||
// }
|
||||
rets << db
|
||||
}
|
||||
return rets
|
||||
}
|
||||
}
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.alibaba.fastjson.JSON
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.temp.shenjian.util.MessageUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.AMQP
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
|
||||
DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "ali_fapai_detail", "192.168.0.235", 27018)
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
|
||||
|
||||
|
||||
//Date date=new Date(1550455200000);
|
||||
//SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
//String str = format.format(date);
|
||||
//System.out.println(str);
|
||||
|
||||
def pmquery = new BasicDBObject()
|
||||
pmquery.put("city","ËÞǨ")
|
||||
pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
|
||||
//pmquery.put("status","Òѳɽ»")
|
||||
//pmquery.put("estateType", null)
|
||||
pmquery.put("url","https://sf-item.taobao.com/sf_item/665574733237.htm")
|
||||
|
||||
def cur = paimai43_COLL.find(pmquery)
|
||||
def urls = []
|
||||
for(def o : cur){
|
||||
urls << o.url
|
||||
}
|
||||
println "´ý½âÎö£º"+urls
|
||||
def index = 0
|
||||
for(def url : urls){
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", url)
|
||||
def dbcursor = collection.find(q)
|
||||
index++
|
||||
println index
|
||||
for(def d : dbcursor){
|
||||
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", "paimai_taobao")
|
||||
json.put("url", d.url)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", d.content)
|
||||
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
|
||||
result.put("trades", bid_content)
|
||||
result.put("condition", d.condition)
|
||||
result.put("sub_count", d.sub_count)
|
||||
def data_json = net.sf.json.JSONObject.fromObject(d.json)
|
||||
result.put("data_json", data_json)
|
||||
result.put("desc_content", d.desc_content)
|
||||
result.put("item_notice_content", d.item_notice_content)
|
||||
result.put("notice_detail_content", d.notice_detail_content)
|
||||
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+1
-3
@@ -44,9 +44,7 @@ class PdsUseStatisticBatch extends BaseBatch{
|
||||
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
|
||||
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000)
|
||||
def static emails = [
|
||||
"hubingliang@fangjia.com",
|
||||
"james.lin@fangjia.com",
|
||||
"panyingjiao@fangjia.com",
|
||||
|
||||
"duweiliang@fangjia.com",
|
||||
"yaojingmeng@fangjia.com",
|
||||
"xuqi@fangjia.com",
|
||||
|
||||
+1
-2
@@ -34,8 +34,7 @@ class DaiQianGuZhiBatch extends BaseBatch{
|
||||
final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files')
|
||||
def static emails = [
|
||||
"hubingliang@fangjia.com",
|
||||
"james.lin@fangjia.com",
|
||||
"panyingjiao@fangjia.com"
|
||||
"james.lin@fangjia.com"
|
||||
]
|
||||
|
||||
static main(args) {
|
||||
|
||||
+1
-2
@@ -34,8 +34,7 @@ class KeHuFenLeiBatch extends BaseBatch{
|
||||
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000)
|
||||
def static emails = [
|
||||
"hubingliang@fangjia.com",
|
||||
"james.lin@fangjia.com",
|
||||
"panyingjiao@fangjia.com"
|
||||
"james.lin@fangjia.com"
|
||||
]
|
||||
|
||||
static main(args) {
|
||||
|
||||
+1
-3
@@ -33,9 +33,7 @@ class KeHuPingJiBatch extends BaseBatch{
|
||||
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
|
||||
final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files')
|
||||
def static emails = [
|
||||
"hubingliang@fangjia.com",
|
||||
"james.lin@fangjia.com",
|
||||
"panyingjiao@fangjia.com"
|
||||
|
||||
]
|
||||
|
||||
static main(args) {
|
||||
|
||||
+2
-2
@@ -34,7 +34,7 @@ sqlInstance.eachRow(sql, { r ->
|
||||
if (year_month == '202205') {
|
||||
fiveMap.put(key, r['avg_price'] + "|" + r['calc_type'])
|
||||
} else if ((year_month == '202206')) {
|
||||
fiveMap.put(key, r['avg_price'] + "|" + r['calc_type'])
|
||||
sixMap.put(key, r['avg_price'] + "|" + r['calc_type'])
|
||||
}
|
||||
})
|
||||
for (String key : sixMap.keySet()) {
|
||||
@@ -47,4 +47,4 @@ for (String key : sixMap.keySet()) {
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("F:\\数据\\价格\\东莞上海价格偏差大.xlsx", result, "统计")
|
||||
ExcelUtils.write("D:\\数据\\价格\\东莞上海价格偏差大.xlsx", result, "统计")
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
package com.fangjia.script.temp.shenjian.poi
|
||||
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.statistics.poi.AmapPoi
|
||||
import com.fangjia.script.statistics.poi.AmapPoiDetail
|
||||
import com.fangjia.script.temp.shenjian.poi.entity.Community
|
||||
import com.fangjia.script.temp.shenjian.util.CommonDtoUtils
|
||||
import com.fangjia.script.temp.shenjian.util.EsUtils
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
import org.apache.commons.collections.CollectionUtils
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.stream.Collectors
|
||||
|
||||
/**
|
||||
* 从ES获取POI
|
||||
*/
|
||||
class GetPoiFromCommunity {
|
||||
|
||||
static final ExecutorService executorService = Executors.newFixedThreadPool(10)
|
||||
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail"
|
||||
// 住宅类
|
||||
static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304";
|
||||
|
||||
static void main(String[] args) {
|
||||
AtomicLong atomicLong = new AtomicLong(0)
|
||||
Map<String, String> map = new ConcurrentHashMap<>()
|
||||
def rets = this.findCommunities()
|
||||
CountDownLatch countDownLatch = new CountDownLatch(rets.size())
|
||||
for (def retTemp : rets) {
|
||||
final def ret = retTemp
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
final String id = ret._id // 索引ID,并非小区ID
|
||||
String city = ret.city
|
||||
String region = ret.region
|
||||
String seaweedName = ret.name
|
||||
String address = ret.address
|
||||
List<AmapPoi> pois = new ArrayList<>();
|
||||
List<AmapPoi> namePois = GaoDeUtils.searchPoi(city, seaweedName, region, typecodes);
|
||||
List<AmapPoi> addressPois = GaoDeUtils.searchPoi(city, address, region, typecodes);
|
||||
if (CollectionUtils.isNotEmpty(namePois)) {
|
||||
pois.addAll(namePois)
|
||||
}
|
||||
if (CollectionUtils.isNotEmpty(addressPois)) {
|
||||
pois.addAll(addressPois)
|
||||
}
|
||||
if (CollectionUtils.isNotEmpty(pois)) {
|
||||
List<AmapPoiDetail> detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois);
|
||||
String jsonStr = JSONObject.toJSONString(detailList)
|
||||
HttpUtils.postForJson(URL, jsonStr)
|
||||
}
|
||||
synchronized (map) {
|
||||
map.put(id, id)
|
||||
countDownLatch.countDown()
|
||||
if (map.size() % 100 == 0) {
|
||||
atomicLong.addAndGet(map.size())
|
||||
List<String> ids = map.values().stream().collect(Collectors.toList())
|
||||
EsUtils.bulkUpdateFieldByIds("community", "crawl_flag", 0, ids)
|
||||
map.clear()
|
||||
println(atomicLong.intValue())
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
countDownLatch.await()
|
||||
if (map.size() > 0) {
|
||||
atomicLong.addAndGet(map.size())
|
||||
List<String> ids = map.values().stream().collect(Collectors.toList())
|
||||
EsUtils.bulkUpdateFieldByIds("community", "crawl_flag", 0, ids)
|
||||
map.clear()
|
||||
println(atomicLong.intValue())
|
||||
}
|
||||
System.exit(1)
|
||||
}
|
||||
|
||||
static def findCommunities(){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
|
||||
.mustNot(QueryBuilders.termQuery("crawl_flag", 0))
|
||||
List<Community> communities = EsUtils.scrollQuery("community", queryBuilder, Community.class)
|
||||
return communities
|
||||
}
|
||||
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package com.fangjia.script.temp.shenjian.poi.entity;
|
||||
|
||||
public class Community extends MyId {
|
||||
private String id;
|
||||
private String city;
|
||||
private String region;
|
||||
private String name;
|
||||
private String address;
|
||||
|
||||
String getId() {
|
||||
return id
|
||||
}
|
||||
|
||||
void setId(String id) {
|
||||
this.id = id
|
||||
}
|
||||
|
||||
String getCity() {
|
||||
return city
|
||||
}
|
||||
|
||||
void setCity(String city) {
|
||||
this.city = city
|
||||
}
|
||||
|
||||
String getRegion() {
|
||||
return region
|
||||
}
|
||||
|
||||
void setRegion(String region) {
|
||||
this.region = region
|
||||
}
|
||||
|
||||
String getName() {
|
||||
return name
|
||||
}
|
||||
|
||||
void setName(String name) {
|
||||
this.name = name
|
||||
}
|
||||
|
||||
String getAddress() {
|
||||
return address
|
||||
}
|
||||
|
||||
void setAddress(String address) {
|
||||
this.address = address
|
||||
}
|
||||
}
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package com.fangjia.script.temp.shenjian.poi.entity
|
||||
|
||||
class MyId {
|
||||
|
||||
/**
|
||||
* Ë÷Òýid
|
||||
*/
|
||||
String _id;
|
||||
|
||||
String get_id() {
|
||||
return _id
|
||||
}
|
||||
|
||||
void set_id(String _id) {
|
||||
this._id = _id
|
||||
}
|
||||
}
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
package com.fangjia.script.temp.shenjian.util
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils
|
||||
import com.fangjia.script.temp.shenjian.poi.entity.MyId
|
||||
import org.elasticsearch.action.DocWriteRequest
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.search.*
|
||||
import org.elasticsearch.action.update.UpdateRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.search.Scroll
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder
|
||||
import org.slf4j.Logger
|
||||
import org.slf4j.LoggerFactory
|
||||
|
||||
/**
|
||||
* @author shenjian
|
||||
* @since 2022/07/02
|
||||
*/
|
||||
class EsUtils {
|
||||
|
||||
private static RestHighLevelClient ES_CLIENT = RestHighLevelClientUtils.getClient()
|
||||
private static final Logger logger = LoggerFactory.getLogger(EsUtils.class)
|
||||
|
||||
/**
|
||||
* 滚动方式查询大批量es数据
|
||||
*
|
||||
* @param index 索引
|
||||
* @param boolQuery 查询语句
|
||||
* @param clazz 要返回的实体类型
|
||||
* @param <T>
|
||||
* @return
|
||||
*/
|
||||
public static <T extends MyId> List<T> scrollQuery(String index, QueryBuilder boolQuery, Class<T> clazz, def limitSize=300000) {
|
||||
List<T> result = new ArrayList<>();
|
||||
// 存活时间,当索引数据量特别大时,出现超时可能性大,此值适当调大
|
||||
Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30L));
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(boolQuery);
|
||||
// TEST 用!!!!!!!!!!!!!!!!!!!!!!
|
||||
//searchSourceBuilder.size(1);
|
||||
searchSourceBuilder.size(10000);
|
||||
SearchRequest searchRequest = new SearchRequest().indices(index).scroll(scroll).source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = null;
|
||||
try {
|
||||
searchResponse = ES_CLIENT.search(searchRequest, RequestOptions.DEFAULT);
|
||||
} catch (IOException e) {
|
||||
logger.error("查询es数据异常:{}", e);
|
||||
}
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits();
|
||||
if (searchHits != null) {
|
||||
for (SearchHit searchHit : searchHits) {
|
||||
T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz);
|
||||
t.set_id(searchHit.getId())
|
||||
result.add(t);
|
||||
}
|
||||
}
|
||||
// 遍历搜索命中的数据,直到没有数据
|
||||
while (searchHits != null && searchHits.length > 0 && result.size() < limitSize) {
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
try {
|
||||
searchResponse = ES_CLIENT.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
} catch (IOException e) {
|
||||
logger.error("查询es数据异常:{}", e);
|
||||
}
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
if (searchHits != null && searchHits.length > 0) {
|
||||
for (SearchHit searchHit : searchHits) {
|
||||
T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz);
|
||||
t.set_id(searchHit.getId())
|
||||
result.add(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
// clean scroll
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
ClearScrollResponse clearScrollResponse = null;
|
||||
try {
|
||||
clearScrollResponse = ES_CLIENT.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
} catch (IOException e) {}
|
||||
boolean succeeded = clearScrollResponse.isSucceeded();
|
||||
if (succeeded) {
|
||||
return result;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 批量更新es某个字段值
|
||||
* @param index 索引名
|
||||
* @param field 字段名
|
||||
* @param val 字段新值
|
||||
* @param ids 要更新的文档id
|
||||
*/
|
||||
public static void bulkUpdateFieldByIds(String index, String field, Object val, List<String> ids) {
|
||||
if (ids.size() > 1000) {
|
||||
throw new RuntimeException("批量更新数量过多")
|
||||
}
|
||||
BulkRequest bulkRequest = new BulkRequest();
|
||||
for (String id : ids) {
|
||||
Map<String, Object> doc = new HashMap<>();
|
||||
doc.put(field, val)
|
||||
bulkRequest.add(new UpdateRequest(index, id).doc(doc));
|
||||
}
|
||||
ES_CLIENT.bulk(bulkRequest, RequestOptions.DEFAULT);
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicInteger
|
||||
class GaoDeUtils {
|
||||
|
||||
// 新获取key
|
||||
def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418"]
|
||||
def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418", "10bcb421d2ee4e51ed9268d6dca58ad1"]
|
||||
def static AtomicInteger keyUsed = new AtomicInteger(0)
|
||||
|
||||
def static codeNameMap = new HashMap<String, String>(){{
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202204);
|
||||
def count2 = findTotalCount(source, 202205);
|
||||
def count1 = findTotalCount(source, 202205);
|
||||
def count2 = findTotalCount(source, 202206);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -132,7 +132,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
||||
q.put("s_date", SHORT_DATE);
|
||||
def ret = []
|
||||
try{
|
||||
def dbcursor = dataProcessColl.find(q).sort(new BasicDBObject("m_date", -1).append("type",1))
|
||||
def dbcursor = dataProcessColl.find(q).sort(new BasicDBObject("type",1).append("m_date", -1))
|
||||
|
||||
if(dbcursor){
|
||||
for(def dbo : dbcursor){
|
||||
|
||||
+2
-2
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20220501", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20220601", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20220601", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20220701", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
|
||||
+1
-1
@@ -62,7 +62,7 @@ class ApiProductFieldStatBatch extends BaseBatch {
|
||||
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
|
||||
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000)
|
||||
def emails = [
|
||||
"yinjihuan@fangjia.com","panyingjiao@fangjia.com","liyan@fangjia.com", "jiangxiaochun@fangjia.com"
|
||||
"jiangxiaochun@fangjia.com"
|
||||
]
|
||||
static def allDis = []
|
||||
static DecimalFormat df = new DecimalFormat("0.00%")
|
||||
|
||||
Reference in New Issue
Block a user