diff --git a/projects/fangjia-coresite/src/main/webapp/header/v5/_header.ftl b/projects/fangjia-coresite/src/main/webapp/header/v5/_header.ftl
index 293708039..cfea9915a 100644
--- a/projects/fangjia-coresite/src/main/webapp/header/v5/_header.ftl
+++ b/projects/fangjia-coresite/src/main/webapp/header/v5/_header.ftl
@@ -58,7 +58,7 @@
-
APP下载
+<#-- APP下载-->
@@ -148,9 +148,9 @@
-
- APP下载
-
+<#-- -->
+<#-- APP下载-->
+<#-- -->
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/auto/CityInfoEmailSender.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/auto/CityInfoEmailSender.groovy
index cb0a3f0f7..323ce415b 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/auto/CityInfoEmailSender.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/auto/CityInfoEmailSender.groovy
@@ -23,14 +23,7 @@ class CityInfoEmailSender extends BaseBatch{
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files')
def emails = [
- "wangsheng@fangjia.com"
- ,"xuyejun@fangjia.com"
- ,"tiki@fangjia.com"
- ,"zhangzx@fangjia.com"
- ,"panyingjiao@fangjia.com"
- ,"yangqixing@fangjia.com"
- ,"james.lin@fangjia.com"
- ,"jhu@fangjia.com"
+ "gaojuan@fangjia.com"
]
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy
index f35522733..406d043a8 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy
@@ -383,7 +383,6 @@ class DealPriceDataProcess {
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
sqlInstance236.executeUpdate(sql)
}
-
}
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy
index 5354eafe8..21ddf507c 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy
@@ -48,13 +48,14 @@ class AnjukeAppCrawl {
// task.run();
// task.run2()
- task.addThirdPrice()
+// task.addThirdPrice()
// task.saveToEs()
// task.saveToEs1()
// task.groupByCity()
// task.selectNoSave()
// task.statCount()
+ task.exportFzDist()
}
def saveToEs1(){
def lists = getAllAppDatas()
@@ -75,6 +76,24 @@ class AnjukeAppCrawl {
})
}
+ def exportFzDist(){
+ def q = new BasicDBObject()
+ q.put("city","¸£ÖÝ")
+ def dbcursor = detail.find(q).batchSize(5000)
+ def index = 0
+ def rets = []
+ rets << ["city","region","name","_id","id","sale_num","city_short","source"]
+ for(def d : dbcursor){
+ def count = 0
+ if(d.propInfo.saleNum){
+ count = d.propInfo.saleNum.toInteger()
+ }
+ rets << [d.base.cityName, d.base.areaName, d.base.name, d._id.toString(),d.base.id, count, "fz","°²¾Ó¿Í"]
+ }
+
+ ExcelUtils.write("D:\\Êý¾Ý\\¸£ÖÝ\\¸£ÖÝÐ¡Çø.xlsx", rets, "Êý¾Ý")
+ }
+
def selectNoSave(){
def esData = findAllEsData()
def q = new BasicDBObject()
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy
index 81056b29d..bd7567c8e 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy
@@ -25,14 +25,14 @@ class AnjukeHouseCrawl {
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
- def href = "https://sh.esf.fang.com/loupan/1210043376.htm"
+ def href = "https://fz.anjuke.com/prop/view/S2512363126645770"
def doc = getDocument(href, driver)
// def doc = ClaABuYun.getResponse(href)
// def newUrl = href.substring(0, hrefs.indexOf("?"))
- sendMq(href, doc, "district_detail_fang")
+ sendMq(href, doc, "sell_anjuke")
-// for(def i=1;i<50;i++){
-// def url = "https://fz.anjuke.com/sale/guloue/p${i}/"
+// for(def i=25;i<50;i++){
+// def url = "https://fz.anjuke.com/sale/jinana/p${i}/"
// def doc = getDocument(url, driver)
// def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc)
// println "Ò³Êý£º"+i+" ¼Ç¼Êý£º"+eles.size()
@@ -203,7 +203,18 @@ class AnjukeHouseCrawl {
}
if(!els){
- els = JQueryUtils.select(".village_info", doc)
+ els = JQueryUtils.select(".detail-content", doc)
+ }
+ if(!els){
+ els = JQueryUtils.select(".content", doc)
+ }
+
+ if(!els){
+ els = JQueryUtils.select(".wrapper", doc)
+ }
+
+ if(!els){
+ els = JQueryUtils.select(".main-wrap", doc)
}
if(els && els.size()>0){
stop = true;
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy
index c9ca234ce..a465c984a 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy
@@ -23,14 +23,14 @@ class ZhugeFangDistCrawl {
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
- def url = "https://fz.esfxiaoqu.zhuge.com/pingtan/"
+ def url = "https://fz.esfxiaoqu.zhuge.com/changle/"
WebDriver driver = new ChromeDriver()
driver.manage().deleteAllCookies()
getRegions(driver, url)
}
def getRegions(driver, url) {
- for(def i=1;i<=4;i++){
+ for(def i=1;i<=7;i++){
def newUrl = url+"page${i}/"
println newUrl
def doc = getDocument(newUrl, driver)
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy
new file mode 100644
index 000000000..0cd120438
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy
@@ -0,0 +1,147 @@
+package com.fangjia.script.gj.paimai
+
+import com.fangjia.script.linxsh.util.AddressUtil
+import com.fangjia.script.util.DateUtils
+import com.fangjia.script.util.ExcelUtils
+import com.fangjia.script.util.MongoUtils
+import com.mongodb.BasicDBObject
+import com.mongodb.DBCollection
+
+class ExportPaimai {
+
+
+ def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
+ DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
+
+ static main(args) {
+ def task = new ExportPaimai();
+ task.run()
+// task.addressStandard()
+// task.addPrice()
+// task.stat()
+ }
+
+ def stat(){
+ def file = "D:\\Êý¾Ý\\ÖØÇìÅ©ÉÌÐÐ\\·¨ÅÄ\\ËÞǨ·¨ÅÄÊý¾Ý.xlsx"
+ def datas = ExcelUtils.read(file, 0, 1)
+ def index1 = 0
+
+ def rates1 = []
+ for(def d : datas){
+ if(d['P'] && d['P']=='Òѳɽ»'){
+ if(d['M'] && d['M']=="סլ")
+// if(d['K']>=150){
+// index1++
+// def sumTradePrice = d['S']
+// def marketPrice = d['Y']
+// rates1 << sumTradePrice/marketPrice
+// }
+ if(d['S']>2000000){
+ index1++
+ def sumTradePrice = d['S']
+ def marketPrice = d['Y']
+ rates1 << sumTradePrice/marketPrice
+ }
+ }
+ }
+ def rate = rates1.sum()/rates1.size()
+ println index1 +" "+ rate
+ }
+
+ def addPrice(){
+ ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\ÖØÇìÅ©ÉÌÐÐ\\·¨ÅÄ\\ËÞǨ·¨ÅÄÊý¾Ý.xlsx",{d->
+ if(!d['Y'] && d['Z']){
+ d['Y'] = d['Z']
+ }
+ })
+ }
+ def addressStandard(){
+ ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\ÖØÇìÅ©ÉÌÐÐ\\·¨ÅÄ\\ÖØÇì·¨ÅÄÊý¾Ý.xlsx",{d->
+ def formatAddress = AddressUtil.formatAddress(d['C'], d['A'])
+ if(formatAddress){
+ d['F'] = formatAddress
+ }
+ })
+ }
+
+ def run(){
+ def datas = findPaimaiData()
+ println "×ÜÁ¿£º"+datas.size()
+ def rows = []
+ def header = ["±êµØÎïµØÖ·","Ê¡","³ÇÊÐ","ÇøÓò","µØÖ·","±êÌâÌáÈ¡µØÖ·","Â¥¶°ºÅ","µ¥Ôª","ÊÒºÅ","Â¥ÅÌÃû³Æ","½¨ÖþÃæ»ý","¿¢¹¤ÈÕÆÚ","·¿ÎÝÓÃ;","×ÜÂ¥²ã",
+ "ÅÄÂô״̬","ÊÇ·ñ³É½»","ÅÄÂô½×¶Î","ÅÄÂô·½Ê½","³É½»¼Û","ÅÄÂô¿ªÊ¼Ê±¼ä","ÅÄÂô½áÊøÊ±¼ä","ÆðÅļÛ","±£Ö¤½ð","¼Ó¼Û·ù¶È","ÆÀ¹À¼Û","Êг¡¼Û",
+ "±äÂôÔ¤½É¿î","¾º¼ÛÖÜÆÚ","ÑÓʱÖÜÆÚ","ÓÅÏȹºÂòȨÈË","±¨ÃûÈËÊý","Χ¹ÛÈËÊý","ÉèÖÃÌáÐÑÈËÊý","ÑÓ³Ù´ÎÊý","URL"]
+ rows << header
+ def set = new HashSet()
+ def index = 0
+ for(def d : datas){
+ if(!set.contains(d.title)){
+ index++
+ println index
+ set << d.title
+ def row = []
+ row << d.title
+ row << d.province
+ row << d.city
+ row << d.region
+ row << d.address
+ row << ""
+// def formatAddress = AddressUtil.formatAddress(d.city, d.title)
+// if(formatAddress){
+// row << formatAddress
+// }else{
+// row << ""
+// }
+ row << d.houseNum
+ row << d.unitNum
+ row << d.roomNum
+ row << d.matchName
+ row << d.area
+ row << d.buildYear
+ row << d.estateType
+ row << ""
+ row << ""
+ row << d.status
+ row << d.stage
+ row << ""
+ row << d.tradePrice
+ row << DateUtils.formatDateByPattern(d.startDate, "yyyy-MM-dd")
+ row << DateUtils.formatDateByPattern(d.finishDate, "yyyy-MM-dd")
+ row << d.startPrice
+ row << d.depositPrice
+ row << d.addPrice
+ row << d.evalatePrice
+ row << d.marketPrice
+ row << d.sellOffAdvancePay
+ row << d.pricePkWeekly
+ row << d.delayWeekly
+ row << d.preemptivePerson
+ row << d.enrollCount
+ row << d.lookCount
+ row << d.callCount
+ row << d.delayCount
+ row << d.url
+
+ rows << row
+ }
+ }
+ ExcelUtils.write("D:\\Êý¾Ý\\ÖØÇìÅ©ÉÌÐÐ\\·¨ÅÄ\\ÖØÇì·¨ÅÄÔÊý¾Ý.xlsx",rows,"Êý¾Ý")
+ }
+
+ def findPaimaiData(){
+ def pmquery = new BasicDBObject()
+ pmquery.put("city","ÖØÇì")
+ pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
+ pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-06-01")))
+ pmquery.put("status","Òѳɽ»")
+ def dbcursor = paimai43_COLL.find(pmquery).sort(new BasicDBObject("finishDate",-1))
+ def rets = []
+ for(def db : dbcursor){
+// if(db.estateType && db.estateType=="³µ¿â"){
+// continue
+// }
+ rets << db
+ }
+ return rets
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy
new file mode 100644
index 000000000..2f0d142d2
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy
@@ -0,0 +1,71 @@
+package com.fangjia.script.gj.paimai
+
+import com.alibaba.fastjson.JSON
+import com.alibaba.fastjson.JSONObject
+import com.fangjia.script.temp.shenjian.util.MessageUtils
+import com.fangjia.script.util.CommonUtils
+import com.fangjia.script.util.DateUtils
+import com.fangjia.script.util.MongoUtils
+import com.mongodb.BasicDBObject
+import com.mongodb.DBCollection
+import com.rabbitmq.client.AMQP
+
+import java.text.SimpleDateFormat
+
+DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "ali_fapai_detail", "192.168.0.235", 27018)
+
+def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
+DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
+
+
+//Date date=new Date(1550455200000);
+//SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+//String str = format.format(date);
+//System.out.println(str);
+
+def pmquery = new BasicDBObject()
+pmquery.put("city","ËÞǨ")
+pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
+//pmquery.put("status","Òѳɽ»")
+//pmquery.put("estateType", null)
+pmquery.put("url","https://sf-item.taobao.com/sf_item/665574733237.htm")
+
+def cur = paimai43_COLL.find(pmquery)
+def urls = []
+for(def o : cur){
+ urls << o.url
+}
+println "´ý½âÎö£º"+urls
+def index = 0
+for(def url : urls){
+ def q = new BasicDBObject()
+ q.put("url", url)
+ def dbcursor = collection.find(q)
+ index++
+ println index
+ for(def d : dbcursor){
+
+ JSON json = new JSONObject()
+ json.put("project", "paimai_taobao")
+ json.put("url", d.url)
+
+ JSON result = new JSONObject()
+ result.put("content", d.content)
+ def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
+ result.put("trades", bid_content)
+ result.put("condition", d.condition)
+ result.put("sub_count", d.sub_count)
+ def data_json = net.sf.json.JSONObject.fromObject(d.json)
+ result.put("data_json", data_json)
+ result.put("desc_content", d.desc_content)
+ result.put("item_notice_content", d.item_notice_content)
+ result.put("notice_detail_content", d.notice_detail_content)
+
+ json.put("result", result)
+
+ byte[] packedMsg = MessageUtils.pack(json)
+ CommonUtils.sendMessage("result2parse2dev", packedMsg)
+ }
+}
+
+
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy
index 98110403e..3579922b1 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy
@@ -44,9 +44,7 @@ class PdsUseStatisticBatch extends BaseBatch{
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000)
def static emails = [
- "hubingliang@fangjia.com",
- "james.lin@fangjia.com",
- "panyingjiao@fangjia.com",
+
"duweiliang@fangjia.com",
"yaojingmeng@fangjia.com",
"xuqi@fangjia.com",
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/DaiQianGuZhiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/DaiQianGuZhiBatch.groovy
index e1f6932eb..49db7393c 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/DaiQianGuZhiBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/DaiQianGuZhiBatch.groovy
@@ -34,8 +34,7 @@ class DaiQianGuZhiBatch extends BaseBatch{
final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files')
def static emails = [
"hubingliang@fangjia.com",
- "james.lin@fangjia.com",
- "panyingjiao@fangjia.com"
+ "james.lin@fangjia.com"
]
static main(args) {
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuFenLeiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuFenLeiBatch.groovy
index 6fdaf20e3..da74f4bd4 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuFenLeiBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuFenLeiBatch.groovy
@@ -34,8 +34,7 @@ class KeHuFenLeiBatch extends BaseBatch{
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000)
def static emails = [
"hubingliang@fangjia.com",
- "james.lin@fangjia.com",
- "panyingjiao@fangjia.com"
+ "james.lin@fangjia.com"
]
static main(args) {
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuPingJiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuPingJiBatch.groovy
index 9aad05cf6..a2219d524 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuPingJiBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuPingJiBatch.groovy
@@ -33,9 +33,7 @@ class KeHuPingJiBatch extends BaseBatch{
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files')
def static emails = [
- "hubingliang@fangjia.com",
- "james.lin@fangjia.com",
- "panyingjiao@fangjia.com"
+
]
static main(args) {
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/oracle/ExportExceptionPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/oracle/ExportExceptionPrice.groovy
index ce6a0b94b..9bb7c8c4b 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/oracle/ExportExceptionPrice.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/oracle/ExportExceptionPrice.groovy
@@ -34,7 +34,7 @@ sqlInstance.eachRow(sql, { r ->
if (year_month == '202205') {
fiveMap.put(key, r['avg_price'] + "|" + r['calc_type'])
} else if ((year_month == '202206')) {
- fiveMap.put(key, r['avg_price'] + "|" + r['calc_type'])
+ sixMap.put(key, r['avg_price'] + "|" + r['calc_type'])
}
})
for (String key : sixMap.keySet()) {
@@ -47,4 +47,4 @@ for (String key : sixMap.keySet()) {
}
}
}
-ExcelUtils.write("F:\\Êý¾Ý\\¼Û¸ñ\\¶«Ý¸ÉϺ£¼Û¸ñÆ«²î´ó.xlsx", result, "ͳ¼Æ")
\ No newline at end of file
+ExcelUtils.write("D:\\Êý¾Ý\\¼Û¸ñ\\¶«Ý¸ÉϺ£¼Û¸ñÆ«²î´ó.xlsx", result, "ͳ¼Æ")
\ No newline at end of file
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromCommunity.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromCommunity.groovy
new file mode 100644
index 000000000..f93b844d7
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromCommunity.groovy
@@ -0,0 +1,94 @@
+package com.fangjia.script.temp.shenjian.poi
+
+
+import com.alibaba.fastjson.JSONObject
+import com.fangjia.script.statistics.poi.AmapPoi
+import com.fangjia.script.statistics.poi.AmapPoiDetail
+import com.fangjia.script.temp.shenjian.poi.entity.Community
+import com.fangjia.script.temp.shenjian.util.CommonDtoUtils
+import com.fangjia.script.temp.shenjian.util.EsUtils
+import com.fangjia.script.temp.shenjian.util.GaoDeUtils
+import com.fangjia.script.temp.shenjian.util.HttpUtils
+import org.apache.commons.collections.CollectionUtils
+import org.elasticsearch.index.query.QueryBuilder
+import org.elasticsearch.index.query.QueryBuilders
+
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.concurrent.atomic.AtomicLong
+import java.util.stream.Collectors
+
+/**
+ * ´ÓES»ñÈ¡POI
+ */
+class GetPoiFromCommunity {
+
+ static final ExecutorService executorService = Executors.newFixedThreadPool(10)
+ static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail"
+ // סլÀà
+ static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304";
+
+ static void main(String[] args) {
+ AtomicLong atomicLong = new AtomicLong(0)
+ Map map = new ConcurrentHashMap<>()
+ def rets = this.findCommunities()
+ CountDownLatch countDownLatch = new CountDownLatch(rets.size())
+ for (def retTemp : rets) {
+ final def ret = retTemp
+ executorService.submit(new Runnable() {
+ @Override
+ void run() {
+ final String id = ret._id // Ë÷ÒýID,²¢·ÇÐ¡ÇøID
+ String city = ret.city
+ String region = ret.region
+ String seaweedName = ret.name
+ String address = ret.address
+ List pois = new ArrayList<>();
+ List namePois = GaoDeUtils.searchPoi(city, seaweedName, region, typecodes);
+ List addressPois = GaoDeUtils.searchPoi(city, address, region, typecodes);
+ if (CollectionUtils.isNotEmpty(namePois)) {
+ pois.addAll(namePois)
+ }
+ if (CollectionUtils.isNotEmpty(addressPois)) {
+ pois.addAll(addressPois)
+ }
+ if (CollectionUtils.isNotEmpty(pois)) {
+ List detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois);
+ String jsonStr = JSONObject.toJSONString(detailList)
+ HttpUtils.postForJson(URL, jsonStr)
+ }
+ synchronized (map) {
+ map.put(id, id)
+ countDownLatch.countDown()
+ if (map.size() % 100 == 0) {
+ atomicLong.addAndGet(map.size())
+ List ids = map.values().stream().collect(Collectors.toList())
+ EsUtils.bulkUpdateFieldByIds("community", "crawl_flag", 0, ids)
+ map.clear()
+ println(atomicLong.intValue())
+ }
+ }
+ }
+ })
+ }
+ countDownLatch.await()
+ if (map.size() > 0) {
+ atomicLong.addAndGet(map.size())
+ List ids = map.values().stream().collect(Collectors.toList())
+ EsUtils.bulkUpdateFieldByIds("community", "crawl_flag", 0, ids)
+ map.clear()
+ println(atomicLong.intValue())
+ }
+ System.exit(1)
+ }
+
+ static def findCommunities(){
+ QueryBuilder queryBuilder = QueryBuilders.boolQuery()
+ .mustNot(QueryBuilders.termQuery("crawl_flag", 0))
+ List communities = EsUtils.scrollQuery("community", queryBuilder, Community.class)
+ return communities
+ }
+
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/Community.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/Community.groovy
new file mode 100644
index 000000000..ba2bfaaea
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/Community.groovy
@@ -0,0 +1,49 @@
+package com.fangjia.script.temp.shenjian.poi.entity;
+
+public class Community extends MyId {
+ private String id;
+ private String city;
+ private String region;
+ private String name;
+ private String address;
+
+ String getId() {
+ return id
+ }
+
+ void setId(String id) {
+ this.id = id
+ }
+
+ String getCity() {
+ return city
+ }
+
+ void setCity(String city) {
+ this.city = city
+ }
+
+ String getRegion() {
+ return region
+ }
+
+ void setRegion(String region) {
+ this.region = region
+ }
+
+ String getName() {
+ return name
+ }
+
+ void setName(String name) {
+ this.name = name
+ }
+
+ String getAddress() {
+ return address
+ }
+
+ void setAddress(String address) {
+ this.address = address
+ }
+}
\ No newline at end of file
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/MyId.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/MyId.groovy
new file mode 100644
index 000000000..53bd1b98b
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/MyId.groovy
@@ -0,0 +1,17 @@
+package com.fangjia.script.temp.shenjian.poi.entity
+
+class MyId {
+
+ /**
+ * Ë÷Òýid
+ */
+ String _id;
+
+ String get_id() {
+ return _id
+ }
+
+ void set_id(String _id) {
+ this._id = _id
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy
new file mode 100644
index 000000000..b158c9351
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy
@@ -0,0 +1,116 @@
+package com.fangjia.script.temp.shenjian.util
+
+import com.alibaba.fastjson.JSONObject
+import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils
+import com.fangjia.script.temp.shenjian.poi.entity.MyId
+import org.elasticsearch.action.DocWriteRequest
+import org.elasticsearch.action.bulk.BulkRequest
+import org.elasticsearch.action.search.*
+import org.elasticsearch.action.update.UpdateRequest
+import org.elasticsearch.client.RequestOptions
+import org.elasticsearch.client.RestHighLevelClient
+import org.elasticsearch.common.unit.TimeValue
+import org.elasticsearch.index.query.QueryBuilder
+import org.elasticsearch.search.Scroll
+import org.elasticsearch.search.SearchHit
+import org.elasticsearch.search.builder.SearchSourceBuilder
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+
+/**
+ * @author shenjian
+ * @since 2022/07/02
+ */
+class EsUtils {
+
+ private static RestHighLevelClient ES_CLIENT = RestHighLevelClientUtils.getClient()
+ private static final Logger logger = LoggerFactory.getLogger(EsUtils.class)
+
+ /**
+ * ¹ö¶¯·½Ê½²éѯ´óÅúÁ¿esÊý¾Ý
+ *
+ * @param index Ë÷Òý
+ * @param boolQuery ²éѯÓï¾ä
+ * @param clazz Òª·µ»ØµÄʵÌåÀàÐÍ
+ * @param
+ * @return
+ */
+ public static List scrollQuery(String index, QueryBuilder boolQuery, Class clazz, def limitSize=300000) {
+ List result = new ArrayList<>();
+ // ´æ»îʱ¼ä£¬µ±Ë÷ÒýÊý¾ÝÁ¿Ìرð´óʱ£¬³öÏÖ³¬Ê±¿ÉÄÜÐԴ󣬴ËÖµÊʵ±µ÷´ó
+ Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30L));
+ SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+ searchSourceBuilder.query(boolQuery);
+ // TEST Óã¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡
+ //searchSourceBuilder.size(1);
+ searchSourceBuilder.size(10000);
+ SearchRequest searchRequest = new SearchRequest().indices(index).scroll(scroll).source(searchSourceBuilder);
+ SearchResponse searchResponse = null;
+ try {
+ searchResponse = ES_CLIENT.search(searchRequest, RequestOptions.DEFAULT);
+ } catch (IOException e) {
+ logger.error("²éѯesÊý¾ÝÒì³£:{}", e);
+ }
+ String scrollId = searchResponse.getScrollId();
+ SearchHit[] searchHits = searchResponse.getHits().getHits();
+ if (searchHits != null) {
+ for (SearchHit searchHit : searchHits) {
+ T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz);
+ t.set_id(searchHit.getId())
+ result.add(t);
+ }
+ }
+ // ±éÀúËÑË÷ÃüÖеÄÊý¾Ý£¬Ö±µ½Ã»ÓÐÊý¾Ý
+ while (searchHits != null && searchHits.length > 0 && result.size() < limitSize) {
+ SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
+ scrollRequest.scroll(scroll);
+ try {
+ searchResponse = ES_CLIENT.scroll(scrollRequest, RequestOptions.DEFAULT);
+ } catch (IOException e) {
+ logger.error("²éѯesÊý¾ÝÒì³£:{}", e);
+ }
+ scrollId = searchResponse.getScrollId();
+ searchHits = searchResponse.getHits().getHits();
+ if (searchHits != null && searchHits.length > 0) {
+ for (SearchHit searchHit : searchHits) {
+ T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz);
+ t.set_id(searchHit.getId())
+ result.add(t);
+ }
+ }
+ }
+ // clean scroll
+ ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
+ clearScrollRequest.addScrollId(scrollId);
+ ClearScrollResponse clearScrollResponse = null;
+ try {
+ clearScrollResponse = ES_CLIENT.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
+ } catch (IOException e) {}
+ boolean succeeded = clearScrollResponse.isSucceeded();
+ if (succeeded) {
+ return result;
+ }
+ return null;
+ }
+
+
+ /**
+ * ÅúÁ¿¸üÐÂesij¸ö×Ö¶ÎÖµ
+ * @param index Ë÷ÒýÃû
+ * @param field ×Ö¶ÎÃû
+ * @param val ×Ö¶ÎÐÂÖµ
+ * @param ids Òª¸üеÄÎĵµid
+ */
+ public static void bulkUpdateFieldByIds(String index, String field, Object val, List ids) {
+ if (ids.size() > 1000) {
+ throw new RuntimeException("ÅúÁ¿¸üÐÂÊýÁ¿¹ý¶à")
+ }
+ BulkRequest bulkRequest = new BulkRequest();
+ for (String id : ids) {
+ Map doc = new HashMap<>();
+ doc.put(field, val)
+ bulkRequest.add(new UpdateRequest(index, id).doc(doc));
+ }
+ ES_CLIENT.bulk(bulkRequest, RequestOptions.DEFAULT);
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy
index 6b880c5ea..af5f5fc8b 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy
@@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicInteger
class GaoDeUtils {
// лñÈ¡key
- def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418"]
+ def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418", "10bcb421d2ee4e51ed9268d6dca58ad1"]
def static AtomicInteger keyUsed = new AtomicInteger(0)
def static codeNameMap = new HashMap(){{
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy
index 4ac878217..383911899 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["·¿¹À¹À","Öî¸ðÕÒ·¿","°²¾Ó¿Í","ËÑ·¿","ÖÐԵزú","¸Ï¼¯","Á´¼Ò","Íò©Ê¿","³ÇÊз¿²ú","±´¿ÇÕÒ·¿","Q·¿","Q·¿(ÐÂ)","ÖÐԵزú(ÐÂ)","ÀÖÓмÒ","ÀÖÓмÒ(ÐÂ)","·¿Ì½","·¿¶à¶à","С×ʼÒ","ÎÒ°®ÎÒ¼Ò","ÂóÌï","·¿Í¾","C21","ÀÖ¾Ó","·¿²ú³¬ÊÐ","58ͬ³Ç","365ÌÔ·¿","¼ªÎÝ"]
for(def source :sources){
- def count1 = findTotalCount(source, 202204);
- def count2 = findTotalCount(source, 202205);
+ def count1 = findTotalCount(source, 202205);
+ def count2 = findTotalCount(source, 202206);
println source +" "+count1 +" "+count2
}
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy
index 3c2bcde38..89e0e9483 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy
@@ -132,7 +132,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
q.put("s_date", SHORT_DATE);
def ret = []
try{
- def dbcursor = dataProcessColl.find(q).sort(new BasicDBObject("m_date", -1).append("type",1))
+ def dbcursor = dataProcessColl.find(q).sort(new BasicDBObject("type",1).append("m_date", -1))
if(dbcursor){
for(def dbo : dbcursor){
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy
index 1297ca3ea..8bca701d7 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
-def startDate = DateUtil.parseDate("20220501", "yyyyMMdd")
-def endDate = DateUtil.parseDate("20220601", "yyyyMMdd")
+def startDate = DateUtil.parseDate("20220601", "yyyyMMdd")
+def endDate = DateUtil.parseDate("20220701", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/batch/ApiProductFieldStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/batch/ApiProductFieldStatBatch.groovy
index 6f120465c..026ec703f 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/batch/ApiProductFieldStatBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/batch/ApiProductFieldStatBatch.groovy
@@ -62,7 +62,7 @@ class ApiProductFieldStatBatch extends BaseBatch {
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000)
def emails = [
- "yinjihuan@fangjia.com","panyingjiao@fangjia.com","liyan@fangjia.com", "jiangxiaochun@fangjia.com"
+ "jiangxiaochun@fangjia.com"
]
static def allDis = []
static DecimalFormat df = new DecimalFormat("0.00%")