diff --git a/projects/fangjia-coresite/src/main/webapp/header/v5/_header.ftl b/projects/fangjia-coresite/src/main/webapp/header/v5/_header.ftl index 293708039..cfea9915a 100644 --- a/projects/fangjia-coresite/src/main/webapp/header/v5/_header.ftl +++ b/projects/fangjia-coresite/src/main/webapp/header/v5/_header.ftl @@ -58,7 +58,7 @@ -
  • APP下载
  • +<#--
  • APP下载
  • --> @@ -148,9 +148,9 @@ -
  • - APP下载 -
  • +<#--
  • --> +<#-- APP下载--> +<#--
  • --> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/auto/CityInfoEmailSender.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/auto/CityInfoEmailSender.groovy index cb0a3f0f7..323ce415b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/auto/CityInfoEmailSender.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/auto/CityInfoEmailSender.groovy @@ -23,14 +23,7 @@ class CityInfoEmailSender extends BaseBatch{ private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files') def emails = [ - "wangsheng@fangjia.com" - ,"xuyejun@fangjia.com" - ,"tiki@fangjia.com" - ,"zhangzx@fangjia.com" - ,"panyingjiao@fangjia.com" - ,"yangqixing@fangjia.com" - ,"james.lin@fangjia.com" - ,"jhu@fangjia.com" + "gaojuan@fangjia.com" ] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index f35522733..406d043a8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -383,7 +383,6 @@ class DealPriceDataProcess { def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}" sqlInstance236.executeUpdate(sql) } - } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy index 5354eafe8..21ddf507c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy @@ -48,13 +48,14 @@ class AnjukeAppCrawl { // task.run(); // task.run2() - task.addThirdPrice() +// task.addThirdPrice() // task.saveToEs() // task.saveToEs1() // task.groupByCity() // task.selectNoSave() // task.statCount() + task.exportFzDist() } def saveToEs1(){ def lists = getAllAppDatas() @@ -75,6 +76,24 @@ class AnjukeAppCrawl { }) } + def exportFzDist(){ + def q = new BasicDBObject() + q.put("city","¸£ÖÝ") + def dbcursor = detail.find(q).batchSize(5000) + def index = 0 + def rets = [] + rets << ["city","region","name","_id","id","sale_num","city_short","source"] + for(def d : dbcursor){ + def count = 0 + if(d.propInfo.saleNum){ + count = d.propInfo.saleNum.toInteger() + } + rets << [d.base.cityName, d.base.areaName, d.base.name, d._id.toString(),d.base.id, count, "fz","°²¾Ó¿Í"] + } + + ExcelUtils.write("D:\\Êý¾Ý\\¸£ÖÝ\\¸£ÖÝÐ¡Çø.xlsx", rets, "Êý¾Ý") + } + def selectNoSave(){ def esData = findAllEsData() def q = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy index 81056b29d..bd7567c8e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy @@ -25,14 +25,14 @@ class AnjukeHouseCrawl { System.setProperty("http.nonProxyHosts", "localhost"); WebDriver driver = new ChromeDriver() - def href = "https://sh.esf.fang.com/loupan/1210043376.htm" + def href = "https://fz.anjuke.com/prop/view/S2512363126645770" def doc = getDocument(href, driver) // def doc = ClaABuYun.getResponse(href) // def newUrl = href.substring(0, hrefs.indexOf("?")) - sendMq(href, doc, "district_detail_fang") + sendMq(href, doc, "sell_anjuke") -// for(def i=1;i<50;i++){ -// def url = "https://fz.anjuke.com/sale/guloue/p${i}/" +// for(def i=25;i<50;i++){ +// def url = "https://fz.anjuke.com/sale/jinana/p${i}/" // def doc = getDocument(url, driver) // def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc) // println "Ò³Êý£º"+i+" ¼Ç¼Êý£º"+eles.size() @@ -203,7 +203,18 @@ class AnjukeHouseCrawl { } if(!els){ - els = JQueryUtils.select(".village_info", doc) + els = JQueryUtils.select(".detail-content", doc) + } + if(!els){ + els = JQueryUtils.select(".content", doc) + } + + if(!els){ + els = JQueryUtils.select(".wrapper", doc) + } + + if(!els){ + els = JQueryUtils.select(".main-wrap", doc) } if(els && els.size()>0){ stop = true; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy index c9ca234ce..a465c984a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy @@ -23,14 +23,14 @@ class ZhugeFangDistCrawl { def run(){ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); - def url = "https://fz.esfxiaoqu.zhuge.com/pingtan/" + def url = "https://fz.esfxiaoqu.zhuge.com/changle/" WebDriver driver = new ChromeDriver() driver.manage().deleteAllCookies() getRegions(driver, url) } def getRegions(driver, url) { - for(def i=1;i<=4;i++){ + for(def i=1;i<=7;i++){ def newUrl = url+"page${i}/" println newUrl def doc = getDocument(newUrl, driver) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy new file mode 100644 index 000000000..0cd120438 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy @@ -0,0 +1,147 @@ +package com.fangjia.script.gj.paimai + +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class ExportPaimai { + + + def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl) + + static main(args) { + def task = new ExportPaimai(); + task.run() +// task.addressStandard() +// task.addPrice() +// task.stat() + } + + def stat(){ + def file = "D:\\Êý¾Ý\\ÖØÇìÅ©ÉÌÐÐ\\·¨ÅÄ\\ËÞǨ·¨ÅÄÊý¾Ý.xlsx" + def datas = ExcelUtils.read(file, 0, 1) + def index1 = 0 + + def rates1 = [] + for(def d : datas){ + if(d['P'] && d['P']=='Òѳɽ»'){ + if(d['M'] && d['M']=="סլ") +// if(d['K']>=150){ +// index1++ +// def sumTradePrice = d['S'] +// def marketPrice = d['Y'] +// rates1 << sumTradePrice/marketPrice +// } + if(d['S']>2000000){ + index1++ + def sumTradePrice = d['S'] + def marketPrice = d['Y'] + rates1 << sumTradePrice/marketPrice + } + } + } + def rate = rates1.sum()/rates1.size() + println index1 +" "+ rate + } + + def addPrice(){ + ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\ÖØÇìÅ©ÉÌÐÐ\\·¨ÅÄ\\ËÞǨ·¨ÅÄÊý¾Ý.xlsx",{d-> + if(!d['Y'] && d['Z']){ + d['Y'] = d['Z'] + } + }) + } + def addressStandard(){ + ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\ÖØÇìÅ©ÉÌÐÐ\\·¨ÅÄ\\ÖØÇì·¨ÅÄÊý¾Ý.xlsx",{d-> + def formatAddress = AddressUtil.formatAddress(d['C'], d['A']) + if(formatAddress){ + d['F'] = formatAddress + } + }) + } + + def run(){ + def datas = findPaimaiData() + println "×ÜÁ¿£º"+datas.size() + def rows = [] + def header = ["±êµØÎïµØÖ·","Ê¡","³ÇÊÐ","ÇøÓò","µØÖ·","±êÌâÌáÈ¡µØÖ·","Â¥¶°ºÅ","µ¥Ôª","ÊÒºÅ","Â¥ÅÌÃû³Æ","½¨ÖþÃæ»ý","¿¢¹¤ÈÕÆÚ","·¿ÎÝÓÃ;","×ÜÂ¥²ã", + "ÅÄÂô״̬","ÊÇ·ñ³É½»","ÅÄÂô½×¶Î","ÅÄÂô·½Ê½","³É½»¼Û","ÅÄÂô¿ªÊ¼Ê±¼ä","ÅÄÂô½áÊøÊ±¼ä","ÆðÅļÛ","±£Ö¤½ð","¼Ó¼Û·ù¶È","ÆÀ¹À¼Û","Êг¡¼Û", + "±äÂôÔ¤½É¿î","¾º¼ÛÖÜÆÚ","ÑÓʱÖÜÆÚ","ÓÅÏȹºÂòȨÈË","±¨ÃûÈËÊý","Χ¹ÛÈËÊý","ÉèÖÃÌáÐÑÈËÊý","ÑÓ³Ù´ÎÊý","URL"] + rows << header + def set = new HashSet() + def index = 0 + for(def d : datas){ + if(!set.contains(d.title)){ + index++ + println index + set << d.title + def row = [] + row << d.title + row << d.province + row << d.city + row << d.region + row << d.address + row << "" +// def formatAddress = AddressUtil.formatAddress(d.city, d.title) +// if(formatAddress){ +// row << formatAddress +// }else{ +// row << "" +// } + row << d.houseNum + row << d.unitNum + row << d.roomNum + row << d.matchName + row << d.area + row << d.buildYear + row << d.estateType + row << "" + row << "" + row << d.status + row << d.stage + row << "" + row << d.tradePrice + row << DateUtils.formatDateByPattern(d.startDate, "yyyy-MM-dd") + row << DateUtils.formatDateByPattern(d.finishDate, "yyyy-MM-dd") + row << d.startPrice + row << d.depositPrice + row << d.addPrice + row << d.evalatePrice + row << d.marketPrice + row << d.sellOffAdvancePay + row << d.pricePkWeekly + row << d.delayWeekly + row << d.preemptivePerson + row << d.enrollCount + row << d.lookCount + row << d.callCount + row << d.delayCount + row << d.url + + rows << row + } + } + ExcelUtils.write("D:\\Êý¾Ý\\ÖØÇìÅ©ÉÌÐÐ\\·¨ÅÄ\\ÖØÇì·¨ÅÄÔ­Êý¾Ý.xlsx",rows,"Êý¾Ý") + } + + def findPaimaiData(){ + def pmquery = new BasicDBObject() + pmquery.put("city","ÖØÇì") + pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01"))) + pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-06-01"))) + pmquery.put("status","Òѳɽ»") + def dbcursor = paimai43_COLL.find(pmquery).sort(new BasicDBObject("finishDate",-1)) + def rets = [] + for(def db : dbcursor){ +// if(db.estateType && db.estateType=="³µ¿â"){ +// continue +// } + rets << db + } + return rets + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy new file mode 100644 index 000000000..2f0d142d2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy @@ -0,0 +1,71 @@ +package com.fangjia.script.gj.paimai + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.rabbitmq.client.AMQP + +import java.text.SimpleDateFormat + +DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "ali_fapai_detail", "192.168.0.235", 27018) + +def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; +DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl) + + +//Date date=new Date(1550455200000); +//SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); +//String str = format.format(date); +//System.out.println(str); + +def pmquery = new BasicDBObject() +pmquery.put("city","ËÞǨ") +pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01"))) +//pmquery.put("status","Òѳɽ»") +//pmquery.put("estateType", null) +pmquery.put("url","https://sf-item.taobao.com/sf_item/665574733237.htm") + +def cur = paimai43_COLL.find(pmquery) +def urls = [] +for(def o : cur){ + urls << o.url +} +println "´ý½âÎö£º"+urls +def index = 0 +for(def url : urls){ + def q = new BasicDBObject() + q.put("url", url) + def dbcursor = collection.find(q) + index++ + println index + for(def d : dbcursor){ + + JSON json = new JSONObject() + json.put("project", "paimai_taobao") + json.put("url", d.url) + + JSON result = new JSONObject() + result.put("content", d.content) + def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content) + result.put("trades", bid_content) + result.put("condition", d.condition) + result.put("sub_count", d.sub_count) + def data_json = net.sf.json.JSONObject.fromObject(d.json) + result.put("data_json", data_json) + result.put("desc_content", d.desc_content) + result.put("item_notice_content", d.item_notice_content) + result.put("notice_detail_content", d.notice_detail_content) + + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse2dev", packedMsg) + } +} + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy index 98110403e..3579922b1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/PdsUseStatisticBatch.groovy @@ -44,9 +44,7 @@ class PdsUseStatisticBatch extends BaseBatch{ private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000) def static emails = [ - "hubingliang@fangjia.com", - "james.lin@fangjia.com", - "panyingjiao@fangjia.com", + "duweiliang@fangjia.com", "yaojingmeng@fangjia.com", "xuqi@fangjia.com", diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/DaiQianGuZhiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/DaiQianGuZhiBatch.groovy index e1f6932eb..49db7393c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/DaiQianGuZhiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/DaiQianGuZhiBatch.groovy @@ -34,8 +34,7 @@ class DaiQianGuZhiBatch extends BaseBatch{ final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files') def static emails = [ "hubingliang@fangjia.com", - "james.lin@fangjia.com", - "panyingjiao@fangjia.com" + "james.lin@fangjia.com" ] static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuFenLeiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuFenLeiBatch.groovy index 6fdaf20e3..da74f4bd4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuFenLeiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuFenLeiBatch.groovy @@ -34,8 +34,7 @@ class KeHuFenLeiBatch extends BaseBatch{ final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000) def static emails = [ "hubingliang@fangjia.com", - "james.lin@fangjia.com", - "panyingjiao@fangjia.com" + "james.lin@fangjia.com" ] static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuPingJiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuPingJiBatch.groovy index 9aad05cf6..a2219d524 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuPingJiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/hubingliang/pds/KeHuPingJiBatch.groovy @@ -33,9 +33,7 @@ class KeHuPingJiBatch extends BaseBatch{ private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files') def static emails = [ - "hubingliang@fangjia.com", - "james.lin@fangjia.com", - "panyingjiao@fangjia.com" + ] static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/oracle/ExportExceptionPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/oracle/ExportExceptionPrice.groovy index ce6a0b94b..9bb7c8c4b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/oracle/ExportExceptionPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/oracle/ExportExceptionPrice.groovy @@ -34,7 +34,7 @@ sqlInstance.eachRow(sql, { r -> if (year_month == '202205') { fiveMap.put(key, r['avg_price'] + "|" + r['calc_type']) } else if ((year_month == '202206')) { - fiveMap.put(key, r['avg_price'] + "|" + r['calc_type']) + sixMap.put(key, r['avg_price'] + "|" + r['calc_type']) } }) for (String key : sixMap.keySet()) { @@ -47,4 +47,4 @@ for (String key : sixMap.keySet()) { } } } -ExcelUtils.write("F:\\Êý¾Ý\\¼Û¸ñ\\¶«Ý¸ÉϺ£¼Û¸ñÆ«²î´ó.xlsx", result, "ͳ¼Æ") \ No newline at end of file +ExcelUtils.write("D:\\Êý¾Ý\\¼Û¸ñ\\¶«Ý¸ÉϺ£¼Û¸ñÆ«²î´ó.xlsx", result, "ͳ¼Æ") \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromCommunity.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromCommunity.groovy new file mode 100644 index 000000000..f93b844d7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromCommunity.groovy @@ -0,0 +1,94 @@ +package com.fangjia.script.temp.shenjian.poi + + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.statistics.poi.AmapPoiDetail +import com.fangjia.script.temp.shenjian.poi.entity.Community +import com.fangjia.script.temp.shenjian.util.CommonDtoUtils +import com.fangjia.script.temp.shenjian.util.EsUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import org.apache.commons.collections.CollectionUtils +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.stream.Collectors + +/** + * ´ÓES»ñÈ¡POI + */ +class GetPoiFromCommunity { + + static final ExecutorService executorService = Executors.newFixedThreadPool(10) + static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail" + // סլÀà + static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304"; + + static void main(String[] args) { + AtomicLong atomicLong = new AtomicLong(0) + Map map = new ConcurrentHashMap<>() + def rets = this.findCommunities() + CountDownLatch countDownLatch = new CountDownLatch(rets.size()) + for (def retTemp : rets) { + final def ret = retTemp + executorService.submit(new Runnable() { + @Override + void run() { + final String id = ret._id // Ë÷ÒýID,²¢·ÇÐ¡ÇøID + String city = ret.city + String region = ret.region + String seaweedName = ret.name + String address = ret.address + List pois = new ArrayList<>(); + List namePois = GaoDeUtils.searchPoi(city, seaweedName, region, typecodes); + List addressPois = GaoDeUtils.searchPoi(city, address, region, typecodes); + if (CollectionUtils.isNotEmpty(namePois)) { + pois.addAll(namePois) + } + if (CollectionUtils.isNotEmpty(addressPois)) { + pois.addAll(addressPois) + } + if (CollectionUtils.isNotEmpty(pois)) { + List detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois); + String jsonStr = JSONObject.toJSONString(detailList) + HttpUtils.postForJson(URL, jsonStr) + } + synchronized (map) { + map.put(id, id) + countDownLatch.countDown() + if (map.size() % 100 == 0) { + atomicLong.addAndGet(map.size()) + List ids = map.values().stream().collect(Collectors.toList()) + EsUtils.bulkUpdateFieldByIds("community", "crawl_flag", 0, ids) + map.clear() + println(atomicLong.intValue()) + } + } + } + }) + } + countDownLatch.await() + if (map.size() > 0) { + atomicLong.addAndGet(map.size()) + List ids = map.values().stream().collect(Collectors.toList()) + EsUtils.bulkUpdateFieldByIds("community", "crawl_flag", 0, ids) + map.clear() + println(atomicLong.intValue()) + } + System.exit(1) + } + + static def findCommunities(){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery() + .mustNot(QueryBuilders.termQuery("crawl_flag", 0)) + List communities = EsUtils.scrollQuery("community", queryBuilder, Community.class) + return communities + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/Community.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/Community.groovy new file mode 100644 index 000000000..ba2bfaaea --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/Community.groovy @@ -0,0 +1,49 @@ +package com.fangjia.script.temp.shenjian.poi.entity; + +public class Community extends MyId { + private String id; + private String city; + private String region; + private String name; + private String address; + + String getId() { + return id + } + + void setId(String id) { + this.id = id + } + + String getCity() { + return city + } + + void setCity(String city) { + this.city = city + } + + String getRegion() { + return region + } + + void setRegion(String region) { + this.region = region + } + + String getName() { + return name + } + + void setName(String name) { + this.name = name + } + + String getAddress() { + return address + } + + void setAddress(String address) { + this.address = address + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/MyId.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/MyId.groovy new file mode 100644 index 000000000..53bd1b98b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/entity/MyId.groovy @@ -0,0 +1,17 @@ +package com.fangjia.script.temp.shenjian.poi.entity + +class MyId { + + /** + * Ë÷Òýid + */ + String _id; + + String get_id() { + return _id + } + + void set_id(String _id) { + this._id = _id + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy new file mode 100644 index 000000000..b158c9351 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/EsUtils.groovy @@ -0,0 +1,116 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils +import com.fangjia.script.temp.shenjian.poi.entity.MyId +import org.elasticsearch.action.DocWriteRequest +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.search.* +import org.elasticsearch.action.update.UpdateRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.client.RestHighLevelClient +import org.elasticsearch.common.unit.TimeValue +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.search.Scroll +import org.elasticsearch.search.SearchHit +import org.elasticsearch.search.builder.SearchSourceBuilder +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * @author shenjian + * @since 2022/07/02 + */ +class EsUtils { + + private static RestHighLevelClient ES_CLIENT = RestHighLevelClientUtils.getClient() + private static final Logger logger = LoggerFactory.getLogger(EsUtils.class) + + /** + * ¹ö¶¯·½Ê½²éѯ´óÅúÁ¿esÊý¾Ý + * + * @param index Ë÷Òý + * @param boolQuery ²éѯÓï¾ä + * @param clazz Òª·µ»ØµÄʵÌåÀàÐÍ + * @param + * @return + */ + public static List scrollQuery(String index, QueryBuilder boolQuery, Class clazz, def limitSize=300000) { + List result = new ArrayList<>(); + // ´æ»îʱ¼ä£¬µ±Ë÷ÒýÊý¾ÝÁ¿Ìرð´óʱ£¬³öÏÖ³¬Ê±¿ÉÄÜÐԴ󣬴ËÖµÊʵ±µ÷´ó + Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30L)); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(boolQuery); + // TEST Óã¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡£¡ + //searchSourceBuilder.size(1); + searchSourceBuilder.size(10000); + SearchRequest searchRequest = new SearchRequest().indices(index).scroll(scroll).source(searchSourceBuilder); + SearchResponse searchResponse = null; + try { + searchResponse = ES_CLIENT.search(searchRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + logger.error("²éѯesÊý¾ÝÒì³£:{}", e); + } + String scrollId = searchResponse.getScrollId(); + SearchHit[] searchHits = searchResponse.getHits().getHits(); + if (searchHits != null) { + for (SearchHit searchHit : searchHits) { + T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz); + t.set_id(searchHit.getId()) + result.add(t); + } + } + // ±éÀúËÑË÷ÃüÖеÄÊý¾Ý£¬Ö±µ½Ã»ÓÐÊý¾Ý + while (searchHits != null && searchHits.length > 0 && result.size() < limitSize) { + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); + scrollRequest.scroll(scroll); + try { + searchResponse = ES_CLIENT.scroll(scrollRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + logger.error("²éѯesÊý¾ÝÒì³£:{}", e); + } + scrollId = searchResponse.getScrollId(); + searchHits = searchResponse.getHits().getHits(); + if (searchHits != null && searchHits.length > 0) { + for (SearchHit searchHit : searchHits) { + T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz); + t.set_id(searchHit.getId()) + result.add(t); + } + } + } + // clean scroll + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(scrollId); + ClearScrollResponse clearScrollResponse = null; + try { + clearScrollResponse = ES_CLIENT.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + } catch (IOException e) {} + boolean succeeded = clearScrollResponse.isSucceeded(); + if (succeeded) { + return result; + } + return null; + } + + + /** + * ÅúÁ¿¸üÐÂesij¸ö×Ö¶ÎÖµ + * @param index Ë÷ÒýÃû + * @param field ×Ö¶ÎÃû + * @param val ×Ö¶ÎÐÂÖµ + * @param ids Òª¸üеÄÎĵµid + */ + public static void bulkUpdateFieldByIds(String index, String field, Object val, List ids) { + if (ids.size() > 1000) { + throw new RuntimeException("ÅúÁ¿¸üÐÂÊýÁ¿¹ý¶à") + } + BulkRequest bulkRequest = new BulkRequest(); + for (String id : ids) { + Map doc = new HashMap<>(); + doc.put(field, val) + bulkRequest.add(new UpdateRequest(index, id).doc(doc)); + } + ES_CLIENT.bulk(bulkRequest, RequestOptions.DEFAULT); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 6b880c5ea..af5f5fc8b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicInteger class GaoDeUtils { // лñÈ¡key - def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418"] + def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418", "10bcb421d2ee4e51ed9268d6dca58ad1"] def static AtomicInteger keyUsed = new AtomicInteger(0) def static codeNameMap = new HashMap(){{ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 4ac878217..383911899 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["·¿¹À¹À","Öî¸ðÕÒ·¿","°²¾Ó¿Í","ËÑ·¿","ÖÐÔ­µØ²ú","¸Ï¼¯","Á´¼Ò","Íò©Ê¿","³ÇÊз¿²ú","±´¿ÇÕÒ·¿","Q·¿","Q·¿(ÐÂ)","ÖÐÔ­µØ²ú(ÐÂ)","ÀÖÓмÒ","ÀÖÓмÒ(ÐÂ)","·¿Ì½","·¿¶à¶à","С×ʼÒ","ÎÒ°®ÎÒ¼Ò","ÂóÌï","·¿Í¾","C21","ÀÖ¾Ó","·¿²ú³¬ÊÐ","58ͬ³Ç","365ÌÔ·¿","¼ªÎÝ"] for(def source :sources){ - def count1 = findTotalCount(source, 202204); - def count2 = findTotalCount(source, 202205); + def count1 = findTotalCount(source, 202205); + def count2 = findTotalCount(source, 202206); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 3c2bcde38..89e0e9483 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -132,7 +132,7 @@ class ThirdDataToGatherBatch extends BaseBatch{ q.put("s_date", SHORT_DATE); def ret = [] try{ - def dbcursor = dataProcessColl.find(q).sort(new BasicDBObject("m_date", -1).append("type",1)) + def dbcursor = dataProcessColl.find(q).sort(new BasicDBObject("type",1).append("m_date", -1)) if(dbcursor){ for(def dbo : dbcursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index 1297ca3ea..8bca701d7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20220501", "yyyyMMdd") -def endDate = DateUtil.parseDate("20220601", "yyyyMMdd") +def startDate = DateUtil.parseDate("20220601", "yyyyMMdd") +def endDate = DateUtil.parseDate("20220701", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/batch/ApiProductFieldStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/batch/ApiProductFieldStatBatch.groovy index 6f120465c..026ec703f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/batch/ApiProductFieldStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/batch/ApiProductFieldStatBatch.groovy @@ -62,7 +62,7 @@ class ApiProductFieldStatBatch extends BaseBatch { private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000) def emails = [ - "yinjihuan@fangjia.com","panyingjiao@fangjia.com","liyan@fangjia.com", "jiangxiaochun@fangjia.com" + "jiangxiaochun@fangjia.com" ] static def allDis = [] static DecimalFormat df = new DecimalFormat("0.00%")