Merge branch 'script_gj' into 'master'

Script gj

See merge request !342
This commit is contained in:
2022-07-06 10:38:21 +08:00
14 changed files with 373 additions and 28 deletions
@@ -23,14 +23,7 @@ class CityInfoEmailSender extends BaseBatch{
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files')
def emails = [
"wangsheng@fangjia.com"
,"xuyejun@fangjia.com"
,"tiki@fangjia.com"
,"zhangzx@fangjia.com"
,"panyingjiao@fangjia.com"
,"yangqixing@fangjia.com"
,"james.lin@fangjia.com"
,"jhu@fangjia.com"
"gaojuan@fangjia.com"
]
@@ -25,14 +25,14 @@ class AnjukeHouseCrawl {
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def href = "https://sh.esf.fang.com/loupan/1210043376.htm"
def href = "https://sh.58.com/hezu/50696364798384x.shtml"
def doc = getDocument(href, driver)
// def doc = ClaABuYun.getResponse(href)
// def newUrl = href.substring(0, hrefs.indexOf("?"))
sendMq(href, doc, "district_detail_fang")
sendMq(href, doc, "rent_58")
// for(def i=1;i<50;i++){
// def url = "https://fz.anjuke.com/sale/guloue/p${i}/"
// for(def i=25;i<50;i++){
// def url = "https://fz.anjuke.com/sale/jinana/p${i}/"
// def doc = getDocument(url, driver)
// def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc)
// println "页数:"+i+" 记录数:"+eles.size()
@@ -203,7 +203,18 @@ class AnjukeHouseCrawl {
}
if(!els){
els = JQueryUtils.select(".village_info", doc)
els = JQueryUtils.select(".detail-content", doc)
}
if(!els){
els = JQueryUtils.select(".content", doc)
}
if(!els){
els = JQueryUtils.select(".wrapper", doc)
}
if(!els){
els = JQueryUtils.select(".main-wrap", doc)
}
if(els && els.size()>0){
stop = true;
@@ -0,0 +1,71 @@
package com.fangjia.script.gj.crawl
import com.alibaba.fastjson.JSON
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.util.MessageUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import java.text.SimpleDateFormat
DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "ali_fapai_detail", "192.168.0.235", 27018)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
//Date date=new Date(1550455200000);
//SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
//String str = format.format(date);
//System.out.println(str);
def pmquery = new BasicDBObject()
//pmquery.put("city","ÖØÇì")
//pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
//pmquery.put("status","Òѳɽ»")
//pmquery.put("estateType", null)
pmquery.put("url","https://susong-item.taobao.com/auction/648798773106.htm")
//def cur = paimai43_COLL.find(pmquery)
//def urls = []
//for(def o : cur){
// urls << o.url
//}
def index = 0
//for(def url : urls){
def q = new BasicDBObject()
// q.put("url", url)
def dbcursor = collection.find(q)
for(def d : dbcursor){
index++
println index
JSON json = new JSONObject()
json.put("project", "paimai_taobao")
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.content)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("trades", bid_content)
result.put("condition", d.condition)
result.put("sub_count", d.sub_count)
def data_json = net.sf.json.JSONObject.fromObject(d.json)
result.put("data_json", data_json)
result.put("desc_content", d.desc_content)
result.put("item_notice_content", d.item_notice_content)
result.put("notice_detail_content", d.notice_detail_content)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse", packedMsg)
}
//}
@@ -44,9 +44,7 @@ class PdsUseStatisticBatch extends BaseBatch{
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000)
def static emails = [
"hubingliang@fangjia.com",
"james.lin@fangjia.com",
"panyingjiao@fangjia.com",
"duweiliang@fangjia.com",
"yaojingmeng@fangjia.com",
"xuqi@fangjia.com",
@@ -34,8 +34,7 @@ class DaiQianGuZhiBatch extends BaseBatch{
final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files')
def static emails = [
"hubingliang@fangjia.com",
"james.lin@fangjia.com",
"panyingjiao@fangjia.com"
"james.lin@fangjia.com"
]
static main(args) {
@@ -34,8 +34,7 @@ class KeHuFenLeiBatch extends BaseBatch{
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000)
def static emails = [
"hubingliang@fangjia.com",
"james.lin@fangjia.com",
"panyingjiao@fangjia.com"
"james.lin@fangjia.com"
]
static main(args) {
@@ -33,9 +33,7 @@ class KeHuPingJiBatch extends BaseBatch{
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
final static DBCollection fsDB = new Mongo("fs.mongo.db.fangjia.com", 27000).getDB('fangjia_fs').getCollection('files')
def static emails = [
"hubingliang@fangjia.com",
"james.lin@fangjia.com",
"panyingjiao@fangjia.com"
]
static main(args) {
@@ -34,7 +34,7 @@ sqlInstance.eachRow(sql, { r ->
if (year_month == '202205') {
fiveMap.put(key, r['avg_price'] + "|" + r['calc_type'])
} else if ((year_month == '202206')) {
fiveMap.put(key, r['avg_price'] + "|" + r['calc_type'])
sixMap.put(key, r['avg_price'] + "|" + r['calc_type'])
}
})
for (String key : sixMap.keySet()) {
@@ -47,4 +47,4 @@ for (String key : sixMap.keySet()) {
}
}
}
ExcelUtils.write("F:\\数据\\价格\\东莞上海价格偏差大.xlsx", result, "统计")
ExcelUtils.write("D:\\数据\\价格\\东莞上海价格偏差大.xlsx", result, "统计")
@@ -0,0 +1,94 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.statistics.poi.AmapPoiDetail
import com.fangjia.script.temp.shenjian.poi.entity.Community
import com.fangjia.script.temp.shenjian.util.CommonDtoUtils
import com.fangjia.script.temp.shenjian.util.EsUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import org.apache.commons.collections.CollectionUtils
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.stream.Collectors
/**
* 从ES获取POI
*/
class GetPoiFromCommunity {
static final ExecutorService executorService = Executors.newFixedThreadPool(10)
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail"
// 住宅类
static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304";
static void main(String[] args) {
AtomicLong atomicLong = new AtomicLong(0)
Map<String, String> map = new ConcurrentHashMap<>()
def rets = this.findCommunities()
CountDownLatch countDownLatch = new CountDownLatch(rets.size())
for (def retTemp : rets) {
final def ret = retTemp
executorService.submit(new Runnable() {
@Override
void run() {
final String id = ret._id // 索引ID,并非小区ID
String city = ret.city
String region = ret.region
String seaweedName = ret.name
String address = ret.address
List<AmapPoi> pois = new ArrayList<>();
List<AmapPoi> namePois = GaoDeUtils.searchPoi(city, seaweedName, region, typecodes);
List<AmapPoi> addressPois = GaoDeUtils.searchPoi(city, address, region, typecodes);
if (CollectionUtils.isNotEmpty(namePois)) {
pois.addAll(namePois)
}
if (CollectionUtils.isNotEmpty(addressPois)) {
pois.addAll(addressPois)
}
if (CollectionUtils.isNotEmpty(pois)) {
List<AmapPoiDetail> detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois);
String jsonStr = JSONObject.toJSONString(detailList)
HttpUtils.postForJson(URL, jsonStr)
}
synchronized (map) {
map.put(id, id)
countDownLatch.countDown()
if (map.size() % 100 == 0) {
atomicLong.addAndGet(map.size())
List<String> ids = map.values().stream().collect(Collectors.toList())
EsUtils.bulkUpdateFieldByIds("community", "crawl_flag", 0, ids)
map.clear()
println(atomicLong.intValue())
}
}
}
})
}
countDownLatch.await()
if (map.size() > 0) {
atomicLong.addAndGet(map.size())
List<String> ids = map.values().stream().collect(Collectors.toList())
EsUtils.bulkUpdateFieldByIds("community", "crawl_flag", 0, ids)
map.clear()
println(atomicLong.intValue())
}
System.exit(1)
}
static def findCommunities(){
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
.mustNot(QueryBuilders.termQuery("crawl_flag", 0))
List<Community> communities = EsUtils.scrollQuery("community", queryBuilder, Community.class)
return communities
}
}
@@ -0,0 +1,49 @@
package com.fangjia.script.temp.shenjian.poi.entity;
public class Community extends MyId {
private String id;
private String city;
private String region;
private String name;
private String address;
String getId() {
return id
}
void setId(String id) {
this.id = id
}
String getCity() {
return city
}
void setCity(String city) {
this.city = city
}
String getRegion() {
return region
}
void setRegion(String region) {
this.region = region
}
String getName() {
return name
}
void setName(String name) {
this.name = name
}
String getAddress() {
return address
}
void setAddress(String address) {
this.address = address
}
}
@@ -0,0 +1,17 @@
package com.fangjia.script.temp.shenjian.poi.entity
class MyId {
/**
* Ë÷Òýid
*/
String _id;
String get_id() {
return _id
}
void set_id(String _id) {
this._id = _id
}
}
@@ -0,0 +1,116 @@
package com.fangjia.script.temp.shenjian.util
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils
import com.fangjia.script.temp.shenjian.poi.entity.MyId
import org.elasticsearch.action.DocWriteRequest
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.search.*
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.search.Scroll
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.builder.SearchSourceBuilder
import org.slf4j.Logger
import org.slf4j.LoggerFactory
/**
* @author shenjian
* @since 2022/07/02
*/
class EsUtils {
private static RestHighLevelClient ES_CLIENT = RestHighLevelClientUtils.getClient()
private static final Logger logger = LoggerFactory.getLogger(EsUtils.class)
/**
* 滚动方式查询大批量es数据
*
* @param index 索引
* @param boolQuery 查询语句
* @param clazz 要返回的实体类型
* @param <T>
* @return
*/
public static <T extends MyId> List<T> scrollQuery(String index, QueryBuilder boolQuery, Class<T> clazz, def limitSize=300000) {
List<T> result = new ArrayList<>();
// 存活时间,当索引数据量特别大时,出现超时可能性大,此值适当调大
Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30L));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(boolQuery);
// TEST 用!!!!!!!!!!!!!!!!!!!!!!
//searchSourceBuilder.size(1);
searchSourceBuilder.size(10000);
SearchRequest searchRequest = new SearchRequest().indices(index).scroll(scroll).source(searchSourceBuilder);
SearchResponse searchResponse = null;
try {
searchResponse = ES_CLIENT.search(searchRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
logger.error("查询es数据异常:{}", e);
}
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits();
if (searchHits != null) {
for (SearchHit searchHit : searchHits) {
T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz);
t.set_id(searchHit.getId())
result.add(t);
}
}
// 遍历搜索命中的数据,直到没有数据
while (searchHits != null && searchHits.length > 0 && result.size() < limitSize) {
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
try {
searchResponse = ES_CLIENT.scroll(scrollRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
logger.error("查询es数据异常:{}", e);
}
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
if (searchHits != null && searchHits.length > 0) {
for (SearchHit searchHit : searchHits) {
T t = JSONObject.parseObject(searchHit.getSourceAsString(), clazz);
t.set_id(searchHit.getId())
result.add(t);
}
}
}
// clean scroll
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
ClearScrollResponse clearScrollResponse = null;
try {
clearScrollResponse = ES_CLIENT.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
} catch (IOException e) {}
boolean succeeded = clearScrollResponse.isSucceeded();
if (succeeded) {
return result;
}
return null;
}
/**
* 批量更新es某个字段值
* @param index 索引名
* @param field 字段名
* @param val 字段新值
* @param ids 要更新的文档id
*/
public static void bulkUpdateFieldByIds(String index, String field, Object val, List<String> ids) {
if (ids.size() > 1000) {
throw new RuntimeException("批量更新数量过多")
}
BulkRequest bulkRequest = new BulkRequest();
for (String id : ids) {
Map<String, Object> doc = new HashMap<>();
doc.put(field, val)
bulkRequest.add(new UpdateRequest(index, id).doc(doc));
}
ES_CLIENT.bulk(bulkRequest, RequestOptions.DEFAULT);
}
}
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20220501", "yyyyMMdd")
def endDate = DateUtil.parseDate("20220601", "yyyyMMdd")
def startDate = DateUtil.parseDate("20220601", "yyyyMMdd")
def endDate = DateUtil.parseDate("20220701", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -62,7 +62,7 @@ class ApiProductFieldStatBatch extends BaseBatch {
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000)
def emails = [
"yinjihuan@fangjia.com","panyingjiao@fangjia.com","liyan@fangjia.com", "jiangxiaochun@fangjia.com"
"jiangxiaochun@fangjia.com"
]
static def allDis = []
static DecimalFormat df = new DecimalFormat("0.00%")