This commit is contained in:
2023-08-17 14:48:36 +08:00
parent af35c80ff0
commit 64c1f4e143
18 changed files with 888 additions and 97 deletions
@@ -217,7 +217,7 @@ class BigdataDealBatch extends BaseBatch{
def price = ccbPriceMap.get(obj._id.toString())
if(price){
def mod = java.lang.Math.abs((trade.price-price)*100/price)
if(mod<=10){
if(mod<=20){
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << trade.city
@@ -3941,5 +3941,125 @@ class CommonDatautils {
return files;
}
static def findLogs(userId, requestUri, startDate, endDate, code){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", userId)).must(QueryBuilders.termQuery("request_uri", requestUri))
if(startDate){
def startTime = DateUtils.parseFromTightYYYYMMDD(startDate)
def endTime = DateUtils.parseFromTightYYYYMMDD(endDate)
def date_range_cdate = QueryBuilders.rangeQuery("c_date").from(startTime.getTime()).to(endTime.getTime()).includeLower(true).includeUpper(false);
queryBuilder.must(date_range_cdate);
}
if(code){
queryBuilder.must(QueryBuilders.termQuery("code", code))
}
def rets = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
return rets.size()
}
static def findLogsBySql(userId, requestUri, startDate, endDate, code){
String sql = "select count(*) as total from open_uri_log where user_id=${userId} and request_uri='${requestUri}' "
if(startDate){
sql = sql + "and c_date >= '${startDate}' and c_date < '${endDate}'"
}
if(code){
sql = sql + " and code=${code}"
}
def total = null
ElasticUtils.executeSqlQuery(sql,{data->
total = data.total
})
return total;
}
static def findAmapDetailBySql(city){
String sql = "select * from amap_poi_detail where cityname='${city}' limit 10"
def rets = []
ElasticUtils.executeSqlQuery(sql,{data->
// if(data.name.contains('华山小区G栋')){
def point = data.location
// def tag = null
// if(data.get("business.keytag")){
// tag = data.get("business.keytag")
// }
def business_area = null
if(data.get("business.business_area")){
business_area = data.get("business.business_area")
}
def location = [:]
location.lon = point.x
location.lat = point.y
rets << ["province":data.pname,"city":data.cityname, "region":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area]
// }
})
return rets;
}
//用sql有些字段取不出来
static def findDetailLogsBySql(userId, requestUri){
String sql = "select * from open_uri_log where user_id=${userId} and request_uri='${requestUri}' and code!=200"
sql = sql + " order by c_date desc"
def total = null
def rets = []
ElasticUtils.executeSqlQuery(sql,{data->
def ret = []
println data.toString()
ret << data.get("parameter_map.city")
// ret << data.get("parameter_map.district")
// ret << data.get("parameter_map.keyword")
// ret << data.get("parameter_map.name")
ret << data.get("parameter_map.address")
// ret << data.get("parameter_map.size")
ret << data.get("msg")
ret << data.get("code")
ret << data.get("c_date").toString()[0..10]
rets << ret
})
return rets;
}
static def findNewAmapDetail(city) {
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fjw_city", city))
def rets = ElasticUtils.executeSimpleQuery("192.168.0.12", "amap_poi", queryBuilder);
def results = []
if(rets) {
for (int k = 0; k < rets.size(); k++) {
def data = rets[k];
println data
}
}
return results;
}
static def findLogs2(userId, requestUri){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", userId)).must(QueryBuilders.termQuery("request_uri", requestUri))
queryBuilder.mustNot(QueryBuilders.termQuery("code", 200))
def rets = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
def results = []
if(rets) {
for (int k = 0; k < rets.size(); k++) {
def data = rets[k];
println data
def ret = []
println data.toString()
ret << data.parameter_map.city
ret << data.parameter_map.district
ret << data.parameter_map.name
ret << data.parameter_map.address
ret << data.msg
ret << data.code
ret << data.c_date.toString()[0..10]
results << ret
}
}
return results
}
}
@@ -47,19 +47,29 @@ class DistrictPriceExportBatch extends BaseBatch{
def ret = createNewDatas("福州")
saveToDB(ret)
def results = selectLastResultDatas(ret);
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".csv"
dataCodeFileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".sha512"
}
def sb = appendResult(headers, ret)
def sb = appendResult(headers, results)
DataHandleUtils.outPutFile(fileName, sb, charset);
FzFtpUtils.ftpUpload(fileName)
//生成加密数据文件并上传
FzFtpUtils.uploadSha512Data2(fileName, dataCodeFileName, charset);
}
def selectLastResultDatas(def rets){
def rows = []
for(def r : rets){
rows << [r[0],r[1], r[2], r[3], r[4],r[5], r[6]]
}
return rows;
}
def appendResult(def headers, def ret){
@@ -147,6 +157,29 @@ class DistrictPriceExportBatch extends BaseBatch{
m << 0
}
m << curDate.toInteger();
if(curPrice && dbo){
if(dbo.value_type == 1){
m << "指导价"
}else if(dbo.value_type == 2){
m << "自身样本"
}else if(dbo.value_type == 3){
m << "第三方"
}else if(dbo.value_type == 4){
m << "交易数据"
}else if(dbo.value_type == 5){
m << "周边楼盘"
}else if(dbo.value_type == 6){
m << "推导"
}else if(dbo.value_type == 9){
m << "神经网络"
}else if(dbo.value_type == 10){
m << "无方式"
}else{
m << ""
}
}else{
m << "无"
}
result << m
}
return result
@@ -300,6 +333,7 @@ class DistrictPriceExportBatch extends BaseBatch{
//按comm_id修改,小区名重新赋值,防止区域和小区名修改
udata.put("region", r[3])
udata.put("name", r[4])
udata.put("value_type", r[7])
FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata))
}else{
def o = new BasicDBObject()
@@ -312,6 +346,7 @@ class DistrictPriceExportBatch extends BaseBatch{
o.put("avg_price", r[5].toInteger())
o.put("month", r[6].toInteger())
o.put("c_date", new Date())
o.put("value_type", r[7])
FZ_DISTRICT_STAT_COLL.save(o)
}
}
@@ -199,6 +199,8 @@ class DistrictSellStatBatch extends BaseBatch {
data.put("mom", mom)
data.put("yoy", yoy)
data.put("avg_price", weekendPriceObj.avg_price)
data.put("value_type", weekendPriceObj.value_type)
data.put("credit", weekendPriceObj.credit)
DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false)
println "[update]"+totalCount.get()+" "+ dist.city+" "+dist.name
}
@@ -210,6 +212,7 @@ class DistrictSellStatBatch extends BaseBatch {
data.put("estate_type", dist.estate_type2)
data.put("date", curMonth)
data.put("avg_price", curPrice)
data.put("value_type", weekendPriceObj.value_type)
data.put("credit", weekendPriceObj.credit)
data.put("m_date", new Date())
data.put("commid", dist._id.toString())
@@ -51,59 +51,61 @@ class FzDataResultStat {
})
}
def priceMarkupRun(){
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")+"15");
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
// List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"])
def idMaps = [:]
for(def d : dists){
idMaps.put(d._id.toString(), d.status==0?"有效":"无效")
}
def list1 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")))
def distkeys = []
def map1 = [:]
for(def s : list1){
def row = s.split("@\\|@")
def key = row[2]+"##"+row[3]+"##"+row[4]
for(def obj : list1){
def key = obj.city + "##" + obj.region + "##" + obj.name
if(!distkeys.contains(key)){
distkeys << key
}else{
println "重复:"+key
}
map1.put(key, row)
map1.put(key, obj)
}
List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
def list2 = findFzDistrictStat(lastMonth)
def map2 = [:]
for(def s : list2) {
def row = s.split("@\\|@")
def key = row[2] + "##" + row[3] + "##" + row[4]
map2.put(key, row)
for(def obj : list2) {
def key = obj.city + "##" + obj.region + "##" + obj.name
map2.put(key, obj)
}
def rows = []
rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"]
map1.each {k,list->
def row = []
def index = 0
map1.each {k,obj1->
def keys = k.split("##")
def lastObj = map2.get(k)
def lastPrice = null
if(lastObj){
lastPrice = lastObj[5]
}
def lastPrice = map2.get(k)
def markup = null
try{
if(list && lastPrice && lastPrice.toInteger()>0){
markup = (list[5].toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger()
if(obj1 && lastPrice && lastPrice.avg_price.toInteger()>0){
markup = (obj1.avg_price.toInteger() - lastPrice.avg_price.toInteger())*100.0/lastPrice.avg_price.toInteger()
}
}catch(Exception e){
e.printStackTrace()
}
def dist = CommonDatautils.findDistrictById(list[1])
def flag = "有效"
if(dist && dist.status!=0){
flag = "失效"
def flag = "无效"
if(obj1){
flag = idMaps.get(obj1.comm_id)
}
rows << [keys[0], keys[1], keys[2], list[5].toString(), lastPrice.toString(), markup, flag]
def row = [obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag]
rows << row
index++
println row
}
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
@@ -255,4 +257,10 @@ class FzDataResultStat {
}
return files;
}
def findFzDistrictStat(lastMonth){
def q = new BasicDBObject()
q.put("month", lastMonth)
return FZ_DISTRICT_STAT_COLL.find(q).toList()
}
}
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202306.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202307.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -14,6 +14,8 @@ import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
class CityStatExport {
@@ -26,10 +28,21 @@ class CityStatExport {
static main(args) {
def task = new CityStatExport();
// task.run()
task.run()
// task.run1()
task.regionStat()
// task.regionStat()
// task.stat()
// task.houseStat()
}
def houseStat(){
def fileName = "D:\\数据\\贝壳\\贝壳城市小区二手房统计V3-已重新补充.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
// def houseCount = findHouseCount(d['F'], 20230201)
// d['H'] = houseCount
d['J'] = findHouse(d['F'])
})
}
def stat(){
@@ -47,7 +60,7 @@ class CityStatExport {
// def citys = CommonDatautils.findAllCity();
def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
def citys = [ "广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"]
def ret = []
ret << ["城市","小区总量"]
// ret << ["城市","楼栋总量", "房号总量"]
@@ -78,7 +91,7 @@ class CityStatExport {
ret << m
println m
}
ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"数据")
ExcelUtils.write("D:\\数据\\广东省城市统计(楼盘量).xlsx",ret,"数据")
}
@@ -124,7 +137,7 @@ class CityStatExport {
def findHouseCount(def city, def startDate){
def total = 0
try{
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} "
String sql = "select count(*) as c from seawater where cat='sell' and city='${city}' and p_date >= ${startDate} "
ElasticUtils.executeSqlQuery(sql,{ data->
total = data.c
@@ -136,6 +149,18 @@ class CityStatExport {
return total
}
def findHouse(city) {
int startDate = 20230201
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell"))
.must(date_range)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
return houses.size()
}
def findLoudongCount(String city) {
return LOUDONG.count(new BasicDBObject('city', city))
}
@@ -90,4 +90,53 @@ class FileUtils {
}
return ret;
}
static def batchCreateFileBySplitChar(ret, fileName, headers,charset, isAppend, splitChar){
File file = new File(fileName);
// if (file.exists()) {
// file.delete();
// }
if (!file.getParentFile().exists()) {
boolean result = file.getParentFile().mkdirs();
if (!result) {
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
}
}
OutputStream out = new FileOutputStream(fileName, isAppend);
def sb = new StringBuffer();
if(headers){
sb.append(headers.join(splitChar))
sb.append("\n");
}
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
def index = 0
for(def row : ret){
if(row){
index++
sb.append(StringUtils.join(row.toArray(), splitChar))
sb.append("\n");
if(index%10000==0){
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
}
}
}
if(sb.length()>0){
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
}
out.close();
}
}
@@ -172,4 +172,32 @@ class FangjiaClientUtils {
}
}
public static def addrEvaluation(city,region,address, size, floor, maxFloor, room, hall){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://localhost:8080/fangjia-open/property/addressEvaluate?city="+URLEncoder.encode(city,"UTF-8")+"&district="+URLEncoder.encode(region,"UTF-8")+"&address="+URLEncoder.encode(address,"UTF-8")+"&size=${size}&sampleCode=1"
if(floor){
url = url + "&floor="+floor.toString().toDouble().toInteger()
}
if(maxFloor){
url = url + "&maxFloor="+maxFloor.toString().toDouble().toInteger()
}
if(room){
url = url + "&room="+room.toString().toDouble().toInteger()
}
if(hall){
url = url + "&hall="+hall.toString().toDouble().toInteger()
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.district
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
@@ -69,14 +70,15 @@ class OfficeFieldCleanBatch extends BaseBatch{
// strandField(offices)
// }
// addBlock()
// addFieldValue()
// updateErrorField()
export()
test()
// export()
// test()
// addOffice()
fileAddBlock()
}
def export(){
@@ -706,7 +708,34 @@ class OfficeFieldCleanBatch extends BaseBatch{
})
}
def fileAddBlock(){
def rets = findAllBlockRange()
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\上海商铺房源.xlsx",{d->
def location = d['T']
if(location && !d['F']){
def arrs = location.split(",")
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), 3)
if(baidu){
def lng2 = baidu.x
def lat2 = baidu.y
def lastObj = null
for (def r : rets) {
def flag = InsidePolygon.isInArea(lng2, lat2, r.points)
if (flag && r.name) {
lastObj = r;
break;
}
}
if (lastObj) {
d['F'] = lastObj.name
}
}
}
})
}
}
@@ -0,0 +1,72 @@
package com.fangjia.script.gj.es
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
def datas = findHouseCount("北京", 20230201, 20230430)
ExcelUtils.write("D:\\数据\\天弘基金.xlsx", datas, "样例")
def findHouseCount(def city, def startDate, def endDate){
def rets = []
try{
// String sql = "select * from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and root='ke.com' order by p_date desc"
def startMdate = DateUtils.parseFromTightByPattern(startDate.toString(),"yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
rets << ["城市","区域","板块","名称","面积","单价","总价","室","厅","卫","装修","朝向","发布时间","物业类型","楼层状况","楼高","建筑结构","建筑年代","标题","URL"]
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
// QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell")).must(QueryBuilders.termQuery("root", "ke.com"))
.must(date_range).must(area_range)//.must(date_range_mdate)
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(6);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
if(houses) {
for (int k = 0; k < houses.size(); k++) {
println k
def data = houses[k];
def r = []
r << data.city
r << data.region
r << data.block
r << data.district_name
r << data.area
r << data.avg_price
r << data.total
r << data.room
r << data.hall
r << data.toilet
r << data.fitment
r << data.direction
r << data.p_date
r << data.estate_type
r << data.floor_info
r << data.height
r << data.build_structure
r << data.build_date
r << data.name
r << data.url
rets << r
if(rets.size()>500){
break;
}
}
}
}catch(Exception e){
e.printStackTrace()
}
return rets
}
@@ -34,7 +34,7 @@ public class BeiKeCrawlByChrome {
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
WebDriver driver = null;
//广州,佛山 重庆
static main(args) {
@@ -50,7 +50,7 @@ public class BeiKeCrawlByChrome {
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
driver = new ChromeDriver()
def citys = getAllCityUrls(driver)
driver = getLogin(driver, "https://sh.ke.com/")
@@ -93,19 +93,28 @@ public class BeiKeCrawlByChrome {
}
def checkDocument(driver, url){
driver.get(url)
def s = driver.getPageSource()
def flag = true
while(flag){
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
try{
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
driver = getLogin(driver, url)
driver.get(url)
Thread.sleep(5000)
s = driver.getPageSource()
}else{
flag = false
}
}catch(Exception e){
driver.close()
driver = new ChromeDriver()
Thread.sleep(15000)
driver = getLogin(driver, url)
driver.get(url)
Thread.sleep(5000)
s = driver.getPageSource()
}else{
flag = false
println "异常Url:"+url
}
}
return Jsoup.parse(s);
return driver;
}
def crawlData(driver, citys){
@@ -130,7 +139,8 @@ public class BeiKeCrawlByChrome {
def newUrl = url+"/chengjiao/"
driver.get(newUrl)
Thread.sleep(10000)
def doc = checkDocument(driver, newUrl);
driver = checkDocument(driver, newUrl);
def doc = Jsoup.parse(driver.getPageSource())
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
if(regions){
for(def r : regions){
@@ -142,8 +152,9 @@ public class BeiKeCrawlByChrome {
regionText = "昆山"
}
def detailUrls = getPageDeail(driver, newCity, href, regionText)
println url +" 详情链接:"+detailUrls.size()
def map = getPageDeail(driver, newCity, href, regionText)
driver = map.driver
println url +" 详情链接:"+map.detailUrls.size()
}
}
FileUtils.batchCreateFile([[city]], "d:\\数据\\成交抓取\\贝壳已抓城市.txt", [], "utf-8", true);
@@ -191,9 +202,9 @@ public class BeiKeCrawlByChrome {
def detailUrls = []
while(!stop){
def url = cityUrl + "pg${page}/"
driver.get(url)
// def doc = Jsoup.parse(driver.getPageSource());
def doc = checkDocument(driver, url);
driver = checkDocument(driver, url);
def doc = Jsoup.parse(driver.getPageSource())
def elements = JQueryUtils.select(".listContent li", doc)
if(elements){
for(def e : elements){
@@ -203,10 +214,11 @@ public class BeiKeCrawlByChrome {
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
println "开始查库 "+new Date().toString()
def dbo = findTrade(durl , "贝壳找房")
println "结束查库"+new Date().toString()
if (!dbo) {
parse(city, e, region, driver, null)
driver = parse(city, e, region, driver, null)
}
// else if(dbo && !dbo.lift){
// parse(city, e, region, driver, dbo._id)
@@ -225,7 +237,10 @@ public class BeiKeCrawlByChrome {
stop = true;
}
}
return detailUrls
def m = [:]
m.driver = driver
m.detailUrls = detailUrls
return m
}
def parse(city,e, region, driver, id){
@@ -233,8 +248,10 @@ public class BeiKeCrawlByChrome {
def detailUrl = e.select(".title a").attr("href")
driver.get(detailUrl)
def detailDoc = checkDocument(driver, detailUrl);
driver = checkDocument(driver, detailUrl);
def detailDoc = Jsoup.parse(driver.getPageSource())
println "打开获取页面 "+new Date().toString()
def title = e.select(".title a").text()
if (title) {
def detail = new BasicDBObject()
@@ -335,18 +352,21 @@ public class BeiKeCrawlByChrome {
detail.create_date = new Date()
detail.m_date = new Date()
println "解析页面完成 "+new Date().toString()
println detail
if(id){
DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail))
}else{
DEAL_PRICE_COLL.save(detail)
}
println "保存修改完成 "+new Date().toString()
}
}
}catch(Exception ex){
ex.printStackTrace()
println e.select(".title a").attr("href")
}
return driver
}
@@ -469,16 +489,16 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = "https:"+e.attr("href")
// if (citys.contains(e.text())){
if (citys.contains(e.text())){
list << [e.text(), href]
// }
}
}
}
return list
}
def getLogin(driver, url){
// WebDriver driver = new ChromeDriver()
def root = url.toString().substring(url.toString().indexOf("//")+2, url.toString().indexOf(".com")+4)
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252F"+root+"%252Fchengjiao%252F");
WebDriverWait wait = new WebDriverWait(driver,20)
@@ -494,17 +514,25 @@ public class BeiKeCrawlByChrome {
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
elemUsername.sendKeys("17148854005");
elemPassword.sendKeys("12qwaszx");
// elemUsername.sendKeys("15021384995");
// elemPassword.sendKeys("12qwaszx");
elemUsername.sendKeys("16283166371");
elemPassword.sendKeys("Goojia123456");
// elemUsername.sendKeys("16283166371");
// elemPassword.sendKeys("Goojia123456");
btn.submit(); // 点击行为;
Thread.sleep(10000)
}catch(Exception e){
driver.close()
driver = new ChromeDriver()
Thread.sleep(15000)
driver = getLogin(driver, url)
println "异常Url:"+url
}
return driver
}
@@ -34,6 +34,7 @@ public class LianjiaCrawlByChrome {
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
WebDriver driver = null;
static main(args) {
def task = new LianjiaCrawlByChrome();
@@ -122,7 +123,7 @@ public class LianjiaCrawlByChrome {
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
driver = new ChromeDriver()
def cityMap = getAllCityUrls(driver)
// cityMap = cityMap.reverse()
driver = getLogin(driver, "https://sh.lianjia.com/")
@@ -206,17 +207,11 @@ public class LianjiaCrawlByChrome {
}
def cityUrl = ret[1]
def newUrl = cityUrl+"chengjiao/"
try{
driver.get(newUrl)
}catch(Exception e){
println newUrl +" 链接有问题!"
continue
}
// driver.get(newUrl)
//def detailUrls = getPageUrls(driver, cityUrl)
Thread.sleep(10000)
def doc = checkDocument(driver, newUrl);
// Thread.sleep(10000)
driver = checkDocument(driver, newUrl);
def doc = Jsoup.parse(driver.getPageSource())
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
if(regions){
for(def r : regions){
@@ -226,8 +221,9 @@ public class LianjiaCrawlByChrome {
regionText = "昆山"
}
println "开始抓取【${city}】【${regionText}】【${href}】"
def detailUrls = getPageDeail(driver, newCity, href, regionText)
println cityUrl +" 详情链接:"+detailUrls.size()
def map = getPageDeail(driver, newCity, href, regionText)
driver = map.driver
println newUrl +" 详情链接:"+map.detailUrls.size()
}
}
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
@@ -267,30 +263,41 @@ public class LianjiaCrawlByChrome {
}
def checkDocument(driver, url){
driver.get(url)
def s = driver.getPageSource()
def flag = true
while(flag){
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
driver = getLogin(driver, url)
driver.get(url)
Thread.sleep(5000)
s = driver.getPageSource()
}else{
flag = false
try{
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
driver = getLogin(driver, url)
driver.get(url)
Thread.sleep(5000)
s = driver.getPageSource()
}else{
flag = false
}
}catch(Exception e){
driver.close()
driver = new ChromeDriver()
Thread.sleep(15000)
driver = getLogin(driver, url)
println "异常Url:"+url
}
}
return Jsoup.parse(s);
return driver;
}
def getPageDeail(driver, city, cityUrl, regionText){
def stop = false
def page = 1
def detailUrls = []
while(!stop){
def url = cityUrl + "pg${page}/"
driver.get(url)
// driver.get(url)
//def detailUrls = getPageUrls(driver, cityUrl)
def doc = checkDocument(driver, url);
driver = checkDocument(driver, url);
def doc = Jsoup.parse(driver.getPageSource())
def elements = JQueryUtils.select(".listContent li", doc)
if(elements){
for(def e : elements){
@@ -303,7 +310,7 @@ public class LianjiaCrawlByChrome {
detailUrls << durl
def dbo = findTrade(durl , "链家在线")
if (!dbo) {
parse(city, e, regionText, driver, null)
driver = parse(city, e, regionText, driver, null)
}
// else if(dbo && !dbo.lift){
// parse(city, e, regionText, driver, dbo._id)
@@ -326,7 +333,10 @@ public class LianjiaCrawlByChrome {
stop = true;
}
}
return detailUrls
def m = [:]
m.driver = driver
m.detailUrls = detailUrls
return m
}
def parse(city,e, regionText, driver, id){
@@ -334,8 +344,11 @@ public class LianjiaCrawlByChrome {
def price = e.select(".totalPrice .number").text()
if(price){
def detailUrl = e.select(".title a").attr("href")
driver.get(detailUrl)
def detailDoc = checkDocument(driver, detailUrl);
// driver.get(detailUrl)
driver = checkDocument(driver, detailUrl);
def detailDoc = Jsoup.parse(driver.getPageSource())
def title = e.select(".title a").text()
if (title) {
def detail = new BasicDBObject()
@@ -450,6 +463,7 @@ public class LianjiaCrawlByChrome {
ex.printStackTrace()
println e.select(".title a").attr("href")
}
return driver;
}
@@ -571,7 +585,7 @@ public class LianjiaCrawlByChrome {
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if (!href.contains("fang.lianjia")){
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
cityList << [e.text(), href]
}
}
@@ -593,16 +607,23 @@ public class LianjiaCrawlByChrome {
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
// elemUsername.sendKeys("15021384995");
// elemPassword.sendKeys("12qwaszx");
elemUsername.sendKeys("17148854005");
elemPassword.sendKeys("12qwaszx");
elemUsername.sendKeys("16283166371");
elemPassword.sendKeys("Goojia123456");
// elemUsername.sendKeys("17740818975");
// elemPassword.sendKeys("13qeadzc");
// elemUsername.sendKeys("16283166371");
// elemPassword.sendKeys("Goojia123456");
btn.submit(); // 点击行为;
Thread.sleep(15000)
Thread.sleep(20000)
}catch(Exception e){
Thread.sleep(10000)
driver.close()
driver = new ChromeDriver()
Thread.sleep(15000)
driver = getLogin(driver, url)
println "异常Url:"+url
}
return driver
}
@@ -0,0 +1,211 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 厦门银行
* 福建省,重庆市poi数据处理
*/
@BatchOwner(Watcher.GaoJuan)
class GaoDePoiDataProcess extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
runBatch(new GaoDePoiDataProcess(), args)
}
@Override
void process(Object args) {
//三明市,宁德市,泉州市,厦门市,福州市,漳州市,南平市,龙岩市,莆田市,重庆市
def citys = ["三明市","漳州市","厦门市","福州市"]
for(def city : citys){
def results = CommonDatautils.findAmapDetailBySql(city)
println results.size()
def rows = running(results)
save(rows)
println city + " 完成"+new Date().toString()
}
}
def running(results){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size())
for(def i=0;i<results.size();i++){
def map = results[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
rets.addAll(createData(map))
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def createData(map){
def fjw_province = null
if(map.city.contains("重庆")){
fjw_province = "重庆"
}else{
fjw_province = "福建"
}
def fjw_city = RegionData.formatCity(map.city)
def fjw_region = RegionData.formatRegion(fjw_city, map.region)
map.id = new ObjectId().toString()
map.fjw_province = fjw_province
map.fjw_city = fjw_city
if(map.cat){
def arrs = map.cat.split(";")
map.type = arrs[0]
if(arrs.size()>=2){
map.secondary_type = arrs[1]
}
if(arrs.size()>=3){
map.small_type = arrs[2]
}
}
//高德转百度
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
if(baiduMap){
def location = [:]
location.lon = baiduMap.lng
location.lat = baiduMap.lat
map.bd_location = location
}
//高德转WGS84
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
if(gpsLoactionMap){
def gpslocation = [:]
gpslocation.lon = gpsLoactionMap.lng
gpslocation.lat = gpsLoactionMap.lat
map.gps_location = gpslocation
}
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
if(gaodeMap && gaodeMap.township){
map.township = gaodeMap.township
}
if(!fjw_region){
if(gaodeMap && gaodeMap.district){
fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district)
}
}
if(!fjw_region){
println fjw_city+" "+map.region+" 无法格式化 "+map.location + " "+gaodeMap
}
map.fjw_region = fjw_region
return map
}
def save(rows){
BulkRequest request = new BulkRequest();
int batchSize = 1000;
int processedCount = 0;
def client = ElasticUtils.getClient("192.168.0.12")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
request.add(new IndexRequest("amap_poi", "_doc", map.id).source(map, XContentType.JSON));
if(i%1000==0){
print "."
}
// 批量提交数据
if (request.numberOfActions() >= batchSize) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
request = new BulkRequest();
}
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
println "结束时间 "+new Date()
}
def getTownshipInfo(def glocation){
def township = null
def district = null
def jsonObject = null
def stop = false
def count = 0
while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
}
}
}
if(district || count>10){
stop = true
}else{
count ++
Thread.sleep(3000)
print "."
}
}
def map = [:]
map.district = district
map.township = township
return map;
}
}
@@ -0,0 +1,35 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
@BatchOwner(Watcher.GaoJuan)
class PoiAndSeaweedExportCSV extends BaseBatch{
static main(args) {
runBatch(new PoiAndSeaweedExportCSV(), args);
}
@Override
public void process(Object args) {
def citys = ["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"]
poiExport(citys)
}
def poiExport(citys){
def fileName = "/home/goojia/xiamen/BIGDATA_POI.csv"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\厦门\\BIGDATA_POI.csv"
}
for(def c : citys){
def rets = CommonDatautils.findNewAmapDetail(c)
FileUtils.batchCreateFileBySplitChar(rets, fileName, null, "utf-8", true, "@|@");
}
}
}
@@ -0,0 +1,44 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.Constants
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.client.MongoCollection
import net.sf.json.JSONObject
import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20230701", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230801", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
def hashSet = []
def data = [["编号", "公司名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "返回参数"]]
while (startDate.before(endDate)) {
calendar.add(Calendar.DATE, 1)
def nextDate = calendar.getTime()
count++
///property/addressEvaluate /address/financialRating /v2/property/detail
OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("request_uri", "/v2/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
// if (!hashSet.contains(result)) {
def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
data << item
// hashSet << result
// } else {
// println("已存在")
// }
}
if (count % 7 == 0) {
hashSet.clear()
}
startDate = nextDate
}
ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信.xlsx", data, "楼盘评情")
@@ -12,7 +12,7 @@ import org.jsoup.Jsoup
import java.util.zip.GZIPInputStream
class GaoDeApiUtils {
def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390"
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca"
/**
* 经纬度转地址<br>
@@ -0,0 +1,83 @@
package com.fangjia.script.zsbank
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
class ZheShangBank {
static main(args) {
def task = new ZheShangBank()
// task.run()
// task.detail()
task.sample()
}
def sample(){
def runedCitys = FileUtils.readTxt("D:\\数据\\浙商\\result.txt")
def rets = []
rets << ["时间","Msg", "城市", "区域", "名称","月份"]
def map = [:]
for(def line : runedCitys){
def ret = []
def date = line.substring(line.indexOf("log.")+4, line.indexOf(":"))
def msg = line.substring(line.lastIndexOf("error :")+7, line.size())
def arrs = msg.split(" ")
ret << date
for(def i=0;i<arrs.size();i++){
def s = arrs[i]
if(i == 0){
ret << s
}else{
ret << s.substring(s.indexOf(":")+1, s.size())
}
}
def month = date.substring(0, 7)
ret << month
rets << ret
println ret.toString()
def count = map.get(ret[1])
if(!count){
map.put(ret[1], 1)
}else{
map.put(ret[1], count+1)
}
// FileUtils.batchCreateFile([ret], "D:\\数据\\浙商\\浙商失败案例.txt",null, "utf-8", true);
}
map.each{m,r->
println m+" "+r
}
// ExcelUtils.write("D:\\数据\\浙商\\浙商失败案例.xlsx", rets, "风险缓释")
}
def run(){
def start = "2021-03-01"
def rets = []
rets << ["月份","总调用量","成功量","失败量"]
while (true){
def end = DateUtils.addMonth(DateUtils.parseFromTightYYYYMMDD(start), 1)
// def total = CommonDatautils.findLogs("1007569", "/address/risk", start, end, null)
// def successCount = CommonDatautils.findLogs("1007569", "/address/risk", start, end, 200)
def total = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, null)
def successCount = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, 200)
def failCount = total - successCount
println start[0..6] +" "+ total +" "+successCount +" "+ failCount
rets << [start[0..6], total, successCount, failCount]
start = end;
if(end == "2023-09-01"){
break;
}
}
ExcelUtils.write("D:\\数据\\浙商\\浙商调用统计.xlsx", rets, "估值地址")
}
def detail(){
def datas = CommonDatautils.findLogs2("1007569", "/property/addressEvaluate")
ExcelUtils.write("D:\\数据\\浙商\\浙商失败案例.xlsx", datas, "估值地址")
}
}