update
This commit is contained in:
+1
-1
@@ -217,7 +217,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def price = ccbPriceMap.get(obj._id.toString())
|
||||
if(price){
|
||||
def mod = java.lang.Math.abs((trade.price-price)*100/price)
|
||||
if(mod<=10){
|
||||
if(mod<=20){
|
||||
ret << cityCodeDatas.get(trade.city)
|
||||
ret << trade._id.toString()
|
||||
ret << trade.city
|
||||
|
||||
+120
@@ -3941,5 +3941,125 @@ class CommonDatautils {
|
||||
return files;
|
||||
}
|
||||
|
||||
static def findLogs(userId, requestUri, startDate, endDate, code){
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", userId)).must(QueryBuilders.termQuery("request_uri", requestUri))
|
||||
if(startDate){
|
||||
def startTime = DateUtils.parseFromTightYYYYMMDD(startDate)
|
||||
def endTime = DateUtils.parseFromTightYYYYMMDD(endDate)
|
||||
def date_range_cdate = QueryBuilders.rangeQuery("c_date").from(startTime.getTime()).to(endTime.getTime()).includeLower(true).includeUpper(false);
|
||||
queryBuilder.must(date_range_cdate);
|
||||
}
|
||||
if(code){
|
||||
queryBuilder.must(QueryBuilders.termQuery("code", code))
|
||||
}
|
||||
def rets = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
|
||||
return rets.size()
|
||||
}
|
||||
|
||||
static def findLogsBySql(userId, requestUri, startDate, endDate, code){
|
||||
String sql = "select count(*) as total from open_uri_log where user_id=${userId} and request_uri='${requestUri}' "
|
||||
if(startDate){
|
||||
sql = sql + "and c_date >= '${startDate}' and c_date < '${endDate}'"
|
||||
}
|
||||
if(code){
|
||||
sql = sql + " and code=${code}"
|
||||
}
|
||||
def total = null
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
total = data.total
|
||||
})
|
||||
return total;
|
||||
}
|
||||
|
||||
static def findAmapDetailBySql(city){
|
||||
String sql = "select * from amap_poi_detail where cityname='${city}' limit 10"
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
// if(data.name.contains('华山小区G栋')){
|
||||
def point = data.location
|
||||
// def tag = null
|
||||
// if(data.get("business.keytag")){
|
||||
// tag = data.get("business.keytag")
|
||||
// }
|
||||
def business_area = null
|
||||
if(data.get("business.business_area")){
|
||||
business_area = data.get("business.business_area")
|
||||
}
|
||||
def location = [:]
|
||||
location.lon = point.x
|
||||
location.lat = point.y
|
||||
rets << ["province":data.pname,"city":data.cityname, "region":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area]
|
||||
// }
|
||||
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
//用sql有些字段取不出来
|
||||
static def findDetailLogsBySql(userId, requestUri){
|
||||
String sql = "select * from open_uri_log where user_id=${userId} and request_uri='${requestUri}' and code!=200"
|
||||
|
||||
sql = sql + " order by c_date desc"
|
||||
def total = null
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
def ret = []
|
||||
println data.toString()
|
||||
ret << data.get("parameter_map.city")
|
||||
// ret << data.get("parameter_map.district")
|
||||
// ret << data.get("parameter_map.keyword")
|
||||
// ret << data.get("parameter_map.name")
|
||||
ret << data.get("parameter_map.address")
|
||||
// ret << data.get("parameter_map.size")
|
||||
ret << data.get("msg")
|
||||
ret << data.get("code")
|
||||
ret << data.get("c_date").toString()[0..10]
|
||||
rets << ret
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
|
||||
static def findNewAmapDetail(city) {
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fjw_city", city))
|
||||
def rets = ElasticUtils.executeSimpleQuery("192.168.0.12", "amap_poi", queryBuilder);
|
||||
def results = []
|
||||
if(rets) {
|
||||
for (int k = 0; k < rets.size(); k++) {
|
||||
def data = rets[k];
|
||||
println data
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
|
||||
static def findLogs2(userId, requestUri){
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", userId)).must(QueryBuilders.termQuery("request_uri", requestUri))
|
||||
|
||||
queryBuilder.mustNot(QueryBuilders.termQuery("code", 200))
|
||||
def rets = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
|
||||
def results = []
|
||||
if(rets) {
|
||||
for (int k = 0; k < rets.size(); k++) {
|
||||
def data = rets[k];
|
||||
println data
|
||||
def ret = []
|
||||
println data.toString()
|
||||
ret << data.parameter_map.city
|
||||
ret << data.parameter_map.district
|
||||
ret << data.parameter_map.name
|
||||
ret << data.parameter_map.address
|
||||
ret << data.msg
|
||||
ret << data.code
|
||||
ret << data.c_date.toString()[0..10]
|
||||
results << ret
|
||||
}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+36
-1
@@ -47,19 +47,29 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
def ret = createNewDatas("福州")
|
||||
saveToDB(ret)
|
||||
|
||||
def results = selectLastResultDatas(ret);
|
||||
|
||||
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
|
||||
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".csv"
|
||||
dataCodeFileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".sha512"
|
||||
}
|
||||
def sb = appendResult(headers, ret)
|
||||
def sb = appendResult(headers, results)
|
||||
DataHandleUtils.outPutFile(fileName, sb, charset);
|
||||
FzFtpUtils.ftpUpload(fileName)
|
||||
//生成加密数据文件并上传
|
||||
FzFtpUtils.uploadSha512Data2(fileName, dataCodeFileName, charset);
|
||||
}
|
||||
|
||||
def selectLastResultDatas(def rets){
|
||||
def rows = []
|
||||
for(def r : rets){
|
||||
rows << [r[0],r[1], r[2], r[3], r[4],r[5], r[6]]
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def appendResult(def headers, def ret){
|
||||
@@ -147,6 +157,29 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
m << 0
|
||||
}
|
||||
m << curDate.toInteger();
|
||||
if(curPrice && dbo){
|
||||
if(dbo.value_type == 1){
|
||||
m << "指导价"
|
||||
}else if(dbo.value_type == 2){
|
||||
m << "自身样本"
|
||||
}else if(dbo.value_type == 3){
|
||||
m << "第三方"
|
||||
}else if(dbo.value_type == 4){
|
||||
m << "交易数据"
|
||||
}else if(dbo.value_type == 5){
|
||||
m << "周边楼盘"
|
||||
}else if(dbo.value_type == 6){
|
||||
m << "推导"
|
||||
}else if(dbo.value_type == 9){
|
||||
m << "神经网络"
|
||||
}else if(dbo.value_type == 10){
|
||||
m << "无方式"
|
||||
}else{
|
||||
m << ""
|
||||
}
|
||||
}else{
|
||||
m << "无"
|
||||
}
|
||||
result << m
|
||||
}
|
||||
return result
|
||||
@@ -300,6 +333,7 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
//按comm_id修改,小区名重新赋值,防止区域和小区名修改
|
||||
udata.put("region", r[3])
|
||||
udata.put("name", r[4])
|
||||
udata.put("value_type", r[7])
|
||||
FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
def o = new BasicDBObject()
|
||||
@@ -312,6 +346,7 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
o.put("avg_price", r[5].toInteger())
|
||||
o.put("month", r[6].toInteger())
|
||||
o.put("c_date", new Date())
|
||||
o.put("value_type", r[7])
|
||||
FZ_DISTRICT_STAT_COLL.save(o)
|
||||
}
|
||||
}
|
||||
|
||||
+3
@@ -199,6 +199,8 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
data.put("mom", mom)
|
||||
data.put("yoy", yoy)
|
||||
data.put("avg_price", weekendPriceObj.avg_price)
|
||||
data.put("value_type", weekendPriceObj.value_type)
|
||||
data.put("credit", weekendPriceObj.credit)
|
||||
DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false)
|
||||
println "[update]"+totalCount.get()+" "+ dist.city+" "+dist.name
|
||||
}
|
||||
@@ -210,6 +212,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
data.put("estate_type", dist.estate_type2)
|
||||
data.put("date", curMonth)
|
||||
data.put("avg_price", curPrice)
|
||||
data.put("value_type", weekendPriceObj.value_type)
|
||||
data.put("credit", weekendPriceObj.credit)
|
||||
data.put("m_date", new Date())
|
||||
data.put("commid", dist._id.toString())
|
||||
|
||||
+35
-27
@@ -51,59 +51,61 @@ class FzDataResultStat {
|
||||
})
|
||||
}
|
||||
def priceMarkupRun(){
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")+"15");
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
|
||||
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
|
||||
// List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
|
||||
def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"])
|
||||
def idMaps = [:]
|
||||
for(def d : dists){
|
||||
idMaps.put(d._id.toString(), d.status==0?"有效":"无效")
|
||||
}
|
||||
|
||||
def list1 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")))
|
||||
def distkeys = []
|
||||
def map1 = [:]
|
||||
for(def s : list1){
|
||||
def row = s.split("@\\|@")
|
||||
def key = row[2]+"##"+row[3]+"##"+row[4]
|
||||
for(def obj : list1){
|
||||
def key = obj.city + "##" + obj.region + "##" + obj.name
|
||||
if(!distkeys.contains(key)){
|
||||
distkeys << key
|
||||
}else{
|
||||
println "重复:"+key
|
||||
}
|
||||
map1.put(key, row)
|
||||
map1.put(key, obj)
|
||||
}
|
||||
|
||||
List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
|
||||
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
|
||||
def list2 = findFzDistrictStat(lastMonth)
|
||||
def map2 = [:]
|
||||
for(def s : list2) {
|
||||
def row = s.split("@\\|@")
|
||||
def key = row[2] + "##" + row[3] + "##" + row[4]
|
||||
map2.put(key, row)
|
||||
for(def obj : list2) {
|
||||
def key = obj.city + "##" + obj.region + "##" + obj.name
|
||||
map2.put(key, obj)
|
||||
}
|
||||
def rows = []
|
||||
rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"]
|
||||
map1.each {k,list->
|
||||
def row = []
|
||||
def index = 0
|
||||
map1.each {k,obj1->
|
||||
def keys = k.split("##")
|
||||
|
||||
def lastObj = map2.get(k)
|
||||
def lastPrice = null
|
||||
if(lastObj){
|
||||
lastPrice = lastObj[5]
|
||||
}
|
||||
def lastPrice = map2.get(k)
|
||||
|
||||
def markup = null
|
||||
try{
|
||||
if(list && lastPrice && lastPrice.toInteger()>0){
|
||||
markup = (list[5].toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger()
|
||||
if(obj1 && lastPrice && lastPrice.avg_price.toInteger()>0){
|
||||
markup = (obj1.avg_price.toInteger() - lastPrice.avg_price.toInteger())*100.0/lastPrice.avg_price.toInteger()
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
def dist = CommonDatautils.findDistrictById(list[1])
|
||||
def flag = "有效"
|
||||
|
||||
if(dist && dist.status!=0){
|
||||
flag = "失效"
|
||||
def flag = "无效"
|
||||
if(obj1){
|
||||
flag = idMaps.get(obj1.comm_id)
|
||||
}
|
||||
|
||||
rows << [keys[0], keys[1], keys[2], list[5].toString(), lastPrice.toString(), markup, flag]
|
||||
def row = [obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag]
|
||||
rows << row
|
||||
index++
|
||||
println row
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
|
||||
|
||||
@@ -255,4 +257,10 @@ class FzDataResultStat {
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
def findFzDistrictStat(lastMonth){
|
||||
def q = new BasicDBObject()
|
||||
q.put("month", lastMonth)
|
||||
return FZ_DISTRICT_STAT_COLL.find(q).toList()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202306.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202307.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+30
-5
@@ -14,6 +14,8 @@ import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
class CityStatExport {
|
||||
|
||||
@@ -26,10 +28,21 @@ class CityStatExport {
|
||||
|
||||
static main(args) {
|
||||
def task = new CityStatExport();
|
||||
// task.run()
|
||||
task.run()
|
||||
// task.run1()
|
||||
task.regionStat()
|
||||
// task.regionStat()
|
||||
// task.stat()
|
||||
// task.houseStat()
|
||||
}
|
||||
|
||||
def houseStat(){
|
||||
def fileName = "D:\\数据\\贝壳\\贝壳城市小区二手房统计V3-已重新补充.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
// def houseCount = findHouseCount(d['F'], 20230201)
|
||||
// d['H'] = houseCount
|
||||
d['J'] = findHouse(d['F'])
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
def stat(){
|
||||
@@ -47,7 +60,7 @@ class CityStatExport {
|
||||
// def citys = CommonDatautils.findAllCity();
|
||||
|
||||
|
||||
def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
|
||||
def citys = [ "广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"]
|
||||
def ret = []
|
||||
ret << ["城市","小区总量"]
|
||||
// ret << ["城市","楼栋总量", "房号总量"]
|
||||
@@ -78,7 +91,7 @@ class CityStatExport {
|
||||
ret << m
|
||||
println m
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"数据")
|
||||
ExcelUtils.write("D:\\数据\\广东省城市统计(楼盘量).xlsx",ret,"数据")
|
||||
|
||||
}
|
||||
|
||||
@@ -124,7 +137,7 @@ class CityStatExport {
|
||||
def findHouseCount(def city, def startDate){
|
||||
def total = 0
|
||||
try{
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} "
|
||||
String sql = "select count(*) as c from seawater where cat='sell' and city='${city}' and p_date >= ${startDate} "
|
||||
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
total = data.c
|
||||
@@ -136,6 +149,18 @@ class CityStatExport {
|
||||
return total
|
||||
}
|
||||
|
||||
|
||||
def findHouse(city) {
|
||||
int startDate = 20230201
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell"))
|
||||
.must(date_range)
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
return houses.size()
|
||||
}
|
||||
|
||||
def findLoudongCount(String city) {
|
||||
return LOUDONG.count(new BasicDBObject('city', city))
|
||||
}
|
||||
|
||||
@@ -90,4 +90,53 @@ class FileUtils {
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
static def batchCreateFileBySplitChar(ret, fileName, headers,charset, isAppend, splitChar){
|
||||
File file = new File(fileName);
|
||||
// if (file.exists()) {
|
||||
// file.delete();
|
||||
// }
|
||||
|
||||
if (!file.getParentFile().exists()) {
|
||||
boolean result = file.getParentFile().mkdirs();
|
||||
if (!result) {
|
||||
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
OutputStream out = new FileOutputStream(fileName, isAppend);
|
||||
def sb = new StringBuffer();
|
||||
|
||||
if(headers){
|
||||
sb.append(headers.join(splitChar))
|
||||
sb.append("\n");
|
||||
}
|
||||
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
|
||||
def index = 0
|
||||
for(def row : ret){
|
||||
if(row){
|
||||
index++
|
||||
sb.append(StringUtils.join(row.toArray(), splitChar))
|
||||
sb.append("\n");
|
||||
|
||||
if(index%10000==0){
|
||||
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(sb.length()>0){
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
}
|
||||
out.close();
|
||||
}
|
||||
}
|
||||
|
||||
+28
@@ -172,4 +172,32 @@ class FangjiaClientUtils {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static def addrEvaluation(city,region,address, size, floor, maxFloor, room, hall){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://localhost:8080/fangjia-open/property/addressEvaluate?city="+URLEncoder.encode(city,"UTF-8")+"&district="+URLEncoder.encode(region,"UTF-8")+"&address="+URLEncoder.encode(address,"UTF-8")+"&size=${size}&sampleCode=1"
|
||||
if(floor){
|
||||
url = url + "&floor="+floor.toString().toDouble().toInteger()
|
||||
}
|
||||
if(maxFloor){
|
||||
url = url + "&maxFloor="+maxFloor.toString().toDouble().toInteger()
|
||||
}
|
||||
if(room){
|
||||
url = url + "&room="+room.toString().toDouble().toInteger()
|
||||
}
|
||||
if(hall){
|
||||
url = url + "&hall="+hall.toString().toDouble().toInteger()
|
||||
}
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+33
-4
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
@@ -69,14 +70,15 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
// strandField(offices)
|
||||
// }
|
||||
|
||||
|
||||
|
||||
// addBlock()
|
||||
|
||||
// addFieldValue()
|
||||
|
||||
// updateErrorField()
|
||||
export()
|
||||
test()
|
||||
// export()
|
||||
// test()
|
||||
// addOffice()
|
||||
fileAddBlock()
|
||||
}
|
||||
|
||||
def export(){
|
||||
@@ -706,7 +708,34 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
})
|
||||
}
|
||||
|
||||
def fileAddBlock(){
|
||||
|
||||
def rets = findAllBlockRange()
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\上海商铺房源.xlsx",{d->
|
||||
def location = d['T']
|
||||
if(location && !d['F']){
|
||||
def arrs = location.split(",")
|
||||
// 高德转百度
|
||||
def baidu = BaiDuApiUtils.geoconv(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), 3)
|
||||
if(baidu){
|
||||
def lng2 = baidu.x
|
||||
def lat2 = baidu.y
|
||||
|
||||
def lastObj = null
|
||||
for (def r : rets) {
|
||||
def flag = InsidePolygon.isInArea(lng2, lat2, r.points)
|
||||
if (flag && r.name) {
|
||||
lastObj = r;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (lastObj) {
|
||||
d['F'] = lastObj.name
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package com.fangjia.script.gj.es
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
def datas = findHouseCount("北京", 20230201, 20230430)
|
||||
ExcelUtils.write("D:\\数据\\天弘基金.xlsx", datas, "样例")
|
||||
|
||||
|
||||
|
||||
def findHouseCount(def city, def startDate, def endDate){
|
||||
def rets = []
|
||||
try{
|
||||
// String sql = "select * from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and root='ke.com' order by p_date desc"
|
||||
def startMdate = DateUtils.parseFromTightByPattern(startDate.toString(),"yyyyMMdd")
|
||||
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
|
||||
|
||||
rets << ["城市","区域","板块","名称","面积","单价","总价","室","厅","卫","装修","朝向","发布时间","物业类型","楼层状况","楼高","建筑结构","建筑年代","标题","URL"]
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
// QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell")).must(QueryBuilders.termQuery("root", "ke.com"))
|
||||
.must(date_range).must(area_range)//.must(date_range_mdate)
|
||||
|
||||
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
|
||||
def index = new Random().nextInt(6);
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
|
||||
|
||||
if(houses) {
|
||||
|
||||
for (int k = 0; k < houses.size(); k++) {
|
||||
println k
|
||||
def data = houses[k];
|
||||
def r = []
|
||||
r << data.city
|
||||
r << data.region
|
||||
r << data.block
|
||||
r << data.district_name
|
||||
r << data.area
|
||||
r << data.avg_price
|
||||
r << data.total
|
||||
r << data.room
|
||||
r << data.hall
|
||||
r << data.toilet
|
||||
r << data.fitment
|
||||
r << data.direction
|
||||
r << data.p_date
|
||||
r << data.estate_type
|
||||
r << data.floor_info
|
||||
r << data.height
|
||||
r << data.build_structure
|
||||
r << data.build_date
|
||||
r << data.name
|
||||
r << data.url
|
||||
rets << r
|
||||
if(rets.size()>500){
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return rets
|
||||
}
|
||||
+51
-23
@@ -34,7 +34,7 @@ public class BeiKeCrawlByChrome {
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
|
||||
WebDriver driver = null;
|
||||
|
||||
//广州,佛山 重庆
|
||||
static main(args) {
|
||||
@@ -50,7 +50,7 @@ public class BeiKeCrawlByChrome {
|
||||
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver = new ChromeDriver()
|
||||
def citys = getAllCityUrls(driver)
|
||||
|
||||
driver = getLogin(driver, "https://sh.ke.com/")
|
||||
@@ -93,19 +93,28 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
|
||||
def checkDocument(driver, url){
|
||||
driver.get(url)
|
||||
def s = driver.getPageSource()
|
||||
def flag = true
|
||||
while(flag){
|
||||
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
|
||||
try{
|
||||
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
|
||||
driver = getLogin(driver, url)
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
}else{
|
||||
flag = false
|
||||
}
|
||||
}catch(Exception e){
|
||||
driver.close()
|
||||
driver = new ChromeDriver()
|
||||
Thread.sleep(15000)
|
||||
driver = getLogin(driver, url)
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
}else{
|
||||
flag = false
|
||||
println "异常Url:"+url
|
||||
}
|
||||
}
|
||||
return Jsoup.parse(s);
|
||||
return driver;
|
||||
}
|
||||
|
||||
def crawlData(driver, citys){
|
||||
@@ -130,7 +139,8 @@ public class BeiKeCrawlByChrome {
|
||||
def newUrl = url+"/chengjiao/"
|
||||
driver.get(newUrl)
|
||||
Thread.sleep(10000)
|
||||
def doc = checkDocument(driver, newUrl);
|
||||
driver = checkDocument(driver, newUrl);
|
||||
def doc = Jsoup.parse(driver.getPageSource())
|
||||
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
@@ -142,8 +152,9 @@ public class BeiKeCrawlByChrome {
|
||||
regionText = "昆山"
|
||||
}
|
||||
|
||||
def detailUrls = getPageDeail(driver, newCity, href, regionText)
|
||||
println url +" 详情链接:"+detailUrls.size()
|
||||
def map = getPageDeail(driver, newCity, href, regionText)
|
||||
driver = map.driver
|
||||
println url +" 详情链接:"+map.detailUrls.size()
|
||||
}
|
||||
}
|
||||
FileUtils.batchCreateFile([[city]], "d:\\数据\\成交抓取\\贝壳已抓城市.txt", [], "utf-8", true);
|
||||
@@ -191,9 +202,9 @@ public class BeiKeCrawlByChrome {
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "pg${page}/"
|
||||
driver.get(url)
|
||||
// def doc = Jsoup.parse(driver.getPageSource());
|
||||
def doc = checkDocument(driver, url);
|
||||
driver = checkDocument(driver, url);
|
||||
def doc = Jsoup.parse(driver.getPageSource())
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
if(elements){
|
||||
for(def e : elements){
|
||||
@@ -203,10 +214,11 @@ public class BeiKeCrawlByChrome {
|
||||
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
|
||||
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
println "开始查库 "+new Date().toString()
|
||||
def dbo = findTrade(durl , "贝壳找房")
|
||||
|
||||
println "结束查库"+new Date().toString()
|
||||
if (!dbo) {
|
||||
parse(city, e, region, driver, null)
|
||||
driver = parse(city, e, region, driver, null)
|
||||
}
|
||||
// else if(dbo && !dbo.lift){
|
||||
// parse(city, e, region, driver, dbo._id)
|
||||
@@ -225,7 +237,10 @@ public class BeiKeCrawlByChrome {
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return detailUrls
|
||||
def m = [:]
|
||||
m.driver = driver
|
||||
m.detailUrls = detailUrls
|
||||
return m
|
||||
}
|
||||
|
||||
def parse(city,e, region, driver, id){
|
||||
@@ -233,8 +248,10 @@ public class BeiKeCrawlByChrome {
|
||||
def detailUrl = e.select(".title a").attr("href")
|
||||
|
||||
driver.get(detailUrl)
|
||||
def detailDoc = checkDocument(driver, detailUrl);
|
||||
driver = checkDocument(driver, detailUrl);
|
||||
def detailDoc = Jsoup.parse(driver.getPageSource())
|
||||
|
||||
println "打开获取页面 "+new Date().toString()
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
@@ -335,18 +352,21 @@ public class BeiKeCrawlByChrome {
|
||||
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println "解析页面完成 "+new Date().toString()
|
||||
println detail
|
||||
if(id){
|
||||
DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail))
|
||||
}else{
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
println "保存修改完成 "+new Date().toString()
|
||||
}
|
||||
}
|
||||
}catch(Exception ex){
|
||||
ex.printStackTrace()
|
||||
println e.select(".title a").attr("href")
|
||||
}
|
||||
return driver
|
||||
}
|
||||
|
||||
|
||||
@@ -469,16 +489,16 @@ public class BeiKeCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = "https:"+e.attr("href")
|
||||
// if (citys.contains(e.text())){
|
||||
if (citys.contains(e.text())){
|
||||
list << [e.text(), href]
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
def getLogin(driver, url){
|
||||
// WebDriver driver = new ChromeDriver()
|
||||
|
||||
def root = url.toString().substring(url.toString().indexOf("//")+2, url.toString().indexOf(".com")+4)
|
||||
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252F"+root+"%252Fchengjiao%252F");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
@@ -494,17 +514,25 @@ public class BeiKeCrawlByChrome {
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
|
||||
elemUsername.sendKeys("17148854005");
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
// elemUsername.sendKeys("15021384995");
|
||||
// elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
elemUsername.sendKeys("16283166371");
|
||||
elemPassword.sendKeys("Goojia123456");
|
||||
// elemUsername.sendKeys("16283166371");
|
||||
// elemPassword.sendKeys("Goojia123456");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(10000)
|
||||
}catch(Exception e){
|
||||
driver.close()
|
||||
driver = new ChromeDriver()
|
||||
Thread.sleep(15000)
|
||||
driver = getLogin(driver, url)
|
||||
println "异常Url:"+url
|
||||
}
|
||||
|
||||
return driver
|
||||
}
|
||||
|
||||
|
||||
+55
-34
@@ -34,6 +34,7 @@ public class LianjiaCrawlByChrome {
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
WebDriver driver = null;
|
||||
|
||||
static main(args) {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
@@ -122,7 +123,7 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver = new ChromeDriver()
|
||||
def cityMap = getAllCityUrls(driver)
|
||||
// cityMap = cityMap.reverse()
|
||||
driver = getLogin(driver, "https://sh.lianjia.com/")
|
||||
@@ -206,17 +207,11 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
def cityUrl = ret[1]
|
||||
def newUrl = cityUrl+"chengjiao/"
|
||||
try{
|
||||
driver.get(newUrl)
|
||||
}catch(Exception e){
|
||||
println newUrl +" 链接有问题!"
|
||||
continue
|
||||
}
|
||||
|
||||
// driver.get(newUrl)
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
Thread.sleep(10000)
|
||||
def doc = checkDocument(driver, newUrl);
|
||||
|
||||
// Thread.sleep(10000)
|
||||
driver = checkDocument(driver, newUrl);
|
||||
def doc = Jsoup.parse(driver.getPageSource())
|
||||
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
@@ -226,8 +221,9 @@ public class LianjiaCrawlByChrome {
|
||||
regionText = "昆山"
|
||||
}
|
||||
println "开始抓取【${city}】【${regionText}】【${href}】"
|
||||
def detailUrls = getPageDeail(driver, newCity, href, regionText)
|
||||
println cityUrl +" 详情链接:"+detailUrls.size()
|
||||
def map = getPageDeail(driver, newCity, href, regionText)
|
||||
driver = map.driver
|
||||
println newUrl +" 详情链接:"+map.detailUrls.size()
|
||||
}
|
||||
}
|
||||
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
|
||||
@@ -267,30 +263,41 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
|
||||
def checkDocument(driver, url){
|
||||
driver.get(url)
|
||||
def s = driver.getPageSource()
|
||||
def flag = true
|
||||
while(flag){
|
||||
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
|
||||
driver = getLogin(driver, url)
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
}else{
|
||||
flag = false
|
||||
try{
|
||||
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
|
||||
driver = getLogin(driver, url)
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
}else{
|
||||
flag = false
|
||||
}
|
||||
}catch(Exception e){
|
||||
driver.close()
|
||||
driver = new ChromeDriver()
|
||||
Thread.sleep(15000)
|
||||
driver = getLogin(driver, url)
|
||||
println "异常Url:"+url
|
||||
}
|
||||
}
|
||||
return Jsoup.parse(s);
|
||||
return driver;
|
||||
}
|
||||
|
||||
|
||||
def getPageDeail(driver, city, cityUrl, regionText){
|
||||
def stop = false
|
||||
def page = 1
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "pg${page}/"
|
||||
driver.get(url)
|
||||
// driver.get(url)
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
def doc = checkDocument(driver, url);
|
||||
driver = checkDocument(driver, url);
|
||||
def doc = Jsoup.parse(driver.getPageSource())
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
if(elements){
|
||||
for(def e : elements){
|
||||
@@ -303,7 +310,7 @@ public class LianjiaCrawlByChrome {
|
||||
detailUrls << durl
|
||||
def dbo = findTrade(durl , "链家在线")
|
||||
if (!dbo) {
|
||||
parse(city, e, regionText, driver, null)
|
||||
driver = parse(city, e, regionText, driver, null)
|
||||
}
|
||||
// else if(dbo && !dbo.lift){
|
||||
// parse(city, e, regionText, driver, dbo._id)
|
||||
@@ -326,7 +333,10 @@ public class LianjiaCrawlByChrome {
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return detailUrls
|
||||
def m = [:]
|
||||
m.driver = driver
|
||||
m.detailUrls = detailUrls
|
||||
return m
|
||||
}
|
||||
|
||||
def parse(city,e, regionText, driver, id){
|
||||
@@ -334,8 +344,11 @@ public class LianjiaCrawlByChrome {
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if(price){
|
||||
def detailUrl = e.select(".title a").attr("href")
|
||||
driver.get(detailUrl)
|
||||
def detailDoc = checkDocument(driver, detailUrl);
|
||||
|
||||
// driver.get(detailUrl)
|
||||
driver = checkDocument(driver, detailUrl);
|
||||
def detailDoc = Jsoup.parse(driver.getPageSource())
|
||||
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
@@ -450,6 +463,7 @@ public class LianjiaCrawlByChrome {
|
||||
ex.printStackTrace()
|
||||
println e.select(".title a").attr("href")
|
||||
}
|
||||
return driver;
|
||||
}
|
||||
|
||||
|
||||
@@ -571,7 +585,7 @@ public class LianjiaCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
if (!href.contains("fang.lianjia")){
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
cityList << [e.text(), href]
|
||||
}
|
||||
}
|
||||
@@ -593,16 +607,23 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
// elemUsername.sendKeys("15021384995");
|
||||
// elemPassword.sendKeys("12qwaszx");
|
||||
elemUsername.sendKeys("17148854005");
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
elemUsername.sendKeys("16283166371");
|
||||
elemPassword.sendKeys("Goojia123456");
|
||||
// elemUsername.sendKeys("17740818975");
|
||||
// elemPassword.sendKeys("13qeadzc");
|
||||
|
||||
// elemUsername.sendKeys("16283166371");
|
||||
// elemPassword.sendKeys("Goojia123456");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(15000)
|
||||
Thread.sleep(20000)
|
||||
}catch(Exception e){
|
||||
Thread.sleep(10000)
|
||||
driver.close()
|
||||
driver = new ChromeDriver()
|
||||
Thread.sleep(15000)
|
||||
driver = getLogin(driver, url)
|
||||
println "异常Url:"+url
|
||||
}
|
||||
return driver
|
||||
}
|
||||
|
||||
+211
@@ -0,0 +1,211 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 厦门银行
|
||||
* 福建省,重庆市poi数据处理
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class GaoDePoiDataProcess extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new GaoDePoiDataProcess(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
//三明市,宁德市,泉州市,厦门市,福州市,漳州市,南平市,龙岩市,莆田市,重庆市
|
||||
def citys = ["三明市","漳州市","厦门市","福州市"]
|
||||
for(def city : citys){
|
||||
def results = CommonDatautils.findAmapDetailBySql(city)
|
||||
println results.size()
|
||||
def rows = running(results)
|
||||
save(rows)
|
||||
println city + " 完成"+new Date().toString()
|
||||
}
|
||||
}
|
||||
|
||||
def running(results){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||
|
||||
for(def i=0;i<results.size();i++){
|
||||
def map = results[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
rets.addAll(createData(map))
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData(map){
|
||||
def fjw_province = null
|
||||
if(map.city.contains("重庆")){
|
||||
fjw_province = "重庆"
|
||||
}else{
|
||||
fjw_province = "福建"
|
||||
}
|
||||
def fjw_city = RegionData.formatCity(map.city)
|
||||
def fjw_region = RegionData.formatRegion(fjw_city, map.region)
|
||||
map.id = new ObjectId().toString()
|
||||
map.fjw_province = fjw_province
|
||||
map.fjw_city = fjw_city
|
||||
|
||||
if(map.cat){
|
||||
def arrs = map.cat.split(";")
|
||||
map.type = arrs[0]
|
||||
if(arrs.size()>=2){
|
||||
map.secondary_type = arrs[1]
|
||||
}
|
||||
if(arrs.size()>=3){
|
||||
map.small_type = arrs[2]
|
||||
}
|
||||
}
|
||||
|
||||
//高德转百度
|
||||
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
|
||||
if(baiduMap){
|
||||
def location = [:]
|
||||
location.lon = baiduMap.lng
|
||||
location.lat = baiduMap.lat
|
||||
map.bd_location = location
|
||||
}
|
||||
|
||||
//高德转WGS84
|
||||
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
|
||||
if(gpsLoactionMap){
|
||||
def gpslocation = [:]
|
||||
gpslocation.lon = gpsLoactionMap.lng
|
||||
gpslocation.lat = gpsLoactionMap.lat
|
||||
map.gps_location = gpslocation
|
||||
}
|
||||
|
||||
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
|
||||
if(gaodeMap && gaodeMap.township){
|
||||
map.township = gaodeMap.township
|
||||
}
|
||||
|
||||
if(!fjw_region){
|
||||
if(gaodeMap && gaodeMap.district){
|
||||
fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district)
|
||||
}
|
||||
}
|
||||
if(!fjw_region){
|
||||
println fjw_city+" "+map.region+" 无法格式化 "+map.location + " "+gaodeMap
|
||||
}
|
||||
map.fjw_region = fjw_region
|
||||
return map
|
||||
}
|
||||
|
||||
def save(rows){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int batchSize = 1000;
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils.getClient("192.168.0.12")
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
request.add(new IndexRequest("amap_poi", "_doc", map.id).source(map, XContentType.JSON));
|
||||
if(i%1000==0){
|
||||
print "."
|
||||
}
|
||||
|
||||
// 批量提交数据
|
||||
if (request.numberOfActions() >= batchSize) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
|
||||
request = new BulkRequest();
|
||||
}
|
||||
}
|
||||
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
|
||||
println "结束时间 "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
}
|
||||
}
|
||||
}
|
||||
if(district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
Thread.sleep(3000)
|
||||
print "."
|
||||
}
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.district = district
|
||||
map.township = township
|
||||
return map;
|
||||
}
|
||||
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class PoiAndSeaweedExportCSV extends BaseBatch{
|
||||
|
||||
static main(args) {
|
||||
runBatch(new PoiAndSeaweedExportCSV(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = ["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"]
|
||||
|
||||
poiExport(citys)
|
||||
|
||||
}
|
||||
|
||||
def poiExport(citys){
|
||||
def fileName = "/home/goojia/xiamen/BIGDATA_POI.csv"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\厦门\\BIGDATA_POI.csv"
|
||||
}
|
||||
|
||||
for(def c : citys){
|
||||
def rets = CommonDatautils.findNewAmapDetail(c)
|
||||
FileUtils.batchCreateFileBySplitChar(rets, fileName, null, "utf-8", true, "@|@");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.client.MongoCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.Document
|
||||
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20230701", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230801", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
def hashSet = []
|
||||
def data = [["编号", "公司名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "返回参数"]]
|
||||
while (startDate.before(endDate)) {
|
||||
calendar.add(Calendar.DATE, 1)
|
||||
def nextDate = calendar.getTime()
|
||||
count++
|
||||
///property/addressEvaluate /address/financialRating /v2/property/detail
|
||||
OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("request_uri", "/v2/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
|
||||
def result = JSONObject.fromObject(doc.res_map.result)
|
||||
// if (!hashSet.contains(result)) {
|
||||
def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
|
||||
data << item
|
||||
// hashSet << result
|
||||
// } else {
|
||||
// println("已存在")
|
||||
// }
|
||||
}
|
||||
if (count % 7 == 0) {
|
||||
hashSet.clear()
|
||||
}
|
||||
|
||||
startDate = nextDate
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信.xlsx", data, "楼盘评情")
|
||||
|
||||
|
||||
+1
-1
@@ -12,7 +12,7 @@ import org.jsoup.Jsoup
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class GaoDeApiUtils {
|
||||
def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390"
|
||||
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca"
|
||||
|
||||
/**
|
||||
* 经纬度转地址<br>
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
package com.fangjia.script.zsbank
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
class ZheShangBank {
|
||||
static main(args) {
|
||||
def task = new ZheShangBank()
|
||||
// task.run()
|
||||
// task.detail()
|
||||
task.sample()
|
||||
}
|
||||
|
||||
def sample(){
|
||||
def runedCitys = FileUtils.readTxt("D:\\数据\\浙商\\result.txt")
|
||||
def rets = []
|
||||
rets << ["时间","Msg", "城市", "区域", "名称","月份"]
|
||||
def map = [:]
|
||||
for(def line : runedCitys){
|
||||
def ret = []
|
||||
def date = line.substring(line.indexOf("log.")+4, line.indexOf(":"))
|
||||
def msg = line.substring(line.lastIndexOf("error :")+7, line.size())
|
||||
def arrs = msg.split(" ")
|
||||
ret << date
|
||||
for(def i=0;i<arrs.size();i++){
|
||||
def s = arrs[i]
|
||||
if(i == 0){
|
||||
ret << s
|
||||
}else{
|
||||
ret << s.substring(s.indexOf(":")+1, s.size())
|
||||
}
|
||||
}
|
||||
def month = date.substring(0, 7)
|
||||
ret << month
|
||||
rets << ret
|
||||
|
||||
println ret.toString()
|
||||
|
||||
def count = map.get(ret[1])
|
||||
if(!count){
|
||||
map.put(ret[1], 1)
|
||||
}else{
|
||||
map.put(ret[1], count+1)
|
||||
}
|
||||
// FileUtils.batchCreateFile([ret], "D:\\数据\\浙商\\浙商失败案例.txt",null, "utf-8", true);
|
||||
}
|
||||
|
||||
map.each{m,r->
|
||||
println m+" "+r
|
||||
}
|
||||
// ExcelUtils.write("D:\\数据\\浙商\\浙商失败案例.xlsx", rets, "风险缓释")
|
||||
}
|
||||
def run(){
|
||||
def start = "2021-03-01"
|
||||
def rets = []
|
||||
rets << ["月份","总调用量","成功量","失败量"]
|
||||
while (true){
|
||||
def end = DateUtils.addMonth(DateUtils.parseFromTightYYYYMMDD(start), 1)
|
||||
// def total = CommonDatautils.findLogs("1007569", "/address/risk", start, end, null)
|
||||
// def successCount = CommonDatautils.findLogs("1007569", "/address/risk", start, end, 200)
|
||||
|
||||
def total = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, null)
|
||||
def successCount = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, 200)
|
||||
|
||||
def failCount = total - successCount
|
||||
println start[0..6] +" "+ total +" "+successCount +" "+ failCount
|
||||
rets << [start[0..6], total, successCount, failCount]
|
||||
start = end;
|
||||
if(end == "2023-09-01"){
|
||||
break;
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\浙商\\浙商调用统计.xlsx", rets, "估值地址")
|
||||
}
|
||||
|
||||
|
||||
def detail(){
|
||||
def datas = CommonDatautils.findLogs2("1007569", "/property/addressEvaluate")
|
||||
ExcelUtils.write("D:\\数据\\浙商\\浙商失败案例.xlsx", datas, "估值地址")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user