diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 1f2867d17..03bc9ebbe 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -217,7 +217,7 @@ class BigdataDealBatch extends BaseBatch{ def price = ccbPriceMap.get(obj._id.toString()) if(price){ def mod = java.lang.Math.abs((trade.price-price)*100/price) - if(mod<=10){ + if(mod<=20){ ret << cityCodeDatas.get(trade.city) ret << trade._id.toString() ret << trade.city diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 5102257a3..2b6c315b3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -3941,5 +3941,125 @@ class CommonDatautils { return files; } + static def findLogs(userId, requestUri, startDate, endDate, code){ + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", userId)).must(QueryBuilders.termQuery("request_uri", requestUri)) + if(startDate){ + def startTime = DateUtils.parseFromTightYYYYMMDD(startDate) + def endTime = DateUtils.parseFromTightYYYYMMDD(endDate) + def date_range_cdate = QueryBuilders.rangeQuery("c_date").from(startTime.getTime()).to(endTime.getTime()).includeLower(true).includeUpper(false); + queryBuilder.must(date_range_cdate); + } + if(code){ + queryBuilder.must(QueryBuilders.termQuery("code", code)) + } + def rets = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder); + return rets.size() + } + + static def findLogsBySql(userId, requestUri, startDate, endDate, code){ + String sql = "select count(*) as total from open_uri_log where user_id=${userId} and request_uri='${requestUri}' " + if(startDate){ + sql = sql + "and c_date >= '${startDate}' and c_date < '${endDate}'" + } + if(code){ + sql = sql + " and code=${code}" + } + def total = null + ElasticUtils.executeSqlQuery(sql,{data-> + total = data.total + }) + return total; + } + + static def findAmapDetailBySql(city){ + String sql = "select * from amap_poi_detail where cityname='${city}' limit 10" + def rets = [] + ElasticUtils.executeSqlQuery(sql,{data-> +// if(data.name.contains('华山小区G栋')){ + def point = data.location +// def tag = null +// if(data.get("business.keytag")){ +// tag = data.get("business.keytag") +// } + def business_area = null + if(data.get("business.business_area")){ + business_area = data.get("business.business_area") + } + def location = [:] + location.lon = point.x + location.lat = point.y + rets << ["province":data.pname,"city":data.cityname, "region":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area] +// } + + }) + return rets; + } + + + //用sql有些字段取不出来 + static def findDetailLogsBySql(userId, requestUri){ + String sql = "select * from open_uri_log where user_id=${userId} and request_uri='${requestUri}' and code!=200" + + sql = sql + " order by c_date desc" + def total = null + def rets = [] + ElasticUtils.executeSqlQuery(sql,{data-> + def ret = [] + println data.toString() + ret << data.get("parameter_map.city") +// ret << data.get("parameter_map.district") +// ret << data.get("parameter_map.keyword") +// ret << data.get("parameter_map.name") + ret << data.get("parameter_map.address") +// ret << data.get("parameter_map.size") + ret << data.get("msg") + ret << data.get("code") + ret << data.get("c_date").toString()[0..10] + rets << ret + }) + return rets; + } + + static def findNewAmapDetail(city) { + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fjw_city", city)) + def rets = ElasticUtils.executeSimpleQuery("192.168.0.12", "amap_poi", queryBuilder); + def results = [] + if(rets) { + for (int k = 0; k < rets.size(); k++) { + def data = rets[k]; + println data + } + } + return results; + } + + + static def findLogs2(userId, requestUri){ + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", userId)).must(QueryBuilders.termQuery("request_uri", requestUri)) + + queryBuilder.mustNot(QueryBuilders.termQuery("code", 200)) + def rets = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder); + def results = [] + if(rets) { + for (int k = 0; k < rets.size(); k++) { + def data = rets[k]; + println data + def ret = [] + println data.toString() + ret << data.parameter_map.city + ret << data.parameter_map.district + ret << data.parameter_map.name + ret << data.parameter_map.address + ret << data.msg + ret << data.code + ret << data.c_date.toString()[0..10] + results << ret + } + } + return results + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index 1e5037735..4ec710a3c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -47,19 +47,29 @@ class DistrictPriceExportBatch extends BaseBatch{ def ret = createNewDatas("福州") saveToDB(ret) + def results = selectLastResultDatas(ret); + def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv" def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".csv" dataCodeFileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".sha512" } - def sb = appendResult(headers, ret) + def sb = appendResult(headers, results) DataHandleUtils.outPutFile(fileName, sb, charset); FzFtpUtils.ftpUpload(fileName) //生成加密数据文件并上传 FzFtpUtils.uploadSha512Data2(fileName, dataCodeFileName, charset); } + def selectLastResultDatas(def rets){ + def rows = [] + for(def r : rets){ + rows << [r[0],r[1], r[2], r[3], r[4],r[5], r[6]] + } + return rows; + } + def appendResult(def headers, def ret){ @@ -147,6 +157,29 @@ class DistrictPriceExportBatch extends BaseBatch{ m << 0 } m << curDate.toInteger(); + if(curPrice && dbo){ + if(dbo.value_type == 1){ + m << "指导价" + }else if(dbo.value_type == 2){ + m << "自身样本" + }else if(dbo.value_type == 3){ + m << "第三方" + }else if(dbo.value_type == 4){ + m << "交易数据" + }else if(dbo.value_type == 5){ + m << "周边楼盘" + }else if(dbo.value_type == 6){ + m << "推导" + }else if(dbo.value_type == 9){ + m << "神经网络" + }else if(dbo.value_type == 10){ + m << "无方式" + }else{ + m << "" + } + }else{ + m << "无" + } result << m } return result @@ -300,6 +333,7 @@ class DistrictPriceExportBatch extends BaseBatch{ //按comm_id修改,小区名重新赋值,防止区域和小区名修改 udata.put("region", r[3]) udata.put("name", r[4]) + udata.put("value_type", r[7]) FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata)) }else{ def o = new BasicDBObject() @@ -312,6 +346,7 @@ class DistrictPriceExportBatch extends BaseBatch{ o.put("avg_price", r[5].toInteger()) o.put("month", r[6].toInteger()) o.put("c_date", new Date()) + o.put("value_type", r[7]) FZ_DISTRICT_STAT_COLL.save(o) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index 459e3e505..6f339f416 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -199,6 +199,8 @@ class DistrictSellStatBatch extends BaseBatch { data.put("mom", mom) data.put("yoy", yoy) data.put("avg_price", weekendPriceObj.avg_price) + data.put("value_type", weekendPriceObj.value_type) + data.put("credit", weekendPriceObj.credit) DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false) println "[update]"+totalCount.get()+" "+ dist.city+" "+dist.name } @@ -210,6 +212,7 @@ class DistrictSellStatBatch extends BaseBatch { data.put("estate_type", dist.estate_type2) data.put("date", curMonth) data.put("avg_price", curPrice) + data.put("value_type", weekendPriceObj.value_type) data.put("credit", weekendPriceObj.credit) data.put("m_date", new Date()) data.put("commid", dist._id.toString()) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 917ddfc5e..b2d9b9338 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -51,59 +51,61 @@ class FzDataResultStat { }) } def priceMarkupRun(){ - def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")+"15"); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); - List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv")) +// List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv")) + def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"]) + def idMaps = [:] + for(def d : dists){ + idMaps.put(d._id.toString(), d.status==0?"有效":"无效") + } + def list1 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"))) def distkeys = [] def map1 = [:] - for(def s : list1){ - def row = s.split("@\\|@") - def key = row[2]+"##"+row[3]+"##"+row[4] + for(def obj : list1){ + def key = obj.city + "##" + obj.region + "##" + obj.name if(!distkeys.contains(key)){ distkeys << key }else{ println "重复:"+key } - map1.put(key, row) + map1.put(key, obj) } - List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv")) +// List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv")) + def list2 = findFzDistrictStat(lastMonth) def map2 = [:] - for(def s : list2) { - def row = s.split("@\\|@") - def key = row[2] + "##" + row[3] + "##" + row[4] - map2.put(key, row) + for(def obj : list2) { + def key = obj.city + "##" + obj.region + "##" + obj.name + map2.put(key, obj) } def rows = [] rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"] - map1.each {k,list-> - def row = [] + def index = 0 + map1.each {k,obj1-> def keys = k.split("##") - def lastObj = map2.get(k) - def lastPrice = null - if(lastObj){ - lastPrice = lastObj[5] - } + def lastPrice = map2.get(k) + def markup = null try{ - if(list && lastPrice && lastPrice.toInteger()>0){ - markup = (list[5].toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger() + if(obj1 && lastPrice && lastPrice.avg_price.toInteger()>0){ + markup = (obj1.avg_price.toInteger() - lastPrice.avg_price.toInteger())*100.0/lastPrice.avg_price.toInteger() } }catch(Exception e){ e.printStackTrace() } - def dist = CommonDatautils.findDistrictById(list[1]) - def flag = "有效" - - if(dist && dist.status!=0){ - flag = "失效" + def flag = "无效" + if(obj1){ + flag = idMaps.get(obj1.comm_id) } - - rows << [keys[0], keys[1], keys[2], list[5].toString(), lastPrice.toString(), markup, flag] + def row = [obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag] + rows << row + index++ + println row } ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据") @@ -255,4 +257,10 @@ class FzDataResultStat { } return files; } + + def findFzDistrictStat(lastMonth){ + def q = new BasicDBObject() + q.put("month", lastMonth) + return FZ_DISTRICT_STAT_COLL.find(q).toList() + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 2220e87ab..8e3147530 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202306.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202307.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy index 5cae7c343..a7b5d4df1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CityStatExport.groovy @@ -14,6 +14,8 @@ import com.mongodb.Bytes import com.mongodb.DBCollection import com.mongodb.DBObject import com.mongodb.QueryOperators +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders class CityStatExport { @@ -26,10 +28,21 @@ class CityStatExport { static main(args) { def task = new CityStatExport(); -// task.run() + task.run() // task.run1() - task.regionStat() +// task.regionStat() // task.stat() +// task.houseStat() + } + + def houseStat(){ + def fileName = "D:\\数据\\贝壳\\贝壳城市小区二手房统计V3-已重新补充.xlsx" + ExcelUtils.multiThreadingModify(fileName, { d -> +// def houseCount = findHouseCount(d['F'], 20230201) +// d['H'] = houseCount + d['J'] = findHouse(d['F']) + }) + } def stat(){ @@ -47,7 +60,7 @@ class CityStatExport { // def citys = CommonDatautils.findAllCity(); - def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"] + def citys = [ "广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"] def ret = [] ret << ["城市","小区总量"] // ret << ["城市","楼栋总量", "房号总量"] @@ -78,7 +91,7 @@ class CityStatExport { ret << m println m } - ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"数据") + ExcelUtils.write("D:\\数据\\广东省城市统计(楼盘量).xlsx",ret,"数据") } @@ -124,7 +137,7 @@ class CityStatExport { def findHouseCount(def city, def startDate){ def total = 0 try{ - String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} " + String sql = "select count(*) as c from seawater where cat='sell' and city='${city}' and p_date >= ${startDate} " ElasticUtils.executeSqlQuery(sql,{ data-> total = data.c @@ -136,6 +149,18 @@ class CityStatExport { return total } + + def findHouse(city) { + int startDate = 20230201 + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell")) + .must(date_range) + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + return houses.size() + } + def findLoudongCount(String city) { return LOUDONG.count(new BasicDBObject('city', city)) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy index 78270dc6c..1a8fb81be 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -90,4 +90,53 @@ class FileUtils { } return ret; } + + + static def batchCreateFileBySplitChar(ret, fileName, headers,charset, isAppend, splitChar){ + File file = new File(fileName); +// if (file.exists()) { +// file.delete(); +// } + + if (!file.getParentFile().exists()) { + boolean result = file.getParentFile().mkdirs(); + if (!result) { + System.out.println("创建目录失败"); + } + } + + + OutputStream out = new FileOutputStream(fileName, isAppend); + def sb = new StringBuffer(); + + if(headers){ + sb.append(headers.join(splitChar)) + sb.append("\n"); + } + + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + + def index = 0 + for(def row : ret){ + if(row){ + index++ + sb.append(StringUtils.join(row.toArray(), splitChar)) + sb.append("\n"); + + if(index%10000==0){ + println index+"开始写入CSV "+new Date().toLocaleString() + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + println index+"结束写入CSV "+new Date().toLocaleString() + } + } + } + + if(sb.length()>0){ + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + } + out.close(); + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index a3236b53a..210fa41c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -172,4 +172,32 @@ class FangjiaClientUtils { } } + + public static def addrEvaluation(city,region,address, size, floor, maxFloor, room, hall){ + def jsonObj = null + def content = null + try { + address = address.replace("{","") + def url = "http://localhost:8080/fangjia-open/property/addressEvaluate?city="+URLEncoder.encode(city,"UTF-8")+"&district="+URLEncoder.encode(region,"UTF-8")+"&address="+URLEncoder.encode(address,"UTF-8")+"&size=${size}&sampleCode=1" + if(floor){ + url = url + "&floor="+floor.toString().toDouble().toInteger() + } + if(maxFloor){ + url = url + "&maxFloor="+maxFloor.toString().toDouble().toInteger() + } + if(room){ + url = url + "&room="+room.toString().toDouble().toInteger() + } + if(hall){ + url = url + "&hall="+hall.toString().toDouble().toInteger() + } + content = new HttpRequest().newGet(url, null, "utf-8") + jsonObj = JSONObject.fromObject(content) + } catch (Exception e) { + println address +" "+content + e.printStackTrace(); + } + return jsonObj; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy index 0af10b4fa..f17e815dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.district +import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher @@ -69,14 +70,15 @@ class OfficeFieldCleanBatch extends BaseBatch{ // strandField(offices) // } + + // addBlock() - // addFieldValue() - // updateErrorField() - export() - test() +// export() +// test() // addOffice() + fileAddBlock() } def export(){ @@ -706,7 +708,34 @@ class OfficeFieldCleanBatch extends BaseBatch{ }) } + def fileAddBlock(){ + def rets = findAllBlockRange() + ExcelUtils.multiThreadingModify("D:\\数据\\其它\\上海商铺房源.xlsx",{d-> + def location = d['T'] + if(location && !d['F']){ + def arrs = location.split(",") + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), 3) + if(baidu){ + def lng2 = baidu.x + def lat2 = baidu.y + def lastObj = null + for (def r : rets) { + def flag = InsidePolygon.isInArea(lng2, lat2, r.points) + if (flag && r.name) { + lastObj = r; + break; + } + } + if (lastObj) { + d['F'] = lastObj.name + } + } + + } + }) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/ExportHouse.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/ExportHouse.groovy new file mode 100644 index 000000000..8dd989f3c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/ExportHouse.groovy @@ -0,0 +1,72 @@ +package com.fangjia.script.gj.es + +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +def datas = findHouseCount("北京", 20230201, 20230430) +ExcelUtils.write("D:\\数据\\天弘基金.xlsx", datas, "样例") + + + +def findHouseCount(def city, def startDate, def endDate){ + def rets = [] + try{ +// String sql = "select * from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and root='ke.com' order by p_date desc" + def startMdate = DateUtils.parseFromTightByPattern(startDate.toString(),"yyyyMMdd") + def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd") + + rets << ["城市","区域","板块","名称","面积","单价","总价","室","厅","卫","装修","朝向","发布时间","物业类型","楼层状况","楼高","建筑结构","建筑年代","标题","URL"] + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + // QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false); + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell")).must(QueryBuilders.termQuery("root", "ke.com")) + .must(date_range).must(area_range)//.must(date_range_mdate) + + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(6); + + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); + + if(houses) { + + for (int k = 0; k < houses.size(); k++) { + println k + def data = houses[k]; + def r = [] + r << data.city + r << data.region + r << data.block + r << data.district_name + r << data.area + r << data.avg_price + r << data.total + r << data.room + r << data.hall + r << data.toilet + r << data.fitment + r << data.direction + r << data.p_date + r << data.estate_type + r << data.floor_info + r << data.height + r << data.build_structure + r << data.build_date + r << data.name + r << data.url + rets << r + if(rets.size()>500){ + break; + } + } + } + }catch(Exception e){ + e.printStackTrace() + } + + return rets +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 88b01f6c1..2003db863 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -34,7 +34,7 @@ public class BeiKeCrawlByChrome { def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") - + WebDriver driver = null; //广州,佛山 重庆 static main(args) { @@ -50,7 +50,7 @@ public class BeiKeCrawlByChrome { System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); - WebDriver driver = new ChromeDriver() + driver = new ChromeDriver() def citys = getAllCityUrls(driver) driver = getLogin(driver, "https://sh.ke.com/") @@ -93,19 +93,28 @@ public class BeiKeCrawlByChrome { } def checkDocument(driver, url){ + driver.get(url) def s = driver.getPageSource() def flag = true while(flag){ - if(s && (s.contains("无法访问此网站") || s.contains("登录"))){ + try{ + if(s && (s.contains("无法访问此网站") || s.contains("登录"))){ + driver = getLogin(driver, url) + driver.get(url) + Thread.sleep(5000) + s = driver.getPageSource() + }else{ + flag = false + } + }catch(Exception e){ + driver.close() + driver = new ChromeDriver() + Thread.sleep(15000) driver = getLogin(driver, url) - driver.get(url) - Thread.sleep(5000) - s = driver.getPageSource() - }else{ - flag = false + println "异常Url:"+url } } - return Jsoup.parse(s); + return driver; } def crawlData(driver, citys){ @@ -130,7 +139,8 @@ public class BeiKeCrawlByChrome { def newUrl = url+"/chengjiao/" driver.get(newUrl) Thread.sleep(10000) - def doc = checkDocument(driver, newUrl); + driver = checkDocument(driver, newUrl); + def doc = Jsoup.parse(driver.getPageSource()) def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc) if(regions){ for(def r : regions){ @@ -142,8 +152,9 @@ public class BeiKeCrawlByChrome { regionText = "昆山" } - def detailUrls = getPageDeail(driver, newCity, href, regionText) - println url +" 详情链接:"+detailUrls.size() + def map = getPageDeail(driver, newCity, href, regionText) + driver = map.driver + println url +" 详情链接:"+map.detailUrls.size() } } FileUtils.batchCreateFile([[city]], "d:\\数据\\成交抓取\\贝壳已抓城市.txt", [], "utf-8", true); @@ -191,9 +202,9 @@ public class BeiKeCrawlByChrome { def detailUrls = [] while(!stop){ def url = cityUrl + "pg${page}/" - driver.get(url) // def doc = Jsoup.parse(driver.getPageSource()); - def doc = checkDocument(driver, url); + driver = checkDocument(driver, url); + def doc = Jsoup.parse(driver.getPageSource()) def elements = JQueryUtils.select(".listContent li", doc) if(elements){ for(def e : elements){ @@ -203,10 +214,11 @@ public class BeiKeCrawlByChrome { def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd") if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl + println "开始查库 "+new Date().toString() def dbo = findTrade(durl , "贝壳找房") - + println "结束查库"+new Date().toString() if (!dbo) { - parse(city, e, region, driver, null) + driver = parse(city, e, region, driver, null) } // else if(dbo && !dbo.lift){ // parse(city, e, region, driver, dbo._id) @@ -225,7 +237,10 @@ public class BeiKeCrawlByChrome { stop = true; } } - return detailUrls + def m = [:] + m.driver = driver + m.detailUrls = detailUrls + return m } def parse(city,e, region, driver, id){ @@ -233,8 +248,10 @@ public class BeiKeCrawlByChrome { def detailUrl = e.select(".title a").attr("href") driver.get(detailUrl) - def detailDoc = checkDocument(driver, detailUrl); + driver = checkDocument(driver, detailUrl); + def detailDoc = Jsoup.parse(driver.getPageSource()) + println "打开获取页面 "+new Date().toString() def title = e.select(".title a").text() if (title) { def detail = new BasicDBObject() @@ -335,18 +352,21 @@ public class BeiKeCrawlByChrome { detail.create_date = new Date() detail.m_date = new Date() + println "解析页面完成 "+new Date().toString() println detail if(id){ DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail)) }else{ DEAL_PRICE_COLL.save(detail) } + println "保存修改完成 "+new Date().toString() } } }catch(Exception ex){ ex.printStackTrace() println e.select(".title a").attr("href") } + return driver } @@ -469,16 +489,16 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = "https:"+e.attr("href") -// if (citys.contains(e.text())){ + if (citys.contains(e.text())){ list << [e.text(), href] -// } + } } } return list } def getLogin(driver, url){ -// WebDriver driver = new ChromeDriver() + def root = url.toString().substring(url.toString().indexOf("//")+2, url.toString().indexOf(".com")+4) driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252F"+root+"%252Fchengjiao%252F"); WebDriverWait wait = new WebDriverWait(driver,20) @@ -494,17 +514,25 @@ public class BeiKeCrawlByChrome { WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 + elemUsername.sendKeys("17148854005"); + elemPassword.sendKeys("12qwaszx"); + // elemUsername.sendKeys("15021384995"); // elemPassword.sendKeys("12qwaszx"); - elemUsername.sendKeys("16283166371"); - elemPassword.sendKeys("Goojia123456"); +// elemUsername.sendKeys("16283166371"); +// elemPassword.sendKeys("Goojia123456"); btn.submit(); // 点击行为; Thread.sleep(10000) }catch(Exception e){ + driver.close() + driver = new ChromeDriver() Thread.sleep(15000) + driver = getLogin(driver, url) + println "异常Url:"+url } + return driver } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 52e1665f4..e43dddbb0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -34,6 +34,7 @@ public class LianjiaCrawlByChrome { def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + WebDriver driver = null; static main(args) { def task = new LianjiaCrawlByChrome(); @@ -122,7 +123,7 @@ public class LianjiaCrawlByChrome { System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); - WebDriver driver = new ChromeDriver() + driver = new ChromeDriver() def cityMap = getAllCityUrls(driver) // cityMap = cityMap.reverse() driver = getLogin(driver, "https://sh.lianjia.com/") @@ -206,17 +207,11 @@ public class LianjiaCrawlByChrome { } def cityUrl = ret[1] def newUrl = cityUrl+"chengjiao/" - try{ - driver.get(newUrl) - }catch(Exception e){ - println newUrl +" 链接有问题!" - continue - } - +// driver.get(newUrl) //def detailUrls = getPageUrls(driver, cityUrl) - Thread.sleep(10000) - def doc = checkDocument(driver, newUrl); - +// Thread.sleep(10000) + driver = checkDocument(driver, newUrl); + def doc = Jsoup.parse(driver.getPageSource()) def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc) if(regions){ for(def r : regions){ @@ -226,8 +221,9 @@ public class LianjiaCrawlByChrome { regionText = "昆山" } println "开始抓取【${city}】【${regionText}】【${href}】" - def detailUrls = getPageDeail(driver, newCity, href, regionText) - println cityUrl +" 详情链接:"+detailUrls.size() + def map = getPageDeail(driver, newCity, href, regionText) + driver = map.driver + println newUrl +" 详情链接:"+map.detailUrls.size() } } // crawlDetail(driver, DEAL_PRICE_COLL, detailUrls) @@ -267,30 +263,41 @@ public class LianjiaCrawlByChrome { } def checkDocument(driver, url){ + driver.get(url) def s = driver.getPageSource() def flag = true while(flag){ - if(s && (s.contains("无法访问此网站") || s.contains("登录"))){ - driver = getLogin(driver, url) - driver.get(url) - Thread.sleep(5000) - s = driver.getPageSource() - }else{ - flag = false + try{ + if(s && (s.contains("无法访问此网站") || s.contains("登录"))){ + driver = getLogin(driver, url) + driver.get(url) + Thread.sleep(5000) + s = driver.getPageSource() + }else{ + flag = false + } + }catch(Exception e){ + driver.close() + driver = new ChromeDriver() + Thread.sleep(15000) + driver = getLogin(driver, url) + println "异常Url:"+url } } - return Jsoup.parse(s); + return driver; } + def getPageDeail(driver, city, cityUrl, regionText){ def stop = false def page = 1 def detailUrls = [] while(!stop){ def url = cityUrl + "pg${page}/" - driver.get(url) +// driver.get(url) //def detailUrls = getPageUrls(driver, cityUrl) - def doc = checkDocument(driver, url); + driver = checkDocument(driver, url); + def doc = Jsoup.parse(driver.getPageSource()) def elements = JQueryUtils.select(".listContent li", doc) if(elements){ for(def e : elements){ @@ -303,7 +310,7 @@ public class LianjiaCrawlByChrome { detailUrls << durl def dbo = findTrade(durl , "链家在线") if (!dbo) { - parse(city, e, regionText, driver, null) + driver = parse(city, e, regionText, driver, null) } // else if(dbo && !dbo.lift){ // parse(city, e, regionText, driver, dbo._id) @@ -326,7 +333,10 @@ public class LianjiaCrawlByChrome { stop = true; } } - return detailUrls + def m = [:] + m.driver = driver + m.detailUrls = detailUrls + return m } def parse(city,e, regionText, driver, id){ @@ -334,8 +344,11 @@ public class LianjiaCrawlByChrome { def price = e.select(".totalPrice .number").text() if(price){ def detailUrl = e.select(".title a").attr("href") - driver.get(detailUrl) - def detailDoc = checkDocument(driver, detailUrl); + +// driver.get(detailUrl) + driver = checkDocument(driver, detailUrl); + def detailDoc = Jsoup.parse(driver.getPageSource()) + def title = e.select(".title a").text() if (title) { def detail = new BasicDBObject() @@ -450,6 +463,7 @@ public class LianjiaCrawlByChrome { ex.printStackTrace() println e.select(".title a").attr("href") } + return driver; } @@ -571,7 +585,7 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia")){ + if (!href.contains("fang.lianjia") && citys.contains(e.text())){ cityList << [e.text(), href] } } @@ -593,16 +607,23 @@ public class LianjiaCrawlByChrome { WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 -// elemUsername.sendKeys("15021384995"); -// elemPassword.sendKeys("12qwaszx"); + elemUsername.sendKeys("17148854005"); + elemPassword.sendKeys("12qwaszx"); - elemUsername.sendKeys("16283166371"); - elemPassword.sendKeys("Goojia123456"); +// elemUsername.sendKeys("17740818975"); +// elemPassword.sendKeys("13qeadzc"); + +// elemUsername.sendKeys("16283166371"); +// elemPassword.sendKeys("Goojia123456"); btn.submit(); // 点击行为; - Thread.sleep(15000) + Thread.sleep(20000) }catch(Exception e){ - Thread.sleep(10000) + driver.close() + driver = new ChromeDriver() + Thread.sleep(15000) + driver = getLogin(driver, url) + println "异常Url:"+url } return driver } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy new file mode 100644 index 000000000..e313fe216 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -0,0 +1,211 @@ +package com.fangjia.script.gj.xiamen + +import com.fangjia.core.RegionData +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import org.bson.types.ObjectId +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.support.WriteRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.common.xcontent.XContentType + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 厦门银行 + * 福建省,重庆市poi数据处理 + */ +@BatchOwner(Watcher.GaoJuan) +class GaoDePoiDataProcess extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + static main(args) { + runBatch(new GaoDePoiDataProcess(), args) + } + + @Override + void process(Object args) { + //三明市,宁德市,泉州市,厦门市,福州市,漳州市,南平市,龙岩市,莆田市,重庆市 + def citys = ["三明市","漳州市","厦门市","福州市"] + for(def city : citys){ + def results = CommonDatautils.findAmapDetailBySql(city) + println results.size() + def rows = running(results) + save(rows) + println city + " 完成"+new Date().toString() + } + } + + def running(results){ + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(results.size()) + + for(def i=0;i=2){ + map.secondary_type = arrs[1] + } + if(arrs.size()>=3){ + map.small_type = arrs[2] + } + } + + //高德转百度 + def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon) + if(baiduMap){ + def location = [:] + location.lon = baiduMap.lng + location.lat = baiduMap.lat + map.bd_location = location + } + + //高德转WGS84 + def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon) + if(gpsLoactionMap){ + def gpslocation = [:] + gpslocation.lon = gpsLoactionMap.lng + gpslocation.lat = gpsLoactionMap.lat + map.gps_location = gpslocation + } + + def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat) + if(gaodeMap && gaodeMap.township){ + map.township = gaodeMap.township + } + + if(!fjw_region){ + if(gaodeMap && gaodeMap.district){ + fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district) + } + } + if(!fjw_region){ + println fjw_city+" "+map.region+" 无法格式化 "+map.location + " "+gaodeMap + } + map.fjw_region = fjw_region + return map + } + + def save(rows){ + BulkRequest request = new BulkRequest(); + int batchSize = 1000; + int processedCount = 0; + def client = ElasticUtils.getClient("192.168.0.12") + for (int i = 0; i < rows.size(); i++) { + def map = rows[i] + request.add(new IndexRequest("amap_poi", "_doc", map.id).source(map, XContentType.JSON)); + if(i%1000==0){ + print "." + } + + // 批量提交数据 + if (request.numberOfActions() >= batchSize) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + print "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + + request = new BulkRequest(); + } + } + + if (request.numberOfActions() > 0) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + } + + println "结束时间 "+new Date() + } + + + def getTownshipInfo(def glocation){ + def township = null + def district = null + def jsonObject = null + def stop = false + def count = 0 + while(!stop){ + if(glocation){ + jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent){ + township = addressComponent.township + district = addressComponent.district + } + } + } + if(district || count>10){ + stop = true + }else{ + count ++ + Thread.sleep(3000) + print "." + } + } + + def map = [:] + map.district = district + map.township = township + return map; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiAndSeaweedExportCSV.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiAndSeaweedExportCSV.groovy new file mode 100644 index 000000000..d691850c2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiAndSeaweedExportCSV.groovy @@ -0,0 +1,35 @@ +package com.fangjia.script.gj.xiamen + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils + +@BatchOwner(Watcher.GaoJuan) +class PoiAndSeaweedExportCSV extends BaseBatch{ + + static main(args) { + runBatch(new PoiAndSeaweedExportCSV(), args); + } + + @Override + public void process(Object args) { + def citys = ["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"] + + poiExport(citys) + + } + + def poiExport(citys){ + def fileName = "/home/goojia/xiamen/BIGDATA_POI.csv" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\厦门\\BIGDATA_POI.csv" + } + + for(def c : citys){ + def rets = CommonDatautils.findNewAmapDetail(c) + FileUtils.batchCreateFileBySplitChar(rets, fileName, null, "utf-8", true, "@|@"); + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ZhuoXin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ZhuoXin.groovy new file mode 100644 index 000000000..8817fefd3 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ZhuoXin.groovy @@ -0,0 +1,44 @@ +package com.fangjia.script.wenyan.temp + +import com.fangjia.core.Constants +import com.fangjia.core.util.DateUtil +import com.fangjia.core.util.Mongo3Utils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.client.MongoCollection +import net.sf.json.JSONObject +import org.bson.Document + +import static com.mongodb.client.model.Filters.* + +MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") +def startDate = DateUtil.parseDate("20230701", "yyyyMMdd") +def endDate = DateUtil.parseDate("20230801", "yyyyMMdd") +Calendar calendar = Calendar.getInstance() +calendar.setTime(startDate) +def count = 0 +def hashSet = [] +def data = [["编号", "公司名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "返回参数"]] +while (startDate.before(endDate)) { + calendar.add(Calendar.DATE, 1) + def nextDate = calendar.getTime() + count++ + ///property/addressEvaluate /address/financialRating /v2/property/detail + OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("request_uri", "/v2/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> + def result = JSONObject.fromObject(doc.res_map.result) +// if (!hashSet.contains(result)) { + def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] + data << item +// hashSet << result +// } else { +// println("已存在") +// } + } + if (count % 7 == 0) { + hashSet.clear() + } + + startDate = nextDate +} +ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信.xlsx", data, "楼盘评情") + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index 4d301536b..ab5c90233 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -12,7 +12,7 @@ import org.jsoup.Jsoup import java.util.zip.GZIPInputStream class GaoDeApiUtils { - def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390" + def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" /** * 经纬度转地址
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/ZheShangBank.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/ZheShangBank.groovy new file mode 100644 index 000000000..b3af27092 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/zsbank/ZheShangBank.groovy @@ -0,0 +1,83 @@ +package com.fangjia.script.zsbank + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils + +class ZheShangBank { + static main(args) { + def task = new ZheShangBank() +// task.run() +// task.detail() + task.sample() + } + + def sample(){ + def runedCitys = FileUtils.readTxt("D:\\数据\\浙商\\result.txt") + def rets = [] + rets << ["时间","Msg", "城市", "区域", "名称","月份"] + def map = [:] + for(def line : runedCitys){ + def ret = [] + def date = line.substring(line.indexOf("log.")+4, line.indexOf(":")) + def msg = line.substring(line.lastIndexOf("error :")+7, line.size()) + def arrs = msg.split(" ") + ret << date + for(def i=0;i + println m+" "+r + } +// ExcelUtils.write("D:\\数据\\浙商\\浙商失败案例.xlsx", rets, "风险缓释") + } + def run(){ + def start = "2021-03-01" + def rets = [] + rets << ["月份","总调用量","成功量","失败量"] + while (true){ + def end = DateUtils.addMonth(DateUtils.parseFromTightYYYYMMDD(start), 1) +// def total = CommonDatautils.findLogs("1007569", "/address/risk", start, end, null) +// def successCount = CommonDatautils.findLogs("1007569", "/address/risk", start, end, 200) + + def total = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, null) + def successCount = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, 200) + + def failCount = total - successCount + println start[0..6] +" "+ total +" "+successCount +" "+ failCount + rets << [start[0..6], total, successCount, failCount] + start = end; + if(end == "2023-09-01"){ + break; + } + } + ExcelUtils.write("D:\\数据\\浙商\\浙商调用统计.xlsx", rets, "估值地址") + } + + + def detail(){ + def datas = CommonDatautils.findLogs2("1007569", "/property/addressEvaluate") + ExcelUtils.write("D:\\数据\\浙商\\浙商失败案例.xlsx", datas, "估值地址") + } +}