Merge branch 'script_gj' into 'master'

Script gj

See merge request !399
This commit is contained in:
2022-10-11 17:36:27 +08:00
15 changed files with 58 additions and 38 deletions
-8
View File
@@ -242,13 +242,5 @@
<version>1.0.0</version>
<classifier>jdk18</classifier>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.springframework.boot</groupId>-->
<!-- <artifactId>spring-boot-devtools</artifactId>-->
<!-- &lt;!&ndash; optional 表示依赖是否向下传递 true表示不向下传递 默认值是false向下传递 &ndash;&gt;-->
<!-- <optional>true</optional>-->
<!-- </dependency>-->
</dependencies>
</project>
@@ -38,9 +38,7 @@
<a href="${wwwUrl}/resource/industry" target="_blank">行业动态</a>
<a href="${wwwUrl}/resource/analysis" target="_blank">数据分析</a>
<a href="${wwwUrl}/resource/case" target="_blank">成功案例</a>
<a href="https://sh.fangjia.com/" target="_blank">各路客</a>
<a href="http://user.fangjia.com/" target="_blank">房价通</a>
<a href="https://www.didatai.com/" target="_blank">地数</a>
<a href="${wwwUrl}/resource/company" target="_blank">公司动态</a>
</li>
</ul>
</div>
@@ -28,7 +28,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
citys = citys.unique()
def rets = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
@@ -17,8 +17,8 @@ class CcbCommonUtils {
//
public static def citys_all = [ "长春","上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
public static def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
// public static def citys = ["武汉"]
public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
@@ -46,6 +46,24 @@ class CcbCommonUtils {
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
public static def getRunningCitys(def args){
def citys = []
println "args 参数:"+args.toString()
if(args){
def countStr = args[0]
if(countStr=="344"){
citys = CcbCommonUtils.citys_all
}else if(countStr=="92"){
citys = CcbCommonUtils.citys_92
}else if(countStr=="252"){
citys = CcbCommonUtils.citys_252
}
}else{
citys = CcbCommonUtils.citys_92
}
println "运行城市数量:"+citys.size()
return citys;
}
public static def findCityCodeValue(){
@@ -24,7 +24,7 @@ class StagingDistrictBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
for(def city : citys){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"])
@@ -22,15 +22,16 @@ class ExportDistrictRongHe extends BaseBatch{
runBatch(new ExportDistrictRongHe(), args);
}
def citys = CcbCommonUtils.citys
def citys = []
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
@Override
void process(Object args) {
citys = CcbCommonUtils.getRunningCitys(args)
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def new_dists = []//["350604","350605","410307","410308","450181"]
//按天导(每天一个文件夹)
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
if(curDate.toString().endsWith("10")){
exportBigdataCommunityGJRH()//
@@ -167,6 +167,7 @@ class DistrictInfoExportBatch extends BaseBatch{
udate.put(k,v)
}
udate.removeField("_id")
udate.put("m_date", new Date())
FZ_DIST_COLL.update(new BasicDBObject('_id',d.get('_id')), new BasicDBObject('$set', udate));
}else{
FZ_DIST_COLL.save(d)
@@ -154,7 +154,11 @@ class DistrictPriceExportBatch extends BaseBatch{
m << rMap.city
m << rMap.region
m << rMap.name
m << curPrice
if(curPrice){
m << curPrice
}else{
m << 0
}
m << curDate.toInteger();
result << m
}
@@ -98,9 +98,11 @@ class FzDataResultStat {
def dist = CommonDatautils.findDistrictById(list[1])
def flag = "有效"
if(dist.status!=0){
if(dist && dist.status!=0){
flag = "失效"
}
rows << [keys[0], keys[1], keys[2], list[5].toString(), lastPrice.toString(), markup, flag]
}
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202208.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202209.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -390,13 +390,13 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
// if(e.text().contains("福州")){
if(e.text().contains("福州")){
def cityMap = [:]
cityMap.city = e.text()
cityMap.url = "https:"+href
list << cityMap
// cityMap.put(e.text(), "https:"+href)
// }
}
}
}
return list
@@ -31,6 +31,7 @@ public class LianjiaCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
static main(args) {
def task = new LianjiaCrawlByChrome();
@@ -250,7 +251,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-07-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e, regionText)
@@ -469,7 +470,7 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
def citys= ["苏州"]
def citys= ["福州"]
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
@@ -335,8 +335,9 @@ class PaimaiDataSend {
def q = new BasicDBObject()
// q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
q.put("province","江苏")
q.put("sub_type","土地")
// q.put("parser_status", new BasicDBObject('$ne',2))
q.put("url","https://sf-item.taobao.com/sf_item/665795393092.htm")
// q.put("url","https://sf-item.taobao.com/sf_item/668496148492.htm")
//https://sf-item.taobao.com/sf_item/657131607486.htm
// q.put("parser_status", new BasicDBObject('$exists',false))
// q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01")))
@@ -420,10 +421,11 @@ class PaimaiDataSend {
// q.put("city",new BasicDBObject('$in', ["苏州","宿迁","南京"]))
// q.put("city", "上海")
q.put("province","江苏")
q.put("sub_type","土地")
// q.put("parser_status", new BasicDBObject('$ne',1))
// q.put("region","姑苏")
q.put("url","https://susong-item.taobao.com/auction/674736290116.htm")
def dbcursor = ali_zc_coll.find(q).batchSize(5000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
// q.put("url","https://susong-item.taobao.com/auction/674736290116.htm")
def dbcursor = ali_zc_coll.find(q).batchSize(1000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def rets = []
for(def d : dbcursor){
index++
@@ -438,6 +440,7 @@ class PaimaiDataSend {
JSON result = new JSONObject()
result.put("content", d.content)
result.put("sub_type", d.sub_type)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("trades", bid_content)
result.put("sub_count", d.sub_count)
@@ -453,7 +456,7 @@ class PaimaiDataSend {
json.put("result", result)
rets << json
if(rets.size()==10000){
if(rets.size()==1000){
println "共:"+rets.size()
send(rets)
rets.clear()
@@ -522,8 +525,8 @@ class PaimaiDataSend {
def getPaimaiDatas(){
def q = new BasicDBObject()
// q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
q.put("city","苏州")
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-10-09")))
q.put("province","江苏")
q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-10-10")))
def dbcursor = paimai43_COLL.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
@@ -73,11 +73,11 @@ class SeaweedAroundPoi extends BaseBatch {
// void run() {
def city = obj.city
if (obj.lng) {
//def lng = SeaweedUtils.formatLatLng(obj.lng)
//def lat = SeaweedUtils.formatLatLng(obj.lat)
def lng = SeaweedUtils.formatLatLng(obj.lng)
def lat = SeaweedUtils.formatLatLng(obj.lat)
// {'lon': 121.810877, 'lat': 31.737642}
def lng = 121.810877
def lat = 31.737642
// def lng = 121.810877
// def lat = 31.737642
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
@@ -146,13 +146,13 @@ class GaoDeUtils {
while (result.size() < count) {
params.page = "${page}".toString()
page++
//params.key = this.getGaodeKey("v3/place/around")
params.key = "25b2d459fd133e22f8abcebeac972692"
params.key = this.getGaodeKey("v3/place/around")
//params.key = "25b2d459fd133e22f8abcebeac972692"
params.offset = "100"
JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params);
if (json == null) {
//params.key = this.getGaodeKey("v3/place/around")
params.key = "25b2d459fd133e22f8abcebeac972692"
params.key = this.getGaodeKey("v3/place/around")
//params.key = "25b2d459fd133e22f8abcebeac972692"
params.offset = "100"
json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params);
}