diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java index 4c6a22236..dd04da3dd 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DistrictController.java @@ -1265,6 +1265,7 @@ public class DistrictController { district.setAddrAlias(new ArrayList(addrAlias)); } district.setStatus(0); + district.setNameCat(0); district.setCategory("district"); district.setPublisherType(11); district.setCreationDate(new Date()); diff --git a/projects/fangjia-bkoff/src/main/webapp/fjt/add.ftl b/projects/fangjia-bkoff/src/main/webapp/fjt/add.ftl index f08cc8a63..4a70b7978 100644 --- a/projects/fangjia-bkoff/src/main/webapp/fjt/add.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/fjt/add.ftl @@ -176,7 +176,7 @@ table{text-align: left;} } }) - if(role==12){ + if(role==12||role==17){ if(userStr.length>0){ console.log(userStr); $("input[name=roleUsers]").val(userStr); @@ -306,7 +306,7 @@ $("input[name=email]").blur(function(){ $("#role").change(function() { var role = $(this).val(); //�对农行生效 - if (role == 12) { + if (role == 12 || role == 17) { $("#show_chk_all").removeAttr("style"); jQuery.ajax({ type: 'POST', @@ -340,12 +340,12 @@ $("input[name=email]").blur(function(){ } return false; } - <#if user?? && user.role == 12> + <#if user?? && (user.role == 12 || user.role == 17)> $("#show_chk_all").removeAttr("style"); jQuery.ajax({ type: 'POST', url: '../fjt/getUsersByRole', - data: {role:12}, + data: {role:${user.role}}, dataType: 'json', timeout: 100000, success: function(model) { diff --git a/projects/fangjia-bkoff/src/main/webapp/yugudan/editEval.ftl b/projects/fangjia-bkoff/src/main/webapp/yugudan/editEval.ftl index eab869c86..53fd8f659 100644 --- a/projects/fangjia-bkoff/src/main/webapp/yugudan/editEval.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/yugudan/editEval.ftl @@ -1339,7 +1339,7 @@ 室� <#if evaluation.buildType!=3 && evaluation.buildType!=8 && evaluation.buildType!=0> - <#if evaluation.floorDesc!=""> + <#if evaluation.floorDesc?? && evaluation.floorDesc!=""> ${evaluation.floorDesc!} diff --git a/projects/fangjia-coresite/src/main/webapp/company/about.ftl b/projects/fangjia-coresite/src/main/webapp/company/about.ftl index 39fee4100..778557075 100644 --- a/projects/fangjia-coresite/src/main/webapp/company/about.ftl +++ b/projects/fangjia-coresite/src/main/webapp/company/about.ftl @@ -54,7 +54,6 @@

房价网�立于2006年,隶属于上海估家网络科技有�公�,致力于建立中国领先的房产大数�库CPDB�推动基于地��置的人工智能的�展和应用。公�以“�现地�价值�为使命,始终秉承客观严谨的�度为政府�银行�金�机构等客户�供大数��建模�咨询等�务。目�,所积累的房产数�已覆盖全国367个城市�地区,技术力�强大,客户查询�行业领先。

公�在贷�估值�内评系统�贷�批��估�信贷授信�地��欺诈�数�治��精准�销等领域都有�丰富的�验,拥有多项专利,被评为高新技术�业,赢得了国家统计局�建设银行�农业银行�平安集团�微众银行等众多客户的信赖。

-

2020年,�京三悦网络科技有�公�与我�达�全�战略�作,并接�我�委托,全�负责我�产�手机版“房价网�公众�和电脑版的“房价通�的市场推广�客户开�与售��务;有�直接与客户签订上述产�的销售��;我�对其所有用户全��供产��级�数�更新�线上售�等�务。

数�为大众�务
diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java index afba53e15..32e526e32 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java @@ -234,6 +234,8 @@ public class District implements SeaweedEntity { private String mainTypeArea; private String createYear; + + private Integer nameCat; @Column(name = "source_web") public String getSourceWeb() { @@ -1879,9 +1881,13 @@ public class District implements SeaweedEntity { public void setTownship(String township) { this.township = township; } - - - - - + + @Column(name = "name_cat") + public Integer getNameCat() { + return nameCat; + } + + public void setNameCat(Integer nameCat) { + this.nameCat = nameCat; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/baidu/CityLatlngBox.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/baidu/CityLatlngBox.groovy index 55204f53b..631b76733 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/baidu/CityLatlngBox.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/baidu/CityLatlngBox.groovy @@ -13,8 +13,9 @@ import com.mongodb.DBObject */ class CityLatlngBox { - public static final DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.105", 27018) - public static final DBCollection LATLNG_COLL = MongoUtils.getCollectionByName("addrepo", "city_latlng_range", "192.168.0.219", 27017) + //public static final DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.105", 27018) + DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.235", 27018) + public static final DBCollection LATLNG_COLL = MongoUtils.getCollectionByName("addrepo", "city_latlng_range", "192.168.0.206", 27020) public static void main(String[] args) { CityLatlngBox sd = new CityLatlngBox(); @@ -22,7 +23,7 @@ class CityLatlngBox { } void process() { - def cities = CommonUtils.findAllCity() + def cities = ["°×ɳ","°¢°ÓÖÝ","¸Ê×ÎÖÝ","°¢ÀÕÌ©","ÁÙ²×","ÄÇÇú","Èýɳ","¿¦Ê²","²ý¼ª","µÏÇìÖÝ","ÆÕ¶ý","µÂºêÖÝ","ÏæÎ÷","Ëþ³Ç","Á¹É½ÖÝ","ÍͲý","ɽÄÏ","º£Î÷ÖÝ","°¢À­ÉÆÃË","ÀÖ¶«","¶¨°²","Å­½­ÖÝ","»ÆÉ½","Í¼Ä¾Êæ¿Ë","³þÐÛ","ÁÙ¸ß","ÎÚÀ¼²ì²¼","ÑÓ±ß","ÎÄɽÖÝ","ÎýÁÖ¹ùÀÕÃË","ÁêË®","ÌìÃÅ","»´±±","Ç­¶«ÄÏ","ÍòÄþ","ÏæÌ¶","º£±±ÖÝ","ÇíÖÐ","ͳ·¬","»ÝÖÝ","°ÍÒô¹ùÀã","¶÷Ê©","Í­ÈÊ","Ç­ÄÏ","Î÷Ë«°æÄÉÖÝ","DZ½­","ÁÖÖ¥","ÎIJý","ÏÉÌÒ","ÉñÅ©¼Ü","ºÍÌï","Ç­Î÷ÄÏ","¹ûÂåÖÝ","ÙÙÖÝ","¹þÃÜ","´óÀí","¶«·½","ÈÕ¿¦Ôò","Ð˰²ÃË","²ý½­","³ÎÂõ","±£Í¤","¼ÃÔ´","Îåָɽ","°¢À­¶û","ÒÁÀç","ÁÙÏÄ","º£ÄÏÖÝ","Çíº£","ʯºÓ×Ó","²ý¶¼","Îå¼ÒÇþ","ÓñÊ÷ÖÝ","²©¶ûËþÀ­","¶«Óª","Í­Áê","»ÆÄÏÖÝ","ÀÈ·»","¼ÎÓø¹Ø","Âí°°É½","Äþ²¨","¹ã°²","ÖÐɽ","¿Ë×ÎÀÕËÕ","ÏÃÃÅ","À³Îß","¶õÖÝ","¶«Ý¸","º£¿Ú","º×±Ú","ÁùÅÌË®","äðºÓ","ÈýÑÇ","Ö麣","ÉîÛÚ","ÎÚº£","»´ÄÏ","±±º£","˫Ѽɽ","ÏåÑô","ÐÂÓà","ÉÇÍ·","¿ËÀ­ÂêÒÀ","ӥ̶","Å̽õ","Õò½­","ʯ×ìɽ","¼ÎÐË","Í­´¨","½¹×÷","Îߺþ","³±ÖÝ","ƼÏç","ºìºÓ","»ÆÊ¯","ÁÉÔ´","ÁÉÑô","Ôæ×¯","±Ï½Ú","Ðí²ý","ÑôȪ","×Ô¹±","ËìÄþ","¾°µÂÕò","Óª¿Ú","ÈÕÕÕ","ÆÎÌï"]//CommonUtils.findAllCity() for (String city : cities) { println city BOUNDS_COLL.remove(new BasicDBObject('city', city)) @@ -70,13 +71,13 @@ class CityLatlngBox { int row = 0 int inx = 0 def buffer = [] - while(seedLatBD < cityBoundBD.getMaxLat()) { + while(seedLatGD < cityBoundGD.getMaxLat()) { int col = 0 Bounds boundsBD = null; Bounds boundsQQ = null; Bounds boundsGD = null; - while (seedLngBD < cityBoundBD.getMaxLng()) { + while (seedLngGD < cityBoundGD.getMaxLng()) { double[] espBD = GeoUtils.getLatLon(seedLngBD, seedLatBD, offset, 90) // ÓÒ²àOFFSET´¦µÄ¾­Î³¶È double[] wnpBD = GeoUtils.getLatLon(seedLngBD, seedLatBD, offset, 360) // ÉÏ·½OFFSET´¦µÄ¾­Î³¶È diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index c1126334f..920c0d4fd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -39,7 +39,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ println city +" "+rows.size() } def headers = ["³ÇÊÐID","Ð¡ÇøID"," ½¨ÖþÀàÐÍ","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾","Êý¾Ýʱ¼ä"] - FileUtils.batchCreateFile(rets, "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\bigdata_commbuildtype.txt",headers, "utf-8", false); + FileUtils.batchCreateFile(rets, "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\BIGDATA_COMMBUILDTYPE_GJ.dat",headers, "utf-8", false); println "txt Éú³É" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index c2efd484b..ef175956f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -12,6 +12,7 @@ import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.OpenApiClient +import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -46,16 +47,19 @@ class BigdataCommunityBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"]) println city +" Ð¡ÇøÁ¿£º"+dists.size() + + def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) + rets = addDeleteDist(dists, ccbMaps, rets) rows.addAll(rets) println city +" "+rets.size() // FileUtils.batchCreateFile(rets, "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); } - def fileName = "/home/goojia/ccb/bigdata_community.dat" + def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\bigdata_community.dat" + fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\BIGDATA_COMMUNITY_GJ.dat" } def allDatas = [] @@ -66,8 +70,27 @@ class BigdataCommunityBatch extends BaseBatch{ // ExcelUtils.write("d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\excel\\bigdata_community.xlsx", allDatas, "Êý¾Ý") } + def addDeleteDist(dists, ccbMaps, rets){ + def fangjiaDists = [] + for(def d : dists){ + fangjiaDists << d._id.toString() + } + println "·¿¼ÛÍøÐ¡Çø£º"+fangjiaDists.size()+" ½¨ÐÐÐ¡Çø£º"+rets.size() + def ccbDists = ccbMaps.keySet() + for(def ccbId : ccbDists){ + if(!fangjiaDists.contains(ccbId)){ + def datas = ccbMaps.get(ccbId) + datas[datas.size()-1] = 1 + rets << datas + } + } + println "Ìí¼Óɾ³ý×´Ì¬Ð¡Çø£º"+rets.size() + return rets + } + def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){ - def rets = Collections.synchronizedList(new ArrayList()); + //def rets = Collections.synchronizedList(new ArrayList()); + def rets = new CopyOnWriteArrayList() def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(dists.size()) for(def i=0;i=startPrice && obj.price<=endPrice){ - rets << obj; - } - - } + if(obj.fjName && ["¹«Ô¢","ÁªÁÐסլ","»¨Ô°×¡Õ¬","±ðÊû","ÀïŪ·¿","й¤·¿","ÆäËü"].contains(obj.houseType)){ + rets << obj; +// def price = statMap.get(obj.fjRegion+"@"+obj.fjName) +// if(price){ +// def startPrice = 0 +// def endPrice= 0 +// if(obj.fjCity=="ÉϺ£"){ +// startPrice = (int)(price - price*0.1) +// endPrice = (int)(price*0.5 + price) +// }else{ +// startPrice = (int)(price - price*0.5) +// endPrice = (int)(price*0.5 + price) +// } +// if(obj.price>=startPrice && obj.price<=endPrice){ +// rets << obj; +// }else{ +// index++ +// println index + " "+obj +// } +// } } } return rets; @@ -486,4 +525,19 @@ class BigdataDealBatch extends BaseBatch{ return map } + def getPpeGTE15Datas(def city){ + def sql = "select unit_price,commid from avm_evaluation_item_gj where data_type='³É½»' and pe>=15 and pe<=20 " + + "and cityname='"+city+"'"; + def map = [:] + try{ + sqlInstance.eachRow(sql,{ + map.put(it.commid, it.unit_price) + }); + }catch(Exception e){ + getPpeGTE15Datas(city) + } + + return map + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy index 815336ae3..c3affa2bd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy @@ -40,8 +40,8 @@ class BigdataDistrictBatch extends BaseBatch{ def rets = [] rets << headers rets.addAll(allDatas) - ExcelUtils.write("d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\excel\\bigdata_district.xlsx", rets, "Êý¾Ý") - FileUtils.batchCreateFile(allDatas, "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\bigdata_district.txt",headers, "utf-8",false); + ExcelUtils.write("d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\excel\\BIGDATA_DISTRICT_GJ.xlsx", rets, "Êý¾Ý") + FileUtils.batchCreateFile(allDatas, "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\BIGDATA_DISTRICT_GJ.dat",headers, "utf-8",false); } def createDatas(rows){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index b8af43278..2cea78fa5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -55,7 +55,7 @@ class BigdataForsaleBatch extends BaseBatch{ } public static final String[] TAGS = ["¶À¶°", "ÁªÅÅ", "µþ¼Ó", "˫ƴ", "»¨Ô°Ãæ»ý", "Ó¾³Ø", "±ðÊû", "´ø»¨Ô°", "Ñó·¿", "µþÆ´", "¶ÀÔº", "ÁªÌå", "΢Êû", "´óƽÊû"]; - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -66,15 +66,16 @@ class BigdataForsaleBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; def index = 0 + def citys = CcbCommonUtils.citys; def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210831); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20211018); + // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); } } @@ -95,9 +96,9 @@ class BigdataForsaleBatch extends BaseBatch{ def commId = rMap.get("_id").toString() def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias) - if(ret && ret.size()<100){ - ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) - } +// if(ret && ret.size()<100){ +// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) +// } if(ret){ def areaData = distCodeMap.get(rMap.region) if(areaData){ @@ -144,9 +145,9 @@ class BigdataForsaleBatch extends BaseBatch{ } latch.await() - def fileName = "/home/goojia/ccb/bigdata_forsale.dat" + def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\bigdata_forsale.dat" + fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\BIGDATA_FORSALE_GJ.dat" } println city + " ·¿Ô´×ÜÁ¿£º"+retMap.size() @@ -158,8 +159,6 @@ class BigdataForsaleBatch extends BaseBatch{ } FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true); println "txtÎļþÉú³É " - - } @@ -284,6 +283,7 @@ class BigdataForsaleBatch extends BaseBatch{ def rets = [] def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + if(houses){ for (int k=0;k=startPrice && d.avg_price<=endPrice){ + def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") + if(!rets.contains(key)){ + rets << key + d.dist_id = dist._id.toString() + d.address = dist.address + d.master_name = dist.name + d.build_type = dist.build_type + d.build_structure = dist.build_structure + d.house_roperty = dist.house_roperty + d.property_life = dist.property_life + d.estate_type2 = dist.estate_type2 + lists << d + } +// } + } + return lists + } + + public static def findTradesDetailByAddress(dist, startDate, endDate, startPrice, endPrice){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + q.put("area", new BasicDBObject('$gte', 20)) + q.put("district_name", dist.address) q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) def dbcursor = DEAL_PRICE_COLL.find(q) def total = dbcursor.count() @@ -168,13 +225,63 @@ class CcbCommonUtils { return dataStoreConnection } + public static def findAllDistrictIdsByOracle(def cityid){ + def sql = "select commid from bigdata_community_gj where cityId=${cityid} and status=0"; + def set = new HashSet() + sqlInstance.eachRow(sql,{r-> + set << r.COMMID + }); + return set + } + + public static def findAllDistrictsByOracle(def cityid){ + def sql = "select CITYID,COMMID,COMMNAME,COMMALIAS,DISTID,STREET,STREETNO,ADDRESS,LANDAREA,BUILDAREA,COMMTYPE,IFNEW,GREENINGRATE,PLOTRATIO,PARKINGCNT,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,WORKDATE,ZTS,LONGITUDE,LATITUDE from bigdata_community_gj where cityId='${cityid}' and status=0 "; + def rows = [] + try{ + sqlInstance.eachRow(sql,{r-> + def map = [:] + map.cityid = r.CITYID + map.commid = r.COMMID + map.commname = r.COMMNAME + map.distid = r.DISTID + map.street = r.STREET + map.streetno = r.STREETNO + map.address = r.ADDRESS + map.landarea = r.LANDAREA + map.buildarea = r.BUILDAREA + map.commtype = r.COMMTYPE + map.ifnew = r.IFNEW + map.greeningrate = r.GREENINGRATE + map.plotratio = r.PLOTRATIO + map.parkingcnt = r.PARKINGCNT + map.introduction = "" + map.remark = r.REMARK + map.createtime = r.CREATETIME + map.updatetime = r.UPDATETIME + map.sourcecode = r.SOURCECODE + map.workdate = r.WORKDATE + map.zts = r.ZTS + map.lng = r.LONGITUDE + map.lat = r.LATITUDE + rows << map + }); + }catch(Exception e){ + sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + findAllDistrictsByOracle(cityid) + println "´íÎó£º"+e.getMessage() + } + + return rows + } + public static def findAllDistrictByCity(def city, def cat, def fields){ DBObject q = new BasicDBObject(); q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name", "ÃÀ³½ÏãÐÑ") +// q.put("name","½­ÄÏÈ˼Ò") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["ÆÕͨסլ","ÉÌס"])) @@ -221,4 +328,48 @@ class CcbCommonUtils { return null; } + + public static def findCcbDistrict(cityid){ + def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" + + "\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj where cityId=${cityid}"; + def map = [:] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.DISTNAME + data << r.COMMNAME + data << r.ADDRESS + data << r.LATITUDE + data << r.LONGITUDE + data << r.LANDAREA + data << r.BUILDAREA + data << r.GREENINGRATE + data << r.PLOTRATIO + data << r.PARKINGCNT + data << r.BUILDYEAR + data << r.HOUSEUSE + data << r.ZTS + data << r.WORKDATE + data << r.UPDATEFLAG + data << r.UPDATETIME + data << r.COMMTYPE + data << r.COMMALIAS + data << r.STREET + data << r.STREETNO + data << r.IFNEW + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.SOURCECODE + data << r.XZJDB + data << r.LJXZQHDM + data << r.LJXZQH + data << r.STATUS + map.put(r.COMMID, data) + }); + return map + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy new file mode 100644 index 000000000..329c4e425 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy @@ -0,0 +1,58 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.jiangnanblank.SFTP +import com.jcraft.jsch.ChannelSftp +import org.apache.commons.codec.digest.DigestUtils + +class CcbFtpUtils { + + static def ftpServer = "192.168.0.45" + static def ftpPort = 17495 + static def ftpUser = "ccb" + static def ftpPassword = "ccb!@#" + static def ftpPath = "/home/ccb/output/" + + static SFTP sftp = new SFTP(); + static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword); + + + static def ftpUpload(outputPath_dist, month){ + //ÉÏ´«µ½ftp + sftpChannel = reConnect(); + sftp.upload(ftpPath+ month+"/", outputPath_dist, sftpChannel); + } + + static def reConnect(){ + if(sftpChannel == null){ + println "sftpChannel Ϊ¿Õ£¬ÐèÖØÁ¬....." + if(sftp == null){ + sftp = new SFTP(); + } + sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword); + }else{ + println "sftpChannel Õý³£....." + } + return sftpChannel; + } + + static def uploadSha512Data(def sb, def dataCodeFileName, def charset){ + String sha512HexStr= DigestUtils.sha512Hex(sb.getBytes())//Éú³É¼ÓÃÜÂëÂë + DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//Éú³É¼ÓÃÜÂëÎļþ + ftpUpload(dataCodeFileName)//ÉÏ´«¼ÓÃÜÂëÎļþµ½FTP + } + + static def uploadSha512Data2(def dataFileName, def dataCodeFileName, def charset){ + + def data = new FileInputStream(new File(dataFileName)) + String sha512HexStr= DigestUtils.sha512Hex(data)//Éú³É¼ÓÃÜÂëÂë + DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//Éú³É¼ÓÃÜÂëÎļþ + ftpUpload(dataCodeFileName)//ÉÏ´«¼ÓÃÜÂëÎļþµ½FTP + } + + + static main(args) { + + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy index f340f72ab..1dbc5cd4d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy @@ -43,7 +43,9 @@ class DistrictDelete { static main(args) { def task = new DistrictDelete() // task.run() - task.gzRun(); +// task.gzRun(); +// task.deleteDistrict() + task.deleteCommPrice() } def run(){ @@ -211,9 +213,9 @@ class DistrictDelete { def gzRun(){ def index = 0 - ExcelUtils.modify("D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\¾­Î³¶ÈÓÐÎÊÌâµÄÐ¡Çø.xlsx",{d-> - if(d['U'] && d['U'].toString().toDouble().toInteger()==1){ - def id = d['E'] + ExcelUtils.modify("D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Ҫɾ³ýÐ¡Çø (5).xlsx",{d-> + if(d['G']){ + def id = d['G'] BasicDBObject q=new BasicDBObject('name_cat',1) q.put("m_date", new Date()) DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q)) @@ -223,4 +225,38 @@ class DistrictDelete { },0) } + def deleteDistrict(){ + ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Ҫɾ³ýÐ¡Çø.xlsx",{d-> + if(d['G']){ + def id = d['G'] + BasicDBObject q=new BasicDBObject('name_cat',1) + q.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q)) + deleteAllRefDistrict(id) + } + },0) + } + + def deleteAllRefDistrict(id){ + def tables = ["bigdata_forsale_gj","avm_evaluation_item_gj","community_base_price_gj","community_poi_gj","community_sale_gj","community_sale_tmp","community_test_price_tmp"] + for(def tableName : tables){ + deleteByCommid(tableName, id) + } + } + + def deleteCommPrice(){ + ExcelUtils.modify("D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\¾ù¼Ûɾ³ý.xlsx",{d-> + if(d['B']){ + def id = d['B'] + println id + deleteByCommid("community_base_price_gj",id) + } + },0) + } + + def deleteByCommid(tableName, commid){ + def sql = "delete from "+tableName+" where commid='${commid}'"; + sqlInstance.execute(sql) + println "ɾ³ý£º"+tableName+" "+commid + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy new file mode 100644 index 000000000..b64fdcd1a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy @@ -0,0 +1,85 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils + + +@BatchOwner(Watcher.GaoJuan) +class DistrictGuidePriceBatch extends BaseBatch{ + + + static main(args) { + runBatch(new DistrictGuidePriceBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.citys; + def rows = [] + + for (def city : citys) { + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + + //Ìí¼Ó×Ô¼º¿âÖеÄÈ˹¤Ö¸µ¼¼Û + def prices = CcbCommonUtils.findGuidePrice(city) + if(prices){ + def distMap = getDistrictMap(city) + def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) + rows.addAll(rets) + println city + " " + rets.size() + } + } + + def headers = ["µÚÈý·½¾ù¼ÛID","Â¥ÅÌID","Ð¡ÇøÃû³Æ","³ÇÊÐID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","Ö¸µ¼¼Û¸ñ","ÀàÐÍ","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾"] + FileUtils.batchCreateFile(rows, "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\COMMUNITY_GUIDE_PRICE_GJ.dat", headers, "utf-8", false); + + } + + + def getDistrictMap(def city){ + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]); + def distMap = [:] + for(def d : districts){ + distMap.put(d.region+"##"+d.name, d) + } + return distMap; + } + + def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){ + def rets = [] + if(prices){ + for(def ps : prices){ + def dist = distMap.get(ps.match_region+"##"+ps.match_name) + if(dist){ + def ret = [] + def areaData = distCodeMap.get(dist.region) + if(areaData){ + ret << ps._id.toString() + ret << dist._id.toString() + ret << dist.name + ret << cityCodeDatas.get(dist.city) + ret << dist.city + ret << areaData.district_code + ret << areaData.district + ret << ps.avg_price + ret << ps.source_type + ret << "" + ret << DateUtils.formatDateByPattern(ps.c_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(ps.c_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret + } + }else{ + println ps.match_region+"##"+ps.match_name + } + } + } + return rets; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy index d64c08dd6..afae30ba4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy @@ -29,8 +29,8 @@ class EvaluateRateBatch extends BaseBatch{ def rets = [] rets << headers rets.addAll(allDatas) - ExcelUtils.write("d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\excel\\evaluate_rate.xlsx", rets, "Êý¾Ý") - FileUtils.batchCreateFile(allDatas, "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\evaluate_rate.txt",headers, "utf-8", false); + ExcelUtils.write("d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\excel\\EVALUATE_RATE_GJ.xlsx", rets, "Êý¾Ý") + FileUtils.batchCreateFile(allDatas, "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\EVALUATE_RATE_GJ.dat",headers, "utf-8", false); } def createDatas(def datas){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy new file mode 100644 index 000000000..792374cd7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -0,0 +1,808 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.ccbevaluation.CcbFtpUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import groovy.sql.Sql + +import java.sql.ResultSet + +class ExportOracleData { + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new ExportOracleData() + task.run() + } + + def run() { +// exportEvaluationRateData() +// exportBigdataPoiData() +// exportCommunityStaging() +// exportCommunityBuileType() +// exportBigdataDistrictData() +// exportBigdataCommunity() +// exportGuidedPrice() + exportBigdataForsale() +// exportThirdPartyPrice() + +// exportThirdPartyPrice2() +// exportBigdataDeal() + +// exportCommunityGuidedPrice() +// exportAvmEvaluationGJNB() +// exportAVMEvaluationItemGJNB() +// exportCommunityBasePrice() +// exportCommunityBasePriceGj() +// exportCityPrice() +// exportRegionPrice() +// exportLastFile() + } + + + def exportEvaluationRateData() { + def sql = "select ID,RATE_TYPE,RATE,RANK,CITY,REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from evaluate_rate_gj"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.RATE_TYPE + data << r.RATE + data << r.RANK + data << r.CITY + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/output/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["ϵÊýID","ϵÊýÀàÐÍ","ϵÊý","ÅÅÐò","³ÇÊÐ","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "ÉÏ´«FtpÍê³É......" + } + + + def exportBigdataPoiData() { + def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITYID + data << r.DISTID + data << r.ID + data << r.POI_NAME + data << r.TYPE + data << r.TYPE_CODE + data << r.LATITUDE + data << r.LONGITUDE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["³ÇÊÐID","ÐÐÕþÇøID","ID","PoiÃû³Æ","ÀàÐÍ","ÀàÐͱàÂë","γ¶È","¾­¶È","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_POI_GJ Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "BIGDATA_POI_GJ ÉÏ´«FtpÍê³É......" + } + + def exportCommunityStaging() { + def sql = "select ID,COMMID,MASTER_COMMID,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_staging_gj"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.MASTER_COMMID + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["·ÖÆÚID","Ð¡ÇøID","Ð¡ÇøÖ÷ÆÚID","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_STAGING_GJ Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "COMMUNITY_STAGING_GJ ÉÏ´«FtpÍê³É......" + } + + def exportCommunityBuileType() { + def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.WORKDATE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["³ÇÊÐID","Ð¡ÇøID"," ½¨ÖþÀàÐÍ","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾","Êý¾Ýʱ¼ä"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_COMMBUILDTYPE_GJ Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "BIGDATA_COMMBUILDTYPE_GJ ÉÏ´«FtpÍê³É......" + } + + def exportBigdataDistrictData() { + + def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.DISTID + data << r.DISTNAME + data << r.DISTSPELL + data << r.CITYID + data << r.DISTCODE + data << r.SEQ + data << r.DISTTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.LJXZQH + data << r.LJXZQHDM + data << r.XZQHDM + data << r.WORKDATE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + } + def headers = [] +//["ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","ÐÐÕþÇøÃû³ÆÆ´Òô","³ÇÊÐID","¹ú±êÂë","ÏÔʾ˳Ðò","ÐÐÕþÇøÀà±ð","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾","Âß¼­ÐÐÕþÇø","Âß¼­ÐÐÕþÇø»®´úÂë","ÐÐÕþÇø»®´úÂë","Êý¾Ýʱ¼ä"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_DISTRICT_GJ Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "BIGDATA_DISTRICT_GJ ÉÏ´«FtpÍê³É......" + } + + + def exportGuidedPrice() { + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj "; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.DEALDATE + data << r.AVG_PRICE + data << r.GUIDING_PERSON + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] +//["µÚÈý·½¾ù¼ÛID","Â¥ÅÌID","Ð¡ÇøÃû³Æ","³ÇÊÐID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","Ö¸µ¼ÈÕÆÚ","Ö¸µ¼¼Û¸ñ","Ìí¼ÓÈË","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "GUIDED_PRICE_GJ Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "GUIDED_PRICE_GJ ÉÏ´«FtpÍê³É......" + } + + def exportCommunityGuidedPrice() { + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.AVGPRICE + data << r.SOURCETYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["µÚÈý·½¾ù¼ÛID","Â¥ÅÌID","Ð¡ÇøÃû³Æ","³ÇÊÐID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","Ö¸µ¼¼Û¸ñ","ÀàÐÍ","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_GUIDE_PRICE_GJ Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "COMMUNITY_GUIDE_PRICE_GJ ÉÏ´«FtpÍê³É......" + } + + def exportBigdataDeal() { + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITYID + data << r.DEALID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.STREET + data << r.STREETNO + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.PERIODNUM + data << r.FLOORNUM + data << r.FLOORTYPE + data << r.FLOORCNT + data << r.BUILDTYPE + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEUSE + data << r.HOUSESTRU + data << r.HOUSEASPECT + data << r.FITMENT + data << r.EVALUATENO + data << r.DEALUNITPRICE + data << r.DEALTOTALPRICE + data << r.DECLAREPRICE + data << r.EVALUATEPRICE + data << r.CONTRACTNO + data << r.DEALDATE + data << r.DEALTYPE + data << r.PAYTYPE + data << r.DEALPERIOD + data << r.IFELEVATOR + data << r.COMPAREAGARDEN + data << r.COMPAREATERRACE + data << r.COMPAREABALCONY + data << r.COMPAREABAYWINDOW + data << r.COMPAREAATTIC + data << r.COMPAREAGARAGE + data << r.COMPAREABASEMENT + data << r.COMPAREAOTHER + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.CJJE + data << r.WORKDATE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] +//["³ÇÊÐID","³É½»ID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","½ÖµÀ","ÃÅÅÆºÅ","Ð¡ÇøID","Ð¡ÇøÃû³Æ","Â¥¶°ID","Â¥¶°±àºÅ","Â¥¶°Ãû³Æ","µ¥ÔªID","µ¥Ôª±àºÅ","µ¥ÔªÃû³Æ","»§ID","ÏêϸµØÖ·","½¨ÖþÄê´ú","ȨÊô","ȨÊôÄêÏÞ","ÆÚÊý","ËùÔÚÂ¥²ã","²ãÀàÐÍ","×ÜÂ¥²ãÊý","½¨ÖþÀàÐÍ","½¨Öþ½á¹¹","½¨ÖþÃæ»ý","Ì×ÄÚÃæ»ý","»§ÐÍ","ÊÒ","Ìü","³ø","ÎÀ","·¿ÎÝÓÃ;","ÊÒÄڽṹ","³¯Ïò","×°ÐÞÇé¿ö","ÆÀ¹À±àºÅ","³É½»µ¥¼Û","³É½»×ܼÛ","É걨×ܼÛ","ÆÀ¹À×ܼÛ","ÍøÇ©ºÏͬ±àºÅ","½»Ò×ÈÕÆÚ","½»Ò×ÀàÐÍ","³öÊÛ¸¶¿î·½Ê½","³É½»ÖÜÆÚ","ÊÇ·ñÓеçÌÝ","ÔùËÍÃæ»ý£¨µØÃ滨԰£©","ÔùËÍÃæ»ý£¨Â¶Ì¨£©","ÔùËÍÃæ»ý£¨Ñǫ̂£©","ÔùËÍÃæ»ý£¨Æ®´°£©","ÔùËÍÃæ»ý£¨¸óÂ¥£©","ÔùËÍÃæ»ý£¨³µ¿â£©","ÔùËÍÃæ»ý£¨µØÏÂÊÒ£©","ÔùËÍÃæ»ý£¨ÆäËû£©","·¿Ô´Êý¾ÝÀ´Ô´","Öн鹫˾ID","Öн鹫˾Ãû³Æ","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾","³É½»½ð¶î","Êý¾Ýʱ¼ä"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_DEAL_GJ Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "BIGDATA_DEAL_GJ ÉÏ´«FtpÍê³É......" + } + + + def exportThirdPartyPrice() { + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, \n" + + "TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE\n" + + "from third_party_price_gj"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.YEAR_MONTH + data << r.ZHUGE_AVG_PRICE + data << r.ANJUKE_AVG_PRICE + data << r.FANG_AVG_PRICE + data << r.LIANJIA_AVG_PRICE + data << r.CITYHOUSE_AVG_PRICE + data << r.MAITIAN_AVG_PRICE + data << r.KE_AVG_PRICE + data << r.TAOFANG_AVG_PRICE + data << r.ZHONGYUAN_AVG_PRICE + data << r.TBS_AVG_PRICE + data << r.QFANG_AVG_PRICE + data << r.FANGTAN_AVG_PRICE + data << r.FANGDUODUO_AVG_PRICE + data << r.WIWJ_AVG_PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE.replaceAll(~'\\r', '') + allRows << data + }) + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] +//["µÚÈý·½¾ù¼ÛID","Â¥ÅÌID","Ð¡ÇøÃû³Æ","³ÇÊÐID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","¾ù¼Û¶ÔÓ¦µÄÔ·Ý","Öî¸ðÕÒ·¿¾ù¼Û","°²¾Ó¿Í¾ù¼Û","·¿ÌìϾù¼Û","Á´¼Ò¾ù¼Û","³ÇÊз¿²ú¾ù¼Û","ÂóÌï¾ù¼Û","±´¿Ç¾ù¼Û","ÌÔ·¿365","ÖÐÔ­¾ù¼Û","Íò©Ê¿¾ù¼Û","Q·¿¾ù¼Û","·¿Ì½¾ù¼Û","·¿¶à¶à¾ù¼Û","ÎÒ°®ÎÒ¼Ò¾ù¼Û","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "THIRD_PARTY_PRICE_GJ Êý¾ÝÎļþÊä³öÍê³É......" + //CcbFtpUtils.ftpUpload(fileName, curDate) + } + + + def exportThirdPartyPrice2() { + def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from third_party_price"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.COMMNAME + data << r.DATA_SOURCE_ID + data << r.DATA_SOURCE + data << r.PRICE_DATE + data << r.COMMUNITY_PRICE + data << r.COMMUNITY_MAX_PRICE + data << r.COMMUNITY_MIN_PRICE + data << r.UPDATETIME + allRows << data + }) + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] +//["µÚÈý·½¾ù¼ÛID","Â¥ÅÌID","Ð¡ÇøÃû³Æ","³ÇÊÐID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","¾ù¼Û¶ÔÓ¦µÄÔ·Ý","Öî¸ðÕÒ·¿¾ù¼Û","°²¾Ó¿Í¾ù¼Û","·¿ÌìϾù¼Û","Á´¼Ò¾ù¼Û","³ÇÊз¿²ú¾ù¼Û","ÂóÌï¾ù¼Û","±´¿Ç¾ù¼Û","ÌÔ·¿365","ÖÐÔ­¾ù¼Û","Íò©Ê¿¾ù¼Û","Q·¿¾ù¼Û","·¿Ì½¾ù¼Û","·¿¶à¶à¾ù¼Û","ÎÒ°®ÎÒ¼Ò¾ù¼Û","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "THIRD_PARTY_PRICE_GJ Êý¾ÝÎļþÊä³öÍê³É......" + //CcbFtpUtils.ftpUpload(fileName, curDate) + } + + + def exportBigdataCommunity(){ + def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" + + "\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj ";//where cityId in (350200,150100) + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.DISTNAME + data << r.COMMNAME + data << r.ADDRESS + data << r.LATITUDE + data << r.LONGITUDE + data << r.LANDAREA + data << r.BUILDAREA + data << r.GREENINGRATE + data << r.PLOTRATIO + data << r.PARKINGCNT + data << r.BUILDYEAR + data << r.HOUSEUSE + data << r.ZTS + data << r.WORKDATE + data << r.UPDATEFLAG + data << r.UPDATETIME + data << r.COMMTYPE + data << r.COMMALIAS + data << r.STREET + data << r.STREETNO + data << r.IFNEW + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.SOURCECODE + data << r.XZJDB + data << r.LJXZQHDM + data << r.LJXZQH + data << r.STATUS + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["³ÇÊÐID","ÐÐÕþÇøID","Ð¡ÇøID","ÐÐÕþÇøÃû³Æ","Ð¡ÇøÃû³Æ","ÏêϸµØÖ·","γ¶È","¾­¶È","Õ¼µØ×ÜÃæ»ý","½¨Öþ×ÜÃæ»ý","ÂÌ»¯ÂÊ","ÈÝ»ýÂÊ","³µÎ»ÊýÁ¿","½¨ÖþÄê´ú","·¿ÎÝÓÃ;", + //"×ÜÌ×Êý","Êý¾Ýʱ¼ä","¸üбê¼Ç","¸üÐÂʱ¼ä","Ð¡ÇøÀàÐÍ","Ð¡ÇøÔøÓÃÃû","½ÖµÀ","ÃÅÅÆºÅ","ÊÇ·ñÐÂÅÌ","Ð¡Çø½éÉÜ","±¸×¢","´´½¨Ê±¼ä","À´Ô´¹«Ë¾", + //"ÏçÕò/½ÖµÀ°ì","Âß¼­ÐÐÕþÇø»®´úÂë","Âß¼­ÐÐÕþÇø»®","״̬"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_COMMUNITY_GJ Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "BIGDATA_COMMUNITY_GJ ÉÏ´«FtpÍê³É......" + } + + def exportBigdataForsale(){ + //["±±¾©","ÉϺ£"] + def citys = ["ÖØÇì","Ìì½ò","³É¶¼","ÉòÑô","´óÁ¬","ÉîÛÚ","º¼ÖÝ","ºÏ·Ê","ÎÞÎý","Î人","¹ãÖÝ"]//CcbCommonUtils.citys; + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0002.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0002.dat" + } + def headers = []//["³ÇÊÐID","³öÊÛID","¹ÒÅÆ·¿ÎÝID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","Ð¡ÇøID","Ð¡ÇøÃû³Æ","Â¥¶°ID","Â¥¶°±àºÅ","Â¥¶°Ãû³Æ","µ¥ÔªID","µ¥Ôª±àºÅ","µ¥ÔªÃû³Æ","»§ID","ÏêϸµØÖ·","½¨ÖþÄê´ú","ȨÊô","ȨÊôÄêÏÞ","µ¥¼Û","×ܼÛ","³öÊÛ¸¶¿î·½Ê½","ËùÔÚÂ¥²ã","×ÜÂ¥²ã","½¨ÖþÀàÐÍ","ÊÒÄڽṹ","½¨Öþ½á¹¹","½¨ÖþÃæ»ý","Ì×ÄÚÃæ»ý","»§ÐÍ","ÊÒ","Ìü","³ø","ÎÀ","³¯Ïò","×°ÐÞÇé¿ö","ÊÇ·ñÓеçÌÝ","¼¸Ìݼ¸»§","·¢²¼Ê±¼ä","µ½ÆÚʱ¼ä","ÊÇ·ñÊ×·¢","·¿Ô´Êý¾ÝÀ´Ô´","Öн鹫˾ID","Öн鹫˾Ãû³Æ","·¢²¼ÈË","·¢²¼È˵绰","ÃèÊöÐÅÏ¢","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾","¸½¼ÓÐÅÏ¢"] + for(def city : citys){ + def cityCode = cityCodeDataMap.get(city) + + def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}'"; + def allRows = [] + def index = 0 + sqlInstance.eachRow(sql,{r-> + index = index+1 + println city +" "+index + def data = [] + data << r.CITYID + data << r.FORSALEID + data << r.PROPCODE + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.UNITPRICE + data << r.TOTALPRICE + data << r.PAYTYPE + data << r.FLOORNUM + data << r.FLOORCNT + data << r.BUILDTYPE + data << r.HOUSESTRU + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEASPECT + data << r.FITMENT + data << r.IFELEVATOR + data << r.ELEVATORHOUSE + data << r.PUBLISHTIME + data << r.EXPIRETIME + data << r.IFFIRST + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.PUBLISHER + data << r.PHONE + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.ATTACH + allRows << data + + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + allRows.clear() + } + }); + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + + } + println "BIGDATA_FORSALE_GJ Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "BIGDATA_FORSALE_GJ ÉÏ´«FtpÍê³É......" + } + + + def exportAvmEvaluationGJNB(){ + def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" + + "ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" + + "DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" + + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH >= 202106"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.COMMUNITY_NUM + data << r.HAS_PRICE_COMMUNITY_NUM + data << r.PRICE_COVERAGE + data << r.ARTIFICIAL_CALC_NUM + data << r.ARTIFICIAL_CALC_COVERAGE + data << r.SAMPLE_CALC_NUM + data << r.SAMPLE_CALC_COVERAGE + data << r.ANN_CALC_NUM + data << r.ANN_CALC_COVERAGE + data << r.TEST_NUM + data << r.TEST_COMM_NUM + data << r.TEST_COMM_COVERAGE + data << r.PPE15_COVERAGE + data << r.RATE_MEDIAN + data << r.COEFFICIENT + data << r.DEAL_TEST_CASE_NUM + data << r.DEAL_TEST_COMM_NUM + data << r.DEAL_TEST_COMM_COVERAGE + data << r.DEAL_PPE15_COVERAGE + data << r.DEAL_COEFFICIENT + data << r.DEAL_RATE_MEDIAN + data << r.THIRD_TEST_CASE_NUM + data << r.THIRD_TEST_COMM_NUM + data << r.THIRD_TEST_COMM_COVERAGE + data << r.THIRD_PPE15_COVERAGE + data << r.THIRD_COEFFICIENT + data << r.THIRD_RATE_MEDIAN + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE", + //"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE", + //"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "AVM_EVALUATION_GJNB Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "AVM_EVALUATION_GJNB ÉÏ´«FtpÍê³É......" + } + + def exportAVMEvaluationItemGJNB(){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" + + "to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" + + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH >= 202106"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.CONFIDENCE + data << r.TARGET_AVG_PRICE + data << r.DEALDATE + data << r.UNIT_PRICE + data << r.BUILDAREA + data << r.FITMENT + data << r.HOUSEASPECT + data << r.FLOORNUM + data << r.FLOORCNT + data << r.DATA_TYPE + data << r.RATE + data << r.PE + data << r.CREATETIME + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","CONFIDENCE","TARGET_AVG_PRICE","DEALDATE","UNIT_PRICE","BUILDAREA","FITMENT","HOUSEASPECT","FLOORNUM","FLOORCNT","DATA_TYPE","RATE","PE","CREATETIME"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "AVM_EVALUATION_ITEM_GJNB Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "AVM_EVALUATION_ITEM_GJNB ÉÏ´«FtpÍê³É......" + } + + def exportCommunityBasePrice(){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.FAKE_RATE + data << r.DIFF_RATE + data << r.CONFI_RATE + data << r.CORR_RATE + data << r.CONFIDENCE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_BASE_PRICE_GJNB Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "COMMUNITY_BASE_PRICE_GJNB ÉÏ´«FtpÍê³É......" + } + + def exportCommunityBasePriceGj(){ + + } + + def exportCityPrice(){ + def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "CITY_PRICE_GJ Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "CITY_PRICE_GJ ÉÏ´«FtpÍê³É......" + } + + def exportRegionPrice(){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "DISTRICT_PRICE_GJ Êý¾ÝÎļþÊä³öÍê³É......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "DISTRICT_PRICE_GJ ÉÏ´«FtpÍê³É......" + } + + def exportLastFile(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def dataCodeFileName = "/ccb/${curDate}.ok" + if(CommonDatautils.isWindow()){ + dataCodeFileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\${curDate}.ok" + } + CcbFtpUtils.ftpUpload(dataCodeFileName, curDate) + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 49c60f702..351473f24 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -4,13 +4,20 @@ import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors - +import java.util.concurrent.atomic.AtomicLong +import org.bson.types.ObjectId /** * ½¨ÐÐÄ£Ð͹ÀÖµÏîÄ¿ * È˹¤Ö¸µ¼¼Ûͬ²½ @@ -20,6 +27,8 @@ import java.util.concurrent.Executors class GuiDingPriceBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol(); static main(args) { runBatch(new GuiDingPriceBatch(), args); @@ -34,6 +43,7 @@ class GuiDingPriceBatch extends BaseBatch{ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + //Ìí¼Ó×Ô¼º¿âÖеÄÈ˹¤Ö¸µ¼¼Û def prices = CcbCommonUtils.findPersonPrice(city, DateUtils.addMonth(new Date(), -6)) if(prices){ def distMap = getDistrictMap(city) @@ -42,18 +52,88 @@ class GuiDingPriceBatch extends BaseBatch{ println city + " " + rets.size() } + //Ìí¼ÓËÑ·¿µÄ²Î¿¼¼Û + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]); +// def datas = findFangDatas(districts) +// def fangRets = createDatas(cityCodeDatas, distCodeMap, datas) +// rows.addAll(fangRets) + + //Ìí¼Ó±£Í¤ÎIJýµÄÖ¸µ¼¼Û + def cityDatas = ExcelUtils.read("D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\ÎIJý-±£Í¤´ýµ÷²éÐ¡Çø¼Û¸ñ - 210902(1).xlsx") + def cityDataMap = [:] + for(def d : cityDatas){ + cityDataMap.put(d['A'], d['E']) + } + def personDatas = findPersonDatas(districts, cityDataMap); + def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas) + rows.addAll(personRets) + } def headers = ["µÚÈý·½¾ù¼ÛID","Â¥ÅÌID","Ð¡ÇøÃû³Æ","³ÇÊÐID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","Ö¸µ¼ÈÕÆÚ","Ö¸µ¼¼Û¸ñ","Ìí¼ÓÈË","±¸×¢","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾"] - FileUtils.batchCreateFile(rows, "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\guiding_price.txt", headers, "utf-8", false); + FileUtils.batchCreateFile(rows, "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false); - def rets = [] - rets << headers - rets.addAll(rows) - ExcelUtils.write("d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\excel\\guiding_price.xlsx", rets, "Êý¾Ý") +// def rets = [] +// rets << headers +// rets.addAll(rows) +// ExcelUtils.write("d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\excel\\guiding_price.xlsx", rets, "Êý¾Ý") } + def findFangDatas(def districts){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + println " Ð¡ÇøÁ¿£º"+districts.size() + def result = Collections.synchronizedList(new ArrayList()); + for(def i=0;i + index++ + if(d['C']){ + + println index +" "+d['A']+" "+d['B']+" "+d['C'] + def region = RegionData.formatRegion(d['A'], d['B']) + if(d['H']){ + def status = d['H'].toString().toDouble().toInteger() + if(status==1){ + def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") + if(obj){ + def gmap = GeocodeUtils.getGaodeToBaidu(d['F'].toString().toDouble(), d['G'].toString().toDouble()) + def lng2 = null + def lat2 = null + if(gmap){ + lng2 = (gmap.get("lng")*10000000000).longValue() + lat2 = (gmap.get("lat")*10000000000).longValue() + } + + BasicDBObject q=new BasicDBObject() + q.put("m_date", new Date()) + q.put("lat2", lat2) + q.put("lng2", lng2) + q.put("lng", (d['F'].toString().toDouble()*10000000000).longValue()) + q.put("lat", (d['G'].toString().toDouble()*10000000000).longValue()) + DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q)) + + }else{ + println d['A']+" "+d['B']+" "+d['C']+" ûÕÒµ½Ð¡Çø" + } + }else{ + def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") + if(obj){ + BasicDBObject q=new BasicDBObject('name_cat',1) + q.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q)) + }else{ + println d['A']+" "+d['B']+" "+d['C']+" ûÕÒµ½Ð¡Çø" + } + } + + }else{ + def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") + if(obj){ + BasicDBObject q=new BasicDBObject('name_cat',1) + q.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q)) + }else{ + println d['A']+" "+d['B']+" "+d['C']+" ûÕÒµ½Ð¡Çø" + } + + } + + } + + }) + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index b003cd950..26db4158d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -40,8 +40,8 @@ class StagingDistrictBatch extends BaseBatch{ results << headers results.addAll(rows) - ExcelUtils.write("d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\excel\\staging_district.xlsx", results, "Êý¾Ý") - FileUtils.batchCreateFile(rows, "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\staging_district.txt",headers, "utf-8", false); + ExcelUtils.write("d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "Êý¾Ý") + FileUtils.batchCreateFile(rows, "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\COMMUNITY_STAGING_GJ.dat",headers, "utf-8", false); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index d7b05ed1b..f9ac2e3fc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -3,6 +3,8 @@ package com.fangjia.script.ccbevaluation import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.looplineinfo.Point import com.fangjia.script.util.DbUtils @@ -14,6 +16,11 @@ import com.mongodb.DBCollection import groovy.sql.Sql import org.bson.types.ObjectId +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong import java.util.regex.*; @@ -42,7 +49,7 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw // println city +" "+count //} -//def fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\²É¹ºÊý¾Ý½»¸¶±ê×¼_0804(3).xlsx" +//def fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\»ù´¡Êý¾Ý½»¸¶Çé¿ö_0927.xlsx" //def datas = ExcelUtils.read("d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\7Ô³ɽ».xlsx",0,1) //def dealMap = [:] //for(def d : datas){ @@ -59,7 +66,7 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw -def fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\²É¹ºÊý¾Ý½»¸¶±ê×¼_0807 - ¸±±¾.xlsx" +//def fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\»ù´¡Êý¾Ý½»¸¶Çé¿ö_0903.xlsx" //def datas = ExcelUtils.read("d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Ð¡Çø×Ö¶Î.xlsx",0,1) //def dealMap = [:] @@ -83,58 +90,71 @@ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() // if (city) { // def cityid = cityCodeDatas.get(city) // def total = findCount(cityid, sqlInstance) -// d['K'] = total +// d['O'] = total // } //},0) -//Ð¡Çø×ֶθ²¸ÇÂÊ -ExcelUtils.multiThreadingModify(fileName,{d-> - def city = d['A'] - if(city){ - def cityid = cityCodeDatas.get(city) - def total = findCount(cityid, sqlInstance) - d['AA'] = total +////Ð¡Çø×ֶθ²¸ÇÂÊ +//ExcelUtils.multiThreadingModify(fileName,{d-> +// def city = d['A'] +// if(city){ +// def cityid = cityCodeDatas.get(city) +// def total = findCount(cityid, sqlInstance) +// d['AA'] = total +// +// def a = findEstateType(cityid, sqlInstance) +// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) +// +// def b = findFiledCount(cityid, 'buildyear', sqlInstance) +// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) +// +// def c = findFiledCount(cityid, 'address', sqlInstance) +// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) +// +// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) +// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) +// +// def e = findFiledCount(cityid, 'landarea', sqlInstance) +// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) +// +// def f = findFiledCount(cityid, 'buildarea', sqlInstance) +// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) +// +// def g = findFiledCount(cityid, 'plotratio', sqlInstance) +// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) +// +// def h = findFiledCount(cityid, 'greeningrate', sqlInstance) +// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) +// +// def i = findFiledCount(cityid, 'latitude', sqlInstance) +// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) +// +// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) +// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) +// println d +// } +//},1) - def a = findEstateType(cityid, sqlInstance) - d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) - - def b = findFiledCount(cityid, 'buildyear', sqlInstance) - d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) - - def c = findFiledCount(cityid, 'address', sqlInstance) - d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) - - def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) - d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) - - def e = findFiledCount(cityid, 'landarea', sqlInstance) - d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) - - def f = findFiledCount(cityid, 'buildarea', sqlInstance) - d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) - - def g = findFiledCount(cityid, 'plotratio', sqlInstance) - d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) - - def h = findFiledCount(cityid, 'greeningrate', sqlInstance) - d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) - - def i = findFiledCount(cityid, 'latitude', sqlInstance) - d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) - - def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) - d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) - println d - } -},1) - -//def fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\²É¹ºÊý¾Ý½»¸¶±ê×¼_0807-1.xlsx" +//def fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\8Ô³ɽ»Í³¼Æ.xlsx" //def dealMap = findDealCount(sqlInstance) //ExcelUtils.multiThreadingModify(fileName,{d-> -// if(d['C']){ -// d['Q'] = dealMap.get(d['C']) +// if(d['A']){ +// d['U'] = dealMap.get(d['A']) // } -//},2) +//},0) + + +def fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\³É½»Î´Èë¿âÈ˹¤²¿·Ö³ÇÊÐͳ¼Æ.xlsx" +def datas = ExcelUtils.read(fileName, 0, 1) +def dealMap = [:] +for(def d : datas){ + dealMap.put(d['A'], d['B']) +} +ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\8Ô³ɽ»Í³¼Æ.xlsx",{d-> + if(d['A']){ + d['W'] = dealMap.get(d['A']) + } +},0) //def fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\¾­Î³¶È´íÎóÊý¾ÝÐÞÕý.xlsx" @@ -170,7 +190,7 @@ ExcelUtils.multiThreadingModify(fileName,{d-> def findDealCount(sqlInstance){ - def sql = "select cityname,count(*) as count from bigdata_deal where dealdate>=to_date('2021-06-01','yyyy-mm-dd') and dealdate @@ -194,7 +214,7 @@ def findEstateType(cityid, sqlInstance){ } def findCount(cityid, sqlInstance){ - def sql = "select count(*) as count from bigdata_community where cityid='${cityid}'"; + def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0"; def count = 0 sqlInstance.eachRow(sql,{r-> @@ -204,7 +224,7 @@ def findCount(cityid, sqlInstance){ } def findFiledCount(cityid, field, sqlInstance){ - def sql = "select count(*) as count from bigdata_community where cityid='${cityid}' and "+field+" is not null"; + def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0 and "+field+" is not null"; def count = 0 sqlInstance.eachRow(sql,{r-> count = r.count @@ -353,3 +373,5 @@ def getStreetInfo(def city, def glocation){ + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TestRepeat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TestRepeat.groovy new file mode 100644 index 000000000..c0833369a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TestRepeat.groovy @@ -0,0 +1,81 @@ +package com.fangjia.script.ccbevaluation + +import groovy.sql.Sql + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class TestRepeat { + + ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new TestRepeat() + task.run() + } + def run(){ + + } + + + def checkUpdateCount(){ + def newHouses = readTxt("C:\\Users\\gaojuan\\Desktop\\BIGDATA_FORSALE_GJ.dat") + + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(newHouses.size()) + println " ·¿Ô´Á¿£º"+newHouses.size() + + def retMap = new ConcurrentHashMap() + for(def i=0;i + obj = r + }); + return obj + } + + + def readTxt(def file){ + def lines = org.apache.commons.io.FileUtils.readLines(new File(file)) + def rets = []; + for(line in lines) { + def arrs = line.split("|@|") + def map = [:] + map.houseId = arrs[1] + map.commId = arrs[6] + map.pdate = arrs[10] + rets << map + } + return rets; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index f14ace3a9..b813a155d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -11,6 +11,8 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import java.util.concurrent.ConcurrentHashMap + /** * ½¨ÐÐÄ£Ð͹ÀÖµÏîÄ¿ * µÚÈý·½¾ù¼ÛÊý¾Ýͬ²½ @@ -18,11 +20,13 @@ import com.mongodb.DBCollection @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def sDate= 202109//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202107 + def endDate = 202109 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") - + DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") static main(args) { runBatch(new ThirdPartyPriceBatch(), args); @@ -33,13 +37,15 @@ class ThirdPartyPriceBatch extends BaseBatch { def citys = CcbCommonUtils.citys; def rows = [] def index = 0 + def weekend = DateUtils.getFirstDayOfWeek(new Date()) for(def city : citys){ index++ + def statMap = getDistrictStatByCity(city, weekend) def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"]) - def thirdPriceMap = findThirdPartyPrice(city) + def thirdPriceMap = findThirdPartyPrice(city, statMap) def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap) rows.addAll(rets) println index+" "+city+" "+rets.size() @@ -47,9 +53,9 @@ class ThirdPartyPriceBatch extends BaseBatch { - def fileName = "/home/goojia/ccb/third_party_price.txt" + def fileName = "/home/goojia/ccb/THIRD_PARTY_PRICE_GJ.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\third_party_price.txt" + fileName = "d:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\THIRD_PARTY_PRICE_GJ.dat" } def header = ["µÚÈý·½¾ù¼ÛID","Â¥ÅÌID","Ð¡ÇøÃû³Æ","³ÇÊÐID","³ÇÊÐÃû³Æ","ÐÐÕþÇøID","ÐÐÕþÇøÃû³Æ","¾ù¼Û¶ÔÓ¦µÄÔ·Ý","Öî¸ðÕÒ·¿¾ù¼Û","°²¾Ó¿Í¾ù¼Û","·¿ÌìϾù¼Û","Á´¼Ò¾ù¼Û","³ÇÊз¿²ú¾ù¼Û","ÂóÌï¾ù¼Û","±´¿Ç¾ù¼Û","ÌÔ·¿365","ÖÐÔ­¾ù¼Û","Íò©Ê¿¾ù¼Û","Q·¿¾ù¼Û","·¿Ì½¾ù¼Û","·¿¶à¶à¾ù¼Û","ÎÒ°®ÎÒ¼Ò¾ù¼Û","´´½¨Ê±¼ä","¸üÐÂʱ¼ä","À´Ô´¹«Ë¾"] @@ -61,6 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch { def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){ def rets = [] for(def d : dists){ + def third = thirdPriceMap.get(d.region+"##"+d.name) if(third){ def data = distCodeMap.get(d.region) @@ -166,10 +173,10 @@ class ThirdPartyPriceBatch extends BaseBatch { } - def findThirdPartyPrice(def city){ + def findThirdPartyPrice(def city, def statMap){ def q = new BasicDBObject() q.put("city", city) - q.put("s_date", new BasicDBObject('$gte', sDate)) + q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def map = [:] for(def db : dbcursor){ @@ -242,13 +249,46 @@ class ThirdPartyPriceBatch extends BaseBatch { def resultMap = [:] map.each {key, value-> if(value && value.size()>1){ - resultMap.put(key, value) + if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){ + def price = statMap.get(key) + if(price){ + def priceMap = [:] + def startPrice = (int)(price - price*0.3) + def endPrice = (int)(price*0.3 + price) + value.each{pk,pv-> + if(pk.contains("price")){ + if(pv>=startPrice && pv<=endPrice){ + priceMap.put(pk,pv) + } + }else{ + priceMap.put(pk,pv) + } + } + if(priceMap.size()>1){ + resultMap.put(key, priceMap) + } + } + }else{ + resultMap.put(key, value) + } + } } return resultMap } + def getDistrictStatByCity(city, weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + def dbcursor = DIST_SELL_STAT.find(q) + def map = new ConcurrentHashMap() + for(def d : dbcursor){ + map.put(d.region+"##"+d.district_name, d.avg_price) + } + return map + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index c2378636d..a35a99a4e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -1,12 +1,22 @@ package com.fangjia.script.ccbevaluation +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import groovy.sql.Sql +import org.bson.types.ObjectId + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong /*** * ½¨Ðгɽ»×¥È¡Í³¼Æ @@ -21,23 +31,34 @@ class TradeStatLog { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + static main(args) { def task = new TradeStatLog() - task.run() +// task.run() // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) +// task.averageStat() + task.exportTrade() +// task.deleteDistrict() + } def run(){ def citys = CcbCommonUtils.citys; def rets = [] - rets << ["³ÇÊÐ","5ÔÂ×ÜÁ¿","6ÔÂ×ÜÁ¿","7ÔÂ×ÜÁ¿","±´¿Ç","Á´¼Ò", "ÎÒ°®ÎÒ¼Ò","·¿ÌìÏÂ","ÖÐÔ­"] + rets << ["³ÇÊÐ","6ÔÂ×ÜÁ¿","7ÔÂ×ÜÁ¿","8ÔÂ×ÜÁ¿","±´¿Ç","Á´¼Ò", "ÎÒ°®ÎÒ¼Ò","·¿ÌìÏÂ","ÖÐÔ­"] def sources = ["±´¿ÇÕÒ·¿","Á´¼ÒÔÚÏß", "ÎÒ°®ÎÒ¼Ò","·¿ÌìÏÂ","ÖÐÔ­µØ²ú"] - def startDate = "2021-05-01" - def date = "2021-06-01" - def endDate = "2021-07-01" - def newDate = "2021-08-01" + //def startDate = "2021-05-01" + def startDate = "2021-06-01" + def date = "2021-07-01" + def endDate = "2021-08-01" + def newDate = "2021-09-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) @@ -64,7 +85,7 @@ class TradeStatLog { ret << lastMonthCount ret << curMonthCount ret << nearMonthCount - if(lastMonthCount){ + if(curMonthCount){ def a = (nearMonthCount-curMonthCount)*100/curMonthCount def pre = DataHandleUtils.getDoubleRound(2, a) ret << pre @@ -87,7 +108,7 @@ class TradeStatLog { } - ExcelUtils.write("D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\³É½»Í³¼Æ.xlsx", rets, "Êý¾Ý") + ExcelUtils.write("D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\8Ô³ɽ»Í³¼Æ.xlsx", rets, "Êý¾Ý") println "±í¸ñÉú³ÉÍê³É" } @@ -103,7 +124,7 @@ class TradeStatLog { rets << [city, districts.size(), m.total, m.count1, m.count2, m.count3, m.count4, m.count5, m.count6, m.count7, m.count8, m.count9, m.count10, m.count11, m.count12, m.count13, m.count14] println rets.get(rets.size()-1) } - ExcelUtils.write("D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\µÚÈý·½¾ù¼Ûͳ¼Æ.xlsx", rets, "Êý¾Ý") + ExcelUtils.write("D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\µÚÈý·½¾ù¼Ûͳ¼Æ0927.xlsx", rets, "Êý¾Ý") println "±í¸ñÉú³ÉÍê³É" } @@ -129,7 +150,7 @@ class TradeStatLog { def getThirdPartyPricePrecent(def city){ - def sql = "SELECT * FROM third_party_price WHERE cityname = '"+city+"'"; + def sql = "SELECT * FROM third_party_price_gj WHERE cityname = '"+city+"' and year_month=202108"; def count0 = 0 def count1 = 0 @@ -299,5 +320,333 @@ class TradeStatLog { } + def averageStat(){ + def citys = ["±±¾©","ÉϺ£","¹ãÖÝ","ÉîÛÚ","ÖØÇì","º¼ÖÝ","ËÕÖÝ","ÏÃÃÅ","´óÁ¬","ºôºÍºÆÌØ"] + + def map = [:] + for(def city : citys){ + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]); + def rets = findThirdPriceByDist(districts) + map.put(city, rets.size()) + map.put(city+"-count", districts.size()) + println districts.size()+" "+ rets.size() + } + println map + + ExcelUtils.modify("D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\¹ÀÖµ²âÊÔÊý¾Ý¸²¸ÇÂÊÕ¼±È.xlsx",{d-> + def city = d['A'] + def count = map.get(city) + def distCount = map.get(city+"-count") + d['H'] = count + d['J'] = distCount + + def pre = DataHandleUtils.getDoubleRound(2, count*100.0/distCount) + d['I'] = pre + },0) + } + + + def findThirdPriceByDist(districts){ + def rets = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + for(def i=0;iendPrice){ +// reason = "¼Û¸ñ¹ýÂË" +// } + if(dist.name_cat != 0 || !dist.lng || !["ÆÕͨסլ","ÉÌס"].contains(dist.estate_type2) || dist.name.contains("Ñó·¿") || dist.name.contains("¹«·¿")){ + reason = "Ð¡Çø¹ýÂË" + } + + if(!reason){ + def key=t.city+"##"+dist.region+"##"+dist.name+"##"+t.area+"##"+t.avg_price+"##"+DateUtils.formatDateByPattern(t.trade_date,"yyyy-MM-dd") + if(details.contains(key)){ + reason = "Êý¾ÝÖØ¸´" + } + } + } + + + def tradeDate = DateUtils.formatDateByPattern(t.trade_date,"yyyy-MM-dd") + //rets << [t._id.toString(), t.city, t.region, t.district_name, t.source, t.avg_price, t.area,t.address, tradeDate, reason, t.estate_type] + rets << [t._id.toString(), t.city, t.region, t.district_name, t.source, t.avg_price, t.area,t.address, tradeDate, reason] + } + } + println city +" "+rets.size() + } + + ExcelUtils.write("d:\\Êý¾Ý\\³É½»Î´Èë¿â¼Ç¼(7ÔÂÈ«).xlsx", rets,"Êý¾Ý") + } + + def deleteDistrict(){ + ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Ҫɾ³ýÐ¡Çø.xlsx",{d-> + if(d['G']){ + def id = d['G'] + BasicDBObject q=new BasicDBObject('name_cat',1) + q.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q)) + deleteAllRefDistrict(id) + } + },0) + } + + def deleteAllRefDistrict(id){ + def tables = ["bigdata_community_gj"] + for(def tableName : tables){ + deleteByCommid(tableName, id) + } + } + + def deleteByCommid(tableName, commid){ + def sql = "delete from "+tableName+" where commid='${commid}'"; + sqlInstance.execute(sql) + println "ɾ³ý£º"+tableName+" "+commid + } + + def getYouDaData() { + def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("2021-07-01", "yyyy-MM-dd")) + def endDate = DateUtils.parseFromTightByPattern("2021-08-01", "yyyy-MM-dd") + def q = new BasicDBObject() +// q.put("premisesName","¹²¸»Ò»´å") + q.put("contractTime", new BasicDBObject('$gte', startDate));//.append('$lt', endDate) + def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000) + def trades = [] + for(def db : dbcursor){ + def map = [:] + map._id=db._id.toString() + map.city = "ÉϺ£" + map.region = db.region + if(db.districtName && db.districtName!="null"){ + map.district_name=db.districtName + }else if(db.address){ + map.district_name=db.address + } + + map.area = db.area + map.trade_date = db.contractTime + map.avg_price= db.price + map.estate_type = db.houseType + trades << map + } + return trades + } + + def findDealIds(city){ + def sql = "select * from bigdata_deal_gj where dealdate>=to_date('2021-07-01','yyyy-mm-dd') and dealdate + if(!ids.contains(r.dealid)){ + ids << r.dealid + + def region = RegionData.formatRegion(r.cityname, r.distname) + def key=r.cityname+"##"+region+"##"+r.commname+"##"+r.buildarea.toDouble()+"##"+r.dealunitprice+"##"+DateUtils.formatDateByPattern(r.dealdate,"yyyy-MM-dd") + if(!detailInfos.contains(key)){ + detailInfos << key + } + } + }); + def rMap = [:] + rMap.ids = ids + rMap.detail = detailInfos + return rMap + } + + + public static def findTradesDetail(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) +// q.put("district_name","°ÙºÏ»¨Ô°") + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def dbcursor = DEAL_PRICE_COLL.find(q) + def rets = [] + for(def d : dbcursor){ + rets << d + } + return rets + } + + def getDistrictStatByCity(city, weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + def dbcursor = DIST_SELL_STAT.find(q) + def map = new ConcurrentHashMap() + for(def d : dbcursor){ + map.put(d.region+"@"+d.district_name, d.avg_price) + } + return map + } + + def getAllDistrict(city){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name", "alias"]) + def map = [:] + for(def d : dists){ + map.put(d.city+"##"+d.region+"##"+d.name, d._id.toString()) + def alias = d.alias + if(alias){ + for(def name : alias){ + map.put(d.city+"##"+d.region+"##"+name, d._id.toString()) + } + } + } + return map + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy new file mode 100644 index 000000000..cb189d420 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy @@ -0,0 +1,392 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.ExcelUtil +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import net.sf.json.JSONObject +import org.bson.types.ObjectId +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.ResultSet +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Pattern + +class UpdateHouseRegion { + + + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + + def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + + static main(args) { + def task = new UpdateHouseRegion() +// task.updateSeawaterRegion() +// task.updateDealRegion() +// task.updateThirdPartyRegion() + task.updateNoMatchDealRegion() +// task.addAlias() +// task.addFenQi() + } + + def addFenQi(){ + def citys = CommonDatautils.findAllCity() + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district",["city","region","name","alias"]) + addFenQiAlias(dists) + println city +" ´¦ÀíÍê³É" + } + } + + def addFenQiAlias(dists){ + for(def dist : dists){ + def s1 = DataUtil.match(dist.name, Pattern.compile("[Ò»|¶þ|Èý|ËÄ|Îå|Áù|Æß|°Ë|¾Å|Ê®|ʮһ|Ê®¶þ|Ê®Èý|Ê®ËÄ|Ê®Îå|Ê®Áù|Ê®Æß|Ê®°Ë|\\w+](ÆÚ|Çø)\$")) + if(s1){ + def newName = dist.name.replace("¢ñ","1").replace("¢ò","2").replace("AÇø","aÇø").replace("BÇø","bÇø").replace("CÇø","cÇø").replace("DÇø","dÇø").replace("EÇø","eÇø").replace("FÇø","fÇø").replace("Ò»ÆÚ","1ÆÚ").replace("¶þÆÚ","2ÆÚ").replace("ÈýÆÚ","3ÆÚ").replace("ËÄÆÚ","4ÆÚ").replace("ÎåÆÚ","5ÆÚ").replace("ÁùÆÚ","6ÆÚ").replace("ÆßÆÚ","7ÆÚ").replace("°ËÆÚ","8ÆÚ").replace("¾ÅÆÚ","9ÆÚ").replace("Ê®ÆÚ","10ÆÚ").replace("ʮһÆÚ","11ÆÚ").replace("Ê®¶þÆÚ","12ÆÚ") + if(newName!=dist.name){ + def alias = dist.alias + if(!alias){ + alias = [] + } + if(!alias.contains(newName)){ + alias << newName + def udata = new BasicDBObject() + udata.put("alias", alias) + DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata)) + println dist.city +" "+dist.name+" "+dist.alias+" Уº"+alias + } + } + } + } + } + + def addAlias(){ + def citys = CommonDatautils.findAllCity() + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district",["city","region","name","alias"]) + addDistricts(dists) + println city +" ´¦ÀíÍê³É" + } + } + + def addDistricts(dists){ + for(def dist : dists){ + def alias = dist.alias + if(alias){ + def oldCount = alias.size() + def newAlias = [] + for(def aliasName : alias){ + if(aliasName.contains("¡¤") || aliasName.contains(".")){ + def newName = aliasName.replace("¡¤","").replace(".","") + newAlias << aliasName + if(!newAlias.contains(newName)){ + if(newName!=dist.name){ + newAlias << newName + } + } + }else{ + if(!newAlias.contains(aliasName)) { + newAlias << aliasName + } + } + } + def newCount = newAlias.size() + if(oldCount && alias!=newAlias){ + def udata = new BasicDBObject() + udata.put("alias", newAlias) + DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata)) + println dist.city +" "+dist.name+" "+dist.alias+" Уº"+newAlias + } + + + } + } + } + + + def updateThirdPartyRegion(){ + def city = "¶«Ý¸" + def rets = findThirdPartyPrice(city) + batchThirdPartyUpdate(city, rets); + + } + def findThirdPartyPrice(city){ + def q = new BasicDBObject() + q.put("city",city) + q.put("s_date",new BasicDBObject('$gte', 202101)) + def dbcursor = thirdPartyColl.find(q) + def rets = [] + for(def d : dbcursor){ + rets << d + } + return rets; + } + + def findDealData(def city){ + def q = new BasicDBObject() + q.put("city", city) + q.put('trade_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-01-01"))); + def cursor = DEAL_PRICE_COLL.find(q).batchSize(2000) + def datas = [] + def index = 0 + for(def obj : cursor){ + index++ + datas << obj + println index + } + return datas + } + + def updateDealRegion(){ + def city = "¶«Ý¸" + def datas = findDealData(city) + batchDealRegion(city, datas) + } + + def batchDealRegion(def city, def datas){ + def updateCount = new AtomicLong() + def curCount = new AtomicLong() + def totalCount = datas.size(); + final CountDownLatch latch = new CountDownLatch(totalCount) + println city +" ³É½»×ÜÁ¿£º"+totalCount + for(def i=0;i + if(!d['J']){ + def city = d['B'] + def region = d['C'] + def name = d['D'] + def rets = CommonDatautils.findDistrictsByName(city, name, "district") + if (!rets) { + rets = CommonDatautils.findDistrictByAlias(city, null, name, "district") + } + + if (rets.size() == 1) { + def dist = rets[0] + if (region != dist.region) { + def udata = new BasicDBObject() + udata.put("region", dist.region) + DEAL_PRICE_COLL.update(new BasicDBObject('_id', new ObjectId(d['A'])), new BasicDBObject('$set', udata)) + d['J'] = "Ô­Êý¾ÝÇøÓòÐ޸ģ¨"+dist.region+"£©" + println d['A']+" "+region +" "+dist.region + } + } + } + + },0) + } + + def updateSeawaterRegion(){ + def city = "¶«Ý¸" + def rets = findHouseSource(city, 20190701, 20211001) + batchUpdate(city, rets); + } + + + def batchThirdPartyUpdate(def city, def thirdDatas){ + def updateCount = new AtomicLong() + def curCount = new AtomicLong() + def totalCount = thirdDatas.size(); + final CountDownLatch latch = new CountDownLatch(totalCount) + println city +" µÚÈý·½×ÜÁ¿£º"+totalCount + for(def i=0;i="+startDate).append(" and p_date <="+endDate) + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + + while (rs.next()) { + def count = rs.getInt("count") + println count + } + } + + def batchUpdate(def city, def houses){ + def updateCount = new AtomicLong() + def curCount = new AtomicLong() + def totalCount = houses.size(); + final CountDownLatch latch = new CountDownLatch(totalCount) + println city +" ·¿Ô´×ÜÁ¿£º"+totalCount + for(def i=0;i="+startDate).append(" and p_date <"+endDate); + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + + def houses = [] + while (rs.next()){ + houses << ["region":rs.getString("region"), "name":rs.getString("district_name"),"url":rs.getString("url")] + } + return houses; + } + + + + def updateHouseRegion2(def url, def region){ + def header = [:] + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("developer:5e8edd851d2fdfbd7415232c67367cc3".getBytes("UTF-8"))}".toString() + String seawaterUrl = "http://192.168.0.206:9200/seawater-2019/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false" + def body = """ + { + "script": { + "source": "ctx._source.region='${region}'", + "lang": "painless" + }, + "query": { + "term": { + "url": "${url}" + } + } + } + """ + HttpRequest request = new HttpRequest() + String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") +// def response = JSONObject.fromObject(res) +// println(response) + } + + + def updateHouseRegion(def url, def region){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("update seawater-2021") + sqlBuffer.append(" set region = '"+region+"'") + sqlBuffer.append(" where url='"+url+"'") + def result = dataSource.connection.createStatement().executeUpdate(sqlBuffer.toString()) + println region +" "+result + } + + def getAllDistricts(def city){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias"]) + def map = [:] + for(def d : dists){ + + map.put() + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy new file mode 100644 index 000000000..12bbc9acc --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -0,0 +1,375 @@ +package com.fangjia.script.ccbevaluation.ronghe + +import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.jusfoun.JusfounCommon +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.OpenApiClient +import groovy.sql.Sql +import org.bson.types.ObjectId + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Pattern + + +/** + * ½¨ÐÐÈںϡª¡ªÐ¡Çø±í + */ +@BatchOwner(Watcher.GaoJuan) +class BigdataCommunityGJRHBatch extends BaseBatch { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + public static def batchsize = 1000 + static main(args) { + runBatch(new BigdataCommunityGJRHBatch(), args); + } + + @Override + public void process(Object args) { + + //"ÏÃÃÅ","±±¾©","ÉϺ£", + def citys = ["¹ãÖÝ","ÉîÛÚ","ÖØÇì","Ö£ÖÝ","³¤É³","Î÷°²","³É¶¼","¶«Ý¸","·ðɽ","º¼ÖÝ","ºÏ·Ê","ÄϾ©","Çൺ","ÉòÑô","ËÕÖÝ","Ìì½ò","Î人"]//CcbCommonUtils.citys; + def rows = [] + +// addBigdataDistrictFJW() + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + for (def city : citys) { + def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) + + def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"]) + + for(def dist : districts){ + def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"]) + if(obj){ + if(obj.alias && obj.alias.size()>0){ + dist.commalias = obj.alias.get(0) + }else{ + dist.commalias = "" + } + dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure) + dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2) + dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date); + dist.houseuse = "11" + dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type) + dist.compid = new ObjectId() //ÎïÒµ¹«Ë¾id + dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2) + } + } + println city +" Ð¡Çø¼ÓÔØÍê³É "+districts.size() + addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city)) + println city +" 1.Ð¡Çø±íÍê³É" + addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city)) + println city +" 2.Ð¡Çø½¨Öþ½á¹¹±í" + addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city)) + println city +" 3.Ð¡Çø·¿ÎÝÀàÐͱí" + addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city)) + println city +" 4.Ð¡Çø½¨ÖþÄê´ú±í" + addBigdataCommfeeGj(districts, cityCodeDataMap.get(city)) + println city +" 6.Ð¡Çø·ÑÓñí" + addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city)) + println city +" 7.Ð¡Çø·¿ÎÝÓÃ;±í" + addBigdataPositionGJ(districts, cityCodeDataMap.get(city)) + println city + " 8.µØÀíλÖñí" + } + } + + def addBigdataCommunityGJRH(def districts, def cityid){ + + sqlInstance.executeUpdate("delete from BIGDATA_COMMUNITY_FJW where CITYID='${cityid}'") + + sqlInstance.withBatch(batchsize) { stmt -> + for(def d : districts){ + def sql = "INSERT INTO BIGDATA_COMMUNITY_FJW (CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,CREATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.distid}','${d.commid}','${d.commname}',${converValue(d.address)},${d.landarea},${d.buildarea},${d.greeningrate},${d.plotratio},${d.parkingcnt}," + if(d.updatetime){ + sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.commtype}','${d.commalias}'," + }else{ + sql = sql + "null,'${d.commtype}','${d.commalias}'," + } + if(d.street && d.street!='null'){ + sql = sql +"'${d.street}'," + }else{ + sql = sql +"null," + } + if(d.streetno && d.streetno!='null'){ + sql = sql + "'${d.streetno}'," + }else{ + sql = sql +"null," + } + + sql = sql + "'${d.ifnew}',null,${d.remark}," + if(d.createtime){ + sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')" + }else{ + sql = sql + "null,'${d.sourcecode}')" + } + + try{ + stmt.addBatch sql + }catch(Exception e){ + println d + println sql + } + + } + + } + } + + def converValue(def val){ + if(val && val!='null'){ + return "'"+val+"'"; + }else{ + return null + } + } + + def addBigdataCommbuildstruGJ(def districts, def cityid){ + sqlInstance.executeUpdate("delete from BIGDATA_COMMBUILDSTRU_FJW where CITYID='${cityid}'") + sqlInstance.withBatch(batchsize) { stmt -> + for(def d : districts){ + def sql = "INSERT INTO BIGDATA_COMMBUILDSTRU_FJW (CITYID,COMMID,BUILDSTRU,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.buildstru}',${d.remark}," + if(d.createtime){ + sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss')," + }else{ + sql = sql + "null," + } + if(d.updatetime){ + sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')" + }else{ + sql = sql + "null,'${d.sourcecode}')" + } + try{ + stmt.addBatch sql + }catch(Exception e){ + println d + println sql + } + } + } + } + + + def addBigdataCommhousetypeGJ(def districts, def cityid){ + sqlInstance.executeUpdate("delete from BIGDATA_COMMHOUSETYPE_FJW where CITYID='${cityid}'") + + sqlInstance.withBatch(batchsize) { stmt -> + for(def d : districts){ + def sql = "INSERT INTO BIGDATA_COMMHOUSETYPE_FJW (CITYID,COMMID,HOUSETYPE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.housetype}',${d.remark}," + if(d.createtime){ + sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss')," + }else{ + sql = sql + "null," + } + if(d.updatetime){ + sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')" + }else{ + sql = sql + "null,'${d.sourcecode}')" + } + try{ + stmt.addBatch sql + }catch(Exception e){ + println d + println sql + } + } + } + } + + def addBigdataCommbuildyearGj(districts, def cityid){ + + sqlInstance.executeUpdate("delete from BIGDATA_COMMBUILDYEAR_FJW where CITYID='${cityid}'") + + sqlInstance.withBatch(batchsize) { stmt -> + for(def d : districts){ + if(d.buildyear){ + def sql = "INSERT INTO BIGDATA_COMMBUILDYEAR_FJW (CITYID,COMMID,BUILDYEAR,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}',${d.buildyear},${d.remark}," + if(d.createtime){ + sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss')," + }else{ + sql = sql + "null," + } + if(d.updatetime){ + sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')" + }else{ + sql = sql + "null,'${d.sourcecode}')" + } + try{ + stmt.addBatch sql + }catch(Exception e){ + println d + println sql + } + } + } + } + } + + def addBigdataCommfeeGj(districts, def cityid){ + + sqlInstance.executeUpdate("delete from BIGDATA_COMMFEE_FJW where CITYID='${cityid}'") + + def sqls = [] + for(def d : districts){ + if(d.estatecharge){ + def sql = "INSERT INTO BIGDATA_COMMFEE_FJW (CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.compid}',${d.estatecharge},'${d.houseuse}','${d.buildtype}','1',${d.note},${d.remark}," + if(d.createtime){ + sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss')," + }else{ + sql = sql + "null," + } + if(d.updatetime){ + sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')" + }else{ + sql = sql + "null,'${d.sourcecode}')" + } + sqls << sql + + if (sqls.size() >= batchsize) { + try { + sqlInstance.withBatch(batchsize, { stmt -> + sqls.each { def s -> + stmt.addBatch s + } + }) + + } catch (Exception e) { + println sqls[0] + e.printStackTrace() + } + sqls.clear() + } + + } + } + + if (sqls.size() > 0) { + sqlInstance.withBatch(100, { stmt -> + sqls.each { def s -> + stmt.addBatch s + } + }) + } + } + + def addBigdataCommhouseuseGJ(districts, def cityid){ + + sqlInstance.executeUpdate("delete from BIGDATA_COMMHOUSEUSE_FJW where CITYID='${cityid}'") + + def sqls = [] + for(def d : districts){ + def sql = "INSERT INTO BIGDATA_COMMHOUSEUSE_FJW (CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.houseuse}',${d.zts},${d.remark}," + if(d.createtime){ + sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss')," + }else{ + sql = sql + "null," + } + if(d.updatetime){ + sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')" + }else{ + sql = sql + "null,'${d.sourcecode}')" + } + sqls << sql + + if (sqls.size() >= batchsize) { + try { + sqlInstance.withBatch(batchsize, { stmt -> + sqls.each { def s -> + stmt.addBatch s + } + }) + + } catch (Exception e) { + e.printStackTrace() + } + sqls.clear() + } + + } + + if (sqls.size() > 0) { + sqlInstance.withBatch(100, { stmt -> + sqls.each { def s -> + stmt.addBatch s + } + }) + } + } + + def addBigdataPositionGJ(districts, def cityid){ + + sqlInstance.executeUpdate("delete from BIGDATA_POSITION_FJW where CITYID='${cityid}'") + + def sqls = [] + def index = 1 + for(def d : districts){ + def sql = "INSERT INTO BIGDATA_POSITION_FJW (CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}','5', '${d.commid}','1',${index},${d.lng},${d.lat},null,${d.remark}," + index++ + if(d.createtime){ + sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss')," + }else{ + sql = sql + "null," + } + if(d.updatetime){ + sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')" + }else{ + sql = sql + "null,'${d.sourcecode}')" + } + sqls << sql + + if (sqls.size() >= batchsize) { + try { + sqlInstance.withBatch(batchsize, { stmt -> + sqls.each { def s -> + stmt.addBatch s + } + }) + + } catch (Exception e) { + e.printStackTrace() + } + sqls.clear() + } + + } + + if (sqls.size() > 0) { + sqlInstance.withBatch(100, { stmt -> + sqls.each { def s -> + stmt.addBatch s + } + }) + } + } + + def addBigdataDistrictFJW(){ + def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from bigdata_district_gj"; + def rows = [] + try{ + sqlInstance.eachRow(sql,{r-> + def insertSql = "INSERT INTO bigdata_district_fjw(DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${r.distid}','${r.distname}','${r.distspell}','${r.cityid}','${r.distcode}','${r.seq}',${r.disttype},null,to_date('${r.createtime}','yyyy-mm-dd hh24:mi:ss'),to_date('${r.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${r.sourcecode}')" + sqlInstance.withBatch(batchsize, { stmt -> + stmt.addBatch insertSql + }) + }); + }catch(Exception e){ + + } + } + +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy new file mode 100644 index 000000000..8dbfd559f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -0,0 +1,239 @@ +package com.fangjia.script.ccbevaluation.ronghe + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import groovy.sql.Sql + +import java.sql.ResultSet +import java.sql.ResultSetMetaData + +@BatchOwner(Watcher.GaoJuan) +class ExportDistrictRongHe extends BaseBatch{ + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static main(args) { + runBatch(new ExportDistrictRongHe(), args); + } + + @Override + void process(Object args) { + exportBigdataCommunityGJRH() + exportBigdataCommbuildstruGJ() + exportBigdataCommhousetypeGJ() + exportBigdataCommbuildyearGj() + exportBigdataCommfeeGj() + exportBigdataCommhouseuseGJ() + exportBigdataPositionGJ() + exportBigdataDistrictFJW() + } + + def exportBigdataCommunityGJRH(){ + def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.COMMNAME + data << r.ADDRESS + data << r.LANDAREA + data << r.BUILDAREA + data << r.GREENINGRATE + data << r.PLOTRATIO + data << r.PARKINGCNT + data << r.UPDATETIME + data << r.COMMTYPE + data << r.COMMALIAS + data << r.STREET + data << r.STREETNO + data << r.IFNEW + data << r.INTRODUCTION + data << r.REMARK + data << r.CREATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18" + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_FJW_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\BIGDATA_COMMUNITY_GJRH_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + } + def exportBigdataCommbuildstruGJ(){ + def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDSTRU + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18" + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\BIGDATA_COMMBUILDSTRU_GJ_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + } + + def exportBigdataCommhousetypeGJ(){ + def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.HOUSETYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18" + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\BIGDATA_COMMHOUSETYPE_GJ_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + } + + def exportBigdataCommbuildyearGj(){ + def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDYEAR + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18" + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\BIGDATA_COMMBUILDYEAR_GJ_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + } + + def exportBigdataCommfeeGj(){ + def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.COMPID + data << r.FEE + data << r.HOUSEUSE + data << r.BUILDTYPE + data << r.FEETYPE + data << r.NOTE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18" + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMFEE_FJW_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\BIGDATA_COMMFEE_GJ_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + } + + + def exportBigdataCommhouseuseGJ(){ + def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.HOUSEUSE + data << r.HOUSECNT + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18" + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\BIGDATA_COMMHOUSEUSE_GJ_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + } + + def exportBigdataPositionGJ(){ + def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.OBJECTTYPE + data << r.OBJECTID + data << r.POSITIONTYPE + data << r.SEQNO + data << r.LONGITUDE + data << r.LATITUDE + data << r.ALTITUDE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18" + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POSITION_FJW_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\BIGDATA_POSITION_GJ_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + } + + def exportBigdataDistrictFJW(){ + def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.DISTID + data << r.DISTNAME + data << r.DISTSPELL + data << r.CITYID + data << r.DISTCODE + data << r.SEQ + data << r.DISTTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18" + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_FJW_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Êý¾ÝÎļþ\\txt\\${curDate}\\BIGDATA_DISTRICT_FJW_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index ca7dada90..821cc9765 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -536,8 +536,8 @@ class CommonDatautils { if(name){ alias << name; } - - + + DBObject q = new BasicDBObject(); q.put("city", city); q.put("region", region); @@ -677,6 +677,28 @@ class CommonDatautils { } + public static def findAllDistricMaptByCity(def city, def cat, def fields){ + + DBObject q = new BasicDBObject(); + q.put("visible",0) + q.put("cat", cat); + q.put("city", city); + + DBObject keys = new BasicDBObject(); + if(fields){ + for(def field : fields){ + keys.put(field, 1); + } + } + def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000) + def retMap = [:] + for(def d : dbsour){ + retMap.put(d._id.toString(), d) + } + return retMap; + } + + public static def findAllDistrictByCity(def city, def cat, def fields){ DBObject q = new BasicDBObject(); @@ -760,6 +782,29 @@ class CommonDatautils { } return ret; } + + public static def findDistrictFieldById(String id, def fields){ + + if(!id){ + return null; + } + DBObject q = new BasicDBObject(); + q.put("_id", new ObjectId(id.toString().replace("[","").replace("]",""))); + + DBObject keys = new BasicDBObject(); + if(fields){ + for(def field : fields){ + keys.put(field, 1); + } + } + def cursor = getDataStoreConnection().find(q, keys).batchSize(2000) + if(cursor){ + for(def obj : cursor){ + return obj; + } + } + return null; + } public static def findDistrictById(String id){ @@ -782,6 +827,7 @@ class CommonDatautils { keys.put("out_pics", 1); keys.put("lat", 1); keys.put("lng", 1); + keys.put("name_cat", 1); DSCursor cursor = getDataStoreConnection().find(q, keys); if(cursor){ for(def obj : cursor){ @@ -889,6 +935,25 @@ class CommonDatautils { } return null; } + + public static def findDistrictsByName(String city, String name, String cat){ + + DBObject q = new BasicDBObject(); + q.put("city", city); + q.put("cat", cat); + q.put("name", name); + q.put("visible", 0); + + def rets = [] + DSCursor cursor = getDataStoreConnection().find(q); + if(cursor){ + for(def obj : cursor){ + rets << obj; + } + } + return rets; + } + public static def getSimpleDistrictMap(def cat, def city){ DBObject q = new BasicDBObject(); q.put("visible", 0); @@ -1211,6 +1276,28 @@ class CommonDatautils { return ret; } + + + public static def findDistrictByAddress2(def city, def region, def address, def cat){ + DBObject q = new BasicDBObject(); + if(region){ + q.put('region', region) + } + + q.put('city',city) + q.put('cat', cat) + q.put('visible',0) + q.put("address", address); + + + def dsCursor = getDataStoreConnection().find(q); + def ret = []; + for(def obj : dsCursor){ + ret << obj; + } + + return ret; + } public static def getCityRegions(city){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index c61861c99..ad5a8babc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //¸£ÖݵÚÈý·½¾ù¼Ûͳ¼Æ,·¢ËͶ¤¶¤ÏûÏ¢ //def datas = ExcelUtils.read("d:\\Êý¾Ý\\¸£ÖÝ\\¸£ÖÝ4ÔÂ2.xlsx",0,1) //def datas = CsvUtils.read("d:\\Êý¾Ý\\¸£ÖÝ\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\Êý¾Ý\\¸£ÖÝ\\FJW_THIRD_PARTY_AVG_PRICE_202107.csv") +def datas = readTxt("d:\\Êý¾Ý\\¸£ÖÝ\\FJW_THIRD_PARTY_AVG_PRICE_202109.csv") def count0 = 0 def count1 = 0 def count2 = 0 @@ -135,7 +135,7 @@ rets << ["9 def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM") ExcelUtils.write("d:\\Êý¾Ý\\¸£ÖÝ\\¶à·½¾ù¼Û¶Ô±Èͳ¼ÆÊý¾Ý"+curMonth+".xlsx",rets,"ͳ¼Æ") def s = "µÚÈý·½¾ù¼Û×ÜÁ¿£º${total}\n6Çø×ÜÁ¿£º${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )" - +println s def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60" DingDingMessageUtil.sendTextMessage(access_token, s) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index 5c5cf081d..bbb18334c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -38,9 +38,7 @@ class GaoDeMapCrawl extends BaseBatch{ DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.235", 27018) def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc" - final def dbUrl = "mongodb://192.168.0.235:27018"; - ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) final AtomicInteger count = new AtomicInteger() def updateLists = [] @@ -50,8 +48,8 @@ class GaoDeMapCrawl extends BaseBatch{ // def categorys = ["141202", "141203"]//ÖÐѧ£¬Ð¡Ñ§ - def prefix = "district_" - def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//סլÏà¹Ø +// def prefix = "district_" +// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//סլÏà¹Ø // def prefix = "road_" // def categorys = ["190301"]//µÀ·Ïà¹Ø @@ -59,6 +57,10 @@ class GaoDeMapCrawl extends BaseBatch{ // def prefix = "village_" // def categorys = ["190108","190109"]//´åׯ + + def prefix = "food_" + def categorys = ["050000"] + // def prefix = "company_" // def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"] def totalCount = new AtomicLong() @@ -119,20 +121,24 @@ class GaoDeMapCrawl extends BaseBatch{ def run(){//,"Î÷°²","ÖØÇì" //def citys = ["ÉϺ£","ÖØÇì","¹þ¶û±õ","ÉòÑô","¶«Ý¸","Äϲý","À¥Ã÷","½ð»ª","ÎÂÖÝ","×Ͳ©","Õ¿½­","ÕØÇì","Ò˲ý","ÔÀÑô","°²Çì","Îߺþ","Î÷Äþ","ÈÕÕÕ","³£µÂ","ÕżҿÚ","±±¾©","¹ãÖÝ","ÉîÛÚ","ÖØÇì","Ö£ÖÝ","³¤É³","Î÷°²","³É¶¼","¶«Ý¸","·ðɽ","º¼ÖÝ","ºÏ·Ê","ÄϾ©","Çൺ","ÉòÑô","ËÕÖÝ","Ìì½ò","Î人","Ö麣","ÖÐɽ","³¤´º","ÑïÖÝ","ÑĮ̀","ÐìÖÝ","ÎÞÎý","ÎÂÖÝ","̫ԭ","̨ÖÝ","ʯ¼Òׯ","ÉÜÐË","ÏÃÃÅ","ȪÖÝ","Äþ²¨","ÄÏͨ","ÄÏÄþ","Äϲý","À¼ÖÝ","³£ÖÝ","´óÁ¬","¸£ÖÝ","¹óÑô","¹þ¶û±õ","º£¿Ú","»ÝÖÝ","¼ÃÄÏ","¼ÎÐË","½ð»ª","À¥Ã÷","×ñÒå","×Ͳ©","ÖêÖÞ","Õò½­","ÕØÇì","ÕÄÖÝ","Õ¿½­","ÔÀÑô","Òø´¨","Ò˲ý","ÑγÇ","Îߺþ","ÎÚ³ľÆë","ÌÆÉ½","Ì©ÖÝ","ÉÇÍ·","ÈýÑÇ","ÇåÔ¶","ÇØ»Êµº","Âí°°É½","ÂåÑô","ÁøÖÝ","Á¬ÔƸÛ","ÀÈ·»","¾Å½­","¼ªÁÖ","»´°²","ºþÖÝ","ºôºÍºÆÌØ","²×ÖÝ","±£¶¨","°²Çì","ÕżҿÚ","Î÷Äþ","ÈÕÕÕ","À­Èø","½õÖÝ","´óͬ","³£µÂ","±±º£","ÎIJý","ÁêË®","±£Í¤"] - def citys =["Äϲý"] -// def citys = CommonUtils.findAllCity() + def citys = ["ÄϾ©","ËÕÖÝ","ÎÞÎý","ÑïÖÝ","³£ÖÝ","ÐìÖÝ","Õò½­","ÄÏͨ","Ì©ÖÝ","Á¬ÔƸÛ","ÑγÇ","»´°²","ËÞǨ","º¼ÖÝ","Äþ²¨","ÎÂÖÝ","¼ÎÐË","̨ÖÝ","ºþÖÝ","ÉÜÐË","½ð»ª","ÖÛɽ","áéÖÝ","ÀöË®"] + def allcitys = ["ÉϺ£"]//CommonUtils.findAllCity() def totalCount = 0; - for(def city : citys){ - def stop = false - while(stop == false){ - def cityLations = findAllLation(city); - if(!cityLations){ - stop = true; - }else{ - println "¿ªÊ¼´¦Àí["+city +"]£º"+cityLations.size() - gaoDeCitySearch(cityLations); + for(def city : allcitys){ + + if(!citys.contains(city)){ + def stop = false + while(stop == false){ + def cityLations = findAllLation(city); + if(!cityLations){ + stop = true; + }else{ + println "¿ªÊ¼´¦Àí["+city +"]£º"+cityLations.size() + gaoDeCitySearch(cityLations); + } } } + } } @@ -144,7 +150,11 @@ class GaoDeMapCrawl extends BaseBatch{ q.put(prefix+"status", new BasicDBObject(QueryOperators.EXISTS, false)) // q.put("status", "ʧ°Ü") // q.put("traffic_status", new BasicDBObject(QueryOperators.EXISTS, false)) - def results = BOUNDS_COLL.find(q).limit(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def dbcursor = BOUNDS_COLL.find(q).batchSize(3000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def results = [] + for(def d : dbcursor){ + results << d + } return results; } @@ -185,14 +195,15 @@ class GaoDeMapCrawl extends BaseBatch{ } def searching(i, location){ - // def bound_gd = location.bound_gd +// def bound_gd = location.bound_gd // def lats = [bound_gd[1], bound_gd[3]] // def lngs = [bound_gd[0], bound_gd[2]] // lats.sort() // lngs.sort() -// println index -// if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){ - search(i, location) + println i+" "+ location +// 120.019556,30.313701 + //if(lngs[0]<=120.019556 && lngs[1]>=120.019556 && lats[0]<=30.313701 && lats[1]>=30.313701){ + search(i, location) // } if(updateLists.size() > 0 && updateLists.size()>1000){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenUserBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenUserBatch.groovy index c322efc25..45c2f60a5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenUserBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenUserBatch.groovy @@ -45,18 +45,20 @@ class OpenUserBatch { } def addShenZhenFjtUser(){ - def datas = ExcelUtils.read("D:\\Êý¾Ý\\Å©ÐÐ\\ÕØÇìÅ©ÐÐÖ§Ðз¿¼ÛͨÕ˺Å.xlsx",0,1) + def datas = ExcelUtils.read("D:\\Êý¾Ý\\²³º£\\¿ªÍ¨ÏßÉÏÆÀ¹À¿Í»§¾­ÀíÃûµ¥.xlsx",0,1) def i = 0; for(def data : datas){ i++; - def id = 252580 + i; - def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+data[2]+"', '"+data[3]+"', '16', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2021-12-10 00:00:00', '0', '', NULL)"; + def id = 252608 + i; + BigDecimal bd = new BigDecimal(data[3].toString()); + String mobile = bd.toPlainString(); + def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '17', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2023-01-01 00:00:00', '0', '', NULL)"; onLineSqlInstance.execute(sql); - def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[1]+"', '"+data[0]+"', NULL, NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', 'Íõ³½á¯')"; + def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', 'Ò¦¾²ÃÎ')"; onLineSqlInstance.execute(sql2) - def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'ÕØÇì','ÕØÇì:')" + def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'ÉϺ£','ÉϺ£:')" onLineSqlInstance.execute(sql3) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index c2c545471..3a182be6a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2021-07-01" + static def statDate = "2021-09-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//ÓÑ´ïÒ»ÊÖ·¿´æÈëmongo -// task.esfParser();//ÓÑ´ï¶þÊÖ·¿´æÈëmongo + task.esfParser();//ÓÑ´ï¶þÊÖ·¿´æÈëmongo // task.newhouseToSaveEs();//з¿±£´æµ½es task.esfhouseToSaveEs();//¶þÊÖ·¿±£´æµ½es // task.updateFinishDateByEsfTrade();//¸ù¾Ý¶þÊֳɽ»µÄ¿¢¹¤Ê±¼äÐÞ¸Äseaweed¿âÖеĿ¢¹¤Ê±¼ä @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\Êý¾Ý\\ÓÑ´ï\\2021Äê8ÔÂ\\¶þÊÖ·¿\\²éѯ1.DBF"] + def files = ["D:\\Êý¾Ý\\ÓÑ´ï\\2021Äê10ÔÂ\\¶þÊÖ·¿\\²éѯ1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy index 7a766b668..a418cebb9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy @@ -1,7 +1,9 @@ package com.fangjia.script.gj.ccb import com.fangjia.core.util.DataUtil +import org.apache.commons.lang.StringUtils +import java.util.regex.Matcher import java.util.regex.Pattern class CcbCodeUtils { @@ -173,19 +175,36 @@ class CcbCodeUtils { } //¸Ö»ì½á¹¹:1 ש»ì½á¹¹:2 ¸Ö½á¹¹:3 ¿ò¼Ü½á¹¹:4 ¿ò¼Ü¼ôÁ¦Ç½½á¹¹:5 ¼ôÁ¦Ç½½á¹¹:6 ͲÌå½á¹¹:7 ¿ò¼ÜͲÌå½á¹¹:8 ͲÖÐͲ:9 שľ½á¹¹:10 ÆäËû:11 - //½¨Öþ½á¹¹ +// ¸Ö¡¢¸Ö½î»ìÄýÍÁ½á¹¹ 002 +// ¸Ö½î»ìÄýÍÁ½á¹¹ 003 +// שľ½á¹¹ 004 +// »ìºÏ½á¹¹ 005 +// ÆäËü½á¹¹ 006 + static def getBuildStructureCode(def buildStructure){ if(buildStructure){ if(buildStructure.contains("¸Ö½á¹¹")){ - return 3; + return "001"; + }else if(buildStructure.contains("¸Ö¡¢¸Ö½î»ìÄýÍÁ½á¹¹")){ + return "002"; }else if(buildStructure.contains("¸Ö½î»ìÄýÍÁ½á¹¹")){ - return 1; + return "003"; }else if(buildStructure.contains("שľ½á¹¹")){ - return 10; + return "004"; + }else if(buildStructure.contains("»ìºÏ½á¹¹")){ + return "005"; } - return 11; } - return null; + return "006"; + } + + static def getDistHouseTypeCode(def estateType){ + if(estateType && estateType=="±ðÊû"){ + return "08" + }else if(estateType && (estateType=="ÆÕͨסլ" || estateType=="ÉÌס")){ + return "01" + } + return "99" } //ÊÒÄڽṹ //ƽ²ã:11 ´í²ã:12 ¸´Ê½:13 ¸óÂ¥:14 ÍŽủ:15 ÀÏʽ·¿:16 µØÏÂÊÒ:17 ´ø¸óÂ¥:21 ´ø³µ¿â:22 ÆäËû:99 Ô¾²ã:18 ´øµØÏÂÊÒ:24 ´ø´¢²ØÊÒ:25 ´ø³µÎ»:23 @@ -221,4 +240,67 @@ class CcbCodeUtils { } return 99 } + + public static Integer getBuildYear(String buildDate){ + if(StringUtils.isNotBlank(buildDate)){ + Pattern p = Pattern.compile("\\d{4}") + Matcher m = p.matcher(buildDate); + while (m.find()) { + return m.group(0).toInteger(); + } + } + return null; + } + + static def processEstateCharge(def city, def estateCharege, def estateType2){ + if(estateCharege){ + try{ + estateCharege = trimChar(estateCharege); + estateCharege = estateCharege.replace("Ôª/ƽÃ×/ÔÂ","").replace("Ôª/ƽ·½Ã×/ÔÂ","").replace("Ôª/?O?ÔÂ","") + estateCharege = estateCharege.toDouble() + if(estateType2=="ÆÕͨסլ"){ + if(city=="ÉϺ£" || city=="±±¾©" || city=="¹ãÖÝ" || city=="º¼ÖÝ"){ + if(estateCharege>=0.6 && estateCharege<=3.5){ + return estateCharege + }else{ + return null; + } + }else{ + if(estateCharege>=0.2 && estateCharege<=3.5){ + return estateCharege + }else{ + return null; + } + } + }else if(estateType2=="±ðÊû"){ + if(estateCharege>=3){ + return estateCharege; + }else{ + return null; + } + }else if(estateType2=="ÉÌס"){ + if(estateCharege>=4){ + return estateCharege; + }else{ + return null; + } + } + }catch (Exception e){ + println "estateCharege ת²»ÁË "+estateCharege + return null; + } + } + return null; + } + + static def trimChar(String c) { + try { + if(c){ + c = new String(c.getBytes("GBK"), "GBK").replace("?", "/"); + } + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + return c; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbXykData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbXykData.groovy new file mode 100644 index 000000000..c90064d97 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbXykData.groovy @@ -0,0 +1,18 @@ +package com.fangjia.script.gj.ccb + +import com.fangjia.script.commData.CommonDatautils + +class CcbXykData { + static main(args) { + def task = new CcbXykData() + task.run() + } + + def run(){ + def districts = CommonDatautils.findAllDistrictByCity("ÉϺ£","district",["city","region","name","estate_charge"]) + for(def d : districts){ + + } + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy index e7232a3c8..3e4981856 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -9,12 +9,20 @@ class FileUtils { // file.delete(); // } + if (!file.getParentFile().exists()) { + boolean result = file.getParentFile().mkdirs(); + if (!result) { + System.out.println("´´½¨Ä¿Â¼Ê§°Ü"); + } + } + + OutputStream out = new FileOutputStream(fileName, isAppend); def sb = new StringBuffer(); if(headers){ sb.append(headers.join("|@|")) - sb.append(System.getProperty("line.separator")); + sb.append("\n"); } out.write(sb.toString().getBytes(charset)); @@ -25,7 +33,7 @@ class FileUtils { if(row){ index++ sb.append(StringUtils.join(row.toArray(), "|@|")) - sb.append(System.getProperty("line.separator")); + sb.append("\n"); if(index%10000==0){ println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy index eb7d4aadd..6b07a6159 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy @@ -14,12 +14,45 @@ class FieldStat { static main(args) { def task = new FieldStat(); // task.run() - task.run1() +// task.run1() // task.run2() + task.run3() } + def run3(){ + def citys = CommonDatautils.findAllCity()//["±±¾©","ÉϺ£","¹ãÖÝ","ÉîÛÚ","ÖØÇì","Ö£ÖÝ","³¤É³","Î÷°²","³É¶¼","¶«Ý¸","·ðɽ","º¼ÖÝ","ºÏ·Ê","ÄϾ©","Çൺ","ÉòÑô","ËÕÖÝ","Ìì½ò","Î人","Ö麣","ÖÐɽ","³¤´º","ÑïÖÝ","ÑĮ̀","ÐìÖÝ","ÎÞÎý","ÎÂÖÝ","̫ԭ","̨ÖÝ","ʯ¼Òׯ","ÉÜÐË","ÏÃÃÅ","ȪÖÝ","Äþ²¨","ÄÏͨ","ÄÏÄþ","Äϲý","À¼ÖÝ","³£ÖÝ","´óÁ¬","¸£ÖÝ","¹óÑô","¹þ¶û±õ","º£¿Ú","»ÝÖÝ","¼ÃÄÏ","¼ÎÐË","½ð»ª","À¥Ã÷","×ñÒå","×Ͳ©","ÖêÖÞ","Õò½­","ÕØÇì","ÕÄÖÝ","Õ¿½­","ÔÀÑô","Òø´¨","Ò˲ý","ÑγÇ","Îߺþ","ÎÚ³ľÆë","ÌÆÉ½","Ì©ÖÝ","ÉÇÍ·","ÈýÑÇ","ÇåÔ¶","ÇØ»Êµº","Âí°°É½","ÂåÑô","ÁøÖÝ","Á¬ÔƸÛ","ÀÈ·»","¾Å½­","¼ªÁÖ","»´°²","ºþÖÝ","ºôºÍºÆÌØ","²×ÖÝ","±£¶¨","°²Çì","ÕżҿÚ","Î÷Äþ","ÈÕÕÕ","À­Èø","½õÖÝ","´óͬ","³£µÂ","±±º£","ÎIJý","ÁêË®","±£Í¤"] + def dist_total = 0 + + def addr_alias_total = 0 + def rets = [] + rets << ["³ÇÊÐ","ÓеØÖ·±ðÃû×ÜÁ¿","Ð¡Çø×ÜÁ¿","ÓеØÖ·±ðÃûÐ¡ÇøÕ¼±È"] + for(def city : citys){ + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "addr_alias"]); + def alias_index = 0 + def alias_total = 0 + + for(def d : districts){ + def alias = d.addr_alias + if(alias){ + alias_index++ + } + } + def precent = 0 + if(districts.size()){ + precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size()) + } + rets << [city, alias_index, districts.size(), precent+"%"] + + } + ExcelUtils.write("d:\\Êý¾Ý\\È«¹ú³ÇÊеØÖ·±ðÃûͳ¼Æ.xlsx", rets, "ͳ¼Æ") +// println dist_total +// println alias_total +// println addr_alias_total + } + + def run1(){ - def citys = ["±±¾©","ÉϺ£","¹ãÖÝ","ÉîÛÚ","ÖØÇì","Ö£ÖÝ","³¤É³","Î÷°²","³É¶¼","¶«Ý¸","·ðɽ","º¼ÖÝ","ºÏ·Ê","ÄϾ©","Çൺ","ÉòÑô","ËÕÖÝ","Ìì½ò","Î人","Ö麣","ÖÐɽ","³¤´º","ÑïÖÝ","ÑĮ̀","ÐìÖÝ","ÎÞÎý","ÎÂÖÝ","̫ԭ","̨ÖÝ","ʯ¼Òׯ","ÉÜÐË","ÏÃÃÅ","ȪÖÝ","Äþ²¨","ÄÏͨ","ÄÏÄþ","Äϲý","À¼ÖÝ","³£ÖÝ","´óÁ¬","¸£ÖÝ","¹óÑô","¹þ¶û±õ","º£¿Ú","»ÝÖÝ","¼ÃÄÏ","¼ÎÐË","½ð»ª","À¥Ã÷","×ñÒå","×Ͳ©","ÖêÖÞ","Õò½­","ÕØÇì","ÕÄÖÝ","Õ¿½­","ÔÀÑô","Òø´¨","Ò˲ý","ÑγÇ","Îߺþ","ÎÚ³ľÆë","ÌÆÉ½","Ì©ÖÝ","ÉÇÍ·","ÈýÑÇ","ÇåÔ¶","ÇØ»Êµº","Âí°°É½","ÂåÑô","ÁøÖÝ","Á¬ÔƸÛ","ÀÈ·»","¾Å½­","¼ªÁÖ","»´°²","ºþÖÝ","ºôºÍºÆÌØ","²×ÖÝ","±£¶¨","°²Çì","ÕżҿÚ","Î÷Äþ","ÈÕÕÕ","À­Èø","½õÖÝ","´óͬ","³£µÂ","±±º£","ÎIJý","ÁêË®","±£Í¤"] + def citys = CommonDatautils.findAllCity()//["±±¾©","ÉϺ£","¹ãÖÝ","ÉîÛÚ","ÖØÇì","Ö£ÖÝ","³¤É³","Î÷°²","³É¶¼","¶«Ý¸","·ðɽ","º¼ÖÝ","ºÏ·Ê","ÄϾ©","Çൺ","ÉòÑô","ËÕÖÝ","Ìì½ò","Î人","Ö麣","ÖÐɽ","³¤´º","ÑïÖÝ","ÑĮ̀","ÐìÖÝ","ÎÞÎý","ÎÂÖÝ","̫ԭ","̨ÖÝ","ʯ¼Òׯ","ÉÜÐË","ÏÃÃÅ","ȪÖÝ","Äþ²¨","ÄÏͨ","ÄÏÄþ","Äϲý","À¼ÖÝ","³£ÖÝ","´óÁ¬","¸£ÖÝ","¹óÑô","¹þ¶û±õ","º£¿Ú","»ÝÖÝ","¼ÃÄÏ","¼ÎÐË","½ð»ª","À¥Ã÷","×ñÒå","×Ͳ©","ÖêÖÞ","Õò½­","ÕØÇì","ÕÄÖÝ","Õ¿½­","ÔÀÑô","Òø´¨","Ò˲ý","ÑγÇ","Îߺþ","ÎÚ³ľÆë","ÌÆÉ½","Ì©ÖÝ","ÉÇÍ·","ÈýÑÇ","ÇåÔ¶","ÇØ»Êµº","Âí°°É½","ÂåÑô","ÁøÖÝ","Á¬ÔƸÛ","ÀÈ·»","¾Å½­","¼ªÁÖ","»´°²","ºþÖÝ","ºôºÍºÆÌØ","²×ÖÝ","±£¶¨","°²Çì","ÕżҿÚ","Î÷Äþ","ÈÕÕÕ","À­Èø","½õÖÝ","´óͬ","³£µÂ","±±º£","ÎIJý","ÁêË®","±£Í¤"] def dist_total = 0 def addr_alias_total = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy index 38c00f546..9219fc438 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy @@ -37,10 +37,11 @@ class SeaweedDataMatchProcess { rMap = OpenApiUtils.matchDistrict(city, name, region, false) } + if(!rMap && address){ - rMap = OpenApiUtils.matchDistrict(city, address, region, true) - if(rMap && rMap.region && !region.contains(rMap.region)){ - rMap = null + def flag = address.matches('^[\u4e00-\u9fa5]+?(·|´óµÀ|½Ö).+[ŪºÅ]$') + if(flag){ + rMap = OpenApiUtils.matchDistrict(city, address, region, true) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/guangfa/GuangfaBlankSelectDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/guangfa/GuangfaBlankSelectDistrict.groovy index 4643a812a..7a00efd46 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/guangfa/GuangfaBlankSelectDistrict.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/guangfa/GuangfaBlankSelectDistrict.groovy @@ -31,18 +31,24 @@ class GuangfaBlankSelectDistrict { static main(args) { def task = new GuangfaBlankSelectDistrict(); - task.run() +// task.run() + task.run2() } def run(){ - def file = "D:\\Êý¾Ý\\¹ã·¢\\»ñȡ¥ÅÌÐÅÏ¢ÐèÇ󣨷¢ËͰ棩.xlsx" + def file = "D:\\Êý¾Ý\\¹ã·¢\\»ñȡ¥ÅÌÐÅÏ¢ÐèÇ󣨶«Ý¸ºÍº¼ÖÝ£©.xlsx" def cityMap = standardRegion(file) def priceMap = getZhiDaoPrice(file) selectDistricts(cityMap, priceMap) println "дÈëÍê³É" } + def run2(){ + selectDistricts2() + println "дÈëÍê³É" + } + def selectDistricts(cityMap, priceMap){ def rets = [] rets << ["³ÇÊÐ","ÇøÓò","Ö¸µ¼¼Û","Â¥ÅÌÃû³Æ","Â¥Å̵ØÖ·","¿ª·¢ÉÌÃû³Æ","ÈÝ»ýÂÊ","ÎïÒµÃû³Æ","*Ì×Êý","*Â¥Áä","»§ÐÍ","*½¨ÖþÃæ»ýÇø¼ä","2021Äê1-6Ô³ɽ»Á¿","*¾ù¼Û","*³É½»Á¿","*¾ù¼Û","*³É½»Á¿","*¾ù¼Û"] @@ -107,6 +113,70 @@ class GuangfaBlankSelectDistrict { ExcelUtils.write("d:\\Êý¾Ý\\¹ã·¢\\Â¥ÅÌ(Î÷°²¶«Ý¸º¼ÖÝ).xlsx", rets, "Êý¾Ý") } + + def selectDistricts2(){ + def rets = [] + rets << ["³ÇÊÐ","ÇøÓò","Ö¸µ¼¼Û","Â¥ÅÌÃû³Æ","Â¥Å̵ØÖ·","¿ª·¢ÉÌÃû³Æ","ÈÝ»ýÂÊ","ÎïÒµÃû³Æ","*Ì×Êý","*Â¥Áä","»§ÐÍ","*½¨ÖþÃæ»ýÇø¼ä","2021Äê1-6Ô³ɽ»Á¿","*¾ù¼Û","*³É½»Á¿","*¾ù¼Û","*³É½»Á¿","*¾ù¼Û"] + def citys = ["¶«Ý¸","º¼ÖÝ"] + citys.each{city-> + def thirdPriceMap = findNearThirdPartyPrice(city) + def retMaps = findCreditHightDistrict3(city, "2021-08-22", thirdPriceMap) + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias","estate_company", "developer","cubage_percent","household_count", "finish_building_date"]) + //»ù±¾Çé¿öºÏ¸ñµÄÐ¡Çø + def distMap = [:] + for(def d : districts){ + // && d.household_count && d.household_count>=50 + def distKeys = retMaps.get(d.region) + if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){ + if(d.finish_building_date){ + def year = CcbCodeUtils.parseBuildYear(d.finish_building_date) + if(year){ + def mod = 2021-year + if(mod <=15){ + d.house_year = mod + }else{ + d.house_year = null + } + }else{ + d.house_year = null + } + + }else{ + d.house_year = null + } + distMap.put(d.city+"##"+d.region+"##"+d.name, d) + } +// if(d.finish_building_date){ +// def year = CcbCodeUtils.parseBuildYear(d.finish_building_date) +// if(year){ +// def mod = 2021-year +// if(mod <=15){ +// def distKeys = retMaps.get(d.region) +// if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){ +// d.house_year = mod +// distMap.put(d.city+"##"+d.region+"##"+d.name, d) +// } +// } +// } +// } + + } + println city +" »ù±¾Çé¿öºÏ¸ñµÄÐ¡Çø:"+distMap.size() + + for(def r : regions){ + def names = retMaps.get(r) + if(names){ + def results = getRoomTypeAndTradeInfo(names, distMap, priceMap) + rets.addAll(results) + } + } + println "½áÊø" + } + + println "¿ªÊ¼Ð´ÈëExcel....."+rets.size() + ExcelUtils.write("d:\\Êý¾Ý\\¹ã·¢\\Â¥ÅÌ(Î÷°²¶«Ý¸º¼ÖÝ).xlsx", rets, "Êý¾Ý") + } + def getRoomTypeAndTradeInfo(def names, def distMap, def priceMap){ def rets = Collections.synchronizedList(new ArrayList()); @@ -332,4 +402,52 @@ class GuangfaBlankSelectDistrict { } return thirdPriceMap } + + + def findNearThirdPartyPrice(def city){ + def q = new BasicDBObject() + q.put("city", city) + q.put("s_date", new BasicDBObject('$gte', 202105)) + q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true)) + def dbcursor = AVERAGE_STATISTICS.find(q) + def thirdPriceMap = [:] + for(def d : dbcursor){ + def key = d.city+"#"+d.region+"#"+d.name + if(!thirdPriceMap.containsKey(key)){ + thirdPriceMap.put(d.city+"#"+d.region+"#"+d.name, d.lianjia_esf_price) + } + } + return thirdPriceMap + } + + def findCreditHightDistrict3(def city, def regions, def weekend, def thirdPriceMap){ + def q = new BasicDBObject() + q.put("city", city) + q.put("value_type", 2) + q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd")) + def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1)) + def retMaps = [:] + for(def d : dbcursor){ + def thirdPrice = thirdPriceMap.get(d.city +"#"+d.region+"#"+d.district_name) + if(thirdPrice){ + def precent = (thirdPrice-d.avg_price)*100/d.avg_price + if(Math.abs(precent)>=0 && Math.abs(precent)<=10){ + def rets = retMaps.get(d.region) + if(!rets){ + rets = [] + retMaps.put(d.region, rets) + } + rets << d.city+"##"+d.region+"##"+d.district_name + } + }else{ + def rets = retMaps.get(d.region) + if(!rets){ + rets = [] + retMaps.put(d.region, rets) + } + rets << d.city+"##"+d.region+"##"+d.district_name + } + } + return retMaps; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy new file mode 100644 index 000000000..0ad5dbfcf --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy @@ -0,0 +1,64 @@ +package com.fangjia.script.gj.location + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.NameProcess +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.ResultSet + +class DistrictLocationUpdate { + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + static main(args) { + def task = new DistrictLocationUpdate() + task.run() + } + def run(){ + def citys = CommonDatautils.findAllCity() + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias", "lng", "lat"]) + + + updateLocation(city, dists); + + } + + } + + def queryPoiMap(def city){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1") + sqlBuffer.append(" and cityname = '" + city + "'") + sqlBuffer.append(" and source = '¸ßµÂ'") + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def retMap = [:] + while (rs.next()) { + def map = [:] + map.city = rs.getString("cityname") + map.region = rs.getString("adname") + map.name = rs.getString("name") + map.address = rs.getString("address") + map.type = rs.getString("type") + def location = rs.getObject("location") + if(location){ + map.lng = location.x + map.lat = location.y + } + + def name = NameProcess.nameStandard(map.city, map.region, map.name) + if(name){ + retMap.put(map.region+"##"+name, map) + } + } + return retMap; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy index 4313382e5..d60d5f02c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy @@ -521,6 +521,10 @@ class GaoDeUtils { return KeyList.get(count); } + static void main(def a){ + println search("Èç¶«ÏØ´óÔ¥ÕòÔ¥¶«´åί»á","ÄÏͨ") + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/MatchDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/MatchDistrict.groovy new file mode 100644 index 000000000..7d52d6a3b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/MatchDistrict.groovy @@ -0,0 +1,45 @@ +package com.fangjia.script.gj.test + +import com.fangjia.core.RegionData +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject + +class MatchDistrict { + static main(args) { + def task = new MatchDistrict() + task.run() + } + def run(){ + def filePath = "D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\Ö¸µ¼¼Û\\ÉîÛÚÖ¸µ¼¼Û - 210909 -.xlsx" + def city = "ÉîÛÚ" + ExcelUtils.multiThreadingModify(filePath, { data -> + if(data['A']){ + def region = RegionData.formatRegion(city, data['A']) + if(region){ + def rMap = SeaweedDataMatchProcess.doMatch(city, data['A'], data['C'], null); + if(rMap){ + data['E'] = rMap.city + data['F'] = rMap.region + data['G'] = rMap.name + } + } + } + }, 0); + } +} + +//{ +// "_id" : ObjectId("607963dc5bb87e2a18f4dde6"), +// "guiding_type" : NumberInt(0), +// "estate_type" : "ÆÕͨסլ", +// "house_type" : NumberInt(0), +// "c_date" : ISODate("2021-04-16T10:15:56.651+0000"), +// "guiding_person" : "ÉîÛÚ·¿¹Ü¾Ö", +// "guiding_date" : ISODate("2021-04-10T16:00:00.000+0000"), +// "category" : "district", +// "guiding_value" : NumberInt(48300), +// "name" : "ÖÐÁ¸Ôƾ°¹ú¼Ê", +// "city" : "ÉîÛÚ", +// "region" : "¹âÃ÷" +//} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy index aa345e412..91498750a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy @@ -1,33 +1,55 @@ package com.fangjia.script.gj.test +import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.Mongo +import groovy.sql.Sql import java.text.SimpleDateFormat //SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); //Date date = new Date(1617211269868); //print dateFormat.format(date); -DBCollection CITY_INFO_CONN = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_data_ana').getCollection('city_info'); -def path = "D:\\Êý¾Ý\\¹ã·¢\\¹ã¶«Ê¡Ð¡Çø¼°¾ù¼Û¸²¸ÇÂÊͳ¼Æ.xlsx" -ExcelUtils.modify(path,{d-> - def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["name"]) - d['G'] = districts.size() +//DBCollection CITY_INFO_CONN = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_data_ana').getCollection('city_info'); +//def path = "D:\\Êý¾Ý\\¹ã·¢\\¹ã¶«Ê¡Ð¡Çø¼°¾ù¼Û¸²¸ÇÂÊͳ¼Æ.xlsx" +//ExcelUtils.modify(path,{d-> +// def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["name"]) +// d['G'] = districts.size() +// +// def q = new BasicDBObject() +// q.put("city", d['B']) +// q.put("weekend", DateUtils.parseFromTightYYYYMMDD("2021-08-08")) +// def obj = CITY_INFO_CONN.findOne(q); +// if(obj){ +// d['H'] = obj.price_coverage*100 +// } +//}) - def q = new BasicDBObject() - q.put("city", d['B']) - q.put("weekend", DateUtils.parseFromTightYYYYMMDD("2021-08-08")) - def obj = CITY_INFO_CONN.findOne(q); - if(obj){ - d['H'] = obj.price_coverage*100 +def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver"); + +def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'"; +def countMap = [:] +onLineSqlInstance.eachRow(sql,{r-> + def addTime = r.visit_time + def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM") + def ips = countMap.get(month) + if(!ips){ + ips = new HashSet(); + countMap.put(month, ips) } + ips << r.ip +}); + +countMap.each {k,v-> + println k+" "+v.size() +} + + - -}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy index 98eca9153..811de10a1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy @@ -2,6 +2,8 @@ package com.fangjia.script.jiangnanblank import com.jcraft.jsch.* +import javax.transaction.SystemException + /** * Created by gaojuan on 2017/12/6. */ @@ -47,12 +49,23 @@ class SFTP { * @param sftp */ public void upload(String directory, String uploadFile, ChannelSftp sftp) { - try { + FileInputStream stream = null + try{ + sftp = createDir(directory, sftp) sftp.cd(directory); File file=new File(uploadFile); - sftp.put(new FileInputStream(file), file.getName()); + stream = new FileInputStream(file) + sftp.put(stream, file.getName()); } catch (Exception e) { e.printStackTrace(); + } finally { + if(stream!=null){ + stream.close(); + } +// if (sftp != null) { +// sftp.disconnect(); +// sftp.exit(); +// } } } @@ -100,4 +113,51 @@ class SFTP { } + /** + * ´´½¨Ò»¸öÎļþĿ¼ + */ + public ChannelSftp createDir(String createpath, ChannelSftp sftp) { + try { + if (isDirExist(createpath, sftp)) { + sftp.cd(createpath); + } + def pathArry = createpath.split("/"); + StringBuffer filePath = new StringBuffer("/"); + for (String path : pathArry) { + if (path.equals("")) { + continue; + } + filePath.append(path + "/"); + if (isDirExist(filePath.toString(), sftp)) { + sftp.cd(filePath.toString()); + } else { + // ½¨Á¢Ä¿Â¼ + sftp.mkdir(filePath.toString()); + // ½øÈë²¢ÉèÖÃΪµ±Ç°Ä¿Â¼ + sftp.cd(filePath.toString()); + } + } + sftp.cd(createpath); + } catch (SftpException e) { + throw new SystemException("´´½¨Â·¾¶´íÎó£º" + createpath); + } + return sftp + } +/** + * ÅжÏĿ¼ÊÇ·ñ´æÔÚ + */ + public boolean isDirExist(String directory, ChannelSftp sftp) { + boolean isDirExistFlag = false; + try { + SftpATTRS sftpATTRS = sftp.lstat(directory); + isDirExistFlag = true; + return sftpATTRS.isDir(); + } catch (Exception e) { + if (e.getMessage().toLowerCase().equals("no such file")) { + isDirExistFlag = false; + } + } + return isDirExistFlag; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/test/RegionalismCodeStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/test/RegionalismCodeStat.groovy new file mode 100644 index 000000000..7f49b80b0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/test/RegionalismCodeStat.groovy @@ -0,0 +1,58 @@ +package com.fangjia.script.test + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class RegionalismCodeStat { + + DBCollection REGIONALISM_CODE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "regionalism_code", "192.168.0.235", 27018) + + + static main(args) { + def task = new RegionalismCodeStat() + task.run() + } + + def run(){ + def provinces = ["ÉϺ£ÊÐ","±±¾©ÊÐ","Ìì½òÊÐ","ÖØÇìÊÐ","¸£½¨Ê¡","¹óÖÝÊ¡","ºþÄÏÊ¡","¹ã¶«Ê¡","ºþ±±Ê¡","°²»ÕÊ¡","ËÄ´¨Ê¡","½­ËÕÊ¡","¼ªÁÖÊ¡","ºÓ±±Ê¡","ºÓÄÏÊ¡","º£ÄÏÊ¡","½­Î÷Ê¡","ÔÆÄÏÊ¡","¸ÊËàÊ¡","ɽ¶«Ê¡","ÉÂÎ÷Ê¡","Õã½­Ê¡","Çຣʡ","ÁÉÄþÊ¡","ºÚÁú½­Ê¡","ɽÎ÷Ê¡","Î÷²Ø×ÔÖÎÇø","н®Î¬Îá¶û×ÔÖÎÇø","ÄþÏÄ»Ø×å×ÔÖÎÇø","ÄÚÃɹÅ×ÔÖÎÇø","¹ãÎ÷׳×å×ÔÖÎÇø"] +// def provinces = ["ÄþÏÄ»Ø×å×ÔÖÎÇø"] + provinces.each {v-> + def sets = findCount(v) + println v +" "+ sets.size() + } + + } + + def findCount(v){ + def q = new BasicDBObject() + q.put("province", v) + def dbcursor = REGIONALISM_CODE_COLL.find(q).batchSize(5000) + def set = new HashSet() + for(def obj : dbcursor){ +// if(obj.city=="Ê¡Ö±Ï½ÏØ¼¶ÐÐÕþÇø»®" || obj.city=="×ÔÖÎÇøÖ±Ï½ÏØ¼¶ÐÐÕþÇø»®"){ +// //set << obj.region +// println "¡¾"+obj.region+"¡¿ ¹ýÂË" +// continue +// }else{ +// if(obj.region && !set.contains(obj.region)){ +// set << obj.region +// println "¡¾"+obj.region+"¡¿ add" +// } +// } + + if(obj.juweihui){ + def k = obj.street+"##"+obj.juweihui + if(!set.contains(k)){ + set << k +// println "¡¾"+k+"¡¿ add" + } + } + } + +// set.each {k-> +// println k +// } + return set; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index c8ab1b6d8..75a441c06 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -4,6 +4,8 @@ import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.NameProcess +import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.linxsh.util.CommonUtil import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.CosineSimilarAlgorithm @@ -59,7 +61,7 @@ class DataStandardBatch extends BaseBatch{ private static def limit = 5000; def static final SHORT_DATE = Utils.getLastMonth() - def static final statusMap = ["³ÇÊÐδƥÅä":1,"ÇøÓòδƥÅä":2,"Ð¡ÇøÆ¥Åäʧ°Ü":3,"Ð¡ÇøÆ¥Åä³É¹¦":4,"Òì³£´íÎó":5, "ÎÞ¼Û¸ñ":6] + def static final statusMap = ["³ÇÊÐδƥÅä":1,"ÇøÓòδƥÅä":2,"Ð¡ÇøÆ¥Åäʧ°Ü":3,"Ð¡ÇøÆ¥Åä³É¹¦":4,"Òì³£´íÎó":5, "ÎÞ¼Û¸ñ":6, "ÎÞÃû³Æ":7] def static final websites = ["·¿¹À¹À","Öî¸ðÕÒ·¿","°²¾Ó¿Í","ËÑ·¿","ÖÐÔ­µØ²ú","¸Ï¼¯", "Á´¼Ò","Íò©Ê¿","³ÇÊз¿²ú","±´¿ÇÕÒ·¿","Q·¿","Q·¿(ÐÂ)","ÖÐÔ­µØ²ú(ÐÂ)","ÎÒ°®ÎÒ¼Ò","ÂóÌï","·¿Í¾","C21","ÀÖ¾Ó","·¿²ú³¬ÊÐ","58ͬ³Ç","365ÌÔ·¿","¼ªÎÝ"] public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) @@ -81,7 +83,6 @@ class DataStandardBatch extends BaseBatch{ @Override void process(Object args) { - def flag = true; // ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl) @@ -120,26 +121,35 @@ class DataStandardBatch extends BaseBatch{ void run() { try{ println "(${rowIndex}-${count}-${count_.incrementAndGet()})¡¾${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}¡¿" -// if(dbo.avg_price){ + if(dbo.name){ //±ê×¼»¯³ÇÊÐ(°üº¬region) def data = standardCity(dbo); if(data){ //Æ¥ÅäÇøÓò - data = standardRegion(data); - - if(data){ - //±ê×¼»¯Ð¡ÇøÃû - def data3 = standardDistrict(data, dbo.avg_price); + if(data.city=="ÈýÑÇ" && data.region=="ÁêË®"){ + data.city = "ÁêË®" + data.region = "" + def data3 = standardDistrictSimple(data, dbo.avg_price); if(data3){ successCount++; } + }else if(data){ + data = standardRegion(data); + if(data){ + //±ê×¼»¯Ð¡ÇøÃû + def data3 = standardDistrict(data, dbo.avg_price); + if(data3){ + successCount++; + } + } } + } -// }else{ -// saveLog(dbo, "ÎÞ¼Û¸ñ"); -// updateStatus(dbo,statusMap.get("ÎÞ¼Û¸ñ")) -// } + }else{ + saveLog(dbo, "ÎÞÃû³Æ"); + updateStatus(dbo,statusMap.get("ÎÞÃû³Æ")) + } count++; }catch (Exception e){ @@ -253,6 +263,104 @@ class DataStandardBatch extends BaseBatch{ return data; } + def standardDistrict2(def data, def avg_price){ + def newName = NameProcess.nameStandard(data.city, data.region, data.name) + def district = null + if(newName){ + district = SeaweedDataMatchProcess.doMatch(data.city, data.region, data.name, data.address); + if(district){ + + } + } + + if(district && avg_price){ + removeErrorLog(data);//ɾ³ý´íÎóÈÕÖ¾±íÖÐÇøÓò²»Æ¥ÅäµÄ£¬ÕýʽÔËÐÐʱɾ³ý + saveSuccessLog(data, district, type); + updateStatus(data,statusMap.get("Ð¡ÇøÆ¥Åä³É¹¦")) + println district.city+" "+district.name+" success" + }else{ + if(district){ +// data.type = type + saveLog(data, "ÎÞ¼Û¸ñ"); + updateStatus(data,statusMap.get("ÎÞ¼Û¸ñ")) + println data.city+" "+data.name+" fail(ÎÞ¼Û¸ñ)" + }else{ + println "[Æ¥Åäʧ°Ü]" + saveLog(data, "Ð¡ÇøÆ¥Åäʧ°Ü"); + updateStatus(data,statusMap.get("Ð¡ÇøÆ¥Åäʧ°Ü")) + println data.city+" "+data.name+" fail" + } + } + return district; + + } + + def standardDistrictSimple(def data, def avg_price){ + def newName1 = null + def newName2 = null + if(data.name.contains("±ðÃû")){ + newName1 = data.name.toString().substring(0, data.name.toString().indexOf("±ðÃû")-1) + newName2 = data.name.toString().substring(data.name.toString().indexOf("±ðÃû")+3) + }else{ + newName1 = NameProcess.nameStandard(data.city, data.region, data.name) + } + def district = null + def type = null + if(newName1){ + district = SeaweedDataMatchProcess.doMatch(data.city, data.region, newName1, data.address); + if(district && (district.name == newName1 || (district.alias && district.alias.contains(newName1)))){ + district._id = data._id + type = 1 + }else{ + if(newName2){ + district = SeaweedDataMatchProcess.doMatch(data.city, data.region, newName2, data.address); + if(district && (district.name == newName2 || (district.alias && district.alias.contains(newName2)))){ + district._id = data._id + type = 1 + }else{ + def score = CosineSimilarAlgorithm.getSimilarity(district.name, newName2) + if(score>0.8){ + district._id = data._id + type = 11 + }else{ + district = null + } + } + }else{ + if(district){ + def score = CosineSimilarAlgorithm.getSimilarity(district.name, newName1) + if(score>0.8){ + district._id = data._id + type = 11 + }else{ + district = null + } + } + } + } + } + + if(district && avg_price){ + removeErrorLog(data);//ɾ³ý´íÎóÈÕÖ¾±íÖÐÇøÓò²»Æ¥ÅäµÄ£¬ÕýʽÔËÐÐʱɾ³ý + saveSuccessLog(data, district, type); + updateStatus(data,statusMap.get("Ð¡ÇøÆ¥Åä³É¹¦")) + println district.city+" "+district.name+" success" + }else{ + if(district){ + data.type = type + saveLog(data, "ÎÞ¼Û¸ñ"); + updateStatus(data,statusMap.get("ÎÞ¼Û¸ñ")) + println data.city+" "+data.name+" fail(ÎÞ¼Û¸ñ)" + }else{ + println "[Æ¥Åäʧ°Ü]" + saveLog(data, "Ð¡ÇøÆ¥Åäʧ°Ü"); + updateStatus(data,statusMap.get("Ð¡ÇøÆ¥Åäʧ°Ü")) + println data.city+" "+data.name+" fail" + } + } + return district; + } + def standardDistrict(def data, def avg_price){ //Ãû³ÆÆ¥Å䣬Ãû³Æ±ðÃûÆ¥Å䣬µØÖ·±ðÃûÆ¥Åä //def district = getMatchName(data.city, data.region, data.name, data.address,"district") @@ -368,30 +476,10 @@ class DataStandardBatch extends BaseBatch{ } if(district){ if(data.region != district.region){ - //type = 12;//ÇøÓò²»Í¬ + type = 12;//ÇøÓò²»Í¬ district = null; }else{ - def score1 = 0; - if(data.address && district.address){ - score1 = CosineSimilarAlgorithm.getSimilarityRatio(data.address, district.address); - } - if(score1 && score1 >=0.8){ - type = 13;//µØÖ·ÏàËÆ¶ÈÔÚ0.8ÒÔÉÏ - }else{ - if(addr && (district.address && addr.indexOf(district.address)>-1||district.address.toString().indexOf(addr)>-1) && (data.name.indexOf(district.name)>-1 || district.name.toString().indexOf(data.name)>-1)){ - type = 13; - }else{ - if(!addr){ - def openDistName = district.name.replace("Ð¡Çø","") - def name = data.name.replace("Ð¡Çø","") - if(openDistName.contains(name) || name.contains(openDistName)){ - type = 15;//Æ¥Åä½Ó¿Úͨ¹ýÃû³ÆÅäÉÏ£¬Ãû³Æ°üº¬ - } - } - } - //type = 11; - district = null; - } + type=15 } } } @@ -495,10 +583,16 @@ class DataStandardBatch extends BaseBatch{ }*/ - def getMatchNameFromMap(def city, def region, def disName, def address){ + def getMatchNameFromMap(def city, def region, def disName, String address){ //²éѯ·¿¼ÛÍøÊÇ·ñ´æÔÚ¸ÃÐ¡Çø def dbo = findSameNameFromMap(city, region, disName); - dbo = addressCheck(dbo, region, address); + def addrFlag = null + if(address){ + addrFlag = address.matches('^[\u4e00-\u9fa5]+?(·|´óµÀ|½Ö).+[ŪºÅ]$') + if(addrFlag){ + dbo = addressCheck(dbo, region, address); + } + } if(!dbo){ //±ðÃû def distrs = findAliasNameFromMap(city, region, disName); @@ -506,7 +600,7 @@ class DataStandardBatch extends BaseBatch{ dbo = distrs[0] }else if(distrs && distrs.size()>1 && address){ for(def d : distrs){ - if(d.address.contains(address)||d.address==address || address.contains(d.address)){ + if(addrFlag && (d.address.contains(address)||d.address==address || address.contains(d.address))){ dbo = d; break; } @@ -570,10 +664,16 @@ class DataStandardBatch extends BaseBatch{ return dbo; }*/ def getMatchAddressFromMap(def city, def region, def address, def name){ - def dbo = findSameNameFromMap(city, region, address); + def dbo = null + if(address && isStandardRoad(address)){ + dbo = findSameNameFromMap(city, region, address); + } if(!dbo){ - def result = findAddressFromMap(city, region, address); - if(result.size()==1){ + def result = null + if(address && isStandardRoad(address)){ + result = findAddressFromMap(city, region, address); + } + if(result && result.size()==1){ dbo = result[0]; }else if(result && result.size()>1 && name){ def score = null; @@ -621,6 +721,15 @@ class DataStandardBatch extends BaseBatch{ return dbo; } + def isStandardRoad(def s){ + def pattern = Pattern.compile("[·|½Ö|´óµÀ|Ïï]\\d+(ºÅ|Ū)"); + def matcher = pattern.matcher(s) + if(matcher.find()){ + return true; + } + return false + } + def addressCheck(def dbo, def region, def address){ if(dbo && !region && address && address.length()>3){ if(dbo.address.contains(address)||address.contains(dbo.address)||dbo.address==address){ @@ -673,7 +782,6 @@ class DataStandardBatch extends BaseBatch{ def findAllNoMatchData(){ DBObject qdata = new BasicDBObject(); qdata.put("s_date", SHORT_DATE) -// qdata.put("name", "ÒøºÓÍ廨԰£¨±ðÊû£©") qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); def ret = [] try{ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 53823ee85..bc1909c72 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -38,6 +38,152 @@ class Esf58Crawl extends BaseBatch{ def static useDistKeys = [] static def totalCount = new AtomicLong() + def cityList = [ + "ºÏ·Ê":"hf|837","Îߺþ":"wuhu|2045","°ö²º":"bengbu|3470","¸·Ñô":"fy|2325","»´ÄÏ":"hn|2319", + "°²Çì":"anqing|3251","ËÞÖÝ":"suzhou|3359","Áù°²":"la|2328","»´±±":"huaibei|9357","³üÖÝ":"chuzhou|10266", + "Âí°°É½":"mas|2039","Í­Áê":"tongling|10285","Ðû³Ç":"xuancheng|5633","ÙñÖÝ":"bozhou|2329","»ÆÉ½":"huangshan|2323", + "³ØÖÝ":"chizhou|10260","³²ºþ":"ch|10229","ºÍÏØ":"hexian|10892","»ôÇñ":"hq|11226","Í©³Ç":"tongcheng|11296", + "Äþ¹ú":"ningguo|5645","Ì쳤":"tianchang|10273","¶«ÖÁ":"dongzhi|10262","ÎÞΪ":"wuweixian|10232", + "¸£ÖÝ":"fz|304","ÏÃÃÅ":"xm|606","ȪÖÝ":"qz|291","ÆÎÌï":"pt|2429","ÕÄÖÝ":"zhangzhou|710", + "ÄþµÂ":"nd|7951","ÈýÃ÷":"sm|2048","ÄÏÆ½":"np|10291","ÁúÑÒ":"ly|6752","ÎäÒÄɽ":"wuyishan|10761", + "ʯʨ":"shishi|296","½ú½­":"jinjiangshi|297","Äϰ²":"nananshi|293","Áúº£":"longhai|713","ÉϺ¼":"shanghangxian|6757", + "¸£°²":"fuanshi|7969","¸£¶¦":"fudingshi|7970","°²Ïª":"anxixian|7100","ÓÀ´º":"yongchunxian|7101","ÓÀ°²":"yongan|2133", + "ÕÄÆÖ":"zhangpu|717", + "ÉîÛÚ":"sz|4","¹ãÖÝ":"gz|3","¶«Ý¸":"dg|413","·ðɽ":"fs|222","ÖÐɽ":"zs|771", + "Ö麣":"zh|910","»ÝÖÝ":"huizhou|722","½­ÃÅ":"jm|629","ÉÇÍ·":"st|783","Õ¿½­":"zhanjiang|791", + "ÕØÇì":"zq|901","ïÃû":"mm|679","½ÒÑô":"jy|927","÷ÖÝ":"mz|9389","ÇåÔ¶":"qingyuan|7303", + "Ñô½­":"yj|2284","ÉØ¹Ø":"sg|2192","ºÓÔ´":"heyuan|10467","ÔÆ¸¡":"yf|10485","ÉÇβ":"sw|9449", + "³±ÖÝ":"chaozhou|10461","̨ɽ":"taishan|11263","Ñô´º":"yangchun|8566","˳µÂ":"sd|8716","»Ý¶«":"huidong|725", + "²©ÂÞ":"boluo|726","º£·á":"haifengxian|9444","¿ªÆ½":"kaipingshi|634","½·á":"lufengshi|9456", + "ÄÏÄþ":"nn|845","ÁøÖÝ":"liuzhou|7133","¹ðÁÖ":"gl|1039","ÓñÁÖ":"yulin|2337","ÎàÖÝ":"wuzhou|2046", + "±±º£":"bh|10536","¹ó¸Û":"gg|6770","ÇÕÖÝ":"qinzhou|2335","°ÙÉ«":"baise|10513","ºÓ³Ø":"hc|2340", + "À´±ö":"lb|10552","ºØÖÝ":"hezhou|10549","·À³Ç¸Û":"fcg|10539","³ç×ó":"chongzuo|10524","¹ðƽ":"guipingqu|6774", + "±±Á÷":"beiliushi|9168","²©°×":"bobaixian|9173","á¯Ïª":"cenxi|2119", + "¹óÑô":"gy|2015","×ñÒå":"zunyi|7620","Ç­¶«ÄÏ":"qdn|9363","Ç­ÄÏ":"qn|10492","ÁùÅÌË®":"lps|10506", + "±Ï½Ú":"bijie|10564","Í­ÈÊ":"tr|10417","°²Ë³":"anshun|7468","Ç­Î÷ÄÏ":"qxn|10434","ÈÊ»³":"renhuaishi|7628", + "ÇåÕò":"qingzhen|12703", + "À¼ÖÝ":"lz|952","ÌìË®":"tianshui|8601","°×Òø":"by|10304","ÇìÑô":"qingyang|10475","ƽÁ¹":"pl|7154", + "¾ÆÈª":"jq|10387","ÕÅÒ´":"zhangye|10454","ÎäÍþ":"wuwei|10448","¶¨Î÷":"dx|10322","½ð²ý":"jinchang|7428", + "¤ÄÏ":"ln|10415","ÁÙÏÄ":"linxia|7112","¼ÎÓø¹Ø":"jyg|10362","¸ÊÄÏ":"gn|10343","¶Ø»Í":"dunhuang|10390", + "º£¿Ú":"haikou|2053","ÈýÑÇ":"sanya|2422","Îåָɽ":"wzs|9952","Èýɳ":"sansha|13722","Çíº£":"qh|10136", + "ÎIJý":"wenchang|9984","ÍòÄþ":"wanning|10022","ÍͲý":"tunchang|10044","ÇíÖÐ":"qiongzhong|10064","ÁêË®":"lingshui|10184", + "¶«·½":"df|10250","¶¨°²":"da|10303","³ÎÂõ":"cm|10331","±£Í¤":"baoting|10367","°×ɳ":"baish|10380", + "ÙÙÖÝ":"danzhou|10394", + "Ö£ÖÝ":"zz|342","ÂåÑô":"luoyang|556","ÐÂÏç":"xx|1016","ÄÏÑô":"ny|592","Ðí²ý":"xc|977", + "ƽ¶¥É½":"pds|1005","°²Ñô":"ay|1096","½¹×÷":"jiaozuo|3266","ÉÌÇð":"sq|1029","¿ª·â":"kaifeng|2342", + "å§Ñô":"puyang|2346","ÖÜ¿Ú":"zk|933","ÐÅÑô":"xy|8694","פÂíµê":"zmd|1067","äðºÓ":"luohe|2347", + "ÈýÃÅÏ¿":"smx|9317","º×±Ú":"hb|2344","¼ÃÔ´":"jiyuan|9918","Ã÷¸Û":"mg|8541","Û³Áê":"yanling|9123", + "ÓíÖÝ":"yuzhou|979","³¤¸ð":"changge|9344","Á鱦":"lingbaoshi|9307","è½ÏØ":"qixianqu|7389","ÈêÖÝ":"ruzhou|1010", + "Ïî³Ç":"xiangchengshi|935","ÙÈʦ":"yanshiqu|7121","³¤Ô«":"changyuan|5936","»¬ÏØ":"huaxian|5405","ÁÖÖÝ":"linzhou|1101", + "ÇßÑô":"qinyang|3268","ÃÏÖÝ":"mengzhou|3267","ÎÂÏØ":"wenxian|7312","ξÊÏ":"weishixian|7391","À¼¿¼":"lankaoxian|7393", + "ͨÐí":"tongxuxian|7390","а²":"lyxinan|11217","ÒÁ´¨":"yichuan|11220","ÃϽò":"mengjinqu|7122","ÒËÑô":"lyyiyang|11219", + "Îè¸Ö":"wugang|1011","ÓÀ³Ç":"yongcheng|1032","î¡ÏØ":"suixian|1038","Â¹ÒØ":"luyi|939","äųØ":"yingchixian|9322", + "ÉòÇð":"shenqiu|942","Ì«¿µ":"taikang|938","ÉÌË®":"shangshui|936","ä¿ÏØ":"qixianq|9186","¿£ÏØ":"junxian|9185", + "·¶ÏØ":"fanxian|7285","¹Ìʼ":"gushixian|8698","»´±õ":"huaibinxian|8702","µËÖÝ":"dengzhou|595","ÐÂÒ°":"xinye|603", + "¹þ¶û±õ":"hrb|202","´óÇì":"dq|375","ÆëÆë¹þ¶û":"qqhr|5853","ĵµ¤½­":"mdj|3489","Ë绯":"suihua|6718", + "¼Ñľ˹":"jms|6776","¼¦Î÷":"jixi|7289","˫Ѽɽ":"sys|9837","º×¸Ú":"hegang|9061","ºÚºÓ":"heihe|9862", + "ÒÁ´º":"yich|9773","Æß̨ºÓ":"qth|9848","´óÐ˰²Áë":"dxal|9878","°²´ï":"shanda|6720","ÕØ¶«":"shzhaodong|6721", + "ÕØÖÝ":"zhaozhou|382", + "Î人":"wh|158","Ò˲ý":"yc|858","ÏåÑô":"xf|891","¾£ÖÝ":"jingzhou|3479","Ê®Ñß":"shiyan|2032", + "»ÆÊ¯":"hshi|1734","Т¸Ð":"xiaogan|3434","»Æ¸Ô":"hg|2299","¶÷Ê©":"es|2302","¾£ÃÅ":"jingmen|2296", + "ÏÌÄþ":"xianning|9617","¶õÖÝ":"ez|9709","ËæÖÝ":"suizhou|9656","DZ½­":"qianjiang|9669","ÌìÃÅ":"tm|9517", + "ÏÉÌÒ":"xiantao|9736","ÉñÅ©¼Ü":"snj|9605","Ò˶¼":"yidou|864","ºº´¨":"hanchuan|3439","ÔæÑô":"zaoyang|896", + "ÎäѨ":"wuxueshi|7362","ÖÓÏé":"zhongxiangshi|9119","¾©É½":"jingshanxian|9117","ɳÑó":"shayangxian|9118","ËÉ×Ì":"songzi|3484", + "¹ãË®":"guangshuishi|9657","³à±Ú":"chibishi|9623","ÀϺӿÚ":"laohekou|895","¹È³Ç":"gucheng|899","Ò˳Ç":"yichengshi|897", + "ÄÏÕÄ":"nanzhang|898","ÔÆÃÎ":"yunmeng|3438","°²Â½":"anlu|3442","´óÎò":"dawu|3437","Т²ý":"xiaochang|3436", + "µ±Ñô":"dangyang|865","Ö¦½­":"zhijiang|866","¼ÎÓã":"jiayuxian|9624","ËæÏØ":"suixia|9660", + "³¤É³":"cs|414","ÖêÖÞ":"zhuzhou|1086","ÒæÑô":"yiyang|10198","³£µÂ":"changde|872","ºâÑô":"hy|914", + "ÏæÌ¶":"xiangtan|2047","ÔÀÑô":"yy|821","³»ÖÝ":"chenzhou|5695","ÉÛÑô":"shaoyang|2303","»³»¯":"hh|5756", + "ÓÀÖÝ":"yongzhou|2307","¦µ×":"ld|9481","ÏæÎ÷":"xiangxi|10219","Õżҽç":"zjj|6788","õ·Áê":"liling|1091", + "å¢ÏØ":"lixian|876","¹ðÑô":"czguiyang|5699","×ÊÐË":"zixing|5698","ÓÀÐË":"yongxing|5701","³£Äþ":"changningshi|921", + "Æî¶«":"qidongxian|5690","ºâ¶«":"hengdong|5693","ÀäË®½­":"lengshuijiangshi|9470","Á°Ô´":"lianyuanshi|9471","Ë«·å":"shuangfengxian|9473", + "ÉÛÑôÏØ":"shaoyangxian|6955","ÉÛ¶«":"shaodongxian|6954","ãä½­":"yuanjiangs|10201","ÄÏÏØ":"nanxian|10202","ÆîÑô":"qiyang|8532", + "ÏæÒõ":"xiangyin|828","»ªÈÝ":"huarong|830","´ÈÀû":"cilixian|6791","ØüÏØ":"zzyouxian|1095", + "ʯ¼Òׯ":"sjz|241","±£¶¨":"bd|424","ÌÆÉ½":"ts|276","ÀÈ·»":"lf|772","ºªµ¦":"hd|572", + "ÇØ»Êµº":"qhd|1078","²×ÖÝ":"cangzhou|652","ÐĮ̈":"xt|751","ºâË®":"hs|993","ÕżҿÚ":"zjk|3328", + "³ÐµÂ":"chengde|6760","¶¨ÖÝ":"dingzhou|8398","¹ÝÌÕ":"gt|8706","Õű±":"zhangbei|11201","ÕÔÏØ":"zx|9048", + "Õý¶¨":"zd|3198","Ǩ°²ÊÐ":"qianan|284","ÈÎÇð":"renqiu|656","ÈýºÓ":"sanhe|776","Îä°²":"wuan|577", + "ÐÛ°²ÐÂÇø":"xionganxinqu|111234","Ñོ":"lfyanjiao|12804","äÃÖÝ":"zhuozhou|428","ºÓ¼ä":"hejian|658","»Ææè":"huanghua|657", + "²×ÏØ":"cangxian|659","´ÅÏØ":"cixian|591","ÉæÏØ":"shexian|14059","°ÔÖÝ":"bazhou|775","ÏãºÓ":"xianghe|5395", + "¹Ì°²":"lfguan|12803","×ñ»¯ÊÐ":"zunhua|283","ǨÎ÷":"qianxixian|7061","ÓñÌï":"yutianxian|7060","ÂÐÄÏ":"luannanxian|7066", + "ɳºÓ":"shaheshi|755", + "ËÕÖÝ":"su|5","ÄϾ©":"nj|172","ÎÞÎý":"wx|93","³£ÖÝ":"cz|463","ÐìÖÝ":"xz|471", + "ÄÏͨ":"nt|394","ÑïÖÝ":"yz|637","ÑγÇ":"yancheng|613","»´°²":"ha|968","Á¬ÔƸÛ":"lyg|2049", + "Ì©ÖÝ":"taizhou|693","ËÞǨ":"suqian|2350","Õò½­":"zj|645","ãðÑô":"shuyang|5772","´ó·á":"dafeng|11279", + "Èç¸Þ":"rugao|397","Æô¶«":"qidong|400","äàÑô":"liyang|469","º£ÃÅ":"haimen|399","¶«º£":"donghai|2147", + "ÑïÖÐ":"yangzhong|649","ÐË»¯":"xinghuashi|699","ÐÂÒÊ":"xinyishi|478","Ì©ÐË":"taixing|696","Èç¶«":"rudong|402", + "ÚüÖÝ":"pizhou|477","ÅæÏØ":"xzpeixian|11349","¾¸½­":"jingjiang|698","½¨ºþ":"jianhu|618","º£°²":"haian|401", + "¶«Ì¨":"dongtai|615","µ¤Ñô":"danyang|648","±¦Ó¦ÏØ":"baoyingx|14451","¹àÄÏ":"guannan|2150","¹àÔÆ":"guanyun|2148", + "½ªÑß":"jiangyan|697","½ð̳":"jintan|468","À¥É½":"szkunshan|16","ãôºé":"sihong|5958","ãôÑô":"siyang|5959", + "¾äÈÝ":"jurong|650","ÉäÑô":"sheyang|621","¸·Äþ":"funingxian|620","ÏìË®":"xiangshui|619","íìíô":"xuyi|976", + "½ðºþ":"jinhu|975","½­Òõ":"jiangyins|34984", + "Äϲý":"nc|669","¸ÓÖÝ":"ganzhou|2363","¾Å½­":"jj|2247","ÒË´º":"yichun|5709","¼ª°²":"ja|2364", + "ÉÏÈÄ":"sr|10120","ƼÏç":"px|2248","¸§ÖÝ":"fuzhou|10134","¾°µÂÕò":"jdz|2360","ÐÂÓà":"xinyu|10115", + "ӥ̶":"yingtan|3209","ÓÀÐÂ":"yxx|11077","ÀÖÆ½":"lepingshi|9048","½øÏÍ":"jinxian|677","·ÖÒË":"fenyi|10118", + "·á³Ç":"fengchengshi|5711","ÕÁÊ÷":"zhangshu|5713","¸ß°²":"gaoan|5712","Ó཭":"yujiang|3210","ÄϳÇ":"nanchengx|10137", + "¸¡Áº":"fuliangxian|9071", + "³¤´º":"cc|319","¼ªÁÖ":"jl|700","ËÄÆ½":"sp|10171","ÑÓ±ß":"yanbian|3184","ËÉÔ­":"songyuan|2315", + "°×³Ç":"bc|5918","ͨ»¯":"th|10159","°×ɽ":"baishan|10179","ÁÉÔ´":"liaoyuan|2501","¹«Ö÷Áë":"gongzhuling|10171", + "÷ºÓ¿Ú":"meihekou|10162","·öÓà":"fuyuxian|9085","³¤Áë":"changlingxian|9084","èëµé":"huadian|706","ÅÍʯ":"panshi|708", + "ÀæÊ÷ÏØ":"lishu|10176", + "ÉòÑô":"sy|188","´óÁ¬":"dl|147","°°É½":"as|523","½õÖÝ":"jinzhou|2354","¸§Ë³":"fushun|5722", + "Óª¿Ú":"yk|5898","Å̽õ":"pj|2041","³¯Ñô":"cy|10106","µ¤¶«":"dandong|3445","ÁÉÑô":"liaoyang|2038", + "±¾Ïª":"benxi|5845","ºù«µº":"hld|10088","ÌúÁë":"tl|6729","¸·ÐÂ":"fx|10097","ׯºÓ":"pld|3306", + "Íß·¿µê":"wfd|3279","µÆËþ":"dengta|2071","·ï³Ç":"fengcheng|3450","±±Æ±":"beipiao|10109","¿ªÔ­":"kaiyuan|6733", + "Òø´¨":"yinchuan|2054","ÎâÖÒ":"wuzhong|9962","ʯ×ìɽ":"szs|9971","ÖÐÎÀ":"zw|9951","¹ÌÔ­":"guyuan|2421", + "ºôºÍºÆÌØ":"hu|811","°üÍ·":"bt|801","³à·å":"chifeng|6700","¶õ¶û¶à˹":"erds|2037","ͨÁÉ":"tongliao|10015", + "ºôÂ×±´¶û":"hlbe|10039","°ÍÑåÄ×¶ûÊÐ":"bycem|10070","ÎÚÀ¼²ì²¼":"wlcb|9993","ÎýÁÖ¹ùÀÕ":"xl|2408","Ð˰²ÃË":"xam|9976", + "ÎÚº£":"wuhai|2404","°¢À­ÉÆÃË":"alsm|10083","º£À­¶û":"hlr|2043", + "Î÷Äþ":"xn|2052","º£Î÷":"hx|9902","º£±±":"haibei|9917","¹ûÂå":"guoluo|9936","º£¶«":"haidong|9909", + "»ÆÄÏ":"huangnan|9896","ÓñÊ÷":"ys|9888","º£ÄÏ":"hainan|10574","¸ñ¶ûľ":"geermushi|9904", + "Çൺ":"qd|122","¼ÃÄÏ":"jn|265","ÑĮ̀":"yt|228","Ϋ·»":"wf|362","ÁÙÒÊ":"linyi|505", + "×Ͳ©":"zb|385","¼ÃÄþ":"jining|450","Ì©°²":"ta|686","ÁijÇ":"lc|882","Íþº£":"weihai|518", + "Ôæ×¯":"zaozhuang|961","µÂÖÝ":"dz|728","ÈÕÕÕ":"rizhao|3177","¶«Óª":"dy|623","ºÊÔó":"heze|5632", + "±õÖÝ":"bz|944","À³Îß":"lw|2292","ÕÂÇð":"zhangqiu|8680","¿ÑÀû":"kl|11313","Öî³Ç":"zc|9146", + "ÊÙ¹â":"shouguang|369","Áú¿Ú":"longkou|233","²ÜÏØ":"caoxian|5638","µ¥ÏØ":"shanxian|5636","·Ê³Ç":"feicheng|690", + "¸ßÃÜ":"gaomi|371","¹ãÈÄ":"guangrao|627","»¸Ì¨":"huantaixian|7335","ÜìÏØ":"juxian|3180","À³ÖÝ":"laizhou|235", + "ÅîÀ³":"penglai|237","ÇàÖÝ":"qingzhou|367","ÈÙ³É":"rongcheng|522","Èéɽ":"rushan|520","ëøÖÝ":"tengzhou|967", + "ÐÂÌ©":"xintai|689","ÕÐÔ¶":"zhaoyuan|3325","×Þ³Ç":"zoucheng|455","×ÞÆ½":"zouping|946","ÁÙÇå":"linqing|884", + "ÜÝÆ½":"chiping|887","Û©³Ç":"hzyc|5637","²©ÐË":"boxing|949","¶«Ã÷":"dongming|5641","¾ÞÒ°":"juye|5640", + "ÎÞé¦":"wudi|951","ÆëºÓ":"qihe|734","΢ɽ":"weishan|459","Óí³Ç":"yuchengshi|731","ÁÙÒØ":"linyixianq|739", + "ÀÖÁê":"leling|730","À³Ñô":"laiyang|234","Äþ½ò":"ningjin|733","¸ßÌÆ":"gaotang|885","Ý·ÏØ":"shenxian|888", + "Ñô¹È":"yanggu|886","¹ÚÏØ":"guanxian|890","Æ½ÒØ":"pingyi|514","Û°³Ç":"tancheng|510","ÒÊÔ´":"yiyuanxian|7334", + "ãëÉÏ":"wenshang|460","ÁºÉ½":"liangshanx|462","Àû½ò":"lijin|628","ÒÊÄÏ":"yinanxian|7301","ÆÜϼ":"qixia|238", + "ÄþÑô":"ningyang|691","¶«Æ½":"dongping|692","²ýÒØ":"changyishi|372","°²Çð":"anqiu|370","²ýÀÖ":"changle|373", + "ÁÙëÔ":"linqu|374","Û²³Ç":"juancheng|5635", + "̫ԭ":"ty|740","ÁÙ·Ú":"linfen|5669","´óͬ":"dt|6964","Ô˳Ç":"yuncheng|5653","½úÖÐ":"jz|8854", + "³¤ÖÎ":"changzhi|6921","½ú³Ç":"jincheng|3350","ÑôȪ":"yq|8760","ÂÀÁº":"lvliang|3222","ÐÃÖÝ":"xinzhou|3453", + "Ë·ÖÝ":"shuozhou|9871","ÁÙâ¢":"linyixian|9193","ÇåÐì":"qingxu|10908","ÁøÁÖ":"liulin|3225","¸ßƽ":"gaoping|3354", + "ÔóÖÝ":"zezhou|3353","ÏåÔ«":"xiangyuanxian|6928","ТÒå":"xiaoyi|3227", + "Î÷°²":"xa|483","ÏÌÑô":"xianyang|7453","±¦¼¦":"baoji|2044","μÄÏ":"wn|5733","ººÖÐ":"hanzhong|3163", + "ÓÜÁÖ":"yl|5942","ÑÓ°²":"yanan|8973","°²¿µ":"ankang|3157","ÉÌÂå":"sl|9854","Í­´¨":"tc|9832", + "Éñľ":"shenmu|5944","º«³Ç":"hancheng|5735","¸®¹È":"fugu|5945","¾¸±ß":"jingbian|5947","¶¨±ß":"dingbian|5948", + "³É¶¼":"cd|102","ÃàÑô":"mianyang|1057","µÂÑô":"deyang|2373","Äϳä":"nanchong|2378","Ò˱ö":"yb|2380", + "×Ô¹±":"zg|6745","ÀÖɽ":"ls|3237","ãòÖÝ":"luzhou|2372","´ïÖÝ":"dazhou|9799","ÄÚ½­":"scnj|5928", + "ËìÄþ":"suining|9688","ÅÊÖ¦»¨":"panzhihua|2371","üɽ":"ms|9704","¹ã°²":"ga|2381","×ÊÑô":"zy|6803", + "Á¹É½":"liangshan|9717","¹ãÔª":"guangyuan|9749","ÑŰ²":"ya|9687","°ÍÖÐ":"bazhong|9811","°¢°Ó":"ab|9817", + "¸Ê×Î":"ganzi|9764","°²ÔÀ":"anyuexian|6806","¹ãºº":"guanghanshi|8719","¼òÑô":"jianyangshi|6805","ÈÊÊÙ":"renshouxian|9706", + "Éäºé":"shehongxian|9694","´óÖñ":"dazu|9806","Ðûºº":"xuanhan|9804","ÇþÏØ":"qux|9807","³¤Äþ":"changningx|7148", + "ÎÚ³ľÆë":"xj|984","²ý¼ª":"changji|8582","°ÍÒô¹ùÀã":"bygl|9530","ÒÁÀç":"yili|9472","°¢¿ËËÕ":"aks|9499", + "¿¦Ê²":"ks|9326","¹þÃÜ":"hami|7452","¿ËÀ­ÂêÒÀ":"klmy|2042","²©¶ûËþÀ­":"betl|9529","ͳ·¬":"tlf|9475", + "ºÍÌï":"ht|9489","ʯºÓ×Ó":"shz|9551","¿Ë×ÎÀÕËÕ":"kzls|9519","°¢À­¶û":"ale|9539","Îå¼ÒÇþ":"wjq|9562", + "Í¼Ä¾Êæ¿Ë":"tmsk|9559","¿â¶ûÀÕ":"kel|7168","°¢ÀÕÌ©":"alt|18837","Ëþ³Ç":"tac|18845", + "À­Èø":"lasa|2055","ÈÕ¿¦Ôò":"rkz|9615","ɽÄÏ":"sn|9576","ÁÖÖ¥":"linzhi|9646","²ý¶¼":"changdu|9648", + "ÄÇÇú":"nq|9618","°¢Àï":"al|9678","ÈÕÍÁ":"rituxian|9682","¸ÄÔò":"gaizexian|9684", + "À¥Ã÷":"km|541","Çú¾¸":"qj|2389","´óÀí":"dali|2398","ºìºÓ":"honghe|2394","ÓñϪ":"yx|2040", + "Àö½­":"lj|2392","ÎÄɽ":"ws|2395","³þÐÛ":"cx|2393","Î÷Ë«°æÄÉ":"bn|2397","ÕÑͨ":"zt|9409", + "µÂºê":"dh|9437","ÆÕ¶ý":"pe|9444","±£É½":"bs|2390","ÁÙ²×":"lincang|9422","µÏÇì":"diqing|9432", + "Å­½­":"nujiang|9462","ÃÖÀÕ":"milexian|8892","°²Äþ":"anningshi|547","ÐûÍþ":"xuanwushi|7533", + "º¼ÖÝ":"hz|79","Äþ²¨":"nb|135","ÎÂÖÝ":"wz|330","½ð»ª":"jh|531","¼ÎÐË":"jx|497", + "̨ÖÝ":"tz|403","ÉÜÐË":"sx|355","ºþÖÝ":"huzhou|831","ÀöË®":"lishui|7921","áéÖÝ":"quzhou|6793", + "ÖÛɽ":"zhoushan|8481","ÀÖÇå":"yueqingcity|13950","Èð°²":"ruiancity|13951","ÒåÎÚ":"yiwu|12291","ÓàÒ¦":"yuyao|5333", + "Öîôß":"zhuji|357","Ïóɽ":"xiangshanxian|6738","ÎÂÁë":"wenling|408","Í©Ïç":"tongxiang|502","´ÈϪ":"cixi|5334", + "³¤ÐË":"changxing|834","¼ÎÉÆ":"jiashanx|14357","º£Äþ":"haining|500","µÂÇå":"deqing|835","¶«Ñô":"dongyang|536", + "°²¼ª":"anji|836","²ÔÄÏ":"cangnanxian|7579","ÁÙº£":"linhai|407","ÓÀ¿µ":"yongkang|537","Óñ»·":"yuhuan|409", + "ƽºþ":"pinghushi|501","º£ÑÎ":"haiyan|504","ÎäÒåÏØ":"wuyix|14528","áÓÖÝ":"shengzhou|359","вý":"xinchang|361", + "½­É½":"jiangshanshi|6796","ƽÑô":"pingyangxian|7575" + ] + static main(args) { runBatch(new Esf58Crawl(), args) } @@ -45,28 +191,41 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("¸£ÖÝ","https://fz.58.com/xiaoqu/", 70); - for(def i=0;i<5;i++){ - def city = "¸£ÖÝ" - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); - distMap.put(city, districts) + def citys = ["³É¶¼","¹ãÖÝ","ÉîÛÚ","º¼ÖÝ","ÄϾ©","Ö£ÖÝ","ËÕÖÝ","Î人","Çൺ","ʯ¼Òׯ","̫ԭ","¼ÃÄÏ","³¤É³","ÉòÑô","Äþ²¨","·ðɽ","Ö麣","À¥Ã÷","ÎÂÖÝ","ÎÞÎý","¹þ¶û±õ","ÄÏÄþ","¶«Ý¸","´óÁ¬","º£¿Ú","Ϋ·»","ÃàÑô","ÎÚ³ľÆë","»ÝÖÝ"] + cityList.each{city, str-> + for(def i=0;i<1;i++){ +// def city = "¸£ÖÝ" + if(citys.contains(city)){ + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); + if(districts){ + def arrs = str.split("\\|") + def url = "https://"+arrs[0]+".58.com/xiaoqu/" + distMap.put(city, districts) - addCrawlKeys(city); - addDetailContent(city, "https://fz.58.com/xiaoqu/"); + addCrawlKeys(city); + addDetailContent(city, url); - Thread.sleep(1000*60*10) +// Thread.sleep(1000*60*10) + } + } + } } } + + def addCrawlKeys(city){ def districts = distMap.get(city); for(def i=0;i - for(def city : citys){ - if(!["ÉϺ£","±±¾©","º£¿Ú"].contains(city)){ - def rets = findCrawlDistrictDetails(city, k, v) - println city +" "+rets.size() - synchData(city, rets, v) - } + def rets = findCrawlDistrictDetails(city, k, v) + println city +" "+rets.size() + synchData(city, rets, v) } } } def findCrawlDistrictDetails(city, root, source){ + + def curMonthFirstDay = DateUtils.getFirstDayOfMonth(new Date()) def q = new BasicDBObject() q.put("city", city) q.put("root", root) - q.put('m_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-07-01"))); +// q.put("name","À¼ÖÝÍò´ï³Ç") + q.put('m_date', new BasicDBObject('$gte', curMonthFirstDay)); def keys = new BasicDBObject() keys.put("city", 1) keys.put("region", 1) @@ -53,6 +53,7 @@ class SynchDistrictPrice { keys.put("address", 1) keys.put("avg_price", 1) keys.put("root", 1) + keys.put("c_date", 1) keys.put("url", 1) def rets = [] def dbcursor = districtDetailColl.find(q, keys) @@ -95,6 +96,8 @@ class SynchDistrictPrice { thirdPartyColl.remove(new BasicDBObject('_id',obj._id)) addIndex++ thirdPartyColl.save(r) + }else{ + thirdPartyColl.save(r) } // if(!obj){ // addIndex++ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index f6887e506..e299c60f6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["·¿¹À¹À","Öî¸ðÕÒ·¿","°²¾Ó¿Í","ËÑ·¿","ÖÐÔ­µØ²ú","¸Ï¼¯","Á´¼Ò","Íò©Ê¿","³ÇÊз¿²ú","±´¿ÇÕÒ·¿","Q·¿","Q·¿(ÐÂ)","ÖÐÔ­µØ²ú(ÐÂ)","ÀÖÓмÒ","ÀÖÓмÒ(ÐÂ)","·¿Ì½","·¿¶à¶à","С×ʼÒ","ÎÒ°®ÎÒ¼Ò","ÂóÌï","·¿Í¾","C21","ÀÖ¾Ó","·¿²ú³¬ÊÐ","58ͬ³Ç","365ÌÔ·¿"] for(def source :sources){ - def count1 = findTotalCount(source, 202106); - def count2 = findTotalCount(source, 202107); + def count1 = findTotalCount(source, 202108); + def count2 = findTotalCount(source, 202109); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 5f96db29c..ddc282cb5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -41,8 +41,10 @@ class ThirdDataToGatherBatch extends BaseBatch{ @Override void process(Object args) { + println SHORT_DATE def citys = CommonDatautils.findAllCity(); def weekend = DateUtils.getFirstDayOfWeek(new Date()); + //def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD("2021-07-15")); def cityIndex = 0; final CountDownLatch latch = new CountDownLatch(citys.size()) citys.each { city -> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy index 91cb7d57e..c30a1353a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy @@ -80,7 +80,7 @@ class ElasticUtils { @Override RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) { requestConfigBuilder.setConnectionRequestTimeout(1000 * 60); - requestConfigBuilder.setSocketTimeout(1000 * 60); + requestConfigBuilder.setSocketTimeout(1000 * 180); return requestConfigBuilder; } }) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/ReCalcAvgPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/ReCalcAvgPrice.groovy new file mode 100644 index 000000000..5fceee582 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/ReCalcAvgPrice.groovy @@ -0,0 +1,32 @@ +package com.fangjia.script.wenyan.auto + +import com.fangjia.core.util.DateUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject + +//def keys = new BasicDBObject() +//keys.city = 1 +//keys.region = 1 +//keys.name = 1 +//keys.estate_type2 = 1 +//["ÖØÇì"].each {def city-> +// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject seaweed-> +// BasicDBObject stat = DbUtils.findDistrictStat(seaweed, DateUtil.getFirstDayOfWeek(new Date())) +// if(!stat){ +// CommonUtils.recalculationAverage(seaweed.city, seaweed.region, seaweed.name, seaweed.estate_type2) +// println(seaweed) +// } +// }, keys) +// +//} +ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\³É½»¼Û¡ª¡ªÔ¤²â¼Ò.xlsx",{List item-> + BasicDBObject query = DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"]) + BasicDBObject seaweed = DbUtils.seaweed.findOne(query) + if(seaweed){ + BasicDBObject stat = DbUtils.findDistrictStat(seaweed, item["E"]) + item["H"] = stat?.credit + } + println(item) +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/SaveMapPoiToSeaweed.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/SaveMapPoiToSeaweed.groovy new file mode 100644 index 000000000..b603f367e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/SaveMapPoiToSeaweed.groovy @@ -0,0 +1,113 @@ +package com.fangjia.script.wenyan.auto + +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils +import org.apache.lucene.index.Term +import org.apache.lucene.search.BooleanClause +import org.apache.lucene.search.BooleanQuery +import org.apache.lucene.search.TermQuery + +/** + * ÒÆ³ýÐ¡ÇøÃûÖеÄÇøÓòÐÅÏ¢ + * @param name + * @param region + * @return + */ +String removeRegion(String name, String region){ + if(name.startsWith(region) && name.length() - region.length() > 2){ + return name.replaceFirst(region, "") + } + return name; +} + +def String getStandardAddress(def city, def addr) { + if (StringUtils.isBlank(addr)) { + return null + } + JSONObject result = OpenApiClient.addressStandardize(city, addr) + if (result.containsKey("standardAddress") && result.getBoolean("standardAddress")) { + def standardAddr = result.getString("street") + if (result.containsKey("streetNumber")) { + standardAddr += result.getString("streetNumber") + } + return standardAddr + } + return null +} +def NOW = new Date() +def getType(String type){ + if(type == null){ + return "ÆÕͨסլ" + } + if(type.contains("ÉÌסÁ½ÓÃ")){ + return "ÉÌס" + }else if(type.contains("±ðÊû")){ + return "±ðÊû" + }else{ + return "ÆÕͨסլ" + } +} + +ExcelUtils.modify("D:\\nextcloud\\¹²ÏíÎĵµ\\ÏîÄ¿Îĵµ\\ÖØÇìÅ©ÉÌÐÐ\\ÐÂÔöÐ¡Çø4.xlsx",{List item-> + if(item["C"] != "Ã÷¼ÑÔ°"){ + return + } + BasicDBObject seaweed = new BasicDBObject() + seaweed.city = item["A"] + seaweed.region = item["B"] + seaweed.name = item["C"] + def query = new BasicDBObject("city", seaweed.city).append("region", seaweed.region) + .append('name', seaweed.name) + .append("visible", 0) + .append("cat", "district"); + BasicDBObject one = DbUtils.seaweed.findOne(query) + if(!one){ + query.removeField("name") + query.append("alias", seaweed.name) + one = DbUtils.seaweed.findOne(query) + } + if (one && one.status == 0) { + item["i"] = "ÒÑ´æÔÚ" + return; + } + seaweed.address = getStandardAddress( seaweed.city, item["D"]) + String source = item["H"] + Double lng = item["E"] + Double lat = item["F"] + if(lng){ + if(source.contains("¸ßµÂ")){ + seaweed.lng = (long)(lng * 10000000000l) + seaweed.lat = (long)(lat * 10000000000l) + JSONObject geo = BaiDuApiUtils.googleToBaidu(lat, lng) + seaweed.lng2 = (long)(geo.lng * 10000000000l) + seaweed.lat2 = (long)(geo.lat * 10000000000l) + }else if(source.contains("°Ù¶È")){ + Map point = GeoUtils.geoconvBaiduToGoogle(lng, lat); + long google_lat = (long)(point.get("lat") * 10000000000l); + long google_lng = (long)(point.get("lng") * 10000000000l); + long baiDu_lat = (long)(lat * 10000000000l) + long baiDu_lng = (long)(lng * 10000000000l) + seaweed.lng = google_lng + seaweed.lat = google_lat + seaweed.lat2 = baiDu_lat + seaweed.lng2 = baiDu_lng + } + } + seaweed.cat = "district" + seaweed.status = 0 + seaweed.c_date = NOW + seaweed.ip = "MACHINE_SAVE" + seaweed.estate_type2 = getType(item["G"]) + Map markMap = new HashMap() + seaweed.mark_map = markMap + markMap.put("data_source", "»úÆ÷Èë¿â") + DbUtils.seaweed.save(seaweed) + println(seaweed) +}) + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Price.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Price.groovy index b245cfec6..a60bd7b2b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Price.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Price.groovy @@ -1,16 +1,42 @@ package com.fangjia.script.wenyan.temp - +import com.fangjia.core.RegionData +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.common.ServerAreaData +import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject +import net.sf.json.JSONObject -def data = [["³ÇÊÐ", "ÇøÓò", "Ð¡ÇøÃû", "±ðÃû", "µØÖ·", "µØÖ·±ðÃû"]] -["ID", "±±¾©", "ÖØÇì", "¹þ¶û±õ"].each {def city-> - DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(city)).each {BasicDBObject district-> - def item = [district._id.toString(), district.city, district.region, district.name, district.alias?.join(","), district.address, district.addr_alias?.join(",")] - println(item) - data << item +//city_sell_stat = MongoUtils.getDbCollection("fangjia_stat", "city_sell_stat") +//data = [["³ÇÊÐ", "202101", "202106"]] +//CommonUtils.findAllCity().each {def city-> +// BasicDBObject serverArea = ServerAreaData.getServerAreaByDisplay(city) +// if(!serverArea.ranking || !serverArea.ranking.toString().contains("ÏØ¼¶ÐÐÕþµ¥Î»")){ +// BasicDBObject pre = city_sell_stat.findOne(new BasicDBObject("city", city).append("date", 202101)) +// BasicDBObject now = city_sell_stat.findOne(new BasicDBObject("city", city).append("date", 202106)) +// println([serverArea.display, pre?.avg_price, now?.avg_price]) +// data << [serverArea.display, pre?.avg_price, now?.avg_price] +// } +//} +//ExcelUtils.write("D:\\³ÇÊоù¼ÛÊý¾Ý.xlsx", data, "³ÇÊоù¼Û") + +//ExcelUtils.multiThreadingModify("C:\\Users\\wenyan\\Downloads\\¾­Î³¶È´íÎóÊý¾ÝÐÞÕý.xlsx",{List item-> +// String city = item["B"] +// String region = RegionData.formatRegion(city, item["D"]) +// String sql = "select count(1) as count from latest_seawater where cat='sell' and city='${item["B"]}' and region='${region}' and district_name='${item["F"]}'" +// ElasticUtils.executeSqlQuery(sql,{Map result-> +// item["P"] = result.get("count") +// }) +// println(item) +//}) +ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\×°ÐÞ°¸Àý.xlsx",{List item-> + JSONObject json = OpenApiClient.addressMatch("Î÷°²", item["H"], "property", false, item["C"]) + if(json.name){ + item["I"] = json.district + item["J"] = json.name } -} -ExcelUtils.write("D:\\´ý×¥È¡Ð¡Çø.xlsx", data, "Êý¾Ý") \ No newline at end of file +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index e2790a624..785addef5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20210401", "yyyyMMdd") -def endDate = DateUtil.parseDate("20210501", "yyyyMMdd") +def startDate = DateUtil.parseDate("20210901", "yyyyMMdd") +def endDate = DateUtil.parseDate("20211001", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 @@ -25,7 +25,7 @@ while (startDate.before(endDate)) { OPEN_URI_LOG.find(and(eq("user_id", "1007705"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> def result = JSONObject.fromObject(doc.res_map.result) if (!hashSet.contains(result)) { - def item = ["5c6535b99c213017e0768ee1", "qlkj", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] + def item = ["60a3202c0b42a00001aad385", "qlkjgx", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] data << item hashSet << result } else { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy index 08bf4c5b7..8243bde22 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20210401", "yyyyMMdd") -def endDate = DateUtil.parseDate("20210501", "yyyyMMdd") +def startDate = DateUtil.parseDate("20210901", "yyyyMMdd") +def endDate = DateUtil.parseDate("20211001", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 @@ -42,7 +42,7 @@ while (startDate.before(endDate)) { keys.addAll(params.keys()) def key = "${params.address}_${params.size}".toString() if (!hashSet.contains(key)) { - def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date.format("yyy"), doc.data_status, doc.code, doc.msg] + def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg] param_keys.each { item << params[it] } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test1122.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test1122.groovy index 6909f23a1..ec3f9affa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test1122.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test1122.groovy @@ -261,15 +261,16 @@ import org.dom4j.io.SAXReader // data << [dbo.parameter_map.city, dbo.parameter_map.address, dbo.res_map.result.province, dbo.res_map.result.city, dbo.res_map.result.district, dbo.res_map.result.developmentZone, dbo.res_map.result.township, dbo.res_map.result.street,dbo.res_map.result.streetNumber,dbo.res_map.result.name,dbo.res_map.result.buildingNumber,dbo.res_map.result.unitNumber,dbo.res_map.result.roomNumber,dbo.res_map.result.formattedAddress] //} //ExcelUtils.write("D:\\½¨ÐвâÊÔµØÖ·.xlsx", data, "½á¹û") -DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com") -def query = new BasicDBObject() -query.cat = "region" -query.city= "ÉϺ£" -query.weekend = Date.parse("yyyyMMdd", "20210228") -def doc = [] -def prices = [] -SQMETRE_RENT_COLL.find(query, new BasicDBObject("region", 1).append("avg_price", 1)).each { - doc.add(new BasicDBObject("name", it.region+"Çø").append("value", it.avg_price)) - prices << it.avg_price -} -println(JSONArray.fromObject(doc).toString()) \ No newline at end of file +//DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com") +//def query = new BasicDBObject() +//query.cat = "region" +//query.city= "ÉϺ£" +//query.weekend = Date.parse("yyyyMMdd", "20210228") +//def doc = [] +//def prices = [] +//SQMETRE_RENT_COLL.find(query, new BasicDBObject("region", 1).append("avg_price", 1)).each { +// doc.add(new BasicDBObject("name", it.region+"Çø").append("value", it.avg_price)) +// prices << it.avg_price +//} +//println(JSONArray.fromObject(doc).toString()) +DbUtils.seaweed.remove(new BasicDBObject("_id", DbUtils.seaweed.findById("6103bf1c64baf60001e7e1d0")._id)) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test7.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test7.groovy index 168eaaf37..91e428cd4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test7.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test7.groovy @@ -1,6 +1,11 @@ +import com.fangjia.core.util.CNNMFilter +import com.fangjia.core.util.HttpRequest import com.fangjia.datastore.driver.DSCollection import com.fangjia.datastore.driver.DataStore +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject +import net.sf.json.JSONObject import org.apache.commons.io.FileUtils // @@ -35,7 +40,79 @@ import org.apache.commons.io.FileUtils // println("copy:" + file.getAbsolutePath() + "\tto:" + ("E:\\³É½»Êý¾Ý\\DBF»ã×Ü\\" + file.getName())) // } //} -DSCollection seawater = new DataStore("192.168.9.118", 27777).getCollection("fangjia.seawater") -seawater.find(new BasicDBObject("cat", "sell").append("city", "ÉϺ£").append("region", "³¤Äþ").append("district_name", "ÐÂãþÈý´å")).each { - println(it) -} \ No newline at end of file +//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\¸±±¾µØÖ·±ê×¼»¯ÑéÖ¤½á¹û·ÖÎö.xlsx",{List item-> +// HttpRequest request = new HttpRequest() +// JSONObject json = request.getJsonContent2("http://localhost:8082/structure", ["address": item["A"]], "UTF-8") +// if(json){ +// item["D"] = json.result.province +// item["E"] = json.result.city +// item["F"] = json.result.district +// item["G"] = json.result.dev +// item["H"] = json.result.township +// item["I"] = json.result.village +// item["J"] = json.result.group +// item["K"] = json.result.street +// item["L"] = json.result.streetNumber +// item["M"] = json.result.name +// item["N"] = json.result.buildingNumber +// item["O"] = json.result.unitNumber +// item["P"] = json.result.roomNumber +// item["q"] = json.result.floor +// item["r"] = json.result.formatAddress +// } +// println(item) +//}) +//for(int i=1; i < 100; i++){ +// println("${i}ºÅÂ¥#@#building_number") +// println("${i}¶°#@#building_number") +// println("${i}´±#@#building_number") +// println("${i}µ¥Ôª#@#unit_number") +// if(i< 20){ +// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}ºÅÂ¥#@#building_number") +// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}¶°#@#building_number") +// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}´±#@#building_number") +// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}µ¥Ôª#@#unit_number") +// } +//} +//nums= "abcdefghijklmn" +//for(def num : nums.toCharArray()){ +// println("${num}¶°#@#building_number") +// println("${num}´±#@#building_number") +// println("${num}µ¥Ôª#@#unit_number") +//} +//nums= "abcdefghijklmn" +//for(def num : nums.toCharArray()){ +// println("${num}×ù#@#building_number") +// for(int i = 1; i <11; i++){ +// println("${num}${i}×ù#@#building_number") +//// println("${num}${i}´±#@#building_number") +//// println("${num}${i}µ¥Ôª#@#unit_number") +// } +//} +//for(int i = 1; i <20; i++){ +// println("${i}Çø#@#part_number") +// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}Çø#@#part_number") +// println("${i}ÆÚ#@#stage_number") +// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}ÆÚ#@#stage_number") +//} +// +//for(def num : "¶«Äϱ±ÖÐÎ÷¼×ÒÒ±û¶¡".toCharArray()){ +// println("${num}Çø#@#part_number") +//} +//for(int i = -5; i < 100; i++){ +// println("${i}Â¥#@#floor") +// if(i < 10 && i > 0){ +// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}Â¥#@#floor") +// } +//} +//for(def num : "abcdefg¶«ÄÏÎ÷±±".toCharArray()){ +// println("${num}»§#@#room") +//} +//for(i in 1.. 40){ +// for(j in 1..9){ +// println("${i}0${j}ÊÒ#@#room") +// } +//} +ElasticUtils.executeSqlQuery("select * from seawater where cat='sell' and district_name='ÐÂãþÈý´å' and city='ÉϺ£' and p_date > 20210101",{Map doc-> + println(doc.avg_price + "\t" + doc.root + "\t" + doc.p_date) +}) \ No newline at end of file diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/sql/EvaluationRateServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/sql/EvaluationRateServiceImpl.java index 3310cce53..4c6147db6 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/sql/EvaluationRateServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/sql/EvaluationRateServiceImpl.java @@ -912,7 +912,6 @@ public class EvaluationRateServiceImpl implements EvaluationRateService { evaluatePrice = (long) (evaluatePrice * houseFloorRate); } }else{ - int mid = evaluation.getHeight()/2; double num = getFloorRate(mid, flag); evaluatePrice = (long) (evaluatePrice * num * houseFloorRate);