diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy
index 0f2519b79..66f2cc491 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy
@@ -36,35 +36,35 @@ class ExportDistrictRongHeTest extends BaseBatch{
// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
-// if(curDate.toString().endsWith("10")){
+ if(curDate.toString().endsWith("10")){
-// exportEvaluateRate()
-// exportBigdataBuildprojFjw()
-// exportBigdataCommlayoutpicFjw()
-//
-//
-// exportBigdataCommunityGJRH()/
-// exportBigdataBuildingFJW()//
-// exportBigdataCommfeeGj()//
-//
-// exportBigdataCommbuildstruGJ()//
-// exportBigdataCommhousetypeGJ()//
-// exportBigdataCommbuildyearGj()//
-// exportBigdataCommhouseuseGJ()//
-// exportBigdataPositionGJ()//
-// exportBigdataDistrictFJW()//
-// exportBigdataAliasFJW()//
-//
-//
-// //本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据
-// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
-// exportBigdataCityPrice(queryDate)
-// exportBigdataDistPrice(queryDate)
-// exportBigdataCommPrice(queryDate)
+ exportEvaluateRate()
+ exportBigdataBuildprojFjw()
+ exportBigdataCommlayoutpicFjw()
+
+
+ exportBigdataCommunityGJRH()/
+ exportBigdataBuildingFJW()//
+ exportBigdataCommfeeGj()//
+
+ exportBigdataCommbuildstruGJ()//
+ exportBigdataCommhousetypeGJ()//
+ exportBigdataCommbuildyearGj()//
+ exportBigdataCommhouseuseGJ()//
+ exportBigdataPositionGJ()//
+ exportBigdataDistrictFJW()//
+ exportBigdataAliasFJW()//
+
+
+ //本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据
+ def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
+ exportBigdataCityPrice(queryDate)
+ exportBigdataDistPrice(queryDate)
+ exportBigdataCommPrice(queryDate)
exportInOk()
-// }
+ }
SSHHelperTest jschCommand = new SSHHelperTest();
System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}"));
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy
index b78e0ec4b..da0c07889 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy
@@ -29,16 +29,16 @@ class EsHouseStat {
DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002)
// DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.206", 27020)
-// public static JdbcDataSource dataSource = new JdbcDataSource()
-// private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
-// public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
-// static {
-// dataSource.setUrl(address)
-// Properties connectionProperties = new Properties()
-// connectionProperties.put("user", "developer")
-// connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
-// dataSource.setProperties(connectionProperties)
-// }
+ public static JdbcDataSource dataSource = new JdbcDataSource()
+ private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
+ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
+ static {
+ dataSource.setUrl(address)
+ Properties connectionProperties = new Properties()
+ connectionProperties.put("user", "developer")
+ connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
+ dataSource.setProperties(connectionProperties)
+ }
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy
index 805c41944..e6cdef564 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy
@@ -285,7 +285,7 @@ class FiledStrand {
}
static def processCarSiteDesc(car_site){
- if(car_site && car_site.size()<=20){
+ if(car_site && car_site.size()<=20 && car_site.size()>=2 && !car_site.contains("待定") && !car_site.contains("暂无")){
return car_site.toString()
}
return null
@@ -337,6 +337,82 @@ class FiledStrand {
return null
}
+ static def processBlock(def block){
+ if(block){
+ if(block.contains("无") || block.toString().length()>10){
+ return null
+ }
+ return block
+ }
+ return null
+ }
+
+ static def processBuildType(def buildType, def estateType){
+ if(buildType){
+ def types = []
+ if(estateType == "别墅"){
+ def arrs = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院"]
+ for(def s : arrs){
+ if(buildType.contains(s)){
+ types << s
+ }
+ }
+ }else if(estateType == "商住"){
+ def arrs = ["独栋","双拼","联排","叠加","老式别墅洋房"]
+ for(def s : arrs){
+ if(buildType.contains(s)){
+ types << s
+ }
+ }
+ }else{
+ def arrs = ["多层","小高层","高层","超高层","SOHO","soho","LOFT","loft","酒店式公寓"]
+ for(def s : arrs){
+ if(buildType.contains(s)){
+ types << s
+ }
+ }
+ }
+ if(types){
+ return types.join(",").toString()
+ }
+ }
+ return null
+ }
+
+ static def processlandProperty(val){
+ if(val){
+ def arrs = ["经适房","限价房","廉租房","宅基地","保障房","动迁房","配套房","商品房"]
+ def property = null
+ for(def s : arrs){
+ if(val.contains(s)){
+ property = s
+ break;
+ }
+ }
+ return property
+ }
+ return null
+ }
+
+ static def processBuildStructure(val) {
+ if (val) {
+ def buildStructure= null
+ if(val.indexOf("砖木")>-1){
+ buildStructure = "砖木结构"
+ }else if(val.indexOf("钢结构")>-1){
+ buildStructure = "钢结构";
+ }else if(val.indexOf("钢、钢筋混凝土")>-1){
+ buildStructure = "钢、钢筋混凝土结构";
+ }else if(val.indexOf("钢筋混凝土")>-1){
+ buildStructure = "钢筋混凝土结构";
+ }else if(val.indexOf("混合")>-1){
+ buildStructure = "混合结构";
+ }
+ return buildStructure
+ }
+ return null
+ }
+
static def isRoad(def s){
def pattern = Pattern.compile("[路|街|大道|巷]");
def matcher = pattern.matcher(s)
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy
index d32f3e459..5495b6bfd 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy
@@ -7,9 +7,11 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.lixue.util.DBUtils
+import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
+import com.fangjia.script.util.Jms
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
@@ -17,6 +19,7 @@ import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import org.bson.types.ObjectId
+import java.text.SimpleDateFormat
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -31,15 +34,32 @@ import java.util.regex.Pattern
class DistrictFieldAddBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
-
+ static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
DBCollection SEAWEEDMAP_COLL = MongoUtils.getCollectionByName("fangjia", "seaweedmap", "192.168.0.235", 27018)
//"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房"
- //,"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家"
- def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家"] //key:穆融合表的来源 valve:es解析表的来源
+ //
+ def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
+ "乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
//"alias":"alias",
- def fieldsMap = ["estate_charge":"property_fee","estate_type2":"estate_type2","build_date":"build_year","estate_company":"property_company","developer":"developer","virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households","car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address"]//key:seaweed表字段 value:es表字段
+ //ownership_type:权属类型 为商品房、经适房、动迁房等
+ //house_property 产权性质:国有产权 "house_property":"网站没有",
+ def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer",
+ "virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households",
+ "car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure",
+ "build_type":"build_type", "land_property":"ownership_type","live_date":"live_date",
+ "loop_line_info":"loop_line"]//key:seaweed表字段 value:es表字段
+
+ def fieldsNameMap = ["estate_charge":"物业费","block":"板块","build_date":"建筑年代","estate_company":"物业公司","developer":"开发商",
+ "virescence_percent":"绿化率","cubage_percent":"容积率","build_area":"建筑面积","all_area":"占地面积","household_count":"户型",
+ "car_site_count":"车位","car_site":"车位描述","property_life":"产权年限","address":"地址","build_structure":"建筑结构",
+ "build_type":"建筑类型", "land_property":"权属类别","live_date":"入住时间","loop_line_info":"环线"]
+
+
+ final static def EMAILS = [
+ "gaojuan@fangjia.com"
+ ]
static main(args) {
runBatch(new DistrictFieldAddBatch(), args)
@@ -47,21 +67,37 @@ class DistrictFieldAddBatch extends BaseBatch{
@Override
void process(Object args) {
- def citys = ["漳州","福州","常州","龙岩","厦门","扬州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","南京","苏州","合肥","无锡","上海","北京","昆明","琼海","呼和浩特","南平","青岛","常德","黄石","黄冈","汉中","安康","榆林","南充","泸州","资阳","克拉玛依","荆门","铜川","内江","达州","巴中","鄂州","十堰","遂宁","随州","广安","荆州","咸宁","阿克苏","喀什","昌吉","吐鲁番","哈密","和田","塔城","阿勒泰","博尔塔拉","巴音郭楞","克孜勒苏","伊犁","恩施","咸阳","宜宾","自贡","攀枝花","延安","德阳","西安","重庆","成都","绵阳","乌鲁木齐","宜昌","武汉","乐山","眉山","宝鸡","襄阳","孝感","拉萨","深圳","三门峡","杭州","怀化","聊城","图木舒克","渭南","金华","宁德","三明","泉州","莆田","营口","牡丹江","盘锦","本溪","铁岭","齐齐哈尔","四平","辽源","锦州","佳木斯","白山","绥化","鸡西","七台河","白城","朝阳","双鸭山","通化","鹤岗","黑河","阜新","抚顺","葫芦岛","伊春","松原","大兴安岭","延边","吉林","沈阳","大连","长春","哈尔滨","鞍山","大庆","丹东"] //"广州", "成都","杭州", "深圳", "重庆","苏州","上海","北京"
-
+ def citys = CommonDatautils.findAllCity()
def analysRets = []
+
+ def rh_analysRets = []
+ def fields = fieldsMap.keySet() as List
+ def headers = ["来源", "城市"]
+ fields.each{it->
+ headers << fieldsNameMap.get(it)
+ }
+
+ headers << "抓取量"
+ headers << "匹配量"
+ headers << "0.9分以上量"
+ headers << "匹配率"
+ headers << "0.9以上匹配率"
+// analysRets << headers
+
for(def c : citys){
def totalCount = CommonDatautils.findAllDistrictCount("district", c)
-// addFieldFromRonghe(c, totalCount)
+ rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)
analysRets = addFieldFromMatch(c, totalCount, analysRets)
}
println analysRets.toString()
-
- ExcelUtils.write("D:\\数据\\小区\\覆盖率.xlsx", analysRets, "数据")
+ sendStatEmail(headers, analysRets, rh_analysRets)
+// ExcelUtils.write("D:\\数据\\小区\\覆盖率.xlsx", analysRets, "数据")
}
def addFieldFromMatch(c, totalCount, analysRets){
+ def fields = fieldsMap.keySet() as List
+
sourceMap.each { seaweedmapSource, esWebSource ->
def esRets = findEsDatas(c, esWebSource)
def matchRets1 = matchApi(esRets)
@@ -80,7 +116,18 @@ class DistrictFieldAddBatch extends BaseBatch{
pre1 = DataHandleUtils.getDoubleRound(2, matchRets1.size()*100/esRets.size())
pre2 = DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/esRets.size())
}
- analysRets << [seaweedmapSource, c, precentMap.get("property_fee"), esRets.size(), matchRets1.size(), matchRets2.size(), pre1, pre2]
+ def analyRet = []
+ analyRet << seaweedmapSource
+ analyRet << c
+ fields.each{key->
+ analyRet << precentMap.get(key)
+ }
+ analyRet << esRets.size()
+ analyRet << matchRets1.size()
+ analyRet << matchRets2.size()
+ analyRet << pre1
+ analyRet << pre2
+ analysRets << analyRet
}
return analysRets;
}
@@ -88,7 +135,8 @@ class DistrictFieldAddBatch extends BaseBatch{
- def addFieldFromRonghe(c, totalCount){
+ def addFieldFromRonghe(c, totalCount, rh_analysRets){
+ def fields = fieldsMap.keySet() as List
sourceMap.each{seaweedmapSource,esWebSource->
def rhRets = findRongheDists(c, seaweedmapSource)
def esRets = findEsDistrict(c, esWebSource)
@@ -99,7 +147,26 @@ class DistrictFieldAddBatch extends BaseBatch{
println c +" "+precentMap.toString()
updateDB(c, updateRets)
println "----------------------------------------------------------------------------------"
+ def pre1 = 0
+ def pre2 = 0
+ if(esRets){
+ pre1 = DataHandleUtils.getDoubleRound(2, rhRets.size()*100/esRets.size())
+ pre2 = DataHandleUtils.getDoubleRound(2, esRets.size()*100/esRets.size())
+ }
+ def analyRet = []
+ analyRet << seaweedmapSource
+ analyRet << c
+ fields.each{key->
+ analyRet << precentMap.get(key)
+ }
+ analyRet << esRets.size()
+ analyRet << rhRets.size()
+ analyRet << esRets.size()
+ analyRet << pre1
+ analyRet << pre2
+ rh_analysRets << analyRet
}
+ return rh_analysRets;
}
def processFieldsFromRh(c, rhRets, esRets){
@@ -116,26 +183,7 @@ class DistrictFieldAddBatch extends BaseBatch{
println c+" "+rhDist.id+" 不存在[ES]"
continue
}
- def newValues = [:]
- fieldsMap.each{seaweedfield,esField->
- def val = dist.get(seaweedfield)
- if(!val){
- def esVal = esDbo.get(esField)
- if(esVal){
- def newVal = null
- if(seaweedfield == "estate_charge"){
- newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
- newVal = newVal.toString()
- }
- if(seaweedfield == "build_date"){
- newVal = FiledStrand.processFinishBuildingDate(esVal)
- }
- if(newVal){
- newValues.put(seaweedfield, newVal)
- }
- }
- }
- }
+ def newValues = strandField(c, dist, esDbo)
if(newValues){
fieldValueMap.put(dist._id.toString(), newValues)
}
@@ -149,77 +197,15 @@ class DistrictFieldAddBatch extends BaseBatch{
def processFieldsFromMatch(c,matchRets){
def fields = fieldsMap.keySet() as List
fields << "city"
+ fields << "estate_type2"
def fieldValueMap = [:]
for(def matchDist : matchRets){
-
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
if(!dist){
println c+" "+matchDist.match_id+" 不存在"
continue
}
-
- def newValues = [:]
- fieldsMap.each{seaweedfield,esField->
- def val = dist.get(seaweedfield)
- if(!val){
- def esVal = matchDist.get(esField)
- if(esVal){
- def newVal = null
- if(seaweedfield == "estate_charge"){
- newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
- newVal = newVal.toString()
- }
- if(seaweedfield == "build_date"){
- newVal = FiledStrand.processFinishBuildingDate(esVal)
- }
- if(seaweedfield == "estate_company" || seaweedfield == "developer"){
- newVal = FiledStrand.processCompany(esVal)
- }
- if(seaweedfield =="virescence_percent"){
- newVal = FiledStrand.processVirescencePercent(esVal)
- }
-
- if(seaweedfield =="cubage_percent"){
- newVal = FiledStrand.processCubagePercent(esVal)
- }
-
- if(seaweedfield =="all_area" || seaweedfield =="build_area"){
- newVal = FiledStrand.processArea(esVal)
- newVal = newVal.toString()
- }
- if(seaweedfield =="household_count"){
- newVal = FiledStrand.processHouseholdCount(esVal, dist.estate_type2)
- }
-
- if(seaweedfield =="loudong_count"){
- newVal = FiledStrand.processLoudongCount(esVal)
- }
-
- if(seaweedfield =="car_site_count"){
- newVal = FiledStrand.processCarSite(esVal)
- }
- if(seaweedfield =="car_site"){
- newVal = FiledStrand.processCarSiteDesc(esVal)
- }
-
- if(seaweedfield =="property_life"){
- newVal = FiledStrand.processPropertyLife(esVal)
- }
-
- if(seaweedfield =="alias"){
- newVal = FiledStrand.processAlias(esVal, dist.name, dist.alias)
- }
-
- if(seaweedfield =="address"){
- newVal = FiledStrand.processAddress(esVal, dist.city)
- }
-
- if(newVal){
- newValues.put(seaweedfield, newVal)
- }
- }
- }
- }
+ def newValues = strandField(c, dist, matchDist)
if(newValues){
println dist._id.toString()+" "+newValues.toString()
fieldValueMap.put(dist._id.toString(), newValues)
@@ -229,6 +215,89 @@ class DistrictFieldAddBatch extends BaseBatch{
return fieldValueMap;
}
+ def strandField(c,dist, matchDist){
+ def newValues = [:]
+ fieldsMap.each{seaweedfield,esField->
+ def val = dist.get(seaweedfield)
+ if(!val){
+ def esVal = matchDist.get(esField)
+ if(esVal){
+ def newVal = null
+ if(seaweedfield == "estate_charge"){
+ newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
+ newVal = newVal.toString()
+ }
+ if(seaweedfield == "build_date" || seaweedfield == "live_date"){
+ newVal = FiledStrand.processFinishBuildingDate(esVal)
+ }
+ if(seaweedfield == "estate_company" || seaweedfield == "developer"){
+ newVal = FiledStrand.processCompany(esVal)
+ }
+ if(seaweedfield =="virescence_percent"){
+ newVal = FiledStrand.processVirescencePercent(esVal)
+ }
+
+ if(seaweedfield =="cubage_percent"){
+ newVal = FiledStrand.processCubagePercent(esVal)
+ }
+
+ if(seaweedfield =="all_area" || seaweedfield =="build_area"){
+ newVal = FiledStrand.processArea(esVal)
+ if(newVal){
+ newVal = newVal.toString()
+ }
+ }
+ if(seaweedfield =="household_count"){
+ newVal = FiledStrand.processHouseholdCount(esVal, dist.estate_type2)
+ }
+
+ if(seaweedfield =="loudong_count"){
+ newVal = FiledStrand.processLoudongCount(esVal)
+ }
+
+ if(seaweedfield =="car_site_count"){
+ newVal = FiledStrand.processCarSite(esVal)
+ }
+ if(seaweedfield =="car_site"){
+ newVal = FiledStrand.processCarSiteDesc(esVal)
+ }
+
+ if(seaweedfield =="property_life"){
+ newVal = FiledStrand.processPropertyLife(esVal)
+ }
+
+ if(seaweedfield =="alias"){
+ newVal = FiledStrand.processAlias(esVal, dist.name, dist.alias)
+ }
+
+ if(seaweedfield =="address"){
+ newVal = FiledStrand.processAddress(esVal, dist.city)
+ }
+ if(seaweedfield =="block"){
+ newVal = FiledStrand.processBlock(esVal)
+ }
+
+ if(seaweedfield == "build_type"){
+ newVal = FiledStrand.processBuildType(esVal, dist.estate_type2)
+ }
+
+ if(seaweedfield == "land_property"){
+ newVal = FiledStrand.processlandProperty(esVal)
+ }
+
+ if(seaweedfield == "build_structure"){
+ newVal = FiledStrand.processBuildStructure(esVal)
+ }
+
+ if(newVal){
+ newValues.put(seaweedfield, newVal)
+ }
+ }
+ }
+ }
+ return newValues;
+ }
+
def findRongheDists(def city, def source){
@@ -285,9 +354,9 @@ class DistrictFieldAddBatch extends BaseBatch{
def precentMap = [:]
fieldsMap.each() {key,skey->
if(countMap.containsKey(key)) {
- precentMap.put(skey, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total))
+ precentMap.put(key, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total))
}else {
- precentMap.put(skey, 0)
+ precentMap.put(key, 0)
}
}
return precentMap;
@@ -414,4 +483,61 @@ class DistrictFieldAddBatch extends BaseBatch{
}
return false
}
+
+ def sendStatEmail(headers, analysRets, rh_nalysRets){
+ Date dateTime = DateUtils.mins(1);
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy.MM.dd");
+ StringBuffer sb = new StringBuffer();
+ if(analysRets){
+ sb.append("
")
+ sb.append("字段补充情况分析
")
+ sb.append("")
+ headers.each{it->
+ if(it=="来源" || it=="城市"){
+ sb.append("| "+ it+" | ")
+ }else{
+ sb.append(""+ it+" | ")
+ }
+ }
+ sb.append("
")
+
+ for(def ret : analysRets){
+ sb.append("");
+ ret.each{it->
+ sb.append("| " + it + " | ")
+ }
+ sb.append("
");
+ }
+ sb.append("
");
+ }
+
+
+ StringBuffer rh_sb = new StringBuffer();
+ if(rh_nalysRets){
+ rh_sb.append("")
+ rh_sb.append("字段补充情况分析
")
+ rh_sb.append("")
+ headers.each{it->
+ if(it=="来源" || it=="城市"){
+ rh_sb.append("| "+ it+" | ")
+ }else{
+ rh_sb.append(""+ it+" | ")
+ }
+ }
+ rh_sb.append("
")
+
+ for(def ret : rh_nalysRets){
+ rh_sb.append("");
+ ret.each{it->
+ rh_sb.append("| " + it + " | ")
+ }
+ rh_sb.append("
");
+ }
+ rh_sb.append("
");
+ }
+
+ for (def email : EMAILS) {
+ jms.sendMail(email, "字段填充情况分析", sb.toString()+rh_sb.toString());
+ }
+ }
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFiledStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFiledStat.groovy
new file mode 100644
index 000000000..47f1563d0
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFiledStat.groovy
@@ -0,0 +1,69 @@
+package com.fangjia.script.gj.district
+
+import com.fangjia.script.commData.CommonDatautils
+import com.fangjia.script.common.DataHandleUtils
+import com.fangjia.script.util.DateUtils
+import com.fangjia.script.util.ExcelUtils
+import com.fangjia.script.util.MongoUtils
+import com.mongodb.BasicDBObject
+import com.mongodb.DBCollection
+
+/**
+ * 字段提升率统计
+ */
+DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo);
+
+def fieldsMap = ["district_count":"小区数量","name":"名称" ,"region":"区域" ,"block":"板块" ,"address":"地址" ,"estateType2":"物业类型" ,"householdCount":"户数" ,"openDate":"开盘时间" ,"liveDate":"入住时间" ,"buildDate":"建筑年代" ,"allArea":"占地面积" ,"buildArea":"建筑面积" ,"buildType":"建筑类型" ,"developer":"开发商" ,"estateCompany":"物业公司" ,"cubagePercent":"容积率" ,"virescencePercent":"绿化率" ,"carSite":"停车位" ,"estateCharge":"物业费" ,"floorplanPics":"户型图" ,"outPics":"外景图" ,"finishBuildingDate":"竣工时间" ,"businessQuarter":"商圈" ,"startBuildingDate":"开建时间" ,"loopLineInfo":"环线" ,"propertyLife":"产权年限" ,"houseProperty":"产权性质" ,"licence":"预售许可证" ,"buildStructure":"建筑结构" ,"lat":"百度纬度" ,"pics":"图片" ,"streetCommunity":"街道" ,"schools":"学校" ,"houseStructure":"房屋结构" ,"alias":"别名" ,"addrAlias":"地址别名" ,"mainTypeArea":"主力面积" ,"loudongCount":"楼栋数" ,"latLng" :"高德经度" ,"rentPrice":"租金" ]
+def weeknd = DateUtils.getFirstDayOfWeek(new Date())
+def lastWeeknd = DateUtils.getFirstDayOfWeek(new Date()-7)
+
+
+def weekendDatas = CITY_INFO_CONN.find(new BasicDBObject("weekend", weeknd))
+def lastWeekendDatas = CITY_INFO_CONN.find(new BasicDBObject("weekend", lastWeeknd))
+
+def weekendMapDatas = findDataByCity(weekendDatas)
+def lastWeekendMapDatas = findDataByCity(lastWeekendDatas)
+
+def citys = CommonDatautils.findAllCity()
+
+def fields = fieldsMap.keySet() as List
+def rets = []
+def headers = []
+headers << "城市"
+fields.each{k->
+ headers << fieldsMap.get(k)
+}
+rets << headers
+for(def c : citys){
+ def weekendDetailMap = weekendMapDatas.get(c)
+ def lastWeekendDetailMap = lastWeekendMapDatas.get(c)
+ def ret = []
+ ret << c
+ fields.each{key->
+ if(weekendDetailMap){
+ def v1 = weekendDetailMap.get(key)
+ def v2 = lastWeekendDetailMap.get(key)
+ if(v1 && v2){
+ ret << DataHandleUtils.getDoubleRound(2, v1-v2)
+ }else{
+ ret << "-"
+ }
+ }else{
+ ret << "-"
+ }
+ }
+ rets << ret
+}
+
+ExcelUtils.write("D:\\数据\\小区\\字段提升率"+DateUtils.formatDateByPattern(weeknd,"yyyy-MM-dd")+".xlsx", rets, "字段提升率统计")
+
+
+
+def findDataByCity(weekendDatas){
+ def map = [:]
+ for(def d : weekendDatas){
+ d.district_detail.district_count = d.existing_district_count
+ map.put(d.city, d.district_detail)
+ }
+ return map
+}
\ No newline at end of file
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy
index a3e22d7a6..9be75aeb0 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy
@@ -1,7 +1,9 @@
package com.fangjia.script.gj.es
+import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
+import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
@@ -19,11 +21,44 @@ class EsMonthHouseStat {
def task = new EsMonthHouseStat()
// task.run()
// task.tradeRun()
- task.houseRun()
+// task.houseRun()
// task.statHouseUpdateCount()
// task.statDistHouse()
+ task.statRent()
+ }
+
+ def statRent(){
+ def index = 0
+ ExcelUtils.multiThreadingModify("D:\\数据\\其它\\租金.xlsx", { d ->
+ index++
+ def region = RegionData.formatRegion(d['A'], d['B'])
+ if(region){
+ def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
+ if(!dist){
+ dist = OpenApiUtils.matchDistrictByAddr(d['A'], d['C'], region)
+ if(dist){
+ dist = CommonDatautils.findDistrictById(dist.id)
+ }
+ }
+ def alias = []
+ if(dist.alias){
+ if(!dist.alias.contains(dist.name)){
+ dist.alias << dist.name
+ }
+ alias = dist.alias
+ }else{
+ alias << dist.name
+ }
+
+ def distCount = findHouseCount(dist.city, dist.region, alias, 20220501, 20230501)
+ d['G'] = distCount
+ d['E'] = dist.region
+ d['F'] = dist.name
+ println index+" "+ d['C'] +" "+distCount
+ }
+ });
}
def statDistHouse(){
@@ -172,7 +207,7 @@ class EsMonthHouseStat {
def total = 0
try{
aliasStr = escapeExprSpecialWord(alias.join("','"))
- String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
+ String sql = "select count(*) as c from latest_seawater where cat='rent' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and root='lianjia.com'"
ElasticUtils.executeSqlQuery(sql,{data->
total = data.c
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy
index 2aae0edb5..916db9961 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy
@@ -239,6 +239,12 @@ public class BeiKeCrawlByChrome {
if (areaStr && areaStr.isNumber()) {
detail.area = areaStr.toDouble()
}
+ if(!detail.area){
+ areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("㎡", "")
+ if (areaStr && areaStr.isNumber()) {
+ detail.area = areaStr.toDouble()
+ }
+ }
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
@@ -254,7 +260,7 @@ public class BeiKeCrawlByChrome {
}
detail.source = "贝壳找房"
- if(detail.total_price){
+ if(detail.total_price && detail.avg_price){
def detailUrl = e.select(".title a").attr("href")
detail.url = detailUrl
def dbo = findTrade(detail)
@@ -404,9 +410,9 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = "https:"+e.attr("href")
- if (citys.contains(e.text())){
+// if (citys.contains(e.text())){
list << [e.text(), href]
- }
+// }
}
}
return list
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy
index 343a4a3a4..6c01e6c11 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy
@@ -333,7 +333,7 @@ public class LianjiaCrawlByChrome {
detail.source = "链家在线"
detail.url = e.select(".title a").attr("href")
- if(detail.total_price){
+ if(detail.total_price && detail.avg_price){
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
@@ -482,9 +482,9 @@ public class LianjiaCrawlByChrome {
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
- if (!href.contains("fang.lianjia") && citys.contains(e.text())){
+// if (!href.contains("fang.lianjia") && citys.contains(e.text())){
cityList << [e.text(), href]
- }
+// }
}
}
return cityList
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/OpenUriUserIdCountDay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/OpenUriUserIdCountDay.groovy
index 4e27d62a5..954fface3 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/OpenUriUserIdCountDay.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/OpenUriUserIdCountDay.groovy
@@ -27,19 +27,11 @@ class OpenUriUserIdCountDay extends BaseBatch {
runBatch(new OpenUriUserIdCountDay(), args)
}
final static def EMAILS = [
- "joe@fangjia.com",
"wangwenyan@fangjia.com",
"xiaohai@fangjia.com",
"caoyuyan@fangjia.com",
"jiangxiaochun@fangjia.com",
- "gaojuan@fangjia.com",
- 'shijing@fangjia.com',
- 'zhongjingyi@fangjia.com',
- "duweiliang@fangjia.com",
- "sunjian@fangjia.com",
- "yaojingmeng@fangjia.com",
- "xuqi@fangjia.com",
- "wangpeili@fangjia.com"
+ "gaojuan@fangjia.com"
]
@Override