Merge branch 'script_gj' into 'master'

update

See merge request !623
This commit is contained in:
2023-05-12 10:24:46 +08:00
9 changed files with 454 additions and 150 deletions
@@ -36,35 +36,35 @@ class ExportDistrictRongHeTest extends BaseBatch{
// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) // exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
// if(curDate.toString().endsWith("10")){ if(curDate.toString().endsWith("10")){
// exportEvaluateRate() exportEvaluateRate()
// exportBigdataBuildprojFjw() exportBigdataBuildprojFjw()
// exportBigdataCommlayoutpicFjw() exportBigdataCommlayoutpicFjw()
//
//
// exportBigdataCommunityGJRH()/ exportBigdataCommunityGJRH()/
// exportBigdataBuildingFJW()// exportBigdataBuildingFJW()//
// exportBigdataCommfeeGj()// exportBigdataCommfeeGj()//
//
// exportBigdataCommbuildstruGJ()// exportBigdataCommbuildstruGJ()//
// exportBigdataCommhousetypeGJ()// exportBigdataCommhousetypeGJ()//
// exportBigdataCommbuildyearGj()// exportBigdataCommbuildyearGj()//
// exportBigdataCommhouseuseGJ()// exportBigdataCommhouseuseGJ()//
// exportBigdataPositionGJ()// exportBigdataPositionGJ()//
// exportBigdataDistrictFJW()// exportBigdataDistrictFJW()//
// exportBigdataAliasFJW()// exportBigdataAliasFJW()//
//
//
// //本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据 //本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据
// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
// exportBigdataCityPrice(queryDate) exportBigdataCityPrice(queryDate)
// exportBigdataDistPrice(queryDate) exportBigdataDistPrice(queryDate)
// exportBigdataCommPrice(queryDate) exportBigdataCommPrice(queryDate)
exportInOk() exportInOk()
// } }
SSHHelperTest jschCommand = new SSHHelperTest(); SSHHelperTest jschCommand = new SSHHelperTest();
System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}")); System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}"));
@@ -29,16 +29,16 @@ class EsHouseStat {
DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002) DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002)
// DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.206", 27020) // DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.206", 27020)
// public static JdbcDataSource dataSource = new JdbcDataSource() public static JdbcDataSource dataSource = new JdbcDataSource()
// private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
// public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
// static { static {
// dataSource.setUrl(address) dataSource.setUrl(address)
// Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
// connectionProperties.put("user", "developer") connectionProperties.put("user", "developer")
// connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
// dataSource.setProperties(connectionProperties) dataSource.setProperties(connectionProperties)
// } }
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -285,7 +285,7 @@ class FiledStrand {
} }
static def processCarSiteDesc(car_site){ static def processCarSiteDesc(car_site){
if(car_site && car_site.size()<=20){ if(car_site && car_site.size()<=20 && car_site.size()>=2 && !car_site.contains("待定") && !car_site.contains("暂无")){
return car_site.toString() return car_site.toString()
} }
return null return null
@@ -337,6 +337,82 @@ class FiledStrand {
return null return null
} }
static def processBlock(def block){
if(block){
if(block.contains("无") || block.toString().length()>10){
return null
}
return block
}
return null
}
static def processBuildType(def buildType, def estateType){
if(buildType){
def types = []
if(estateType == "别墅"){
def arrs = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院"]
for(def s : arrs){
if(buildType.contains(s)){
types << s
}
}
}else if(estateType == "商住"){
def arrs = ["独栋","双拼","联排","叠加","老式别墅洋房"]
for(def s : arrs){
if(buildType.contains(s)){
types << s
}
}
}else{
def arrs = ["多层","小高层","高层","超高层","SOHO","soho","LOFT","loft","酒店式公寓"]
for(def s : arrs){
if(buildType.contains(s)){
types << s
}
}
}
if(types){
return types.join(",").toString()
}
}
return null
}
static def processlandProperty(val){
if(val){
def arrs = ["经适房","限价房","廉租房","宅基地","保障房","动迁房","配套房","商品房"]
def property = null
for(def s : arrs){
if(val.contains(s)){
property = s
break;
}
}
return property
}
return null
}
static def processBuildStructure(val) {
if (val) {
def buildStructure= null
if(val.indexOf("砖木")>-1){
buildStructure = "砖木结构"
}else if(val.indexOf("钢结构")>-1){
buildStructure = "钢结构";
}else if(val.indexOf("钢、钢筋混凝土")>-1){
buildStructure = "钢、钢筋混凝土结构";
}else if(val.indexOf("钢筋混凝土")>-1){
buildStructure = "钢筋混凝土结构";
}else if(val.indexOf("混合")>-1){
buildStructure = "混合结构";
}
return buildStructure
}
return null
}
static def isRoad(def s){ static def isRoad(def s){
def pattern = Pattern.compile("[路|街|大道|巷]"); def pattern = Pattern.compile("[路|街|大道|巷]");
def matcher = pattern.matcher(s) def matcher = pattern.matcher(s)
@@ -7,9 +7,11 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.FiledStrand import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.Jms
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
@@ -17,6 +19,7 @@ import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl import com.rabbitmq.client.impl.AMQImpl
import org.bson.types.ObjectId import org.bson.types.ObjectId
import java.text.SimpleDateFormat
import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService import java.util.concurrent.ExecutorService
@@ -31,15 +34,32 @@ import java.util.regex.Pattern
class DistrictFieldAddBatch extends BaseBatch{ class DistrictFieldAddBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
DBCollection SEAWEEDMAP_COLL = MongoUtils.getCollectionByName("fangjia", "seaweedmap", "192.168.0.235", 27018) DBCollection SEAWEEDMAP_COLL = MongoUtils.getCollectionByName("fangjia", "seaweedmap", "192.168.0.235", 27018)
//"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房" //"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房"
//,"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家" //
def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家"] //key:穆融合表的来源 valve:es解析表的来源 def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
"乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
//"alias":"alias", //"alias":"alias",
def fieldsMap = ["estate_charge":"property_fee","estate_type2":"estate_type2","build_date":"build_year","estate_company":"property_company","developer":"developer","virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households","car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address"]//key:seaweed表字段 value:es表字段 //ownership_type:权属类型 为商品房、经适房、动迁房等
//house_property 产权性质:国有产权 "house_property":"网站没有",
def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer",
"virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households",
"car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure",
"build_type":"build_type", "land_property":"ownership_type","live_date":"live_date",
"loop_line_info":"loop_line"]//key:seaweed表字段 value:es表字段
def fieldsNameMap = ["estate_charge":"物业费","block":"板块","build_date":"建筑年代","estate_company":"物业公司","developer":"开发商",
"virescence_percent":"绿化率","cubage_percent":"容积率","build_area":"建筑面积","all_area":"占地面积","household_count":"户型",
"car_site_count":"车位","car_site":"车位描述","property_life":"产权年限","address":"地址","build_structure":"建筑结构",
"build_type":"建筑类型", "land_property":"权属类别","live_date":"入住时间","loop_line_info":"环线"]
final static def EMAILS = [
"gaojuan@fangjia.com"
]
static main(args) { static main(args) {
runBatch(new DistrictFieldAddBatch(), args) runBatch(new DistrictFieldAddBatch(), args)
@@ -47,21 +67,37 @@ class DistrictFieldAddBatch extends BaseBatch{
@Override @Override
void process(Object args) { void process(Object args) {
def citys = ["漳州","福州","常州","龙岩","厦门","扬州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","南京","苏州","合肥","无锡","上海","北京","昆明","琼海","呼和浩特","南平","青岛","常德","黄石","黄冈","汉中","安康","榆林","南充","泸州","资阳","克拉玛依","荆门","铜川","内江","达州","巴中","鄂州","十堰","遂宁","随州","广安","荆州","咸宁","阿克苏","喀什","昌吉","吐鲁番","哈密","和田","塔城","阿勒泰","博尔塔拉","巴音郭楞","克孜勒苏","伊犁","恩施","咸阳","宜宾","自贡","攀枝花","延安","德阳","西安","重庆","成都","绵阳","乌鲁木齐","宜昌","武汉","乐山","眉山","宝鸡","襄阳","孝感","拉萨","深圳","三门峡","杭州","怀化","聊城","图木舒克","渭南","金华","宁德","三明","泉州","莆田","营口","牡丹江","盘锦","本溪","铁岭","齐齐哈尔","四平","辽源","锦州","佳木斯","白山","绥化","鸡西","七台河","白城","朝阳","双鸭山","通化","鹤岗","黑河","阜新","抚顺","葫芦岛","伊春","松原","大兴安岭","延边","吉林","沈阳","大连","长春","哈尔滨","鞍山","大庆","丹东"] //"广州", "成都","杭州", "深圳", "重庆","苏州","上海","北京" def citys = CommonDatautils.findAllCity()
def analysRets = [] def analysRets = []
def rh_analysRets = []
def fields = fieldsMap.keySet() as List
def headers = ["来源", "城市"]
fields.each{it->
headers << fieldsNameMap.get(it)
}
headers << "抓取量"
headers << "匹配量"
headers << "0.9分以上量"
headers << "匹配率"
headers << "0.9以上匹配率"
// analysRets << headers
for(def c : citys){ for(def c : citys){
def totalCount = CommonDatautils.findAllDistrictCount("district", c) def totalCount = CommonDatautils.findAllDistrictCount("district", c)
// addFieldFromRonghe(c, totalCount) rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)
analysRets = addFieldFromMatch(c, totalCount, analysRets) analysRets = addFieldFromMatch(c, totalCount, analysRets)
} }
println analysRets.toString() println analysRets.toString()
sendStatEmail(headers, analysRets, rh_analysRets)
ExcelUtils.write("D:\\数据\\小区\\覆盖率.xlsx", analysRets, "数据") // ExcelUtils.write("D:\\数据\\小区\\覆盖率.xlsx", analysRets, "数据")
} }
def addFieldFromMatch(c, totalCount, analysRets){ def addFieldFromMatch(c, totalCount, analysRets){
def fields = fieldsMap.keySet() as List
sourceMap.each { seaweedmapSource, esWebSource -> sourceMap.each { seaweedmapSource, esWebSource ->
def esRets = findEsDatas(c, esWebSource) def esRets = findEsDatas(c, esWebSource)
def matchRets1 = matchApi(esRets) def matchRets1 = matchApi(esRets)
@@ -80,7 +116,18 @@ class DistrictFieldAddBatch extends BaseBatch{
pre1 = DataHandleUtils.getDoubleRound(2, matchRets1.size()*100/esRets.size()) pre1 = DataHandleUtils.getDoubleRound(2, matchRets1.size()*100/esRets.size())
pre2 = DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/esRets.size()) pre2 = DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/esRets.size())
} }
analysRets << [seaweedmapSource, c, precentMap.get("property_fee"), esRets.size(), matchRets1.size(), matchRets2.size(), pre1, pre2] def analyRet = []
analyRet << seaweedmapSource
analyRet << c
fields.each{key->
analyRet << precentMap.get(key)
}
analyRet << esRets.size()
analyRet << matchRets1.size()
analyRet << matchRets2.size()
analyRet << pre1
analyRet << pre2
analysRets << analyRet
} }
return analysRets; return analysRets;
} }
@@ -88,7 +135,8 @@ class DistrictFieldAddBatch extends BaseBatch{
def addFieldFromRonghe(c, totalCount){ def addFieldFromRonghe(c, totalCount, rh_analysRets){
def fields = fieldsMap.keySet() as List
sourceMap.each{seaweedmapSource,esWebSource-> sourceMap.each{seaweedmapSource,esWebSource->
def rhRets = findRongheDists(c, seaweedmapSource) def rhRets = findRongheDists(c, seaweedmapSource)
def esRets = findEsDistrict(c, esWebSource) def esRets = findEsDistrict(c, esWebSource)
@@ -99,7 +147,26 @@ class DistrictFieldAddBatch extends BaseBatch{
println c +" "+precentMap.toString() println c +" "+precentMap.toString()
updateDB(c, updateRets) updateDB(c, updateRets)
println "----------------------------------------------------------------------------------" println "----------------------------------------------------------------------------------"
def pre1 = 0
def pre2 = 0
if(esRets){
pre1 = DataHandleUtils.getDoubleRound(2, rhRets.size()*100/esRets.size())
pre2 = DataHandleUtils.getDoubleRound(2, esRets.size()*100/esRets.size())
}
def analyRet = []
analyRet << seaweedmapSource
analyRet << c
fields.each{key->
analyRet << precentMap.get(key)
}
analyRet << esRets.size()
analyRet << rhRets.size()
analyRet << esRets.size()
analyRet << pre1
analyRet << pre2
rh_analysRets << analyRet
} }
return rh_analysRets;
} }
def processFieldsFromRh(c, rhRets, esRets){ def processFieldsFromRh(c, rhRets, esRets){
@@ -116,26 +183,7 @@ class DistrictFieldAddBatch extends BaseBatch{
println c+" "+rhDist.id+" 不存在[ES]" println c+" "+rhDist.id+" 不存在[ES]"
continue continue
} }
def newValues = [:] def newValues = strandField(c, dist, esDbo)
fieldsMap.each{seaweedfield,esField->
def val = dist.get(seaweedfield)
if(!val){
def esVal = esDbo.get(esField)
if(esVal){
def newVal = null
if(seaweedfield == "estate_charge"){
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
newVal = newVal.toString()
}
if(seaweedfield == "build_date"){
newVal = FiledStrand.processFinishBuildingDate(esVal)
}
if(newVal){
newValues.put(seaweedfield, newVal)
}
}
}
}
if(newValues){ if(newValues){
fieldValueMap.put(dist._id.toString(), newValues) fieldValueMap.put(dist._id.toString(), newValues)
} }
@@ -149,77 +197,15 @@ class DistrictFieldAddBatch extends BaseBatch{
def processFieldsFromMatch(c,matchRets){ def processFieldsFromMatch(c,matchRets){
def fields = fieldsMap.keySet() as List def fields = fieldsMap.keySet() as List
fields << "city" fields << "city"
fields << "estate_type2"
def fieldValueMap = [:] def fieldValueMap = [:]
for(def matchDist : matchRets){ for(def matchDist : matchRets){
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields) def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
if(!dist){ if(!dist){
println c+" "+matchDist.match_id+" 不存在" println c+" "+matchDist.match_id+" 不存在"
continue continue
} }
def newValues = strandField(c, dist, matchDist)
def newValues = [:]
fieldsMap.each{seaweedfield,esField->
def val = dist.get(seaweedfield)
if(!val){
def esVal = matchDist.get(esField)
if(esVal){
def newVal = null
if(seaweedfield == "estate_charge"){
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
newVal = newVal.toString()
}
if(seaweedfield == "build_date"){
newVal = FiledStrand.processFinishBuildingDate(esVal)
}
if(seaweedfield == "estate_company" || seaweedfield == "developer"){
newVal = FiledStrand.processCompany(esVal)
}
if(seaweedfield =="virescence_percent"){
newVal = FiledStrand.processVirescencePercent(esVal)
}
if(seaweedfield =="cubage_percent"){
newVal = FiledStrand.processCubagePercent(esVal)
}
if(seaweedfield =="all_area" || seaweedfield =="build_area"){
newVal = FiledStrand.processArea(esVal)
newVal = newVal.toString()
}
if(seaweedfield =="household_count"){
newVal = FiledStrand.processHouseholdCount(esVal, dist.estate_type2)
}
if(seaweedfield =="loudong_count"){
newVal = FiledStrand.processLoudongCount(esVal)
}
if(seaweedfield =="car_site_count"){
newVal = FiledStrand.processCarSite(esVal)
}
if(seaweedfield =="car_site"){
newVal = FiledStrand.processCarSiteDesc(esVal)
}
if(seaweedfield =="property_life"){
newVal = FiledStrand.processPropertyLife(esVal)
}
if(seaweedfield =="alias"){
newVal = FiledStrand.processAlias(esVal, dist.name, dist.alias)
}
if(seaweedfield =="address"){
newVal = FiledStrand.processAddress(esVal, dist.city)
}
if(newVal){
newValues.put(seaweedfield, newVal)
}
}
}
}
if(newValues){ if(newValues){
println dist._id.toString()+" "+newValues.toString() println dist._id.toString()+" "+newValues.toString()
fieldValueMap.put(dist._id.toString(), newValues) fieldValueMap.put(dist._id.toString(), newValues)
@@ -229,6 +215,89 @@ class DistrictFieldAddBatch extends BaseBatch{
return fieldValueMap; return fieldValueMap;
} }
def strandField(c,dist, matchDist){
def newValues = [:]
fieldsMap.each{seaweedfield,esField->
def val = dist.get(seaweedfield)
if(!val){
def esVal = matchDist.get(esField)
if(esVal){
def newVal = null
if(seaweedfield == "estate_charge"){
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
newVal = newVal.toString()
}
if(seaweedfield == "build_date" || seaweedfield == "live_date"){
newVal = FiledStrand.processFinishBuildingDate(esVal)
}
if(seaweedfield == "estate_company" || seaweedfield == "developer"){
newVal = FiledStrand.processCompany(esVal)
}
if(seaweedfield =="virescence_percent"){
newVal = FiledStrand.processVirescencePercent(esVal)
}
if(seaweedfield =="cubage_percent"){
newVal = FiledStrand.processCubagePercent(esVal)
}
if(seaweedfield =="all_area" || seaweedfield =="build_area"){
newVal = FiledStrand.processArea(esVal)
if(newVal){
newVal = newVal.toString()
}
}
if(seaweedfield =="household_count"){
newVal = FiledStrand.processHouseholdCount(esVal, dist.estate_type2)
}
if(seaweedfield =="loudong_count"){
newVal = FiledStrand.processLoudongCount(esVal)
}
if(seaweedfield =="car_site_count"){
newVal = FiledStrand.processCarSite(esVal)
}
if(seaweedfield =="car_site"){
newVal = FiledStrand.processCarSiteDesc(esVal)
}
if(seaweedfield =="property_life"){
newVal = FiledStrand.processPropertyLife(esVal)
}
if(seaweedfield =="alias"){
newVal = FiledStrand.processAlias(esVal, dist.name, dist.alias)
}
if(seaweedfield =="address"){
newVal = FiledStrand.processAddress(esVal, dist.city)
}
if(seaweedfield =="block"){
newVal = FiledStrand.processBlock(esVal)
}
if(seaweedfield == "build_type"){
newVal = FiledStrand.processBuildType(esVal, dist.estate_type2)
}
if(seaweedfield == "land_property"){
newVal = FiledStrand.processlandProperty(esVal)
}
if(seaweedfield == "build_structure"){
newVal = FiledStrand.processBuildStructure(esVal)
}
if(newVal){
newValues.put(seaweedfield, newVal)
}
}
}
}
return newValues;
}
def findRongheDists(def city, def source){ def findRongheDists(def city, def source){
@@ -285,9 +354,9 @@ class DistrictFieldAddBatch extends BaseBatch{
def precentMap = [:] def precentMap = [:]
fieldsMap.each() {key,skey-> fieldsMap.each() {key,skey->
if(countMap.containsKey(key)) { if(countMap.containsKey(key)) {
precentMap.put(skey, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total)) precentMap.put(key, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total))
}else { }else {
precentMap.put(skey, 0) precentMap.put(key, 0)
} }
} }
return precentMap; return precentMap;
@@ -414,4 +483,61 @@ class DistrictFieldAddBatch extends BaseBatch{
} }
return false return false
} }
def sendStatEmail(headers, analysRets, rh_nalysRets){
Date dateTime = DateUtils.mins(1);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy.MM.dd");
StringBuffer sb = new StringBuffer();
if(analysRets){
sb.append("<table border='1' style='border-collapse:collapse; border-spacing:0'>")
sb.append("<tr><font style='size:15px;color:grey;'>字段补充情况分析</tr>")
sb.append("<tr style='color:black;font-size:18px;cellspacing:1px;'>")
headers.each{it->
if(it=="来源" || it=="城市"){
sb.append("<th style='width:80px;'>"+ it+"</th>")
}else{
sb.append("<th >"+ it+"</th>")
}
}
sb.append("</tr>")
for(def ret : analysRets){
sb.append("<tr style='text-align:center;font-color:gray;cellspacing:1px;'>");
ret.each{it->
sb.append("<td>" + it + "</td>")
}
sb.append("</tr>");
}
sb.append("</table><br/>");
}
StringBuffer rh_sb = new StringBuffer();
if(rh_nalysRets){
rh_sb.append("<table border='1' style='border-collapse:collapse; border-spacing:0'>")
rh_sb.append("<tr><font style='size:15px;color:grey;'>字段补充情况分析</tr>")
rh_sb.append("<tr style='color:black;font-size:18px;cellspacing:1px;'>")
headers.each{it->
if(it=="来源" || it=="城市"){
rh_sb.append("<th style='width:80px;'>"+ it+"</th>")
}else{
rh_sb.append("<th >"+ it+"</th>")
}
}
rh_sb.append("</tr>")
for(def ret : rh_nalysRets){
rh_sb.append("<tr style='text-align:center;font-color:gray;cellspacing:1px;'>");
ret.each{it->
rh_sb.append("<td>" + it + "</td>")
}
rh_sb.append("</tr>");
}
rh_sb.append("</table><br/>");
}
for (def email : EMAILS) {
jms.sendMail(email, "字段填充情况分析", sb.toString()+rh_sb.toString());
}
}
} }
@@ -0,0 +1,69 @@
package com.fangjia.script.gj.district
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 字段提升率统计
*/
DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo);
def fieldsMap = ["district_count":"小区数量","name":"名称" ,"region":"区域" ,"block":"板块" ,"address":"地址" ,"estateType2":"物业类型" ,"householdCount":"户数" ,"openDate":"开盘时间" ,"liveDate":"入住时间" ,"buildDate":"建筑年代" ,"allArea":"占地面积" ,"buildArea":"建筑面积" ,"buildType":"建筑类型" ,"developer":"开发商" ,"estateCompany":"物业公司" ,"cubagePercent":"容积率" ,"virescencePercent":"绿化率" ,"carSite":"停车位" ,"estateCharge":"物业费" ,"floorplanPics":"户型图" ,"outPics":"外景图" ,"finishBuildingDate":"竣工时间" ,"businessQuarter":"商圈" ,"startBuildingDate":"开建时间" ,"loopLineInfo":"环线" ,"propertyLife":"产权年限" ,"houseProperty":"产权性质" ,"licence":"预售许可证" ,"buildStructure":"建筑结构" ,"lat":"百度纬度" ,"pics":"图片" ,"streetCommunity":"街道" ,"schools":"学校" ,"houseStructure":"房屋结构" ,"alias":"别名" ,"addrAlias":"地址别名" ,"mainTypeArea":"主力面积" ,"loudongCount":"楼栋数" ,"latLng" :"高德经度" ,"rentPrice":"租金" ]
def weeknd = DateUtils.getFirstDayOfWeek(new Date())
def lastWeeknd = DateUtils.getFirstDayOfWeek(new Date()-7)
def weekendDatas = CITY_INFO_CONN.find(new BasicDBObject("weekend", weeknd))
def lastWeekendDatas = CITY_INFO_CONN.find(new BasicDBObject("weekend", lastWeeknd))
def weekendMapDatas = findDataByCity(weekendDatas)
def lastWeekendMapDatas = findDataByCity(lastWeekendDatas)
def citys = CommonDatautils.findAllCity()
def fields = fieldsMap.keySet() as List
def rets = []
def headers = []
headers << "城市"
fields.each{k->
headers << fieldsMap.get(k)
}
rets << headers
for(def c : citys){
def weekendDetailMap = weekendMapDatas.get(c)
def lastWeekendDetailMap = lastWeekendMapDatas.get(c)
def ret = []
ret << c
fields.each{key->
if(weekendDetailMap){
def v1 = weekendDetailMap.get(key)
def v2 = lastWeekendDetailMap.get(key)
if(v1 && v2){
ret << DataHandleUtils.getDoubleRound(2, v1-v2)
}else{
ret << "-"
}
}else{
ret << "-"
}
}
rets << ret
}
ExcelUtils.write("D:\\数据\\小区\\字段提升率"+DateUtils.formatDateByPattern(weeknd,"yyyy-MM-dd")+".xlsx", rets, "字段提升率统计")
def findDataByCity(weekendDatas){
def map = [:]
for(def d : weekendDatas){
d.district_detail.district_count = d.existing_district_count
map.put(d.city, d.district_detail)
}
return map
}
@@ -1,7 +1,9 @@
package com.fangjia.script.gj.es package com.fangjia.script.gj.es
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
@@ -19,11 +21,44 @@ class EsMonthHouseStat {
def task = new EsMonthHouseStat() def task = new EsMonthHouseStat()
// task.run() // task.run()
// task.tradeRun() // task.tradeRun()
task.houseRun() // task.houseRun()
// task.statHouseUpdateCount() // task.statHouseUpdateCount()
// task.statDistHouse() // task.statDistHouse()
task.statRent()
}
def statRent(){
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\租金.xlsx", { d ->
index++
def region = RegionData.formatRegion(d['A'], d['B'])
if(region){
def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
if(!dist){
dist = OpenApiUtils.matchDistrictByAddr(d['A'], d['C'], region)
if(dist){
dist = CommonDatautils.findDistrictById(dist.id)
}
}
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
}
def distCount = findHouseCount(dist.city, dist.region, alias, 20220501, 20230501)
d['G'] = distCount
d['E'] = dist.region
d['F'] = dist.name
println index+" "+ d['C'] +" "+distCount
}
});
} }
def statDistHouse(){ def statDistHouse(){
@@ -172,7 +207,7 @@ class EsMonthHouseStat {
def total = 0 def total = 0
try{ try{
aliasStr = escapeExprSpecialWord(alias.join("','")) aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')" String sql = "select count(*) as c from latest_seawater where cat='rent' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and root='lianjia.com'"
ElasticUtils.executeSqlQuery(sql,{data-> ElasticUtils.executeSqlQuery(sql,{data->
total = data.c total = data.c
@@ -239,6 +239,12 @@ public class BeiKeCrawlByChrome {
if (areaStr && areaStr.isNumber()) { if (areaStr && areaStr.isNumber()) {
detail.area = areaStr.toDouble() detail.area = areaStr.toDouble()
} }
if(!detail.area){
areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("㎡", "")
if (areaStr && areaStr.isNumber()) {
detail.area = areaStr.toDouble()
}
}
def price = e.select(".totalPrice .number").text() def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) { if (price && price.isNumber()) {
@@ -254,7 +260,7 @@ public class BeiKeCrawlByChrome {
} }
detail.source = "贝壳找房" detail.source = "贝壳找房"
if(detail.total_price){ if(detail.total_price && detail.avg_price){
def detailUrl = e.select(".title a").attr("href") def detailUrl = e.select(".title a").attr("href")
detail.url = detailUrl detail.url = detailUrl
def dbo = findTrade(detail) def dbo = findTrade(detail)
@@ -404,9 +410,9 @@ public class BeiKeCrawlByChrome {
if(cityEles){ if(cityEles){
for(def e : cityEles){ for(def e : cityEles){
def href = "https:"+e.attr("href") def href = "https:"+e.attr("href")
if (citys.contains(e.text())){ // if (citys.contains(e.text())){
list << [e.text(), href] list << [e.text(), href]
} // }
} }
} }
return list return list
@@ -333,7 +333,7 @@ public class LianjiaCrawlByChrome {
detail.source = "链家在线" detail.source = "链家在线"
detail.url = e.select(".title a").attr("href") detail.url = e.select(".title a").attr("href")
if(detail.total_price){ if(detail.total_price && detail.avg_price){
def dbo = findTrade(detail) def dbo = findTrade(detail)
if (!dbo) { if (!dbo) {
def houseInfo = e.select(".houseInfo").text() def houseInfo = e.select(".houseInfo").text()
@@ -482,9 +482,9 @@ public class LianjiaCrawlByChrome {
if(cityEles){ if(cityEles){
for(def e : cityEles) { for(def e : cityEles) {
def href = e.attr("href") def href = e.attr("href")
if (!href.contains("fang.lianjia") && citys.contains(e.text())){ // if (!href.contains("fang.lianjia") && citys.contains(e.text())){
cityList << [e.text(), href] cityList << [e.text(), href]
} // }
} }
} }
return cityList return cityList
@@ -27,19 +27,11 @@ class OpenUriUserIdCountDay extends BaseBatch {
runBatch(new OpenUriUserIdCountDay(), args) runBatch(new OpenUriUserIdCountDay(), args)
} }
final static def EMAILS = [ final static def EMAILS = [
"joe@fangjia.com",
"wangwenyan@fangjia.com", "wangwenyan@fangjia.com",
"xiaohai@fangjia.com", "xiaohai@fangjia.com",
"caoyuyan@fangjia.com", "caoyuyan@fangjia.com",
"jiangxiaochun@fangjia.com", "jiangxiaochun@fangjia.com",
"gaojuan@fangjia.com", "gaojuan@fangjia.com"
'shijing@fangjia.com',
'zhongjingyi@fangjia.com',
"duweiliang@fangjia.com",
"sunjian@fangjia.com",
"yaojingmeng@fangjia.com",
"xuqi@fangjia.com",
"wangpeili@fangjia.com"
] ]
@Override @Override