This commit is contained in:
2023-05-15 11:36:49 +08:00
12 changed files with 519 additions and 165 deletions
@@ -42,7 +42,9 @@ public enum NewResourceRecommendEnum {
GOV("1034", "政府机构"),
BOND("1033", "证券"),
COST("1035", "消费金融"),
ZIXUN_APP("1036", "业务咨询app");
ZIXUN_APP("1036", "业务咨询app"),
ARTICLE_APP("1037", "公众号推文"),
MENU_APP("1038", "公众号菜单");
NewResourceRecommendEnum(String id, String value) {
this.id = id;
@@ -38,7 +38,9 @@ public enum SalesLeadEnum {
GOV("1034", "政府机构"),
BOND("1033", "证券"),
COST("1035", "消费金融"),
ZIXUN_APP("1036", "业务咨询app");
ZIXUN_APP("1036", "业务咨询app"),
ARTICLE_APP("1037", "公众号推文"),
MENU_APP("1038", "公众号菜单");
SalesLeadEnum(String id, String value) {
this.id = id;
@@ -36,35 +36,35 @@ class ExportDistrictRongHeTest extends BaseBatch{
// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
// if(curDate.toString().endsWith("10")){
if(curDate.toString().endsWith("10")){
// exportEvaluateRate()
// exportBigdataBuildprojFjw()
// exportBigdataCommlayoutpicFjw()
//
//
// exportBigdataCommunityGJRH()/
// exportBigdataBuildingFJW()//
// exportBigdataCommfeeGj()//
//
// exportBigdataCommbuildstruGJ()//
// exportBigdataCommhousetypeGJ()//
// exportBigdataCommbuildyearGj()//
// exportBigdataCommhouseuseGJ()//
// exportBigdataPositionGJ()//
// exportBigdataDistrictFJW()//
// exportBigdataAliasFJW()//
//
//
// //本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据
// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
// exportBigdataCityPrice(queryDate)
// exportBigdataDistPrice(queryDate)
// exportBigdataCommPrice(queryDate)
exportEvaluateRate()
exportBigdataBuildprojFjw()
exportBigdataCommlayoutpicFjw()
exportBigdataCommunityGJRH()/
exportBigdataBuildingFJW()//
exportBigdataCommfeeGj()//
exportBigdataCommbuildstruGJ()//
exportBigdataCommhousetypeGJ()//
exportBigdataCommbuildyearGj()//
exportBigdataCommhouseuseGJ()//
exportBigdataPositionGJ()//
exportBigdataDistrictFJW()//
exportBigdataAliasFJW()//
//本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
exportBigdataCityPrice(queryDate)
exportBigdataDistPrice(queryDate)
exportBigdataCommPrice(queryDate)
exportInOk()
// }
}
SSHHelperTest jschCommand = new SSHHelperTest();
System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}"));
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202303.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202304.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -29,16 +29,16 @@ class EsHouseStat {
DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002)
// DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.206", 27020)
// public static JdbcDataSource dataSource = new JdbcDataSource()
// private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
// public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
// static {
// dataSource.setUrl(address)
// Properties connectionProperties = new Properties()
// connectionProperties.put("user", "developer")
// connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
// dataSource.setProperties(connectionProperties)
// }
public static JdbcDataSource dataSource = new JdbcDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -285,7 +285,7 @@ class FiledStrand {
}
static def processCarSiteDesc(car_site){
if(car_site && car_site.size()<=20){
if(car_site && car_site.size()<=20 && car_site.size()>=2 && !car_site.contains("待定") && !car_site.contains("暂无")){
return car_site.toString()
}
return null
@@ -337,6 +337,82 @@ class FiledStrand {
return null
}
static def processBlock(def block){
if(block){
if(block.contains("无") || block.toString().length()>10){
return null
}
return block
}
return null
}
static def processBuildType(def buildType, def estateType){
if(buildType){
def types = []
if(estateType == "别墅"){
def arrs = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院"]
for(def s : arrs){
if(buildType.contains(s)){
types << s
}
}
}else if(estateType == "商住"){
def arrs = ["独栋","双拼","联排","叠加","老式别墅洋房"]
for(def s : arrs){
if(buildType.contains(s)){
types << s
}
}
}else{
def arrs = ["多层","小高层","高层","超高层","SOHO","soho","LOFT","loft","酒店式公寓"]
for(def s : arrs){
if(buildType.contains(s)){
types << s
}
}
}
if(types){
return types.join(",").toString()
}
}
return null
}
static def processlandProperty(val){
if(val){
def arrs = ["经适房","限价房","廉租房","宅基地","保障房","动迁房","配套房","商品房"]
def property = null
for(def s : arrs){
if(val.contains(s)){
property = s
break;
}
}
return property
}
return null
}
static def processBuildStructure(val) {
if (val) {
def buildStructure= null
if(val.indexOf("砖木")>-1){
buildStructure = "砖木结构"
}else if(val.indexOf("钢结构")>-1){
buildStructure = "钢结构";
}else if(val.indexOf("钢、钢筋混凝土")>-1){
buildStructure = "钢、钢筋混凝土结构";
}else if(val.indexOf("钢筋混凝土")>-1){
buildStructure = "钢筋混凝土结构";
}else if(val.indexOf("混合")>-1){
buildStructure = "混合结构";
}
return buildStructure
}
return null
}
static def isRoad(def s){
def pattern = Pattern.compile("[路|街|大道|巷]");
def matcher = pattern.matcher(s)
@@ -7,9 +7,11 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.Jms
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
@@ -17,6 +19,7 @@ import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import org.bson.types.ObjectId
import java.text.SimpleDateFormat
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -31,15 +34,32 @@ import java.util.regex.Pattern
class DistrictFieldAddBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
DBCollection SEAWEEDMAP_COLL = MongoUtils.getCollectionByName("fangjia", "seaweedmap", "192.168.0.235", 27018)
//"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房"
//,"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家"
def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家"] //key:穆融合表的来源 valve:es解析表的来源
//
def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
"乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
//"alias":"alias",
def fieldsMap = ["estate_charge":"property_fee","estate_type2":"estate_type2","build_date":"build_year","estate_company":"property_company","developer":"developer","virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households","car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address"]//key:seaweed表字段 value:es表字段
//ownership_type:权属类型 为商品房、经适房、动迁房等
//house_property 产权性质:国有产权 "house_property":"网站没有",
def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer",
"virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households",
"car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure",
"build_type":"build_type", "land_property":"ownership_type","live_date":"live_date",
"loop_line_info":"loop_line"]//key:seaweed表字段 value:es表字段
def fieldsNameMap = ["estate_charge":"物业费","block":"板块","build_date":"建筑年代","estate_company":"物业公司","developer":"开发商",
"virescence_percent":"绿化率","cubage_percent":"容积率","build_area":"建筑面积","all_area":"占地面积","household_count":"户型",
"car_site_count":"车位","car_site":"车位描述","property_life":"产权年限","address":"地址","build_structure":"建筑结构",
"build_type":"建筑类型", "land_property":"权属类别","live_date":"入住时间","loop_line_info":"环线"]
final static def EMAILS = [
"gaojuan@fangjia.com"
]
static main(args) {
runBatch(new DistrictFieldAddBatch(), args)
@@ -47,21 +67,39 @@ class DistrictFieldAddBatch extends BaseBatch{
@Override
void process(Object args) {
def citys = ["漳州","福州","常州","龙岩","厦门","扬州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","南京","苏州","合肥","无锡","上海","北京","昆明","琼海","呼和浩特","南平","青岛","常德","黄石","黄冈","汉中","安康","榆林","南充","泸州","资阳","克拉玛依","荆门","铜川","内江","达州","巴中","鄂州","十堰","遂宁","随州","广安","荆州","咸宁","阿克苏","喀什","昌吉","吐鲁番","哈密","和田","塔城","阿勒泰","博尔塔拉","巴音郭楞","克孜勒苏","伊犁","恩施","咸阳","宜宾","自贡","攀枝花","延安","德阳","西安","重庆","成都","绵阳","乌鲁木齐","宜昌","武汉","乐山","眉山","宝鸡","襄阳","孝感","拉萨","深圳","三门峡","杭州","怀化","聊城","图木舒克","渭南","金华","宁德","三明","泉州","莆田","营口","牡丹江","盘锦","本溪","铁岭","齐齐哈尔","四平","辽源","锦州","佳木斯","白山","绥化","鸡西","七台河","白城","朝阳","双鸭山","通化","鹤岗","黑河","阜新","抚顺","葫芦岛","伊春","松原","大兴安岭","延边","吉林","沈阳","大连","长春","哈尔滨","鞍山","大庆","丹东"] //"广州", "成都","杭州", "深圳", "重庆","苏州","上海","北京"
def citys = CommonDatautils.findAllCity().reverse()
def analysRets = []
def rh_analysRets = []
def fields = fieldsMap.keySet() as List
def headers = ["来源", "城市"]
fields.each{it->
headers << fieldsNameMap.get(it)
}
headers << "抓取量"
headers << "匹配量"
headers << "0.9分以上量"
headers << "匹配率"
headers << "0.9以上匹配率"
// analysRets << headers
for(def c : citys){
def totalCount = CommonDatautils.findAllDistrictCount("district", c)
// addFieldFromRonghe(c, totalCount)
analysRets = addFieldFromMatch(c, totalCount, analysRets)
if(totalCount){
// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)
analysRets = addFieldFromMatch(c, totalCount, analysRets)
}
}
println analysRets.toString()
ExcelUtils.write("D:\\数据\\小区\\覆盖率.xlsx", analysRets, "数据")
sendStatEmail(headers, analysRets, rh_analysRets)
// ExcelUtils.write("D:\\数据\\小区\\覆盖率.xlsx", analysRets, "数据")
}
def addFieldFromMatch(c, totalCount, analysRets){
def fields = fieldsMap.keySet() as List
sourceMap.each { seaweedmapSource, esWebSource ->
def esRets = findEsDatas(c, esWebSource)
def matchRets1 = matchApi(esRets)
@@ -80,7 +118,18 @@ class DistrictFieldAddBatch extends BaseBatch{
pre1 = DataHandleUtils.getDoubleRound(2, matchRets1.size()*100/esRets.size())
pre2 = DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/esRets.size())
}
analysRets << [seaweedmapSource, c, precentMap.get("property_fee"), esRets.size(), matchRets1.size(), matchRets2.size(), pre1, pre2]
def analyRet = []
analyRet << seaweedmapSource
analyRet << c
fields.each{key->
analyRet << precentMap.get(key)
}
analyRet << esRets.size()
analyRet << matchRets1.size()
analyRet << matchRets2.size()
analyRet << pre1
analyRet << pre2
analysRets << analyRet
}
return analysRets;
}
@@ -88,7 +137,8 @@ class DistrictFieldAddBatch extends BaseBatch{
def addFieldFromRonghe(c, totalCount){
def addFieldFromRonghe(c, totalCount, rh_analysRets){
def fields = fieldsMap.keySet() as List
sourceMap.each{seaweedmapSource,esWebSource->
def rhRets = findRongheDists(c, seaweedmapSource)
def esRets = findEsDistrict(c, esWebSource)
@@ -99,13 +149,35 @@ class DistrictFieldAddBatch extends BaseBatch{
println c +" "+precentMap.toString()
updateDB(c, updateRets)
println "----------------------------------------------------------------------------------"
def pre1 = 0
def pre2 = 0
if(esRets){
pre1 = DataHandleUtils.getDoubleRound(2, rhRets.size()*100/esRets.size())
pre2 = DataHandleUtils.getDoubleRound(2, esRets.size()*100/esRets.size())
}
def analyRet = []
analyRet << seaweedmapSource
analyRet << c
fields.each{key->
analyRet << precentMap.get(key)
}
analyRet << esRets.size()
analyRet << rhRets.size()
analyRet << esRets.size()
analyRet << pre1
analyRet << pre2
rh_analysRets << analyRet
}
return rh_analysRets;
}
def processFieldsFromRh(c, rhRets, esRets){
def fields = fieldsMap.keySet() as List
def fieldValueMap = [:]
for(def rhDist : rhRets){
if(rhDist.commid && rhDist.commid!="null"){
continue
}
def dist = CommonDatautils.findDistrictFieldById(rhDist.commid, fields)
if(!dist){
println c+" "+rhDist.commid+" 不存在"
@@ -116,26 +188,7 @@ class DistrictFieldAddBatch extends BaseBatch{
println c+" "+rhDist.id+" 不存在[ES]"
continue
}
def newValues = [:]
fieldsMap.each{seaweedfield,esField->
def val = dist.get(seaweedfield)
if(!val){
def esVal = esDbo.get(esField)
if(esVal){
def newVal = null
if(seaweedfield == "estate_charge"){
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
newVal = newVal.toString()
}
if(seaweedfield == "build_date"){
newVal = FiledStrand.processFinishBuildingDate(esVal)
}
if(newVal){
newValues.put(seaweedfield, newVal)
}
}
}
}
def newValues = strandField(c, dist, esDbo)
if(newValues){
fieldValueMap.put(dist._id.toString(), newValues)
}
@@ -149,77 +202,15 @@ class DistrictFieldAddBatch extends BaseBatch{
def processFieldsFromMatch(c,matchRets){
def fields = fieldsMap.keySet() as List
fields << "city"
fields << "estate_type2"
def fieldValueMap = [:]
for(def matchDist : matchRets){
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
if(!dist){
println c+" "+matchDist.match_id+" 不存在"
continue
}
def newValues = [:]
fieldsMap.each{seaweedfield,esField->
def val = dist.get(seaweedfield)
if(!val){
def esVal = matchDist.get(esField)
if(esVal){
def newVal = null
if(seaweedfield == "estate_charge"){
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
newVal = newVal.toString()
}
if(seaweedfield == "build_date"){
newVal = FiledStrand.processFinishBuildingDate(esVal)
}
if(seaweedfield == "estate_company" || seaweedfield == "developer"){
newVal = FiledStrand.processCompany(esVal)
}
if(seaweedfield =="virescence_percent"){
newVal = FiledStrand.processVirescencePercent(esVal)
}
if(seaweedfield =="cubage_percent"){
newVal = FiledStrand.processCubagePercent(esVal)
}
if(seaweedfield =="all_area" || seaweedfield =="build_area"){
newVal = FiledStrand.processArea(esVal)
newVal = newVal.toString()
}
if(seaweedfield =="household_count"){
newVal = FiledStrand.processHouseholdCount(esVal, dist.estate_type2)
}
if(seaweedfield =="loudong_count"){
newVal = FiledStrand.processLoudongCount(esVal)
}
if(seaweedfield =="car_site_count"){
newVal = FiledStrand.processCarSite(esVal)
}
if(seaweedfield =="car_site"){
newVal = FiledStrand.processCarSiteDesc(esVal)
}
if(seaweedfield =="property_life"){
newVal = FiledStrand.processPropertyLife(esVal)
}
if(seaweedfield =="alias"){
newVal = FiledStrand.processAlias(esVal, dist.name, dist.alias)
}
if(seaweedfield =="address"){
newVal = FiledStrand.processAddress(esVal, dist.city)
}
if(newVal){
newValues.put(seaweedfield, newVal)
}
}
}
}
def newValues = strandField(c, dist, matchDist)
if(newValues){
println dist._id.toString()+" "+newValues.toString()
fieldValueMap.put(dist._id.toString(), newValues)
@@ -229,6 +220,89 @@ class DistrictFieldAddBatch extends BaseBatch{
return fieldValueMap;
}
def strandField(c,dist, matchDist){
def newValues = [:]
fieldsMap.each{seaweedfield,esField->
def val = dist.get(seaweedfield)
if(!val){
def esVal = matchDist.get(esField)
if(esVal){
def newVal = null
if(seaweedfield == "estate_charge"){
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
newVal = newVal.toString()
}
if(seaweedfield == "build_date" || seaweedfield == "live_date"){
newVal = FiledStrand.processFinishBuildingDate(esVal)
}
if(seaweedfield == "estate_company" || seaweedfield == "developer"){
newVal = FiledStrand.processCompany(esVal)
}
if(seaweedfield =="virescence_percent"){
newVal = FiledStrand.processVirescencePercent(esVal)
}
if(seaweedfield =="cubage_percent"){
newVal = FiledStrand.processCubagePercent(esVal)
}
if(seaweedfield =="all_area" || seaweedfield =="build_area"){
newVal = FiledStrand.processArea(esVal)
if(newVal){
newVal = newVal.toString()
}
}
if(seaweedfield =="household_count"){
newVal = FiledStrand.processHouseholdCount(esVal, dist.estate_type2)
}
if(seaweedfield =="loudong_count"){
newVal = FiledStrand.processLoudongCount(esVal)
}
if(seaweedfield =="car_site_count"){
newVal = FiledStrand.processCarSite(esVal)
}
if(seaweedfield =="car_site"){
newVal = FiledStrand.processCarSiteDesc(esVal)
}
if(seaweedfield =="property_life"){
newVal = FiledStrand.processPropertyLife(esVal)
}
if(seaweedfield =="alias"){
newVal = FiledStrand.processAlias(esVal, dist.name, dist.alias)
}
if(seaweedfield =="address"){
newVal = FiledStrand.processAddress(esVal, dist.city)
}
if(seaweedfield =="block"){
newVal = FiledStrand.processBlock(esVal)
}
if(seaweedfield == "build_type"){
newVal = FiledStrand.processBuildType(esVal, dist.estate_type2)
}
if(seaweedfield == "land_property"){
newVal = FiledStrand.processlandProperty(esVal)
}
if(seaweedfield == "build_structure"){
newVal = FiledStrand.processBuildStructure(esVal)
}
if(newVal){
newValues.put(seaweedfield, newVal)
}
}
}
}
return newValues;
}
def findRongheDists(def city, def source){
@@ -285,9 +359,9 @@ class DistrictFieldAddBatch extends BaseBatch{
def precentMap = [:]
fieldsMap.each() {key,skey->
if(countMap.containsKey(key)) {
precentMap.put(skey, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total))
precentMap.put(key, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total))
}else {
precentMap.put(skey, 0)
precentMap.put(key, 0)
}
}
return precentMap;
@@ -414,4 +488,61 @@ class DistrictFieldAddBatch extends BaseBatch{
}
return false
}
def sendStatEmail(headers, analysRets, rh_nalysRets){
Date dateTime = DateUtils.mins(1);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy.MM.dd");
StringBuffer sb = new StringBuffer();
if(analysRets){
sb.append("<table border='1' style='border-collapse:collapse; border-spacing:0'>")
sb.append("<tr><font style='size:15px;color:grey;'>字段补充情况分析</tr>")
sb.append("<tr style='color:black;font-size:18px;cellspacing:1px;'>")
headers.each{it->
if(it=="来源" || it=="城市"){
sb.append("<th style='width:80px;'>"+ it+"</th>")
}else{
sb.append("<th >"+ it+"</th>")
}
}
sb.append("</tr>")
for(def ret : analysRets){
sb.append("<tr style='text-align:center;font-color:gray;cellspacing:1px;'>");
ret.each{it->
sb.append("<td>" + it + "</td>")
}
sb.append("</tr>");
}
sb.append("</table><br/>");
}
StringBuffer rh_sb = new StringBuffer();
if(rh_nalysRets){
rh_sb.append("<table border='1' style='border-collapse:collapse; border-spacing:0'>")
rh_sb.append("<tr><font style='size:15px;color:grey;'>字段补充情况分析</tr>")
rh_sb.append("<tr style='color:black;font-size:18px;cellspacing:1px;'>")
headers.each{it->
if(it=="来源" || it=="城市"){
rh_sb.append("<th style='width:80px;'>"+ it+"</th>")
}else{
rh_sb.append("<th >"+ it+"</th>")
}
}
rh_sb.append("</tr>")
for(def ret : rh_nalysRets){
rh_sb.append("<tr style='text-align:center;font-color:gray;cellspacing:1px;'>");
ret.each{it->
rh_sb.append("<td>" + it + "</td>")
}
rh_sb.append("</tr>");
}
rh_sb.append("</table><br/>");
}
for (def email : EMAILS) {
jms.sendMail(email, "字段填充情况分析", sb.toString()+rh_sb.toString());
}
}
}
@@ -0,0 +1,69 @@
package com.fangjia.script.gj.district
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 字段提升率统计
*/
DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo);
def fieldsMap = ["district_count":"小区数量","name":"名称" ,"region":"区域" ,"block":"板块" ,"address":"地址" ,"estateType2":"物业类型" ,"householdCount":"户数" ,"openDate":"开盘时间" ,"liveDate":"入住时间" ,"buildDate":"建筑年代" ,"allArea":"占地面积" ,"buildArea":"建筑面积" ,"buildType":"建筑类型" ,"developer":"开发商" ,"estateCompany":"物业公司" ,"cubagePercent":"容积率" ,"virescencePercent":"绿化率" ,"carSite":"停车位" ,"estateCharge":"物业费" ,"floorplanPics":"户型图" ,"outPics":"外景图" ,"finishBuildingDate":"竣工时间" ,"businessQuarter":"商圈" ,"startBuildingDate":"开建时间" ,"loopLineInfo":"环线" ,"propertyLife":"产权年限" ,"houseProperty":"产权性质" ,"licence":"预售许可证" ,"buildStructure":"建筑结构" ,"lat":"百度纬度" ,"pics":"图片" ,"streetCommunity":"街道" ,"schools":"学校" ,"houseStructure":"房屋结构" ,"alias":"别名" ,"addrAlias":"地址别名" ,"mainTypeArea":"主力面积" ,"loudongCount":"楼栋数" ,"latLng" :"高德经度" ,"rentPrice":"租金" ]
def weeknd = DateUtils.getFirstDayOfWeek(new Date())
def lastWeeknd = DateUtils.getFirstDayOfWeek(new Date()-7)
def weekendDatas = CITY_INFO_CONN.find(new BasicDBObject("weekend", weeknd))
def lastWeekendDatas = CITY_INFO_CONN.find(new BasicDBObject("weekend", lastWeeknd))
def weekendMapDatas = findDataByCity(weekendDatas)
def lastWeekendMapDatas = findDataByCity(lastWeekendDatas)
def citys = CommonDatautils.findAllCity()
def fields = fieldsMap.keySet() as List
def rets = []
def headers = []
headers << "城市"
fields.each{k->
headers << fieldsMap.get(k)
}
rets << headers
for(def c : citys){
def weekendDetailMap = weekendMapDatas.get(c)
def lastWeekendDetailMap = lastWeekendMapDatas.get(c)
def ret = []
ret << c
fields.each{key->
if(weekendDetailMap){
def v1 = weekendDetailMap.get(key)
def v2 = lastWeekendDetailMap.get(key)
if(v1 && v2){
ret << DataHandleUtils.getDoubleRound(2, v1-v2)
}else{
ret << "-"
}
}else{
ret << "-"
}
}
rets << ret
}
ExcelUtils.write("D:\\数据\\小区\\字段提升率"+DateUtils.formatDateByPattern(weeknd,"yyyy-MM-dd")+".xlsx", rets, "字段提升率统计")
def findDataByCity(weekendDatas){
def map = [:]
for(def d : weekendDatas){
d.district_detail.district_count = d.existing_district_count
map.put(d.city, d.district_detail)
}
return map
}
@@ -1,7 +1,9 @@
package com.fangjia.script.gj.es
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
@@ -19,11 +21,44 @@ class EsMonthHouseStat {
def task = new EsMonthHouseStat()
// task.run()
// task.tradeRun()
task.houseRun()
// task.houseRun()
// task.statHouseUpdateCount()
// task.statDistHouse()
task.statRent()
}
def statRent(){
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\租金.xlsx", { d ->
index++
def region = RegionData.formatRegion(d['A'], d['B'])
if(region){
def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
if(!dist){
dist = OpenApiUtils.matchDistrictByAddr(d['A'], d['C'], region)
if(dist){
dist = CommonDatautils.findDistrictById(dist.id)
}
}
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
}
def distCount = findHouseCount(dist.city, dist.region, alias, 20220501, 20230501)
d['G'] = distCount
d['E'] = dist.region
d['F'] = dist.name
println index+" "+ d['C'] +" "+distCount
}
});
}
def statDistHouse(){
@@ -172,7 +207,7 @@ class EsMonthHouseStat {
def total = 0
try{
aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
String sql = "select count(*) as c from latest_seawater where cat='rent' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and root='lianjia.com'"
ElasticUtils.executeSqlQuery(sql,{data->
total = data.c
@@ -69,7 +69,7 @@ public class BeiKeCrawlByChrome {
//
// }
// citys = citys[(start+1)..(citys.size()-1)]
getLogin(driver)
driver = getLogin()
crawlData(driver, citys)
@@ -108,15 +108,32 @@ public class BeiKeCrawlByChrome {
}
}
def checkDocument(driver, url){
def s = driver.getPageSource()
def flag = true
while(flag){
if(s && s.contains("无法访问此网站")){
driver = getLogin()
driver.get(url)
Thread.sleep(5000)
s = driver.getPageSource()
}else{
flag = false
}
}
return Jsoup.parse(s);
}
def crawlData(driver, citys){
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
for(def m : citys){
def city = m[0]
def url = m[1]
driver.get(url+"/chengjiao/")
def newUrl = url+"/chengjiao/"
driver.get(newUrl)
Thread.sleep(10000)
def doc = Jsoup.parse(driver.getPageSource());
def doc = checkDocument(driver, newUrl);
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
if(regions){
for(def r : regions){
@@ -220,6 +237,9 @@ public class BeiKeCrawlByChrome {
}
def roomStr = arrs[1]
if(roomStr && !roomStr.contains("室")){
roomStr = arrs[2]
}
if (roomStr) {
if (roomStr.indexOf("室") > 0) {
roomStr = roomStr.substring(0, roomStr.indexOf("室"))
@@ -235,7 +255,13 @@ public class BeiKeCrawlByChrome {
}
}
def areaStr = arrs[2].replace("平米", "")
def areaStr = arrs[2]
if(areaStr && !areaStr.contains("平米")){
if(arrs.size()>=4){
areaStr = arrs[3]
}
}
areaStr = areaStr.replace("平米","")
if (areaStr && areaStr.isNumber()) {
detail.area = areaStr.toDouble()
}
@@ -254,7 +280,7 @@ public class BeiKeCrawlByChrome {
}
detail.source = "贝壳找房"
if(detail.total_price){
if(detail.total_price && detail.avg_price){
def detailUrl = e.select(".title a").attr("href")
detail.url = detailUrl
def dbo = findTrade(detail)
@@ -404,15 +430,16 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = "https:"+e.attr("href")
if (citys.contains(e.text())){
// if (citys.contains(e.text())){
list << [e.text(), href]
}
// }
}
}
return list
}
def getLogin(driver){
def getLogin(){
WebDriver driver = new ChromeDriver()
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F");
WebDriverWait wait = new WebDriverWait(driver,20)
Thread.sleep(10000)
@@ -432,6 +459,7 @@ public class BeiKeCrawlByChrome {
btn.submit(); // 点击行为;
Thread.sleep(10000)
return driver
}
}
@@ -122,7 +122,7 @@ public class LianjiaCrawlByChrome {
WebDriver driver = new ChromeDriver()
def cityMap = getAllCityUrls(driver)
// cityMap = cityMap.reverse()
getLogin(driver)
driver = getLogin()
crawlData(driver, cityMap)
@@ -191,10 +191,12 @@ public class LianjiaCrawlByChrome {
for(def ret : cityList){
def city = ret[0]
def cityUrl = ret[1]
def newUrl = cityUrl+"chengjiao/"
driver.get(newUrl)
//def detailUrls = getPageUrls(driver, cityUrl)
driver.get(cityUrl+"chengjiao/")
Thread.sleep(10000)
def doc = Jsoup.parse(driver.getPageSource());
def doc = checkDocument(driver, newUrl);
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
if(regions){
for(def r : regions){
@@ -240,6 +242,21 @@ public class LianjiaCrawlByChrome {
return detailUrls
}
def checkDocument(driver, url){
def s = driver.getPageSource()
def flag = true
while(flag){
if(s && s.contains("无法访问此网站")){
driver = getLogin()
driver.get(url)
Thread.sleep(5000)
s = driver.getPageSource()
}else{
flag = false
}
}
return Jsoup.parse(s);
}
def getPageDeail(driver, city, cityUrl, regionText){
def stop = false
@@ -278,7 +295,7 @@ public class LianjiaCrawlByChrome {
def parse(city,e, regionText){
try {
def title = e.select(".title a").text()
def title = e.select(".title a").text()
if (title) {
def detail = new BasicDBObject()
def arrs = title.split(" ")
@@ -318,7 +335,7 @@ public class LianjiaCrawlByChrome {
}
}
def price = e.select(".totalPrice .number").text()
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = (int)(price.toDouble() * 10000)
}
@@ -333,7 +350,7 @@ public class LianjiaCrawlByChrome {
detail.source = "链家在线"
detail.url = e.select(".title a").attr("href")
if(detail.total_price){
if(detail.total_price && detail.avg_price){
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
@@ -482,15 +499,16 @@ public class LianjiaCrawlByChrome {
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
// if (!href.contains("fang.lianjia") && citys.contains(e.text())){
cityList << [e.text(), href]
}
// }
}
}
return cityList
}
def getLogin(driver){
def getLogin(){
WebDriver driver = new ChromeDriver()
driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.lianjia.com%252Fchengjiao%252F");
WebDriverWait wait = new WebDriverWait(driver,20)
Thread.sleep(10000)
@@ -510,6 +528,7 @@ public class LianjiaCrawlByChrome {
btn.submit(); // 点击行为;
Thread.sleep(20000)
return driver
}
}
@@ -27,19 +27,11 @@ class OpenUriUserIdCountDay extends BaseBatch {
runBatch(new OpenUriUserIdCountDay(), args)
}
final static def EMAILS = [
"joe@fangjia.com",
"wangwenyan@fangjia.com",
"xiaohai@fangjia.com",
"caoyuyan@fangjia.com",
"jiangxiaochun@fangjia.com",
"gaojuan@fangjia.com",
'shijing@fangjia.com',
'zhongjingyi@fangjia.com',
"duweiliang@fangjia.com",
"sunjian@fangjia.com",
"yaojingmeng@fangjia.com",
"xuqi@fangjia.com",
"wangpeili@fangjia.com"
"gaojuan@fangjia.com"
]
@Override