This commit is contained in:
2021-10-13 17:43:30 +08:00
parent 48c505166d
commit 66460b027f
28 changed files with 1084 additions and 96 deletions
@@ -13,8 +13,9 @@ import com.mongodb.DBObject
*/
class CityLatlngBox {
public static final DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.105", 27018)
public static final DBCollection LATLNG_COLL = MongoUtils.getCollectionByName("addrepo", "city_latlng_range", "192.168.0.219", 27017)
//public static final DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.105", 27018)
DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.235", 27018)
public static final DBCollection LATLNG_COLL = MongoUtils.getCollectionByName("addrepo", "city_latlng_range", "192.168.0.206", 27020)
public static void main(String[] args) {
CityLatlngBox sd = new CityLatlngBox();
@@ -22,7 +23,7 @@ class CityLatlngBox {
}
void process() {
def cities = CommonUtils.findAllCity()
def cities = ["白沙","阿坝州","甘孜州","阿勒泰","临沧","那曲","三沙","喀什","昌吉","迪庆州","普洱","德宏州","湘西","塔城","凉山州","屯昌","山南","海西州","阿拉善盟","乐东","定安","怒江州","黄山","图木舒克","楚雄","临高","乌兰察布","延边","文山州","锡林郭勒盟","陵水","天门","淮北","黔东南","万宁","湘潭","海北州","琼中","吐鲁番","惠州","巴音郭楞","恩施","铜仁","黔南","西双版纳州","潜江","林芝","文昌","仙桃","神农架","和田","黔西南","果洛州","儋州","哈密","大理","东方","日喀则","兴安盟","昌江","澄迈","保亭","济源","五指山","阿拉尔","伊犁","临夏","海南州","琼海","石河子","昌都","五家渠","玉树州","博尔塔拉","东营","铜陵","黄南州","廊坊","嘉峪关","马鞍山","宁波","广安","中山","克孜勒苏","厦门","莱芜","鄂州","东莞","海口","鹤壁","六盘水","漯河","三亚","珠海","深圳","乌海","淮南","北海","双鸭山","襄阳","新余","汕头","克拉玛依","鹰潭","盘锦","镇江","石嘴山","嘉兴","铜川","焦作","芜湖","潮州","萍乡","红河","黄石","辽源","辽阳","枣庄","毕节","许昌","阳泉","自贡","遂宁","景德镇","营口","日照","莆田"]//CommonUtils.findAllCity()
for (String city : cities) {
println city
BOUNDS_COLL.remove(new BasicDBObject('city', city))
@@ -70,13 +71,13 @@ class CityLatlngBox {
int row = 0
int inx = 0
def buffer = []
while(seedLatBD < cityBoundBD.getMaxLat()) {
while(seedLatGD < cityBoundGD.getMaxLat()) {
int col = 0
Bounds boundsBD = null;
Bounds boundsQQ = null;
Bounds boundsGD = null;
while (seedLngBD < cityBoundBD.getMaxLng()) {
while (seedLngGD < cityBoundGD.getMaxLng()) {
double[] espBD = GeoUtils.getLatLon(seedLngBD, seedLatBD, offset, 90) // 右侧OFFSET处的经纬度
double[] wnpBD = GeoUtils.getLatLon(seedLngBD, seedLatBD, offset, 360) // 上方OFFSET处的经纬度
@@ -39,7 +39,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
println city +" "+rows.size()
}
def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_commbuildtype.dat",headers, "utf-8", false);
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMBUILDTYPE_GJ.dat",headers, "utf-8", false);
println "txt 生成"
}
@@ -40,8 +40,8 @@ class BigdataDistrictBatch extends BaseBatch{
def rets = []
rets << headers
rets.addAll(allDatas)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_district.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_district.dat",headers, "utf-8",false);
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\BIGDATA_DISTRICT_GJ.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.dat",headers, "utf-8",false);
}
def createDatas(rows){
@@ -65,9 +65,9 @@ class BigdataPoiBatch extends BaseBatch{
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"]
def fileName = "/home/goojia/ccb/bigdata_poi.txt"
def fileName = "/home/goojia/ccb/BIGDATA_POI_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_poi.txt"
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_POI_GJ.dat"
}
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false);
println "txt 生成"
@@ -107,8 +107,8 @@ class CcbCommonUtils {
alias << dist.name
}
if(dist.name.endsWith("弄小区")){
def name = dist.name.replace("弄小区","弄")
if(dist.name.endsWith("弄小区") || dist.name.endsWith("小区") ){
def name = dist.name.replace("弄小区","弄").replace("小区","")
if(!alias.contains(name)){
alias << name
}
@@ -123,10 +123,10 @@ class CcbCommonUtils {
def rets = new HashSet()
def lists = []
for(def d : dbcursor){
if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){
//def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
//if(!rets.contains(key)){
// rets << key
// if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
d.dist_id = dist._id.toString()
d.address = dist.address
d.master_name = dist.name
@@ -136,8 +136,8 @@ class CcbCommonUtils {
d.property_life = dist.property_life
d.estate_type2 = dist.estate_type2
lists << d
//}
}
}
// }
}
return lists
}
@@ -229,7 +229,7 @@ class CcbCommonUtils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name", "海昌欣城")
// q.put("name", "东特金星公馆")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -43,7 +43,9 @@ class DistrictDelete {
static main(args) {
def task = new DistrictDelete()
// task.run()
task.gzRun();
// task.gzRun();
// task.deleteDistrict()
task.deleteCommPrice()
}
def run(){
@@ -211,9 +213,9 @@ class DistrictDelete {
def gzRun(){
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\经纬度有问题的小区.xlsx",{d->
if(d['U'] && d['U'].toString().toDouble().toInteger()==1){
def id = d['E']
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\要删除小区 (5).xlsx",{d->
if(d['G']){
def id = d['G']
BasicDBObject q=new BasicDBObject('name_cat',1)
q.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q))
@@ -223,4 +225,38 @@ class DistrictDelete {
},0)
}
def deleteDistrict(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d->
if(d['G']){
def id = d['G']
BasicDBObject q=new BasicDBObject('name_cat',1)
q.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q))
deleteAllRefDistrict(id)
}
},0)
}
def deleteAllRefDistrict(id){
def tables = ["bigdata_forsale_gj","avm_evaluation_item_gj","community_base_price_gj","community_poi_gj","community_sale_gj","community_sale_tmp","community_test_price_tmp"]
for(def tableName : tables){
deleteByCommid(tableName, id)
}
}
def deleteCommPrice(){
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\均价删除.xlsx",{d->
if(d['B']){
def id = d['B']
println id
deleteByCommid("community_base_price_gj",id)
}
},0)
}
def deleteByCommid(tableName, commid){
def sql = "delete from "+tableName+" where commid='${commid}'";
sqlInstance.execute(sql)
println "删除:"+tableName+" "+commid
}
}
@@ -17,7 +17,7 @@ class DistrictGuidePriceBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = ["深圳"]//CcbCommonUtils.citys;
def citys = CcbCommonUtils.citys;
def rows = []
for (def city : citys) {
@@ -34,8 +34,8 @@ class DistrictGuidePriceBatch extends BaseBatch{
}
}
// def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"]
// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\district_guied_price.dat", headers, "utf-8", false);
def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_GUIDE_PRICE_GJ.dat", headers, "utf-8", false);
}
@@ -29,8 +29,8 @@ class EvaluateRateBatch extends BaseBatch{
def rets = []
rets << headers
rets.addAll(allDatas)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\evaluate_rate.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluate_rate.dat",headers, "utf-8", false);
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\EVALUATE_RATE_GJ.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\EVALUATE_RATE_GJ.dat",headers, "utf-8", false);
}
def createDatas(def datas){
@@ -25,18 +25,24 @@ class ExportOracleData {
// exportCommunityBuileType()
// exportBigdataDistrictData()
// exportGuidedPrice()
// exportCommunityGuidedPrice()
exportCommunityGuidedPrice()
// exportBigdataDeal()
// exportThirdPartyPrice()
// exportBigdataCommunity()
exportBigdataForsale()
// exportBigdataForsale()
// exportAvmEvaluationGJNB()
// exportAVMEvaluationItemGJNB()
// exportCommunityBasePrice()
// exportCommunityBasePriceGj()
// exportCityPrice()
// exportRegionPrice()
// exportLastFile()
}
def exportEvaluationRateData(){
def sql = "select ID,RATE_TYPE,RATE,RANK,CITY,REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from evaluate_rate_gj";
def allRows = []
@@ -213,7 +219,7 @@ class ExportOracleData {
def exportGuidedPrice(){
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where cityId in (350200,150100)";
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where dealdate>=TO_DATE('2021-08-01', 'yyyy-MM-dd') and dealdate<TO_DATE('2021-09-01', 'yyyy-MM-dd') and cityname='上海'";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
@@ -237,20 +243,20 @@ class ExportOracleData {
def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat"
def dataCodeFileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.ok"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.ok"
}
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
println "GUIDED_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, "202107")
CcbFtpUtils.ftpUpload(dataCodeFileName, "202107")
CcbFtpUtils.ftpUpload(fileName, "202108")
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
println "GUIDED_PRICE_GJ 上传Ftp完成......"
}
def exportCommunityGuidedPrice(){
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where cityname='厦门' or cityname='呼和浩特'";
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where cityname='西安'";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
@@ -273,16 +279,16 @@ class ExportOracleData {
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.ok"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.ok"
}
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
println "COMMUNITY_GUIDE_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, "202108")
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
CcbFtpUtils.ftpUpload(fileName, "202107")
CcbFtpUtils.ftpUpload(dataCodeFileName, "202107")
println "COMMUNITY_GUIDE_PRICE_GJ 上传Ftp完成......"
}
@@ -726,5 +732,88 @@ class ExportOracleData {
println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......"
}
def exportCommunityBasePriceGj(){
}
def exportCityPrice(){
def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where (cityId=350200 or cityId=150100) and year_month>=202108";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.PRICE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_CITY_PRICE_GJ_ADD_${curDate}_0001.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.ok"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
println "CITY_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, "202108")
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
println "CITY_PRICE_GJ 上传Ftp完成......"
}
def exportRegionPrice(){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where (cityId=350200 or cityId=150100) and year_month>=202108";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.PRICE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
def dataCodeFileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.ok"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
println "DISTRICT_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, "202108")
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
println "DISTRICT_PRICE_GJ 上传Ftp完成......"
}
def exportLastFile(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM")
def dataCodeFileName = "/ccb/${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curMonth}\\${curDate}.ok"
}
CcbFtpUtils.ftpUpload(dataCodeFileName, "${curMonth}")
}
}
@@ -71,7 +71,7 @@ class GuiDingPriceBatch extends BaseBatch{
}
def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.dat", headers, "utf-8", false);
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
// def rets = []
// rets << headers
@@ -0,0 +1,78 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
class LocationUpdate {
static main(args) {
def task = new LocationUpdate();
task.run()
}
def run(){
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\经纬度\\经纬度人工核实结果_0923.xlsx",{d->
index++
if(d['C']){
println index +" "+d['A']+" "+d['B']+" "+d['C']
def region = RegionData.formatRegion(d['A'], d['B'])
if(d['H']){
def status = d['H'].toString().toDouble().toInteger()
if(status==1){
def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
if(obj){
def gmap = GeocodeUtils.getGaodeToBaidu(d['F'].toString().toDouble(), d['G'].toString().toDouble())
def lng2 = null
def lat2 = null
if(gmap){
lng2 = (gmap.get("lng")*10000000000).longValue()
lat2 = (gmap.get("lat")*10000000000).longValue()
}
BasicDBObject q=new BasicDBObject()
q.put("m_date", new Date())
q.put("lat2", lat2)
q.put("lng2", lng2)
q.put("lng", (d['F'].toString().toDouble()*10000000000).longValue())
q.put("lat", (d['G'].toString().toDouble()*10000000000).longValue())
DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q))
}else{
println d['A']+" "+d['B']+" "+d['C']+" 没找到小区"
}
}else{
def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
if(obj){
BasicDBObject q=new BasicDBObject('name_cat',1)
q.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q))
}else{
println d['A']+" "+d['B']+" "+d['C']+" 没找到小区"
}
}
}else{
def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
if(obj){
BasicDBObject q=new BasicDBObject('name_cat',1)
q.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q))
}else{
println d['A']+" "+d['B']+" "+d['C']+" 没找到小区"
}
}
}
})
}
}
@@ -40,8 +40,8 @@ class StagingDistrictBatch extends BaseBatch{
results << headers
results.addAll(rows)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING.xlsx", results, "数据")
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_STAGING.dat",headers, "utf-8", false);
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "数据")
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_STAGING_GJ.dat",headers, "utf-8", false);
}
@@ -3,6 +3,8 @@ package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.Point
import com.fangjia.script.util.DbUtils
@@ -14,6 +16,11 @@ import com.mongodb.DBCollection
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.*;
@@ -42,7 +49,7 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
// println city +" "+count
//}
//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx"
//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0927.xlsx"
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\7月成交.xlsx",0,1)
//def dealMap = [:]
//for(def d : datas){
@@ -59,7 +66,7 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0903.xlsx"
//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0903.xlsx"
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1)
//def dealMap = [:]
@@ -83,7 +90,7 @@ def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
// if (city) {
// def cityid = cityCodeDatas.get(city)
// def total = findCount(cityid, sqlInstance)
// d['M'] = total
// d['O'] = total
// }
//},0)
@@ -128,13 +135,26 @@ def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
// }
//},1)
//def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交统计.xlsx"
def dealMap = findDealCount(sqlInstance)
ExcelUtils.multiThreadingModify(fileName,{d->
if(d['C']){
d['K'] = dealMap.get(d['C'])
//def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx"
//def dealMap = findDealCount(sqlInstance)
//ExcelUtils.multiThreadingModify(fileName,{d->
// if(d['A']){
// d['U'] = dealMap.get(d['A'])
// }
//},0)
def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交未入库人工部分城市统计.xlsx"
def datas = ExcelUtils.read(fileName, 0, 1)
def dealMap = [:]
for(def d : datas){
dealMap.put(d['A'], d['B'])
}
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx",{d->
if(d['A']){
d['W'] = dealMap.get(d['A'])
}
},2)
},0)
//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx"
@@ -170,7 +190,7 @@ ExcelUtils.multiThreadingModify(fileName,{d->
def findDealCount(sqlInstance){
def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('2021-06-01','yyyy-mm-dd') and dealdate<to_date('2021-07-01','yyyy-mm-dd') group by cityname";
def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('2021-08-01','yyyy-mm-dd') and dealdate<to_date('2021-09-01','yyyy-mm-dd') group by cityname";
def count = 0
def map = [:]
@@ -194,7 +214,7 @@ def findEstateType(cityid, sqlInstance){
}
def findCount(cityid, sqlInstance){
def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}'";
def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0";
def count = 0
sqlInstance.eachRow(sql,{r->
@@ -204,7 +224,7 @@ def findCount(cityid, sqlInstance){
}
def findFiledCount(cityid, field, sqlInstance){
def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and "+field+" is not null";
def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0 and "+field+" is not null";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
@@ -353,3 +373,5 @@ def getStreetInfo(def city, def glocation){
@@ -0,0 +1,81 @@
package com.fangjia.script.ccbevaluation
import groovy.sql.Sql
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class TestRepeat {
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new TestRepeat()
task.run()
}
def run(){
}
def checkUpdateCount(){
def newHouses = readTxt("C:\\Users\\gaojuan\\Desktop\\BIGDATA_FORSALE_GJ.dat")
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(newHouses.size())
println " ·¿Ô´Á¿£º"+newHouses.size()
def retMap = new ConcurrentHashMap()
for(def i=0;i<newHouses.size();i++){
final def map = newHouses[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def obj = findHouse(map.houseId, map.commId)
if(obj){
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
}
def findHouse(def houseId, def commId){
def sql = "select * from bigdata_forsale_gj where commid='${commId}' and forsaleid='${houseId}'";
def obj = null
sqlInstance.eachRow(sql,{r->
obj = r
});
return obj
}
def readTxt(def file){
def lines = org.apache.commons.io.FileUtils.readLines(new File(file))
def rets = [];
for(line in lines) {
def arrs = line.split("|@|")
def map = [:]
map.houseId = arrs[1]
map.commId = arrs[6]
map.pdate = arrs[10]
rets << map
}
return rets;
}
}
@@ -20,9 +20,9 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202107//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202105
def endDate = 202107
def sDate= 202108//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202106
def endDate = 202108
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -53,13 +53,13 @@ class ThirdPartyPriceBatch extends BaseBatch {
def fileName = "/home/goojia/ccb/third_party_price.dat"
def fileName = "/home/goojia/ccb/THIRD_PARTY_PRICE_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\third_party_price.dat"
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\THIRD_PARTY_PRICE_GJ.dat"
}
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true);
def header = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
}
@@ -67,6 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){
def rets = []
for(def d : dists){
def third = thirdPriceMap.get(d.region+"##"+d.name)
if(third){
def data = distCodeMap.get(d.region)
@@ -1,12 +1,22 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/***
* 建行成交抓取统计
@@ -21,23 +31,34 @@ class TradeStatLog {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new TradeStatLog()
task.run()
// task.run()
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
task.exportTrade()
// task.deleteDistrict()
}
def run(){
def citys = CcbCommonUtils.citys;
def rets = []
rets << ["城市","5月总量","6月总量","7月总量","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","6月总量","7月总量","8月总量","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
def startDate = "2021-05-01"
def date = "2021-06-01"
def endDate = "2021-07-01"
def newDate = "2021-08-01"
//def startDate = "2021-05-01"
def startDate = "2021-06-01"
def date = "2021-07-01"
def endDate = "2021-08-01"
def newDate = "2021-09-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -64,7 +85,7 @@ class TradeStatLog {
ret << lastMonthCount
ret << curMonthCount
ret << nearMonthCount
if(lastMonthCount){
if(curMonthCount){
def a = (nearMonthCount-curMonthCount)*100/curMonthCount
def pre = DataHandleUtils.getDoubleRound(2, a)
ret << pre
@@ -87,7 +108,7 @@ class TradeStatLog {
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\成交统计.xlsx", rets, "数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx", rets, "数据")
println "表格生成完成"
}
@@ -103,7 +124,7 @@ class TradeStatLog {
rets << [city, districts.size(), m.total, m.count1, m.count2, m.count3, m.count4, m.count5, m.count6, m.count7, m.count8, m.count9, m.count10, m.count11, m.count12, m.count13, m.count14]
println rets.get(rets.size()-1)
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\第三方均价统计.xlsx", rets, "数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\第三方均价统计0927.xlsx", rets, "数据")
println "表格生成完成"
}
@@ -129,7 +150,7 @@ class TradeStatLog {
def getThirdPartyPricePrecent(def city){
def sql = "SELECT * FROM third_party_price WHERE cityname = '"+city+"'";
def sql = "SELECT * FROM third_party_price_gj WHERE cityname = '"+city+"' and year_month=202108";
def count0 = 0
def count1 = 0
@@ -299,5 +320,333 @@ class TradeStatLog {
}
def averageStat(){
def citys = ["北京","上海","广州","深圳","重庆","杭州","苏州","厦门","大连","呼和浩特"]
def map = [:]
for(def city : citys){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]);
def rets = findThirdPriceByDist(districts)
map.put(city, rets.size())
map.put(city+"-count", districts.size())
println districts.size()+" "+ rets.size()
}
println map
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\估值测试数据覆盖率占比.xlsx",{d->
def city = d['A']
def count = map.get(city)
def distCount = map.get(city+"-count")
d['H'] = count
d['J'] = distCount
def pre = DataHandleUtils.getDoubleRound(2, count*100.0/distCount)
d['I'] = pre
},0)
}
def findThirdPriceByDist(districts){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
for(def i=0;i<districts.size();i++){
final def dist = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def count = findAverageStat(dist)
if(count){
rets << dist
println totalCount.get() +"/${districts.size()} "+dist.region+" "+dist.name
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def findAverageStat(dist){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
q.put("name", dist.name)
q.put("category","district")
q.put("s_date", new BasicDBObject('$gte', 202105).append('$lt',202108))
def dbcursor = AVERAGE_STATISTICS.find(q)
def flag = 0
for(def d : dbcursor){
if(!flag){
if(d.zhugezhaofang_esf_price){
flag = 1
continue
}
if(d.anjuke_esf_price){
flag = 1
continue
}
if(d.get("58tc_esf_price")){
flag = 1
continue
}
if(d.soufang_esf_price){
flag = 1
}
if(d.lianjia_esf_price){
flag = 1
continue
}
if(d.cityhouse_esf_price){
continue
flag = 1
continue
}
if(d.maitian_esf_price){
flag = 1
continue
}
if(d.beike_esf_price){
flag = 1
continue
}
if(d.get("365house_esf_price")){
flag = 1
continue
}
if(d.zhongyuan_esf_price){
flag = 1
continue
}
if(d.tuboshi_esf_price){
flag = 1
continue
}
if(d.qfang_esf_price){
flag = 1
continue
}
if(d.fangtan_esf_price){
flag = 1
continue
}
if(d.fangduoduo_esf_price){
flag = 1
continue
}
if(d.get("5i5j_esf_price")){
flag = 1
continue
}
}
}
return flag
}
def exportTrade(){
def citys = CcbCommonUtils.citys;
// def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
def rets = []
rets << headers
for(def city : citys){
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern("2021-07-01","yyyy-MM-dd"), DateUtils.parseFromTightByPattern("2021-08-01","yyyy-MM-dd"))
// def trades = getYouDaData()
//def statMap = getDistrictStatByCity(city, weekend)
def nameMap = getAllDistrict(city)
def rMap = findDealIds(city);
def ids = rMap.ids
def details = rMap.detail
for(def t : trades){
if(!ids.contains(t._id.toString())){
def distId = nameMap.get(t.city +"##"+t.region+"##"+t.district_name)
if(!distId && (t.district_name.toString().endsWith("弄") || t.district_name.toString().endsWith("号"))){
distId = nameMap.get(t.city +"##"+t.region+"##"+t.district_name+"小区")
}
if(!distId){
def flag = t.district_name.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(flag){
def datas = CommonDatautils.findDistrictByAddress2(t.city, t.region, t.district_name, "district")
if(datas && datas.size()==1){
distId = datas.get(0)._id.toString()
}
}
}
def reason = ""
if(distId){
def dist = CommonDatautils.findDistrictById(distId)
// def price = statMap.get(dist.region+"@"+dist.name)
// def startPrice = null
// def endPrice = null
// if(price){
// if(dist.city=="上海"){
// startPrice = (int)(price - price*0.1)
// endPrice = (int)(price*0.5 + price)
// }else if(dist.city=="杭州"){
// startPrice = (int)(price - price*0.15)
// endPrice = (int)(price*0.15 + price)
// }else{
// startPrice = (int)(price - price*0.5)
// endPrice = (int)(price*0.5 + price)
// }
// }
// if(t.avg_price<startPrice || t.avg_price>endPrice){
// reason = "价格过滤"
// }
if(dist.name_cat != 0 || !dist.lng || !["普通住宅","商住"].contains(dist.estate_type2) || dist.name.contains("洋房") || dist.name.contains("公房")){
reason = "小区过滤"
}
if(!reason){
def key=t.city+"##"+dist.region+"##"+dist.name+"##"+t.area+"##"+t.avg_price+"##"+DateUtils.formatDateByPattern(t.trade_date,"yyyy-MM-dd")
if(details.contains(key)){
reason = "数据重复"
}
}
}
def tradeDate = DateUtils.formatDateByPattern(t.trade_date,"yyyy-MM-dd")
//rets << [t._id.toString(), t.city, t.region, t.district_name, t.source, t.avg_price, t.area,t.address, tradeDate, reason, t.estate_type]
rets << [t._id.toString(), t.city, t.region, t.district_name, t.source, t.avg_price, t.area,t.address, tradeDate, reason]
}
}
println city +" "+rets.size()
}
ExcelUtils.write("d:\\数据\\成交未入库记录(7月全).xlsx", rets,"数据")
}
def deleteDistrict(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d->
if(d['G']){
def id = d['G']
BasicDBObject q=new BasicDBObject('name_cat',1)
q.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q))
deleteAllRefDistrict(id)
}
},0)
}
def deleteAllRefDistrict(id){
def tables = ["bigdata_community_gj"]
for(def tableName : tables){
deleteByCommid(tableName, id)
}
}
def deleteByCommid(tableName, commid){
def sql = "delete from "+tableName+" where commid='${commid}'";
sqlInstance.execute(sql)
println "删除:"+tableName+" "+commid
}
def getYouDaData() {
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("2021-07-01", "yyyy-MM-dd"))
def endDate = DateUtils.parseFromTightByPattern("2021-08-01", "yyyy-MM-dd")
def q = new BasicDBObject()
// q.put("premisesName","共富一村")
q.put("contractTime", new BasicDBObject('$gte', startDate));//.append('$lt', endDate)
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
def trades = []
for(def db : dbcursor){
def map = [:]
map._id=db._id.toString()
map.city = "上海"
map.region = db.region
if(db.districtName && db.districtName!="null"){
map.district_name=db.districtName
}else if(db.address){
map.district_name=db.address
}
map.area = db.area
map.trade_date = db.contractTime
map.avg_price= db.price
map.estate_type = db.houseType
trades << map
}
return trades
}
def findDealIds(city){
def sql = "select * from bigdata_deal_gj where dealdate>=to_date('2021-07-01','yyyy-mm-dd') and dealdate<to_date('2021-08-01','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd')
def ids = []
def detailInfos = []
sqlInstance.eachRow(sql,{r->
if(!ids.contains(r.dealid)){
ids << r.dealid
def region = RegionData.formatRegion(r.cityname, r.distname)
def key=r.cityname+"##"+region+"##"+r.commname+"##"+r.buildarea.toDouble()+"##"+r.dealunitprice+"##"+DateUtils.formatDateByPattern(r.dealdate,"yyyy-MM-dd")
if(!detailInfos.contains(key)){
detailInfos << key
}
}
});
def rMap = [:]
rMap.ids = ids
rMap.detail = detailInfos
return rMap
}
public static def findTradesDetail(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
// q.put("district_name","百合花园")
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def dbcursor = DEAL_PRICE_COLL.find(q)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets
}
def getDistrictStatByCity(city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DIST_SELL_STAT.find(q)
def map = new ConcurrentHashMap()
for(def d : dbcursor){
map.put(d.region+"@"+d.district_name, d.avg_price)
}
return map
}
def getAllDistrict(city){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name", "alias"])
def map = [:]
for(def d : dists){
map.put(d.city+"##"+d.region+"##"+d.name, d._id.toString())
def alias = d.alias
if(alias){
for(def name : alias){
map.put(d.city+"##"+d.region+"##"+name, d._id.toString())
}
}
}
return map
}
}
@@ -1,13 +1,18 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.ExcelUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
@@ -16,6 +21,7 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
class UpdateHouseRegion {
@@ -37,11 +43,89 @@ class UpdateHouseRegion {
static main(args) {
def task = new UpdateHouseRegion()
task.updateSeawaterRegion()
// task.updateSeawaterRegion()
// task.updateDealRegion()
// task.updateThirdPartyRegion()
task.updateNoMatchDealRegion()
// task.addAlias()
// task.addFenQi()
}
def addFenQi(){
def citys = CommonDatautils.findAllCity()
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district",["city","region","name","alias"])
addFenQiAlias(dists)
println city +" 处理完成"
}
}
def addFenQiAlias(dists){
for(def dist : dists){
def s1 = DataUtil.match(dist.name, Pattern.compile("[一|二|三|四|五|六|七|八|九|十|十一|十二|十三|十四|十五|十六|十七|十八|\\w+](期|区)\$"))
if(s1){
def newName = dist.name.replace("Ⅰ","1").replace("Ⅱ","2").replace("A区","a区").replace("B区","b区").replace("C区","c区").replace("D区","d区").replace("E区","e区").replace("F区","f区").replace("一期","1期").replace("二期","2期").replace("三期","3期").replace("四期","4期").replace("五期","5期").replace("六期","6期").replace("七期","7期").replace("八期","8期").replace("九期","9期").replace("十期","10期").replace("十一期","11期").replace("十二期","12期")
if(newName!=dist.name){
def alias = dist.alias
if(!alias){
alias = []
}
if(!alias.contains(newName)){
alias << newName
def udata = new BasicDBObject()
udata.put("alias", alias)
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
println dist.city +" "+dist.name+" "+dist.alias+" 新:"+alias
}
}
}
}
}
def addAlias(){
def citys = CommonDatautils.findAllCity()
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district",["city","region","name","alias"])
addDistricts(dists)
println city +" 处理完成"
}
}
def addDistricts(dists){
for(def dist : dists){
def alias = dist.alias
if(alias){
def oldCount = alias.size()
def newAlias = []
for(def aliasName : alias){
if(aliasName.contains("·") || aliasName.contains(".")){
def newName = aliasName.replace("·","").replace(".","")
newAlias << aliasName
if(!newAlias.contains(newName)){
if(newName!=dist.name){
newAlias << newName
}
}
}else{
if(!newAlias.contains(aliasName)) {
newAlias << aliasName
}
}
}
def newCount = newAlias.size()
if(oldCount && alias!=newAlias){
def udata = new BasicDBObject()
udata.put("alias", newAlias)
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
println dist.city +" "+dist.name+" "+dist.alias+" 新:"+newAlias
}
}
}
}
def updateThirdPartyRegion(){
def city = "东莞"
def rets = findThirdPartyPrice(city)
@@ -121,6 +205,32 @@ class UpdateHouseRegion {
latch.await()
}
def updateNoMatchDealRegion(){
ExcelUtils.multiThreadingModify("D:\\数据\\成交未入库记录(7月全).xlsx",{d->
if(!d['J']){
def city = d['B']
def region = d['C']
def name = d['D']
def rets = CommonDatautils.findDistrictsByName(city, name, "district")
if (!rets) {
rets = CommonDatautils.findDistrictByAlias(city, null, name, "district")
}
if (rets.size() == 1) {
def dist = rets[0]
if (region != dist.region) {
def udata = new BasicDBObject()
udata.put("region", dist.region)
DEAL_PRICE_COLL.update(new BasicDBObject('_id', new ObjectId(d['A'])), new BasicDBObject('$set', udata))
d['J'] = "原数据区域修改("+dist.region+")"
println d['A']+" "+region +" "+dist.region
}
}
}
},0)
}
def updateSeawaterRegion(){
def city = "东莞"
def rets = findHouseSource(city, 20190701, 20211001)
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202107.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202109.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -135,7 +135,7 @@ rets << ["9
def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM")
ExcelUtils.write("d:\\数据\\福州\\多方均价对比统计数据"+curMonth+".xlsx",rets,"统计")
def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )"
println s
def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60"
DingDingMessageUtil.sendTextMessage(access_token, s)
@@ -45,18 +45,20 @@ class OpenUserBatch {
}
def addShenZhenFjtUser(){
def datas = ExcelUtils.read("D:\\数据\\农行\\肇庆农行支行房价通账号.xlsx",0,1)
def datas = ExcelUtils.read("D:\\数据\\渤海\\开通线上评估客户经理名单.xlsx",0,1)
def i = 0;
for(def data : datas){
i++;
def id = 252580 + i;
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+data[2]+"', '"+data[3]+"', '16', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2021-12-10 00:00:00', '0', '', NULL)";
def id = 252608 + i;
BigDecimal bd = new BigDecimal(data[3].toString());
String mobile = bd.toPlainString();
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '17', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2023-01-01 00:00:00', '0', '', NULL)";
onLineSqlInstance.execute(sql);
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[1]+"', '"+data[0]+"', NULL, NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '王辰岑')";
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '姚静梦')";
onLineSqlInstance.execute(sql2)
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'肇庆','肇庆:')"
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'上海','上海:')"
onLineSqlInstance.execute(sql3)
}
}
@@ -0,0 +1,18 @@
package com.fangjia.script.gj.ccb
import com.fangjia.script.commData.CommonDatautils
class CcbXykData {
static main(args) {
def task = new CcbXykData()
task.run()
}
def run(){
def districts = CommonDatautils.findAllDistrictByCity("ÉϺ£","district",["city","region","name","estate_charge"])
for(def d : districts){
}
}
}
@@ -9,6 +9,14 @@ class FileUtils {
// file.delete();
// }
if (!file.getParentFile().exists()) {
boolean result = file.getParentFile().mkdirs();
if (!result) {
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
}
}
OutputStream out = new FileOutputStream(fileName, isAppend);
def sb = new StringBuffer();
@@ -0,0 +1,64 @@
package com.fangjia.script.gj.location
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.NameProcess
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
class DistrictLocationUpdate {
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
static main(args) {
def task = new DistrictLocationUpdate()
task.run()
}
def run(){
def citys = CommonDatautils.findAllCity()
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias", "lng", "lat"])
updateLocation(city, dists);
}
}
def queryPoiMap(def city){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1")
sqlBuffer.append(" and cityname = '" + city + "'")
sqlBuffer.append(" and source = '¸ßµÂ'")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def retMap = [:]
while (rs.next()) {
def map = [:]
map.city = rs.getString("cityname")
map.region = rs.getString("adname")
map.name = rs.getString("name")
map.address = rs.getString("address")
map.type = rs.getString("type")
def location = rs.getObject("location")
if(location){
map.lng = location.x
map.lat = location.y
}
def name = NameProcess.nameStandard(map.city, map.region, map.name)
if(name){
retMap.put(map.region+"##"+name, map)
}
}
return retMap;
}
}
@@ -521,6 +521,10 @@ class GaoDeUtils {
return KeyList.get(count);
}
static void main(def a){
println search("如东县大豫镇豫东村委会","南通")
}
}
@@ -0,0 +1,45 @@
package com.fangjia.script.gj.test
import com.fangjia.core.RegionData
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
class MatchDistrict {
static main(args) {
def task = new MatchDistrict()
task.run()
}
def run(){
def filePath = "D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx"
def city = "深圳"
ExcelUtils.multiThreadingModify(filePath, { data ->
if(data['A']){
def region = RegionData.formatRegion(city, data['A'])
if(region){
def rMap = SeaweedDataMatchProcess.doMatch(city, data['A'], data['C'], null);
if(rMap){
data['E'] = rMap.city
data['F'] = rMap.region
data['G'] = rMap.name
}
}
}
}, 0);
}
}
//{
// "_id" : ObjectId("607963dc5bb87e2a18f4dde6"),
// "guiding_type" : NumberInt(0),
// "estate_type" : "普通住宅",
// "house_type" : NumberInt(0),
// "c_date" : ISODate("2021-04-16T10:15:56.651+0000"),
// "guiding_person" : "深圳房管局",
// "guiding_date" : ISODate("2021-04-10T16:00:00.000+0000"),
// "category" : "district",
// "guiding_value" : NumberInt(48300),
// "name" : "中粮云景国际",
// "city" : "深圳",
// "region" : "光明"
//}
@@ -1,33 +1,55 @@
package com.fangjia.script.gj.test
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.Mongo
import groovy.sql.Sql
import java.text.SimpleDateFormat
//SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
//Date date = new Date(1617211269868);
//print dateFormat.format(date);
DBCollection CITY_INFO_CONN = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_data_ana').getCollection('city_info');
def path = "D:\\数据\\广发\\广东省小区及均价覆盖率统计.xlsx"
ExcelUtils.modify(path,{d->
def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["name"])
d['G'] = districts.size()
//DBCollection CITY_INFO_CONN = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_data_ana').getCollection('city_info');
//def path = "D:\\数据\\广发\\广东省小区及均价覆盖率统计.xlsx"
//ExcelUtils.modify(path,{d->
// def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["name"])
// d['G'] = districts.size()
//
// def q = new BasicDBObject()
// q.put("city", d['B'])
// q.put("weekend", DateUtils.parseFromTightYYYYMMDD("2021-08-08"))
// def obj = CITY_INFO_CONN.findOne(q);
// if(obj){
// d['H'] = obj.price_coverage*100
// }
//})
def q = new BasicDBObject()
q.put("city", d['B'])
q.put("weekend", DateUtils.parseFromTightYYYYMMDD("2021-08-08"))
def obj = CITY_INFO_CONN.findOne(q);
if(obj){
d['H'] = obj.price_coverage*100
def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'";
def countMap = [:]
onLineSqlInstance.eachRow(sql,{r->
def addTime = r.visit_time
def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM")
def ips = countMap.get(month)
if(!ips){
ips = new HashSet();
countMap.put(month, ips)
}
ips << r.ip
});
countMap.each {k,v->
println k+" "+v.size()
}
})
@@ -0,0 +1,58 @@
package com.fangjia.script.test
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class RegionalismCodeStat {
DBCollection REGIONALISM_CODE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "regionalism_code", "192.168.0.235", 27018)
static main(args) {
def task = new RegionalismCodeStat()
task.run()
}
def run(){
def provinces = ["上海市","北京市","天津市","重庆市","福建省","贵州省","湖南省","广东省","湖北省","安徽省","四川省","江苏省","吉林省","河北省","河南省","海南省","江西省","云南省","甘肃省","山东省","陕西省","浙江省","青海省","辽宁省","黑龙江省","山西省","西藏自治区","新疆维吾尔自治区","宁夏回族自治区","内蒙古自治区","广西壮族自治区"]
// def provinces = ["宁夏回族自治区"]
provinces.each {v->
def sets = findCount(v)
println v +" "+ sets.size()
}
}
def findCount(v){
def q = new BasicDBObject()
q.put("province", v)
def dbcursor = REGIONALISM_CODE_COLL.find(q).batchSize(5000)
def set = new HashSet()
for(def obj : dbcursor){
// if(obj.city=="省直辖县级行政区划" || obj.city=="自治区直辖县级行政区划"){
// //set << obj.region
// println "【"+obj.region+"】 过滤"
// continue
// }else{
// if(obj.region && !set.contains(obj.region)){
// set << obj.region
// println "【"+obj.region+"】 add"
// }
// }
if(obj.juweihui){
def k = obj.street+"##"+obj.juweihui
if(!set.contains(k)){
set << k
// println "【"+k+"】 add"
}
}
}
// set.each {k->
// println k
// }
return set;
}
}
@@ -17,7 +17,7 @@ class SynchDistrictPrice {
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产"]
def sourceMap = ["zhuge.com":"诸葛找房"]
def sourceMap = ["cityhouse.cn":"城市房产"]//["zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
for(def source :sources){
def count1 = findTotalCount(source, 202107);
def count2 = findTotalCount(source, 202108);
def count1 = findTotalCount(source, 202108);
def count2 = findTotalCount(source, 202109);
println source +" "+count1 +" "+count2
}
}