+6
-5
@@ -13,8 +13,9 @@ import com.mongodb.DBObject
|
||||
*/
|
||||
|
||||
class CityLatlngBox {
|
||||
public static final DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.105", 27018)
|
||||
public static final DBCollection LATLNG_COLL = MongoUtils.getCollectionByName("addrepo", "city_latlng_range", "192.168.0.219", 27017)
|
||||
//public static final DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.105", 27018)
|
||||
DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.235", 27018)
|
||||
public static final DBCollection LATLNG_COLL = MongoUtils.getCollectionByName("addrepo", "city_latlng_range", "192.168.0.206", 27020)
|
||||
|
||||
public static void main(String[] args) {
|
||||
CityLatlngBox sd = new CityLatlngBox();
|
||||
@@ -22,7 +23,7 @@ class CityLatlngBox {
|
||||
}
|
||||
|
||||
void process() {
|
||||
def cities = CommonUtils.findAllCity()
|
||||
def cities = ["白沙","阿坝州","甘孜州","阿勒泰","临沧","那曲","三沙","喀什","昌吉","迪庆州","普洱","德宏州","湘西","塔城","凉山州","屯昌","山南","海西州","阿拉善盟","乐东","定安","怒江州","黄山","图木舒克","楚雄","临高","乌兰察布","延边","文山州","锡林郭勒盟","陵水","天门","淮北","黔东南","万宁","湘潭","海北州","琼中","吐鲁番","惠州","巴音郭楞","恩施","铜仁","黔南","西双版纳州","潜江","林芝","文昌","仙桃","神农架","和田","黔西南","果洛州","儋州","哈密","大理","东方","日喀则","兴安盟","昌江","澄迈","保亭","济源","五指山","阿拉尔","伊犁","临夏","海南州","琼海","石河子","昌都","五家渠","玉树州","博尔塔拉","东营","铜陵","黄南州","廊坊","嘉峪关","马鞍山","宁波","广安","中山","克孜勒苏","厦门","莱芜","鄂州","东莞","海口","鹤壁","六盘水","漯河","三亚","珠海","深圳","乌海","淮南","北海","双鸭山","襄阳","新余","汕头","克拉玛依","鹰潭","盘锦","镇江","石嘴山","嘉兴","铜川","焦作","芜湖","潮州","萍乡","红河","黄石","辽源","辽阳","枣庄","毕节","许昌","阳泉","自贡","遂宁","景德镇","营口","日照","莆田"]//CommonUtils.findAllCity()
|
||||
for (String city : cities) {
|
||||
println city
|
||||
BOUNDS_COLL.remove(new BasicDBObject('city', city))
|
||||
@@ -70,13 +71,13 @@ class CityLatlngBox {
|
||||
int row = 0
|
||||
int inx = 0
|
||||
def buffer = []
|
||||
while(seedLatBD < cityBoundBD.getMaxLat()) {
|
||||
while(seedLatGD < cityBoundGD.getMaxLat()) {
|
||||
int col = 0
|
||||
Bounds boundsBD = null;
|
||||
Bounds boundsQQ = null;
|
||||
Bounds boundsGD = null;
|
||||
|
||||
while (seedLngBD < cityBoundBD.getMaxLng()) {
|
||||
while (seedLngGD < cityBoundGD.getMaxLng()) {
|
||||
double[] espBD = GeoUtils.getLatLon(seedLngBD, seedLatBD, offset, 90) // 右侧OFFSET处的经纬度
|
||||
double[] wnpBD = GeoUtils.getLatLon(seedLngBD, seedLatBD, offset, 360) // 上方OFFSET处的经纬度
|
||||
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
|
||||
println city +" "+rows.size()
|
||||
}
|
||||
def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
|
||||
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_commbuildtype.dat",headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMBUILDTYPE_GJ.dat",headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
|
||||
}
|
||||
|
||||
+2
-2
@@ -40,8 +40,8 @@ class BigdataDistrictBatch extends BaseBatch{
|
||||
def rets = []
|
||||
rets << headers
|
||||
rets.addAll(allDatas)
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_district.xlsx", rets, "数据")
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_district.dat",headers, "utf-8",false);
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\BIGDATA_DISTRICT_GJ.xlsx", rets, "数据")
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.dat",headers, "utf-8",false);
|
||||
}
|
||||
|
||||
def createDatas(rows){
|
||||
|
||||
+2
-2
@@ -65,9 +65,9 @@ class BigdataPoiBatch extends BaseBatch{
|
||||
|
||||
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"]
|
||||
|
||||
def fileName = "/home/goojia/ccb/bigdata_poi.txt"
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_POI_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_poi.txt"
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_POI_GJ.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
|
||||
+9
-9
@@ -107,8 +107,8 @@ class CcbCommonUtils {
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
if(dist.name.endsWith("弄小区")){
|
||||
def name = dist.name.replace("弄小区","弄")
|
||||
if(dist.name.endsWith("弄小区") || dist.name.endsWith("小区") ){
|
||||
def name = dist.name.replace("弄小区","弄").replace("小区","")
|
||||
if(!alias.contains(name)){
|
||||
alias << name
|
||||
}
|
||||
@@ -123,10 +123,10 @@ class CcbCommonUtils {
|
||||
def rets = new HashSet()
|
||||
def lists = []
|
||||
for(def d : dbcursor){
|
||||
if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){
|
||||
//def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
//if(!rets.contains(key)){
|
||||
// rets << key
|
||||
// if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){
|
||||
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
if(!rets.contains(key)){
|
||||
rets << key
|
||||
d.dist_id = dist._id.toString()
|
||||
d.address = dist.address
|
||||
d.master_name = dist.name
|
||||
@@ -136,8 +136,8 @@ class CcbCommonUtils {
|
||||
d.property_life = dist.property_life
|
||||
d.estate_type2 = dist.estate_type2
|
||||
lists << d
|
||||
//}
|
||||
}
|
||||
}
|
||||
// }
|
||||
}
|
||||
return lists
|
||||
}
|
||||
@@ -229,7 +229,7 @@ class CcbCommonUtils {
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name", "海昌欣城")
|
||||
// q.put("name", "东特金星公馆")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
||||
|
||||
|
||||
+40
-4
@@ -43,7 +43,9 @@ class DistrictDelete {
|
||||
static main(args) {
|
||||
def task = new DistrictDelete()
|
||||
// task.run()
|
||||
task.gzRun();
|
||||
// task.gzRun();
|
||||
// task.deleteDistrict()
|
||||
task.deleteCommPrice()
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -211,9 +213,9 @@ class DistrictDelete {
|
||||
|
||||
def gzRun(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\经纬度有问题的小区.xlsx",{d->
|
||||
if(d['U'] && d['U'].toString().toDouble().toInteger()==1){
|
||||
def id = d['E']
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\要删除小区 (5).xlsx",{d->
|
||||
if(d['G']){
|
||||
def id = d['G']
|
||||
BasicDBObject q=new BasicDBObject('name_cat',1)
|
||||
q.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q))
|
||||
@@ -223,4 +225,38 @@ class DistrictDelete {
|
||||
},0)
|
||||
}
|
||||
|
||||
def deleteDistrict(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d->
|
||||
if(d['G']){
|
||||
def id = d['G']
|
||||
BasicDBObject q=new BasicDBObject('name_cat',1)
|
||||
q.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q))
|
||||
deleteAllRefDistrict(id)
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def deleteAllRefDistrict(id){
|
||||
def tables = ["bigdata_forsale_gj","avm_evaluation_item_gj","community_base_price_gj","community_poi_gj","community_sale_gj","community_sale_tmp","community_test_price_tmp"]
|
||||
for(def tableName : tables){
|
||||
deleteByCommid(tableName, id)
|
||||
}
|
||||
}
|
||||
|
||||
def deleteCommPrice(){
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\均价删除.xlsx",{d->
|
||||
if(d['B']){
|
||||
def id = d['B']
|
||||
println id
|
||||
deleteByCommid("community_base_price_gj",id)
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def deleteByCommid(tableName, commid){
|
||||
def sql = "delete from "+tableName+" where commid='${commid}'";
|
||||
sqlInstance.execute(sql)
|
||||
println "删除:"+tableName+" "+commid
|
||||
}
|
||||
}
|
||||
|
||||
+3
-3
@@ -17,7 +17,7 @@ class DistrictGuidePriceBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = ["深圳"]//CcbCommonUtils.citys;
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
|
||||
for (def city : citys) {
|
||||
@@ -34,8 +34,8 @@ class DistrictGuidePriceBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
|
||||
// def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"]
|
||||
// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\district_guied_price.dat", headers, "utf-8", false);
|
||||
def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_GUIDE_PRICE_GJ.dat", headers, "utf-8", false);
|
||||
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -29,8 +29,8 @@ class EvaluateRateBatch extends BaseBatch{
|
||||
def rets = []
|
||||
rets << headers
|
||||
rets.addAll(allDatas)
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\evaluate_rate.xlsx", rets, "数据")
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluate_rate.dat",headers, "utf-8", false);
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\EVALUATE_RATE_GJ.xlsx", rets, "数据")
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\EVALUATE_RATE_GJ.dat",headers, "utf-8", false);
|
||||
}
|
||||
|
||||
def createDatas(def datas){
|
||||
|
||||
+101
-12
@@ -25,18 +25,24 @@ class ExportOracleData {
|
||||
// exportCommunityBuileType()
|
||||
// exportBigdataDistrictData()
|
||||
// exportGuidedPrice()
|
||||
// exportCommunityGuidedPrice()
|
||||
exportCommunityGuidedPrice()
|
||||
// exportBigdataDeal()
|
||||
// exportThirdPartyPrice()
|
||||
|
||||
// exportBigdataCommunity()
|
||||
exportBigdataForsale()
|
||||
// exportBigdataForsale()
|
||||
|
||||
// exportAvmEvaluationGJNB()
|
||||
// exportAVMEvaluationItemGJNB()
|
||||
// exportCommunityBasePrice()
|
||||
|
||||
// exportCommunityBasePriceGj()
|
||||
// exportCityPrice()
|
||||
// exportRegionPrice()
|
||||
// exportLastFile()
|
||||
}
|
||||
|
||||
|
||||
def exportEvaluationRateData(){
|
||||
def sql = "select ID,RATE_TYPE,RATE,RANK,CITY,REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from evaluate_rate_gj";
|
||||
def allRows = []
|
||||
@@ -213,7 +219,7 @@ class ExportOracleData {
|
||||
|
||||
|
||||
def exportGuidedPrice(){
|
||||
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where cityId in (350200,150100)";
|
||||
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where dealdate>=TO_DATE('2021-08-01', 'yyyy-MM-dd') and dealdate<TO_DATE('2021-09-01', 'yyyy-MM-dd') and cityname='上海'";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
@@ -237,20 +243,20 @@ class ExportOracleData {
|
||||
def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def dataCodeFileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.ok"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.ok"
|
||||
}
|
||||
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
println "GUIDED_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, "202107")
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, "202107")
|
||||
CcbFtpUtils.ftpUpload(fileName, "202108")
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
|
||||
println "GUIDED_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityGuidedPrice(){
|
||||
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where cityname='厦门' or cityname='呼和浩特'";
|
||||
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where cityname='西安'";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
@@ -273,16 +279,16 @@ class ExportOracleData {
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.ok"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.ok"
|
||||
}
|
||||
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"]
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
println "COMMUNITY_GUIDE_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, "202108")
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
|
||||
CcbFtpUtils.ftpUpload(fileName, "202107")
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, "202107")
|
||||
println "COMMUNITY_GUIDE_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
@@ -726,5 +732,88 @@ class ExportOracleData {
|
||||
println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityBasePriceGj(){
|
||||
|
||||
}
|
||||
|
||||
def exportCityPrice(){
|
||||
def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where (cityId=350200 or cityId=150100) and year_month>=202108";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.PRICE
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_CITY_PRICE_GJ_ADD_${curDate}_0001.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.ok"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
println "CITY_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, "202108")
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
|
||||
println "CITY_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportRegionPrice(){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where (cityId=350200 or cityId=150100) and year_month>=202108";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.PRICE
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def dataCodeFileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.ok"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
println "DISTRICT_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(fileName, "202108")
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, "202108")
|
||||
println "DISTRICT_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportLastFile(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
|
||||
def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM")
|
||||
def dataCodeFileName = "/ccb/${curDate}.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curMonth}\\${curDate}.ok"
|
||||
}
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, "${curMonth}")
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+1
-1
@@ -71,7 +71,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.dat", headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
|
||||
|
||||
// def rets = []
|
||||
// rets << headers
|
||||
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
class LocationUpdate {
|
||||
|
||||
static main(args) {
|
||||
def task = new LocationUpdate();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\经纬度\\经纬度人工核实结果_0923.xlsx",{d->
|
||||
index++
|
||||
if(d['C']){
|
||||
|
||||
println index +" "+d['A']+" "+d['B']+" "+d['C']
|
||||
def region = RegionData.formatRegion(d['A'], d['B'])
|
||||
if(d['H']){
|
||||
def status = d['H'].toString().toDouble().toInteger()
|
||||
if(status==1){
|
||||
def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
|
||||
if(obj){
|
||||
def gmap = GeocodeUtils.getGaodeToBaidu(d['F'].toString().toDouble(), d['G'].toString().toDouble())
|
||||
def lng2 = null
|
||||
def lat2 = null
|
||||
if(gmap){
|
||||
lng2 = (gmap.get("lng")*10000000000).longValue()
|
||||
lat2 = (gmap.get("lat")*10000000000).longValue()
|
||||
}
|
||||
|
||||
BasicDBObject q=new BasicDBObject()
|
||||
q.put("m_date", new Date())
|
||||
q.put("lat2", lat2)
|
||||
q.put("lng2", lng2)
|
||||
q.put("lng", (d['F'].toString().toDouble()*10000000000).longValue())
|
||||
q.put("lat", (d['G'].toString().toDouble()*10000000000).longValue())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q))
|
||||
|
||||
}else{
|
||||
println d['A']+" "+d['B']+" "+d['C']+" 没找到小区"
|
||||
}
|
||||
}else{
|
||||
def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
|
||||
if(obj){
|
||||
BasicDBObject q=new BasicDBObject('name_cat',1)
|
||||
q.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q))
|
||||
}else{
|
||||
println d['A']+" "+d['B']+" "+d['C']+" 没找到小区"
|
||||
}
|
||||
}
|
||||
|
||||
}else{
|
||||
def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
|
||||
if(obj){
|
||||
BasicDBObject q=new BasicDBObject('name_cat',1)
|
||||
q.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q))
|
||||
}else{
|
||||
println d['A']+" "+d['B']+" "+d['C']+" 没找到小区"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -40,8 +40,8 @@ class StagingDistrictBatch extends BaseBatch{
|
||||
results << headers
|
||||
results.addAll(rows)
|
||||
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING.xlsx", results, "数据")
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_STAGING.dat",headers, "utf-8", false);
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "数据")
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_STAGING_GJ.dat",headers, "utf-8", false);
|
||||
|
||||
}
|
||||
|
||||
|
||||
+34
-12
@@ -3,6 +3,8 @@ package com.fangjia.script.ccbevaluation
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.looplineinfo.Point
|
||||
import com.fangjia.script.util.DbUtils
|
||||
@@ -14,6 +16,11 @@ import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.*;
|
||||
|
||||
|
||||
@@ -42,7 +49,7 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
|
||||
// println city +" "+count
|
||||
//}
|
||||
|
||||
//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx"
|
||||
//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0927.xlsx"
|
||||
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\7月成交.xlsx",0,1)
|
||||
//def dealMap = [:]
|
||||
//for(def d : datas){
|
||||
@@ -59,7 +66,7 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
|
||||
|
||||
|
||||
|
||||
def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0903.xlsx"
|
||||
//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0903.xlsx"
|
||||
|
||||
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1)
|
||||
//def dealMap = [:]
|
||||
@@ -83,7 +90,7 @@ def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
// if (city) {
|
||||
// def cityid = cityCodeDatas.get(city)
|
||||
// def total = findCount(cityid, sqlInstance)
|
||||
// d['M'] = total
|
||||
// d['O'] = total
|
||||
// }
|
||||
//},0)
|
||||
|
||||
@@ -128,13 +135,26 @@ def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
// }
|
||||
//},1)
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交统计.xlsx"
|
||||
def dealMap = findDealCount(sqlInstance)
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
if(d['C']){
|
||||
d['K'] = dealMap.get(d['C'])
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx"
|
||||
//def dealMap = findDealCount(sqlInstance)
|
||||
//ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
// if(d['A']){
|
||||
// d['U'] = dealMap.get(d['A'])
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交未入库人工部分城市统计.xlsx"
|
||||
def datas = ExcelUtils.read(fileName, 0, 1)
|
||||
def dealMap = [:]
|
||||
for(def d : datas){
|
||||
dealMap.put(d['A'], d['B'])
|
||||
}
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx",{d->
|
||||
if(d['A']){
|
||||
d['W'] = dealMap.get(d['A'])
|
||||
}
|
||||
},2)
|
||||
},0)
|
||||
|
||||
|
||||
//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx"
|
||||
@@ -170,7 +190,7 @@ ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
|
||||
|
||||
def findDealCount(sqlInstance){
|
||||
def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('2021-06-01','yyyy-mm-dd') and dealdate<to_date('2021-07-01','yyyy-mm-dd') group by cityname";
|
||||
def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('2021-08-01','yyyy-mm-dd') and dealdate<to_date('2021-09-01','yyyy-mm-dd') group by cityname";
|
||||
def count = 0
|
||||
|
||||
def map = [:]
|
||||
@@ -194,7 +214,7 @@ def findEstateType(cityid, sqlInstance){
|
||||
}
|
||||
|
||||
def findCount(cityid, sqlInstance){
|
||||
def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}'";
|
||||
def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0";
|
||||
def count = 0
|
||||
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
@@ -204,7 +224,7 @@ def findCount(cityid, sqlInstance){
|
||||
}
|
||||
|
||||
def findFiledCount(cityid, field, sqlInstance){
|
||||
def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and "+field+" is not null";
|
||||
def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0 and "+field+" is not null";
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
@@ -353,3 +373,5 @@ def getStreetInfo(def city, def glocation){
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class TestRepeat {
|
||||
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new TestRepeat()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
|
||||
}
|
||||
|
||||
|
||||
def checkUpdateCount(){
|
||||
def newHouses = readTxt("C:\\Users\\gaojuan\\Desktop\\BIGDATA_FORSALE_GJ.dat")
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(newHouses.size())
|
||||
println " ·¿Ô´Á¿£º"+newHouses.size()
|
||||
|
||||
def retMap = new ConcurrentHashMap()
|
||||
for(def i=0;i<newHouses.size();i++){
|
||||
final def map = newHouses[i];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def obj = findHouse(map.houseId, map.commId)
|
||||
if(obj){
|
||||
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
|
||||
}
|
||||
|
||||
def findHouse(def houseId, def commId){
|
||||
def sql = "select * from bigdata_forsale_gj where commid='${commId}' and forsaleid='${houseId}'";
|
||||
def obj = null
|
||||
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
obj = r
|
||||
});
|
||||
return obj
|
||||
}
|
||||
|
||||
|
||||
def readTxt(def file){
|
||||
def lines = org.apache.commons.io.FileUtils.readLines(new File(file))
|
||||
def rets = [];
|
||||
for(line in lines) {
|
||||
def arrs = line.split("|@|")
|
||||
def map = [:]
|
||||
map.houseId = arrs[1]
|
||||
map.commId = arrs[6]
|
||||
map.pdate = arrs[10]
|
||||
rets << map
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
}
|
||||
+8
-7
@@ -20,9 +20,9 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= 202107//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202105
|
||||
def endDate = 202107
|
||||
def sDate= 202108//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202106
|
||||
def endDate = 202108
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
@@ -53,13 +53,13 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
|
||||
|
||||
def fileName = "/home/goojia/ccb/third_party_price.dat"
|
||||
def fileName = "/home/goojia/ccb/THIRD_PARTY_PRICE_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\third_party_price.dat"
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\THIRD_PARTY_PRICE_GJ.dat"
|
||||
}
|
||||
|
||||
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true);
|
||||
def header = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
|
||||
|
||||
}
|
||||
|
||||
@@ -67,6 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){
|
||||
def rets = []
|
||||
for(def d : dists){
|
||||
|
||||
def third = thirdPriceMap.get(d.region+"##"+d.name)
|
||||
if(third){
|
||||
def data = distCodeMap.get(d.region)
|
||||
|
||||
+359
-10
@@ -1,12 +1,22 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/***
|
||||
* 建行成交抓取统计
|
||||
@@ -21,23 +31,34 @@ class TradeStatLog {
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new TradeStatLog()
|
||||
task.run()
|
||||
// task.run()
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
// task.averageStat()
|
||||
task.exportTrade()
|
||||
// task.deleteDistrict()
|
||||
|
||||
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
rets << ["城市","5月总量","6月总量","7月总量","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","6月总量","7月总量","8月总量","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
|
||||
def startDate = "2021-05-01"
|
||||
def date = "2021-06-01"
|
||||
def endDate = "2021-07-01"
|
||||
def newDate = "2021-08-01"
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2021-06-01"
|
||||
def date = "2021-07-01"
|
||||
def endDate = "2021-08-01"
|
||||
def newDate = "2021-09-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
@@ -64,7 +85,7 @@ class TradeStatLog {
|
||||
ret << lastMonthCount
|
||||
ret << curMonthCount
|
||||
ret << nearMonthCount
|
||||
if(lastMonthCount){
|
||||
if(curMonthCount){
|
||||
def a = (nearMonthCount-curMonthCount)*100/curMonthCount
|
||||
def pre = DataHandleUtils.getDoubleRound(2, a)
|
||||
ret << pre
|
||||
@@ -87,7 +108,7 @@ class TradeStatLog {
|
||||
}
|
||||
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\成交统计.xlsx", rets, "数据")
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx", rets, "数据")
|
||||
println "表格生成完成"
|
||||
}
|
||||
|
||||
@@ -103,7 +124,7 @@ class TradeStatLog {
|
||||
rets << [city, districts.size(), m.total, m.count1, m.count2, m.count3, m.count4, m.count5, m.count6, m.count7, m.count8, m.count9, m.count10, m.count11, m.count12, m.count13, m.count14]
|
||||
println rets.get(rets.size()-1)
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\第三方均价统计.xlsx", rets, "数据")
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\第三方均价统计0927.xlsx", rets, "数据")
|
||||
println "表格生成完成"
|
||||
}
|
||||
|
||||
@@ -129,7 +150,7 @@ class TradeStatLog {
|
||||
|
||||
|
||||
def getThirdPartyPricePrecent(def city){
|
||||
def sql = "SELECT * FROM third_party_price WHERE cityname = '"+city+"'";
|
||||
def sql = "SELECT * FROM third_party_price_gj WHERE cityname = '"+city+"' and year_month=202108";
|
||||
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
@@ -299,5 +320,333 @@ class TradeStatLog {
|
||||
}
|
||||
|
||||
|
||||
def averageStat(){
|
||||
def citys = ["北京","上海","广州","深圳","重庆","杭州","苏州","厦门","大连","呼和浩特"]
|
||||
|
||||
def map = [:]
|
||||
for(def city : citys){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]);
|
||||
def rets = findThirdPriceByDist(districts)
|
||||
map.put(city, rets.size())
|
||||
map.put(city+"-count", districts.size())
|
||||
println districts.size()+" "+ rets.size()
|
||||
}
|
||||
println map
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\估值测试数据覆盖率占比.xlsx",{d->
|
||||
def city = d['A']
|
||||
def count = map.get(city)
|
||||
def distCount = map.get(city+"-count")
|
||||
d['H'] = count
|
||||
d['J'] = distCount
|
||||
|
||||
def pre = DataHandleUtils.getDoubleRound(2, count*100.0/distCount)
|
||||
d['I'] = pre
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findThirdPriceByDist(districts){
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def dist = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def count = findAverageStat(dist)
|
||||
if(count){
|
||||
rets << dist
|
||||
println totalCount.get() +"/${districts.size()} "+dist.region+" "+dist.name
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def findAverageStat(dist){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
q.put("name", dist.name)
|
||||
q.put("category","district")
|
||||
q.put("s_date", new BasicDBObject('$gte', 202105).append('$lt',202108))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q)
|
||||
def flag = 0
|
||||
for(def d : dbcursor){
|
||||
if(!flag){
|
||||
if(d.zhugezhaofang_esf_price){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.anjuke_esf_price){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.get("58tc_esf_price")){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.soufang_esf_price){
|
||||
flag = 1
|
||||
}
|
||||
if(d.lianjia_esf_price){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.cityhouse_esf_price){
|
||||
continue
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.maitian_esf_price){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.beike_esf_price){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.get("365house_esf_price")){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.zhongyuan_esf_price){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.tuboshi_esf_price){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.qfang_esf_price){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
|
||||
if(d.fangtan_esf_price){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.fangduoduo_esf_price){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
if(d.get("5i5j_esf_price")){
|
||||
flag = 1
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
return flag
|
||||
}
|
||||
|
||||
def exportTrade(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
|
||||
// def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")
|
||||
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
|
||||
def rets = []
|
||||
rets << headers
|
||||
for(def city : citys){
|
||||
|
||||
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern("2021-07-01","yyyy-MM-dd"), DateUtils.parseFromTightByPattern("2021-08-01","yyyy-MM-dd"))
|
||||
// def trades = getYouDaData()
|
||||
|
||||
//def statMap = getDistrictStatByCity(city, weekend)
|
||||
def nameMap = getAllDistrict(city)
|
||||
|
||||
|
||||
def rMap = findDealIds(city);
|
||||
def ids = rMap.ids
|
||||
def details = rMap.detail
|
||||
|
||||
|
||||
for(def t : trades){
|
||||
if(!ids.contains(t._id.toString())){
|
||||
|
||||
def distId = nameMap.get(t.city +"##"+t.region+"##"+t.district_name)
|
||||
if(!distId && (t.district_name.toString().endsWith("弄") || t.district_name.toString().endsWith("号"))){
|
||||
distId = nameMap.get(t.city +"##"+t.region+"##"+t.district_name+"小区")
|
||||
}
|
||||
if(!distId){
|
||||
def flag = t.district_name.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||
if(flag){
|
||||
def datas = CommonDatautils.findDistrictByAddress2(t.city, t.region, t.district_name, "district")
|
||||
if(datas && datas.size()==1){
|
||||
distId = datas.get(0)._id.toString()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def reason = ""
|
||||
if(distId){
|
||||
def dist = CommonDatautils.findDistrictById(distId)
|
||||
// def price = statMap.get(dist.region+"@"+dist.name)
|
||||
// def startPrice = null
|
||||
// def endPrice = null
|
||||
// if(price){
|
||||
// if(dist.city=="上海"){
|
||||
// startPrice = (int)(price - price*0.1)
|
||||
// endPrice = (int)(price*0.5 + price)
|
||||
// }else if(dist.city=="杭州"){
|
||||
// startPrice = (int)(price - price*0.15)
|
||||
// endPrice = (int)(price*0.15 + price)
|
||||
// }else{
|
||||
// startPrice = (int)(price - price*0.5)
|
||||
// endPrice = (int)(price*0.5 + price)
|
||||
// }
|
||||
// }
|
||||
// if(t.avg_price<startPrice || t.avg_price>endPrice){
|
||||
// reason = "价格过滤"
|
||||
// }
|
||||
if(dist.name_cat != 0 || !dist.lng || !["普通住宅","商住"].contains(dist.estate_type2) || dist.name.contains("洋房") || dist.name.contains("公房")){
|
||||
reason = "小区过滤"
|
||||
}
|
||||
|
||||
if(!reason){
|
||||
def key=t.city+"##"+dist.region+"##"+dist.name+"##"+t.area+"##"+t.avg_price+"##"+DateUtils.formatDateByPattern(t.trade_date,"yyyy-MM-dd")
|
||||
if(details.contains(key)){
|
||||
reason = "数据重复"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def tradeDate = DateUtils.formatDateByPattern(t.trade_date,"yyyy-MM-dd")
|
||||
//rets << [t._id.toString(), t.city, t.region, t.district_name, t.source, t.avg_price, t.area,t.address, tradeDate, reason, t.estate_type]
|
||||
rets << [t._id.toString(), t.city, t.region, t.district_name, t.source, t.avg_price, t.area,t.address, tradeDate, reason]
|
||||
}
|
||||
}
|
||||
println city +" "+rets.size()
|
||||
}
|
||||
|
||||
ExcelUtils.write("d:\\数据\\成交未入库记录(7月全).xlsx", rets,"数据")
|
||||
}
|
||||
|
||||
def deleteDistrict(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d->
|
||||
if(d['G']){
|
||||
def id = d['G']
|
||||
BasicDBObject q=new BasicDBObject('name_cat',1)
|
||||
q.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q))
|
||||
deleteAllRefDistrict(id)
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def deleteAllRefDistrict(id){
|
||||
def tables = ["bigdata_community_gj"]
|
||||
for(def tableName : tables){
|
||||
deleteByCommid(tableName, id)
|
||||
}
|
||||
}
|
||||
|
||||
def deleteByCommid(tableName, commid){
|
||||
def sql = "delete from "+tableName+" where commid='${commid}'";
|
||||
sqlInstance.execute(sql)
|
||||
println "删除:"+tableName+" "+commid
|
||||
}
|
||||
|
||||
def getYouDaData() {
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("2021-07-01", "yyyy-MM-dd"))
|
||||
def endDate = DateUtils.parseFromTightByPattern("2021-08-01", "yyyy-MM-dd")
|
||||
def q = new BasicDBObject()
|
||||
// q.put("premisesName","共富一村")
|
||||
q.put("contractTime", new BasicDBObject('$gte', startDate));//.append('$lt', endDate)
|
||||
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
|
||||
def trades = []
|
||||
for(def db : dbcursor){
|
||||
def map = [:]
|
||||
map._id=db._id.toString()
|
||||
map.city = "上海"
|
||||
map.region = db.region
|
||||
if(db.districtName && db.districtName!="null"){
|
||||
map.district_name=db.districtName
|
||||
}else if(db.address){
|
||||
map.district_name=db.address
|
||||
}
|
||||
|
||||
map.area = db.area
|
||||
map.trade_date = db.contractTime
|
||||
map.avg_price= db.price
|
||||
map.estate_type = db.houseType
|
||||
trades << map
|
||||
}
|
||||
return trades
|
||||
}
|
||||
|
||||
def findDealIds(city){
|
||||
def sql = "select * from bigdata_deal_gj where dealdate>=to_date('2021-07-01','yyyy-mm-dd') and dealdate<to_date('2021-08-01','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd')
|
||||
def ids = []
|
||||
def detailInfos = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
if(!ids.contains(r.dealid)){
|
||||
ids << r.dealid
|
||||
|
||||
def region = RegionData.formatRegion(r.cityname, r.distname)
|
||||
def key=r.cityname+"##"+region+"##"+r.commname+"##"+r.buildarea.toDouble()+"##"+r.dealunitprice+"##"+DateUtils.formatDateByPattern(r.dealdate,"yyyy-MM-dd")
|
||||
if(!detailInfos.contains(key)){
|
||||
detailInfos << key
|
||||
}
|
||||
}
|
||||
});
|
||||
def rMap = [:]
|
||||
rMap.ids = ids
|
||||
rMap.detail = detailInfos
|
||||
return rMap
|
||||
}
|
||||
|
||||
|
||||
public static def findTradesDetail(city, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
// q.put("district_name","百合花园")
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def getDistrictStatByCity(city, weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
def dbcursor = DIST_SELL_STAT.find(q)
|
||||
def map = new ConcurrentHashMap()
|
||||
for(def d : dbcursor){
|
||||
map.put(d.region+"@"+d.district_name, d.avg_price)
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def getAllDistrict(city){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name", "alias"])
|
||||
def map = [:]
|
||||
for(def d : dists){
|
||||
map.put(d.city+"##"+d.region+"##"+d.name, d._id.toString())
|
||||
def alias = d.alias
|
||||
if(alias){
|
||||
for(def name : alias){
|
||||
map.put(d.city+"##"+d.region+"##"+name, d._id.toString())
|
||||
}
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+111
-1
@@ -1,13 +1,18 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.ExcelUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
@@ -16,6 +21,7 @@ import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class UpdateHouseRegion {
|
||||
|
||||
@@ -37,11 +43,89 @@ class UpdateHouseRegion {
|
||||
|
||||
static main(args) {
|
||||
def task = new UpdateHouseRegion()
|
||||
task.updateSeawaterRegion()
|
||||
// task.updateSeawaterRegion()
|
||||
// task.updateDealRegion()
|
||||
// task.updateThirdPartyRegion()
|
||||
task.updateNoMatchDealRegion()
|
||||
// task.addAlias()
|
||||
// task.addFenQi()
|
||||
}
|
||||
|
||||
def addFenQi(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district",["city","region","name","alias"])
|
||||
addFenQiAlias(dists)
|
||||
println city +" 处理完成"
|
||||
}
|
||||
}
|
||||
|
||||
def addFenQiAlias(dists){
|
||||
for(def dist : dists){
|
||||
def s1 = DataUtil.match(dist.name, Pattern.compile("[一|二|三|四|五|六|七|八|九|十|十一|十二|十三|十四|十五|十六|十七|十八|\\w+](期|区)\$"))
|
||||
if(s1){
|
||||
def newName = dist.name.replace("Ⅰ","1").replace("Ⅱ","2").replace("A区","a区").replace("B区","b区").replace("C区","c区").replace("D区","d区").replace("E区","e区").replace("F区","f区").replace("一期","1期").replace("二期","2期").replace("三期","3期").replace("四期","4期").replace("五期","5期").replace("六期","6期").replace("七期","7期").replace("八期","8期").replace("九期","9期").replace("十期","10期").replace("十一期","11期").replace("十二期","12期")
|
||||
if(newName!=dist.name){
|
||||
def alias = dist.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", alias)
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
|
||||
println dist.city +" "+dist.name+" "+dist.alias+" 新:"+alias
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def addAlias(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district",["city","region","name","alias"])
|
||||
addDistricts(dists)
|
||||
println city +" 处理完成"
|
||||
}
|
||||
}
|
||||
|
||||
def addDistricts(dists){
|
||||
for(def dist : dists){
|
||||
def alias = dist.alias
|
||||
if(alias){
|
||||
def oldCount = alias.size()
|
||||
def newAlias = []
|
||||
for(def aliasName : alias){
|
||||
if(aliasName.contains("·") || aliasName.contains(".")){
|
||||
def newName = aliasName.replace("·","").replace(".","")
|
||||
newAlias << aliasName
|
||||
if(!newAlias.contains(newName)){
|
||||
if(newName!=dist.name){
|
||||
newAlias << newName
|
||||
}
|
||||
}
|
||||
}else{
|
||||
if(!newAlias.contains(aliasName)) {
|
||||
newAlias << aliasName
|
||||
}
|
||||
}
|
||||
}
|
||||
def newCount = newAlias.size()
|
||||
if(oldCount && alias!=newAlias){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", newAlias)
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
|
||||
println dist.city +" "+dist.name+" "+dist.alias+" 新:"+newAlias
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateThirdPartyRegion(){
|
||||
def city = "东莞"
|
||||
def rets = findThirdPartyPrice(city)
|
||||
@@ -121,6 +205,32 @@ class UpdateHouseRegion {
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def updateNoMatchDealRegion(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\成交未入库记录(7月全).xlsx",{d->
|
||||
if(!d['J']){
|
||||
def city = d['B']
|
||||
def region = d['C']
|
||||
def name = d['D']
|
||||
def rets = CommonDatautils.findDistrictsByName(city, name, "district")
|
||||
if (!rets) {
|
||||
rets = CommonDatautils.findDistrictByAlias(city, null, name, "district")
|
||||
}
|
||||
|
||||
if (rets.size() == 1) {
|
||||
def dist = rets[0]
|
||||
if (region != dist.region) {
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", dist.region)
|
||||
DEAL_PRICE_COLL.update(new BasicDBObject('_id', new ObjectId(d['A'])), new BasicDBObject('$set', udata))
|
||||
d['J'] = "原数据区域修改("+dist.region+")"
|
||||
println d['A']+" "+region +" "+dist.region
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
def updateSeawaterRegion(){
|
||||
def city = "东莞"
|
||||
def rets = findHouseSource(city, 20190701, 20211001)
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202107.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202109.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
@@ -135,7 +135,7 @@ rets << ["9
|
||||
def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM")
|
||||
ExcelUtils.write("d:\\数据\\福州\\多方均价对比统计数据"+curMonth+".xlsx",rets,"统计")
|
||||
def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )"
|
||||
|
||||
println s
|
||||
def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60"
|
||||
DingDingMessageUtil.sendTextMessage(access_token, s)
|
||||
|
||||
|
||||
@@ -45,18 +45,20 @@ class OpenUserBatch {
|
||||
}
|
||||
|
||||
def addShenZhenFjtUser(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\农行\\肇庆农行支行房价通账号.xlsx",0,1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\渤海\\开通线上评估客户经理名单.xlsx",0,1)
|
||||
def i = 0;
|
||||
for(def data : datas){
|
||||
i++;
|
||||
def id = 252580 + i;
|
||||
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+data[2]+"', '"+data[3]+"', '16', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2021-12-10 00:00:00', '0', '', NULL)";
|
||||
def id = 252608 + i;
|
||||
BigDecimal bd = new BigDecimal(data[3].toString());
|
||||
String mobile = bd.toPlainString();
|
||||
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '17', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2023-01-01 00:00:00', '0', '', NULL)";
|
||||
onLineSqlInstance.execute(sql);
|
||||
|
||||
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[1]+"', '"+data[0]+"', NULL, NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '王辰岑')";
|
||||
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '姚静梦')";
|
||||
onLineSqlInstance.execute(sql2)
|
||||
|
||||
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'肇庆','肇庆:')"
|
||||
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'上海','上海:')"
|
||||
onLineSqlInstance.execute(sql3)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
package com.fangjia.script.gj.ccb
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
|
||||
class CcbXykData {
|
||||
static main(args) {
|
||||
def task = new CcbXykData()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def districts = CommonDatautils.findAllDistrictByCity("ÉϺ£","district",["city","region","name","estate_charge"])
|
||||
for(def d : districts){
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,14 @@ class FileUtils {
|
||||
// file.delete();
|
||||
// }
|
||||
|
||||
if (!file.getParentFile().exists()) {
|
||||
boolean result = file.getParentFile().mkdirs();
|
||||
if (!result) {
|
||||
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
OutputStream out = new FileOutputStream(fileName, isAppend);
|
||||
def sb = new StringBuffer();
|
||||
|
||||
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
package com.fangjia.script.gj.location
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.NameProcess
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
class DistrictLocationUpdate {
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictLocationUpdate()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias", "lng", "lat"])
|
||||
|
||||
|
||||
updateLocation(city, dists);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def queryPoiMap(def city){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1")
|
||||
sqlBuffer.append(" and cityname = '" + city + "'")
|
||||
sqlBuffer.append(" and source = '¸ßµÂ'")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def retMap = [:]
|
||||
while (rs.next()) {
|
||||
def map = [:]
|
||||
map.city = rs.getString("cityname")
|
||||
map.region = rs.getString("adname")
|
||||
map.name = rs.getString("name")
|
||||
map.address = rs.getString("address")
|
||||
map.type = rs.getString("type")
|
||||
def location = rs.getObject("location")
|
||||
if(location){
|
||||
map.lng = location.x
|
||||
map.lat = location.y
|
||||
}
|
||||
|
||||
def name = NameProcess.nameStandard(map.city, map.region, map.name)
|
||||
if(name){
|
||||
retMap.put(map.region+"##"+name, map)
|
||||
}
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -521,6 +521,10 @@ class GaoDeUtils {
|
||||
return KeyList.get(count);
|
||||
}
|
||||
|
||||
static void main(def a){
|
||||
println search("如东县大豫镇豫东村委会","南通")
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
class MatchDistrict {
|
||||
static main(args) {
|
||||
def task = new MatchDistrict()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx"
|
||||
def city = "深圳"
|
||||
ExcelUtils.multiThreadingModify(filePath, { data ->
|
||||
if(data['A']){
|
||||
def region = RegionData.formatRegion(city, data['A'])
|
||||
if(region){
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(city, data['A'], data['C'], null);
|
||||
if(rMap){
|
||||
data['E'] = rMap.city
|
||||
data['F'] = rMap.region
|
||||
data['G'] = rMap.name
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 0);
|
||||
}
|
||||
}
|
||||
|
||||
//{
|
||||
// "_id" : ObjectId("607963dc5bb87e2a18f4dde6"),
|
||||
// "guiding_type" : NumberInt(0),
|
||||
// "estate_type" : "普通住宅",
|
||||
// "house_type" : NumberInt(0),
|
||||
// "c_date" : ISODate("2021-04-16T10:15:56.651+0000"),
|
||||
// "guiding_person" : "深圳房管局",
|
||||
// "guiding_date" : ISODate("2021-04-10T16:00:00.000+0000"),
|
||||
// "category" : "district",
|
||||
// "guiding_value" : NumberInt(48300),
|
||||
// "name" : "中粮云景国际",
|
||||
// "city" : "深圳",
|
||||
// "region" : "光明"
|
||||
//}
|
||||
@@ -1,33 +1,55 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.Mongo
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
|
||||
//SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
||||
//Date date = new Date(1617211269868);
|
||||
//print dateFormat.format(date);
|
||||
DBCollection CITY_INFO_CONN = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_data_ana').getCollection('city_info');
|
||||
def path = "D:\\数据\\广发\\广东省小区及均价覆盖率统计.xlsx"
|
||||
ExcelUtils.modify(path,{d->
|
||||
def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["name"])
|
||||
d['G'] = districts.size()
|
||||
//DBCollection CITY_INFO_CONN = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_data_ana').getCollection('city_info');
|
||||
//def path = "D:\\数据\\广发\\广东省小区及均价覆盖率统计.xlsx"
|
||||
//ExcelUtils.modify(path,{d->
|
||||
// def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["name"])
|
||||
// d['G'] = districts.size()
|
||||
//
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("city", d['B'])
|
||||
// q.put("weekend", DateUtils.parseFromTightYYYYMMDD("2021-08-08"))
|
||||
// def obj = CITY_INFO_CONN.findOne(q);
|
||||
// if(obj){
|
||||
// d['H'] = obj.price_coverage*100
|
||||
// }
|
||||
//})
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['B'])
|
||||
q.put("weekend", DateUtils.parseFromTightYYYYMMDD("2021-08-08"))
|
||||
def obj = CITY_INFO_CONN.findOne(q);
|
||||
if(obj){
|
||||
d['H'] = obj.price_coverage*100
|
||||
def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
|
||||
|
||||
def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'";
|
||||
def countMap = [:]
|
||||
onLineSqlInstance.eachRow(sql,{r->
|
||||
def addTime = r.visit_time
|
||||
def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM")
|
||||
def ips = countMap.get(month)
|
||||
if(!ips){
|
||||
ips = new HashSet();
|
||||
countMap.put(month, ips)
|
||||
}
|
||||
ips << r.ip
|
||||
});
|
||||
|
||||
countMap.each {k,v->
|
||||
println k+" "+v.size()
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
})
|
||||
|
||||
|
||||
|
||||
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
package com.fangjia.script.test
|
||||
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class RegionalismCodeStat {
|
||||
|
||||
DBCollection REGIONALISM_CODE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "regionalism_code", "192.168.0.235", 27018)
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new RegionalismCodeStat()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def provinces = ["上海市","北京市","天津市","重庆市","福建省","贵州省","湖南省","广东省","湖北省","安徽省","四川省","江苏省","吉林省","河北省","河南省","海南省","江西省","云南省","甘肃省","山东省","陕西省","浙江省","青海省","辽宁省","黑龙江省","山西省","西藏自治区","新疆维吾尔自治区","宁夏回族自治区","内蒙古自治区","广西壮族自治区"]
|
||||
// def provinces = ["宁夏回族自治区"]
|
||||
provinces.each {v->
|
||||
def sets = findCount(v)
|
||||
println v +" "+ sets.size()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def findCount(v){
|
||||
def q = new BasicDBObject()
|
||||
q.put("province", v)
|
||||
def dbcursor = REGIONALISM_CODE_COLL.find(q).batchSize(5000)
|
||||
def set = new HashSet()
|
||||
for(def obj : dbcursor){
|
||||
// if(obj.city=="省直辖县级行政区划" || obj.city=="自治区直辖县级行政区划"){
|
||||
// //set << obj.region
|
||||
// println "【"+obj.region+"】 过滤"
|
||||
// continue
|
||||
// }else{
|
||||
// if(obj.region && !set.contains(obj.region)){
|
||||
// set << obj.region
|
||||
// println "【"+obj.region+"】 add"
|
||||
// }
|
||||
// }
|
||||
|
||||
if(obj.juweihui){
|
||||
def k = obj.street+"##"+obj.juweihui
|
||||
if(!set.contains(k)){
|
||||
set << k
|
||||
// println "【"+k+"】 add"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// set.each {k->
|
||||
// println k
|
||||
// }
|
||||
return set;
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -17,7 +17,7 @@ class SynchDistrictPrice {
|
||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产"]
|
||||
def sourceMap = ["zhuge.com":"诸葛找房"]
|
||||
def sourceMap = ["cityhouse.cn":"城市房产"]//["zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
task.run()
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202107);
|
||||
def count2 = findTotalCount(source, 202108);
|
||||
def count1 = findTotalCount(source, 202108);
|
||||
def count2 = findTotalCount(source, 202109);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user