修改代码字符编码

This commit is contained in:
2026-02-24 17:53:50 +08:00
parent 2495fe5f6d
commit c8418c7d4b
5195 changed files with 121754 additions and 121754 deletions
+2 -2
View File
@@ -7,7 +7,7 @@
<version>0.0.1-SNAPSHOT</version>
<name>fangjia-script</name>
<properties>
<project.build.sourceEncoding>GBK</project.build.sourceEncoding>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<build>
<finalName>fangjia-script</finalName>
@@ -19,7 +19,7 @@
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>GBK</encoding>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<plugin>
@@ -10,17 +10,17 @@ import org.bson.types.ObjectId
import org.springframework.core.annotation.AnnotationUtils
/**
* 继承这个类,实现process(args)方法
* 写main方法
* 继承这个类,实现process(args)方法
* 写main方法
* static void main(args){
* runBatch(new MyBatch(), args)
* }
* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
* !!!!!任何由process方法抛出的异常都会被监测到,并发出错误警报!!!!!
* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
* !!!!!任何由process方法抛出的异常都会被监测到,并发出错误警报!!!!!
* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
*
* 也可以手动调用jms.monitor(name, message, exception)方法发出错误警报
* 在启动时添加 -DuseRollingPrintLog=true 的java参数可以生成滚动日志
* 也可以手动调用jms.monitor(name, message, exception)方法发出错误警报
* 在启动时添加 -DuseRollingPrintLog=true 的java参数可以生成滚动日志
* @author velna
*
*/
@@ -29,11 +29,11 @@ abstract class BaseBatch {
private static final Jms __jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
//private static final NacosProperties properties = NacosProperties.getInstance()
private static final BATCH_CONFIG = [singleProcess : false, //是否只可单进程运行,默认true
reprocessOnException : false, //在发生异常后是否重新执行,默认false
exceptionDelay : 10000, //在发生异常后延迟多少毫秒重新执行,默认10000
warnIfProcessNotExit : false, //对singleProcess的程序,如果发现有一个进程还在运行,则发出一个警报
isSystemExit : true //运行之后是否执行System.exit();
private static final BATCH_CONFIG = [singleProcess : false, //是否只可单进程运行,默认true
reprocessOnException : false, //在发生异常后是否重新执行,默认false
exceptionDelay : 10000, //在发生异常后延迟多少毫秒重新执行,默认10000
warnIfProcessNotExit : false, //对singleProcess的程序,如果发现有一个进程还在运行,则发出一个警报
isSystemExit : true //运行之后是否执行System.exit();
]
def logger = new PrintLogger(this.getClass())
@@ -41,16 +41,16 @@ abstract class BaseBatch {
private static final DBCollection TASK = MongoUtils.getCollectionByName("fangjia_batch", "task", "log.mongo.db.fangjia.com", 52002)
/**
* 执行batch
* @param args main方法的参数
* 执行batch
* @param args main方法的参数
*/
abstract void process(args);
/**
* 运行batch
* @param batch 要运行的batch
* @param args 传给process方法的参数,通常为main方法的参数
* @param batchConfig 见BATCH_CONFIG定义
* 运行batch
* @param batch 要运行的batch
* @param args 传给process方法的参数,通常为main方法的参数
* @param batchConfig 见BATCH_CONFIG定义
*/
static def runBatch(BaseBatch batch, args, batchConfig = BATCH_CONFIG){
def myBatchConfig = [:]
@@ -74,7 +74,7 @@ abstract class BaseBatch {
}
}
task.describe = getBatchDescribe(batch)
task.status = "运行中"
task.status = "运行中"
TASK.save(task)
checkUseRollingPrintLog(batch.getClass().getSimpleName())
if(myBatchConfig.singleProcess){
@@ -96,7 +96,7 @@ abstract class BaseBatch {
exception = true
e.printStackTrace()
__jms.monitor(batchName, '', extraData, e)
def msg = "脚本发生异常错误 BatchName:${batchName}\t"
def msg = "脚本发生异常错误 BatchName:${batchName}\t"
def batchOwner = getBatchOwner(batch)
if(batchOwner && batchOwner.size() > 0){
def owner = null
@@ -134,9 +134,9 @@ abstract class BaseBatch {
sleep(myBatchConfig.exceptionDelay)
}
if(exception){
task.status = "发生异常"
task.status = "发生异常"
}else{
task.status = "成功"
task.status = "成功"
}
task.finish_date = new Date()
TASK.save(task)
@@ -167,10 +167,10 @@ abstract class BaseBatch {
ret.put(it.name(), it.toMap())
}
}else{
bb.logWarn('!!!请使用'+BatchOwner.class.getName()+'注解标明本Batch的拥有者!!!')
bb.logWarn('!!!请使用'+BatchOwner.class.getName()+'注解标明本Batch的拥有者!!!')
}
}catch(Exception e) {
bb.logWarn('!!!请使用'+BatchOwner.class.getName()+'注解标明本Batch的拥有者!!!', e)
bb.logWarn('!!!请使用'+BatchOwner.class.getName()+'注解标明本Batch的拥有者!!!', e)
}
return ret
}
@@ -187,10 +187,10 @@ abstract class BaseBatch {
}
}
} else {
bb.logWarn('!!!请使用' + BatchOwner.class.getName() + '注解标明本Batch的拥有者!!!')
bb.logWarn('!!!请使用' + BatchOwner.class.getName() + '注解标明本Batch的拥有者!!!')
}
} catch (Exception e) {
bb.logWarn('!!!解析BatchOwner失败!!!', e)
bb.logWarn('!!!解析BatchOwner失败!!!', e)
}
return phones[0] ? phones[0] : ""
}
@@ -203,7 +203,7 @@ abstract class BaseBatch {
return annotation.describe()
}
}catch(Exception e) {
bb.logWarn('!!!请使用'+BatchOwner.class.getName()+'注解标明本Batch的拥有者!!!', e)
bb.logWarn('!!!请使用'+BatchOwner.class.getName()+'注解标明本Batch的拥有者!!!', e)
}
return null
}
@@ -111,7 +111,7 @@ import org.apache.poi.hssf.usermodel.HSSFSheet
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Row
import org.apache.poi.ss.usermodel.Sheet;
//导出开盘时间需要调查的数据
//导出开盘时间需要调查的数据
class ExportDistrictData {
private final static String datastoreDomain = "seaweed.db.fangjia.com"
@@ -167,13 +167,13 @@ class ExportDistrictData {
}
if(site.size() > 50){
sbu.append("总量"+"\t");
sbu.append("总量"+"\t");
sbu.append(total+"\t");
sbu.append("开盘时间不存在数量"+"\t");
sbu.append("开盘时间不存在数量"+"\t");
sbu.append(site.size()+"\n");
sbu2.append("总量"+"\t");
sbu2.append("总量"+"\t");
sbu2.append(total+"\t");
sbu2.append("开盘时间不存在数量"+"\t");
sbu2.append("开盘时间不存在数量"+"\t");
sbu2.append(site.size()+"\n");
for(int i = 0; i<50 ; i++){
int max = site.size()-1;
@@ -213,7 +213,7 @@ class ExportDistrictData {
public static def getAllCitiesObject() {
DBObject q = new BasicDBObject()
q.put('status', new BasicDBObject('$lte', 2))
q.put('display', "荆州")
q.put('display', "荆州")
DBObject sq = new BasicDBObject()
sq.put('city_grade', 1)
sq.put('sequence', 1)
@@ -76,10 +76,10 @@ class FccsNewHouse {
BasicDBObject query = new BasicDBObject()
query.put("url", detailUrl)
query.put("city", "银川")
query.put("city", "银川")
query.put("category", "district")
query.put("site_name", "中国房产超市网")
query.put("page_name", "简介")
query.put("site_name", "中国房产超市网")
query.put("page_name", "简介")
DBObject entity = webpage.findOne(query)
if (entity != null){
DBObject data = new BasicDBObject()
@@ -90,10 +90,10 @@ class FccsNewHouse {
} else {
DBObject data = new BasicDBObject()
data.put("url", detailUrl)
data.put("city", "银川")
data.put("city", "银川")
data.put("category", "district")
data.put("site_name", "中国房产超市网")
data.put("page_name", "简介")
data.put("site_name", "中国房产超市网")
data.put("page_name", "简介")
data.put("content", DataUtil.zip(detailContent))
data.put("c_date", new Date())
data.put("m_date", new Date())
@@ -107,6 +107,6 @@ class FccsNewHouse {
e.printStackTrace();
}
}
println "总共新增${countSave}总共更新${countUpdate}"
println "总共新增${countSave}总共更新${countUpdate}"
}
}
@@ -61,10 +61,10 @@ class FccsSell {
// file << detailContent
BasicDBObject query = new BasicDBObject()
query.put("url", detailUrl)
query.put("city", "银川")
query.put("city", "银川")
query.put("category", "district")
query.put("site_name", "中国房产超市网")
query.put("page_name", "概况")
query.put("site_name", "中国房产超市网")
query.put("page_name", "概况")
DBObject entity = webpage.findOne(query)
if (entity != null){
DBObject data = new BasicDBObject()
@@ -75,10 +75,10 @@ class FccsSell {
} else {
DBObject data = new BasicDBObject()
data.put("url", detailUrl)
data.put("city", "银川")
data.put("city", "银川")
data.put("category", "district")
data.put("site_name", "中国房产超市网")
data.put("page_name", "概况")
data.put("site_name", "中国房产超市网")
data.put("page_name", "概况")
data.put("content", DataUtil.zip(detailContent))
data.put("c_date", new Date())
data.put("m_date", new Date())
@@ -92,7 +92,7 @@ class FccsSell {
e.printStackTrace();
}
}
println "总共新增${countSave}总共更新${countUpdate}"
println "总共新增${countSave}总共更新${countUpdate}"
}
}
@@ -16,23 +16,23 @@ class Enlight {
private static final DBCollection COMMAND_COLL = MongoUtils.getCollectionByName("fangjia_log", "crawl_command", "log.mongo.db.fangjia.com");
/*
public static final def JOBS = [
'5588f2f7e4b0a7ef70e98edb' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'赶集', cat:'rent'],//赶集
'5588ec3be4b0ffa7d5e23dad' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'58同城', cat:'rent'],//58同城
'5588fbfce4b0a7ef70e98ef3' : [isEncodingUrl:true, encoding:'GBK', enable:true, jobName:'搜房', cat:'rent'],//搜房
'5589057ee4b0a7ef70e98f48' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'安居客', cat:'rent'],//安居客
'55890362e4b0a7ef70e98f36' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'我爱我家', cat:'rent'],//我爱我家
'55890f74e4b0a7ef70e98f87' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'515房产网', cat:'rent'],//515房产网
'55892150e4b0a7ef70e98fc4' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'带看网', cat:'rent'],//带看网
'558923f5e4b0a7ef70e98fdb' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'购房者', cat:'rent'],//购房者
'5589256de4b0a7ef70e98ff3' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'城市房产网', cat:'rent'],//城市房产网
'558925bce4b0a7ef70e9903b' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'百姓网', cat:'rent'],//百姓网
'563c202ee4b0126639288c3f' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'新浪', cat:'rent'],//新浪
'5588f2f7e4b0a7ef70e98edb' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'赶集', cat:'rent'],//赶集
'5588ec3be4b0ffa7d5e23dad' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'58同城', cat:'rent'],//58同城
'5588fbfce4b0a7ef70e98ef3' : [isEncodingUrl:true, encoding:'GBK', enable:true, jobName:'搜房', cat:'rent'],//搜房
'5589057ee4b0a7ef70e98f48' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'安居客', cat:'rent'],//安居客
'55890362e4b0a7ef70e98f36' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'我爱我家', cat:'rent'],//我爱我家
'55890f74e4b0a7ef70e98f87' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'515房产网', cat:'rent'],//515房产网
'55892150e4b0a7ef70e98fc4' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'带看网', cat:'rent'],//带看网
'558923f5e4b0a7ef70e98fdb' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'购房者', cat:'rent'],//购房者
'5589256de4b0a7ef70e98ff3' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'城市房产网', cat:'rent'],//城市房产网
'558925bce4b0a7ef70e9903b' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'百姓网', cat:'rent'],//百姓网
'563c202ee4b0126639288c3f' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'新浪', cat:'rent'],//新浪
]
*/
static main(args) {
/*
File f = new File("D:/乌鲁木齐.txt")
File f = new File("D:/乌鲁木齐.txt")
JOBS.each {jobId, conf->
if(!conf.get('enable')) {
return false;
@@ -49,29 +49,29 @@ class Enlight {
vars = "keyword:${line}";
}
String conditions = "city:乌鲁木齐";
String conditions = "city:乌鲁木齐";
JSONObject res = addTemplateJobUrl(jobId, vars, conditions)
}
}
*/
def list = ['昆重小区',
'邮政小区',
'中央华冠',
'瑞源棕榈泉',
'卧云仙居',
'隆格兰花园',
'和风颐园',
'半山七哩溪',
'培星东岸',
'小吉坡小区',
'水电设计院小区',
def list = ['昆重小区',
'邮政小区',
'中央华冠',
'瑞源棕榈泉',
'卧云仙居',
'隆格兰花园',
'和风颐园',
'半山七哩溪',
'培星东岸',
'小吉坡小区',
'水电设计院小区',
]
list.each {item->
def encoded = URLEncoder.encode(item, "UTF-8")
addTemplateJobUrl("5588ec3be4b0ffa7d5e23dad","keyword:${encoded}","city:昆明")
addTemplateJobUrl("5588ec3be4b0ffa7d5e23dad","keyword:${encoded}","city:昆明")
}
@@ -79,11 +79,11 @@ class Enlight {
/**
* 向抓取系统添加抓取任务
* @param jobId 任务号
* @param vars 变量
* @param conditions 查询条件
* @return 200:成功 100:任务已停止 >400任务号错误或不存在 -1:其他错误
* 向抓取系统添加抓取任务
* @param jobId 任务号
* @param vars 变量
* @param conditions 查询条件
* @return 200:成功 100:任务已停止 >400任务号错误或不存在 -1:其他错误
*/
protected static JSONObject addTemplateJobUrl(String jobId, String vars, String conditions) {
def result = ATTEMIS_HTTP.invoke("/api/addTemplateJobUrl", [jobId : jobId, vars: vars, conditions:conditions], 'UTF-8')
@@ -16,8 +16,8 @@ class HeExtends {
static main(args) {
DBCollection COMMAND_COLL = MongoUtils.getCollectionByName("fangjia_log", "crawl_command_monitor", "log.mongo.db.fangjia.com");
DBObject fields = new BasicDBObject()
fields.put("city","³¤É³")
fields.put("keyword", "ÁúÌÁÐ¡Çø")
fields.put("city","长沙")
fields.put("keyword", "龙塘小区")
fields.put("job_id", "559f237de4b0e6f33e2bbb68")
fields.put("session_id", "56726578e4b07da06f2efd6b")
DBCursor cursor = COMMAND_COLL.find(fields)
@@ -12,34 +12,34 @@ class HeTests {
public static final def JOBS = [
'559f237de4b0e6f33e2bbb68' : [isEncodingUrl:true, encoding:'GBK', enable:true, jobName:'搜房', cat:'sell'],//搜房
'559f277ee4b0133c041de396' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'58同城', cat:'sell'],//58同城
'559f288ee4b0133c041de3b3' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'赶集网', cat:'sell'],//赶集网
'559f2a7be4b0133c041de3d6' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'安居客', cat:'sell'],//安居客
'559f2dc6e4b0133c041de42a' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'搜狐焦点', cat:'sell'],//搜狐焦点
'559f309ce4b0133c041dea84' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'链家在线', cat:'sell'],//链家在线
'559f31c8e4b0133c041df42a' : [isEncodingUrl:true, encoding:'GBK', enable:true, jobName:'新浪乐居', cat:'sell'],//新浪乐居
'559f32bbe4b0133c041dfaf7' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'吉屋网', cat:'sell'],//吉屋网
'559f37b6e4b0133c041e0e2d' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'515房产网', cat:'sell'],//515房产网
'559f3918e4b0133c041e1521' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'我爱我家', cat:'sell'],//我爱我家
'559f3a7de4b0133c041e1c41' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'楼盘网', cat:'sell'],//楼盘网
'559f3b52e4b0133c041e1f0f' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'好房在线', cat:'sell'],//好房在线
'559f3ca8e4b0133c041e247f' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'百度乐居', cat:'sell'],//百度乐居
'559f3d91e4b0133c041e27ae' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'带看网', cat:'sell'],//带看网
'559f3e6ce4b0133c041e29f4' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'百姓网', cat:'sell'],//百姓网
'559f3f61e4b0133c041e2cce' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'城市房产网', cat:'sell'],//城市房产网
'559f237de4b0e6f33e2bbb68' : [isEncodingUrl:true, encoding:'GBK', enable:true, jobName:'搜房', cat:'sell'],//搜房
'559f277ee4b0133c041de396' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'58同城', cat:'sell'],//58同城
'559f288ee4b0133c041de3b3' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'赶集网', cat:'sell'],//赶集网
'559f2a7be4b0133c041de3d6' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'安居客', cat:'sell'],//安居客
'559f2dc6e4b0133c041de42a' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'搜狐焦点', cat:'sell'],//搜狐焦点
'559f309ce4b0133c041dea84' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'链家在线', cat:'sell'],//链家在线
'559f31c8e4b0133c041df42a' : [isEncodingUrl:true, encoding:'GBK', enable:true, jobName:'新浪乐居', cat:'sell'],//新浪乐居
'559f32bbe4b0133c041dfaf7' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'吉屋网', cat:'sell'],//吉屋网
'559f37b6e4b0133c041e0e2d' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'515房产网', cat:'sell'],//515房产网
'559f3918e4b0133c041e1521' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'我爱我家', cat:'sell'],//我爱我家
'559f3a7de4b0133c041e1c41' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'楼盘网', cat:'sell'],//楼盘网
'559f3b52e4b0133c041e1f0f' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'好房在线', cat:'sell'],//好房在线
'559f3ca8e4b0133c041e247f' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'百度乐居', cat:'sell'],//百度乐居
'559f3d91e4b0133c041e27ae' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'带看网', cat:'sell'],//带看网
'559f3e6ce4b0133c041e29f4' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'百姓网', cat:'sell'],//百姓网
'559f3f61e4b0133c041e2cce' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'城市房产网', cat:'sell'],//城市房产网
'5588f2f7e4b0a7ef70e98edb' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'赶集', cat:'rent'],//赶集
'5588ec3be4b0ffa7d5e23dad' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'58同城', cat:'rent'],//58同城
'5588fbfce4b0a7ef70e98ef3' : [isEncodingUrl:true, encoding:'GBK', enable:true, jobName:'搜房', cat:'rent'],//搜房
'5589057ee4b0a7ef70e98f48' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'安居客', cat:'rent'],//安居客
'55890362e4b0a7ef70e98f36' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'我爱我家', cat:'rent'],//我爱我家
'55890f74e4b0a7ef70e98f87' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'515房产网', cat:'rent'],//515房产网
'55892150e4b0a7ef70e98fc4' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'带看网', cat:'rent'],//带看网
'558923f5e4b0a7ef70e98fdb' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'购房者', cat:'rent'],//购房者
'5589256de4b0a7ef70e98ff3' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'城市房产网', cat:'rent'],//城市房产网
'558925bce4b0a7ef70e9903b' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'百姓网', cat:'rent'],//百姓网
'563c202ee4b0126639288c3f' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'新浪', cat:'rent'],//新浪
'5588f2f7e4b0a7ef70e98edb' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'赶集', cat:'rent'],//赶集
'5588ec3be4b0ffa7d5e23dad' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'58同城', cat:'rent'],//58同城
'5588fbfce4b0a7ef70e98ef3' : [isEncodingUrl:true, encoding:'GBK', enable:true, jobName:'搜房', cat:'rent'],//搜房
'5589057ee4b0a7ef70e98f48' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'安居客', cat:'rent'],//安居客
'55890362e4b0a7ef70e98f36' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'我爱我家', cat:'rent'],//我爱我家
'55890f74e4b0a7ef70e98f87' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'515房产网', cat:'rent'],//515房产网
'55892150e4b0a7ef70e98fc4' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'带看网', cat:'rent'],//带看网
'558923f5e4b0a7ef70e98fdb' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'购房者', cat:'rent'],//购房者
'5589256de4b0a7ef70e98ff3' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'城市房产网', cat:'rent'],//城市房产网
'558925bce4b0a7ef70e9903b' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'百姓网', cat:'rent'],//百姓网
'563c202ee4b0126639288c3f' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'新浪', cat:'rent'],//新浪
]
public static void main(String[] args) {
@@ -19,7 +19,7 @@ class IntroTest {
// }
// newList.each { item -> println item }
Table table = DatabaseBuilder.open(new File("d:\\profile.accdb")).getTable("»ù±¾²éѯ");
Table table = DatabaseBuilder.open(new File("d:\\profile.accdb")).getTable("基本查询");
for(Row row : table) {
String name = row.get("name").toString();
@@ -20,7 +20,7 @@ class LatitudeTest {
def suspectedListNames = []
def filePath = "d:\\suspected.xls"
field.append("name",1)
query.append("city","ÉϺ£")
query.append("city","上海")
field.append("region", 1)
field.append("value_type", 5)
// seaweed.find(query, field).limit(10).each {item ->
@@ -167,7 +167,7 @@ class LoudongOnce {
Iterator<Element> iter = elements.iterator()
while(iter.hasNext()) {
Element ele = iter.next()
if(ele.html().indexOf("ÏêÇé") > -1) {
if(ele.html().indexOf("详情") > -1) {
continue;
}
String a = ele.getElementsByAttribute("href").toString();
File diff suppressed because it is too large Load Diff
@@ -125,23 +125,23 @@ public class Polygon {
int nCross = 0;
for (int i = 0; i < verticesCount; ++ i) {
Mpoint p1 = vertices.get(i);
Mpoint p2 = vertices.get((i + 1) % verticesCount);// 求解 y=p.y 与 p1 p2 的交点
if ( p1.getMapLoationY() == p2.getMapLoationY() ) { // p1p2 与 y=p0.y平行
Mpoint p2 = vertices.get((i + 1) % verticesCount);// 求解 y=p.y 与 p1 p2 的交点
if ( p1.getMapLoationY() == p2.getMapLoationY() ) { // p1p2 与 y=p0.y平行
continue;
}
if ( lat < Math.min(p1.getY(), p2.getY()) ) { // 交点在p1p2延长线上
if ( lat < Math.min(p1.getY(), p2.getY()) ) { // 交点在p1p2延长线上
continue;
}
if ( lat >= Math.max(p1.getY(), p2.getY()) ) { // 交点在p1p2延长线上
if ( lat >= Math.max(p1.getY(), p2.getY()) ) { // 交点在p1p2延长线上
continue;
}
// 求交点的 X 坐标
// 求交点的 X 坐标
double x = (lat - p1.getMapLoationY()) * (p2.getMapLoationX() - p1.getMapLoationX()) / (p2.getMapLoationY() - p1.getMapLoationY()) + p1.getMapLoationX();
if ( x > lng) { // 只统计单边交点
if ( x > lng) { // 只统计单边交点
nCross++;
}
}
// 单边交点为偶数,点在多边形之外
// 单边交点为偶数,点在多边形之外
return (nCross%2==1);
}
@@ -13,25 +13,25 @@ class PoseidonOnce {
static def poseidonConn = Sql.newInstance("jdbc:mysql://192.168.0.132:4306/poseidon?useUnicode=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver")
static def soufunRegionMap = [
"030" : "宝山",
"020" : "长宁",
"0996" : "崇明",
"032" : "奉贤",
"023" : "虹口",
"024" : "黄埔",
"029" : "嘉定",
"035" : "金山",
"021" : "静安",
"022" : "卢湾",
"018" : "闵行",
"031" : "青浦",
"0586" : "松江",
"019" : "徐汇",
"026" : "杨浦",
"027" : "闸北",
"01046" : "上海周边",
"028" : "普陀",
"025" : "浦东"
"030" : "宝山",
"020" : "长宁",
"0996" : "崇明",
"032" : "奉贤",
"023" : "虹口",
"024" : "黄埔",
"029" : "嘉定",
"035" : "金山",
"021" : "静安",
"022" : "卢湾",
"018" : "闵行",
"031" : "青浦",
"0586" : "松江",
"019" : "徐汇",
"026" : "杨浦",
"027" : "闸北",
"01046" : "上海周边",
"028" : "普陀",
"025" : "浦东"
]
public static void main(String[] args) {
@@ -111,7 +111,7 @@ class PoseidonOnce {
}
/**
* 从crawl_request移除数据
* 从crawl_request移除数据
* @return
*/
public static def removeDataFromCrawlRequest() {
@@ -19,16 +19,16 @@ class SaveToWebpage {
println url
saveToWebpage(url,fileContents)
}
println "总共新增${countSave}总共更新${countUpdate}"
println "总共新增${countSave}总共更新${countUpdate}"
}
static void saveToWebpage(String detailUrl, String detailContent){
BasicDBObject query = new BasicDBObject()
query.put("url", detailUrl)
query.put("city", "嘉兴")
query.put("city", "嘉兴")
query.put("category", "district")
query.put("site_name", "中国房产超市网")
query.put("page_name", "概况")
query.put("site_name", "中国房产超市网")
query.put("page_name", "概况")
DBObject entity = webpage.findOne(query)
if (entity != null){
DBObject data = new BasicDBObject()
@@ -39,10 +39,10 @@ class SaveToWebpage {
} else {
DBObject data = new BasicDBObject()
data.put("url", detailUrl)
data.put("city", "嘉兴")
data.put("city", "嘉兴")
data.put("category", "district")
data.put("site_name", "中国房产超市网")
data.put("page_name", "概况")
data.put("site_name", "中国房产超市网")
data.put("page_name", "概况")
data.put("content", DataUtil.zip(detailContent))
data.put("c_date", new Date())
data.put("m_date", new Date())
@@ -11,7 +11,7 @@ class SellDataAddSighterBatch extends BaseBatch{
private static final DBCollection HOUSE_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
private static final DBCollection TOBEGRABBED = MongoUtils.getCollectionByName("fangjia_log", "artemis_message", "mongo.db.fangjia.com")
//假设在周一时运行
//假设在周一时运行
Date d = new Date() - 6;
@@ -26,12 +26,12 @@ class SellDataAddSighterBatch extends BaseBatch{
// String[] attrs = line.split("\t")
// println attrs[0] + ":" + attrs[1]
// DBObject addDbo = new BasicDBObject()
// addDbo.put("city","绍兴")
// addDbo.put("city","绍兴")
// addDbo.put("c_date", new Date())
// addDbo.put("name", attrs[1])
// addDbo.put("region",attrs[0])
// addDbo.put("status", 0)
// addDbo.put("user_name", "机器")
// addDbo.put("user_name", "机器")
// addDbo.put("category", "sell")
// TOBEGRABBED.save(addDbo)
// }
@@ -57,7 +57,7 @@ class SellDataAddSighterBatch extends BaseBatch{
addDbo.put("name", dbo.get('district_name'))
addDbo.put("region",dbo.get('region'))
addDbo.put("status", 0)
addDbo.put("user_name", "贺绿源")
addDbo.put("user_name", "贺绿源")
addDbo.put("category", "sell")
TOBEGRABBED.save(addDbo)
i++
@@ -21,7 +21,7 @@ class TransferTest {
static main(args) {
int c = 0
DBObject query = new BasicDBObject()
query.put("city","ÉîÛÚ")
query.put("city","深圳")
DBCursor dbCursor=LOUDONG_CONN_TEST.find(query)
dbCursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
while (dbCursor.hasNext()) {
@@ -29,7 +29,7 @@ class TransferTest {
c++
DBObject obj = dbCursor.next()
DBObject queryOnline = new BasicDBObject()
queryOnline.put("city","ÉîÛÚ")
queryOnline.put("city","深圳")
queryOnline.put("region", obj.get("region"))
queryOnline.put("name", obj.get("name"))
queryOnline.put("house_num", obj.get("house_num"))
@@ -60,7 +60,7 @@ class TransferTest {
}else
{
DBObject insertObj = new BasicDBObject()
insertObj.put("city", 'ÉîÛÚ')
insertObj.put("city", '深圳')
insertObj.put("name", obj.get("name"))
insertObj.put("region", obj.get("region"))
insertObj.put("c_date", obj.get("c_date"))
@@ -9,20 +9,20 @@ import com.mongodb.DBObject
class UpdateData {
public static final String[] estateType2s = [
"商业街商铺",
"市场类商铺",
"社区内铺",
"住宅底商",
"百货购物中心商铺",
"商业综合体",
"商务区商铺",
"交通设施商铺"
"商业街商铺",
"市场类商铺",
"社区内铺",
"住宅底商",
"百货购物中心商铺",
"商业综合体",
"商务区商铺",
"交通设施商铺"
];
static main(args) {
DSCollection ddColl = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
DBObject query = new BasicDBObject()
// query.put('city', '上海')
// query.put('city', '上海')
query.put('cat', 'shop')
query.put('visible', 0)
@@ -33,8 +33,8 @@ class UpdateData {
DBObject dbo = iter.next()
String et2 = dbo.get('estate_type2')
println dbo.get('name')
// if(et2.equals('内铺')) {
// et2 = '社区内铺'
// if(et2.equals('内铺')) {
// et2 = '社区内铺'
// }
if(estateType2Compare(et2)) {
DBObject uq = new BasicDBObject()
@@ -1,7 +1,7 @@
package com.fangjia.script
public enum Watcher {
//ÀëְͬÊÂ
//离职同事
LiuPeiPei('13621626685', 'wangwenyan@fangjia.com'),
YanZhou('13621626685','wangwenyan@fangjia.com'),
HuangXiaoHu('13621626685', 'wangwenyan@fangjia.com'),
@@ -18,7 +18,7 @@ public enum Watcher {
FangRui('17717680589','wangwenyan@fangjia.com'),
DuXiaoYu('13621626685', 'wangwenyan@fangjia.com'),
//ÔÚְͬÊÂ
//在职同事
LiXue('13311649646','wangwenyan@fangjia.com'),
WangWenYan('13916076378','wangwenyan@fangjia.com'),
GaoJuan('15021384995','gaojuan@fangjia.com'),
@@ -14,7 +14,7 @@ class AddMapData {
static main(args) {
DBObject obj = new BasicDBObject()
obj.put("cityCode",21);
obj.put("name", "Èýɳ");
obj.put("name", "三沙");
obj.put("lat",16.838987d);
obj.put("lng",112.340047d);
obj.put("zoom", 15);
@@ -43,16 +43,16 @@ class AddressParserService {
ret.cat = djson.cat
if (djson.precise == 0) {
// 匹配到小区
// 匹配到小区
ret.cla = djson.cat
ret.qw = djson.cat
if (ret.cat[0] == 'district') {
ret.catStr = '小区'
ret.catStr = '小区'
} else if (ret.cat[0] == 'office') {
ret.catStr = '写字楼'
ret.catStr = '写字楼'
} else if (ret.cat[0] == 'shop') {
ret.catStr = '商铺'
ret.catStr = '商铺'
}
} else if (djson.precise == 1) {
ret.cla = djson.cla
@@ -60,13 +60,13 @@ class AddressParserService {
for (c in ret.cla) {
if (isDistrict(c)) {
ret.catStr = '小区'
ret.catStr = '小区'
break
} else if (isOffice(c)) {
ret.catStr = '写字楼'
ret.catStr = '写字楼'
break
} else if (isShop(c)) {
ret.catStr = '商铺'
ret.catStr = '商铺'
break
}
}
@@ -80,9 +80,9 @@ class AddressParserService {
return ret
}
private final static List<String> DISTRICT_CLAS = Arrays.asList("地产小区", "小区/楼盘", "别墅");
private final static List<String> OFFICE_CLAS = Arrays.asList("商务大厦", "办公大厦");
private final static List<String> SHOP_CLAS = Arrays.asList("综合商场/购物中心", "购物");
private final static List<String> DISTRICT_CLAS = Arrays.asList("地产小区", "小区/楼盘", "别墅");
private final static List<String> OFFICE_CLAS = Arrays.asList("商务大厦", "办公大厦");
private final static List<String> SHOP_CLAS = Arrays.asList("综合商场/购物中心", "购物");
private boolean isDistrict(String cla) {
for (String c : DISTRICT_CLAS) {
@@ -129,10 +129,10 @@ class BaiduAPIService {
static main(args) {
BaiduAPIService s = new BaiduAPIService()
// println s.geocoder('上海', '一二八纪念路933弄')
// println s.geocoder('上海', '一二八纪念路933弄')
// println s.geoconv6To5('13510836.5,3661517.3')
println s.roundSearch(121.48175256075, 31.25314063543, 300, '学校', 10, 0, 1)
println s.roundSearch(121.48175256075, 31.25314063543, 300, '学校', 10, 0, 1)
}
}
@@ -11,13 +11,13 @@ class Bianmen {
static DSCollection ADDREPO = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.addrepo");
static main(args) {
// 处理边门
def p0 = Pattern.compile('.*[0-9一二三四五六七八九]号门[))]?$')
def p1 = Pattern.compile('.*[东南西北]+[1-9一二三四五六七八九]?[大小]?门[))]?$')
def p2 = Pattern.compile('.*[前后正边侧大小路]门[))]?$')
def p = Pattern.compile('.*[东南西北]+[1-9一二三四五六七八九]?[大小]?[前后]?(号)?(边)?门$')
println p.matcher("XXXX南一门").matches()
// DBObject d = new BasicDBObject('$set', new BasicDBObject('std_tag', '出入口'))
// 处理边门
def p0 = Pattern.compile('.*[0-9一二三四五六七八九]号门[))]?$')
def p1 = Pattern.compile('.*[东南西北]+[1-9一二三四五六七八九]?[大小]?门[))]?$')
def p2 = Pattern.compile('.*[前后正边侧大小路]门[))]?$')
def p = Pattern.compile('.*[东南西北]+[1-9一二三四五六七八九]?[大小]?[前后]?(号)?(边)?门$')
println p.matcher("XXXX南一门").matches()
// DBObject d = new BasicDBObject('$set', new BasicDBObject('std_tag', '出入口'))
//
// def index = 0
// ADDREPO.find().batchSize(100).bulk(false).each {
@@ -18,7 +18,7 @@ class CnTokenizer2 extends CnTokenizer {
private static Logger LOGGER = LoggerFactory.getLogger(CnTokenizer2.class);
/**
* 分词结果
* 分词结果
*
* @param k
* @return
@@ -83,7 +83,7 @@ class CnTokenizer2 extends CnTokenizer {
return Collections.EMPTY_LIST;
}
private final static Pattern ALPHA_PATTERN = Pattern.compile("[a-zA-Z\\d\\-负一二三四五六七八九十零两个百千万亿]+(#|弄|号楼?|[甲乙丙丁]?室|幢|橦|栋|座|口|楼|房|单元|区|层)?")
private final static Pattern ALPHA_PATTERN = Pattern.compile("[a-zA-Z\\d\\-负一二三四五六七八九十零两个百千万亿]+(#|弄|号楼?|[甲乙丙丁]?室|幢|橦|栋|座|口|楼|房|单元|区|层)?")
private static String fetchAlpha(List<Token> ret, String t) {
Matcher m = ALPHA_PATTERN.matcher(t);
def value = ""
@@ -7,7 +7,7 @@ import org.apache.poi.xssf.usermodel.XSSFFormulaEvaluator
import org.apache.poi.xssf.usermodel.XSSFWorkbook
/**
* apache poi excel解析器
* apache poi excel解析器
* Created by yk on 2014/4/24.
*/
class ExcelParser {
@@ -89,12 +89,12 @@ class ExcelParser {
try {
CellValue cellValue;
cellValue = evaluator.evaluate(cell);
switch (cellValue.getCellType()) { //判断公式类型
switch (cellValue.getCellType()) { //判断公式类型
case Cell.CELL_TYPE_BOOLEAN:
value = cellValue.getBooleanValue();
break;
case Cell.CELL_TYPE_NUMERIC:
// 处理日期
// 处理日期
if (DateUtil.isCellDateFormatted(cell)) {
value = cell.getDateCellValue();
} else {
@@ -136,7 +136,7 @@ class ExcelParser {
}
public static void main(String[]args) {
def filename = 'C:\\Users\\yk\\Desktop\\yongjia\\永嘉地址数据(上海)_V1.xlsx'
def filename = 'C:\\Users\\yk\\Desktop\\yongjia\\永嘉地址数据(上海)_V1.xlsx'
ExcelParser parser = new ExcelParser(filename)
def (headers, rows) = parser.parse()
println 'Headers'
@@ -23,8 +23,8 @@ class ExportBaiDuMapCrawlarResult {
}
def run(){
def citys = readTxt("e:\\数据导出\\城市列表\\citys.txt")
def keyword = "住宅区,小区,公寓,别墅"
def citys = readTxt("e:\\数据导出\\城市列表\\citys.txt")
def keyword = "住宅区,小区,公寓,别墅"
def rets = [];
def index = 0;
for(def city : citys){
@@ -39,7 +39,7 @@ class ExportBaiDuMapCrawlarResult {
rets << ret
println index +" " + ret
}
export(rets, "e:\\数据导出\\百度抓取\\百度抓取结果.xlsx")
export(rets, "e:\\数据导出\\百度抓取\\百度抓取结果.xlsx")
}
@@ -87,7 +87,7 @@ class ExportBaiDuMapCrawlarResult {
def WORK_BOOK = null;
try {
if (file.exists()) {
FileInputStream fs=new FileInputStream(path); //获取d://test.xls
FileInputStream fs=new FileInputStream(path); //获取d://test.xls
WORK_BOOK = WorkbookFactory.create(fs)
WORK_BOOK.setForceFormulaRecalculation(true)
sheet = WORK_BOOK.getSheetAt(0);
@@ -21,12 +21,12 @@ class InvokeHttpRequest {
private final static int reCount = 10
private final static List PROXY_HOSTS = [
// 物理机
// 物理机
new HttpHost("192.168.0.99", 3234),
new HttpHost("192.168.0.98", 3234),
new HttpHost("192.168.0.96", 3234)
// 虚拟机
// 虚拟机
// "192.168.0.102",
// "192.168.0.101",
// "192.168.0.103",
@@ -44,7 +44,7 @@ class InvokeHttpRequest {
private static synchronized HttpHost changeProxyHost() {
def now = new Date().getTime()
if (null != lastChangeTime && now - lastChangeTime < 120000) {
// 间隔小于2分钟的不处理
// 间隔小于2分钟的不处理
return
}
@@ -16,7 +16,7 @@ class Publish {
static main(args) {
def index = 0
// 按城市上线 289是城市代码 上海
// 按城市上线 289是城市代码 上海
DBCursor cursor = DB_3.find(new BasicDBObject('cityCode', 131)).batchSize(100)
cursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
cursor.each {
@@ -25,7 +25,7 @@ class Publish {
return
}
it.city = '北京'
it.city = '北京'
def orgRegion = it.region
it.region = RegionData.formatRegion(it.city, orgRegion)
@@ -8,14 +8,14 @@ class Test {
static main(args) {
// CnDictionary.init()
// List<Token> tokens = CnTokenizer2.tokenizer('上海市闵行长宁罗阳路莲花河畔景苑19号801室', true)
// List<Token> tokens = CnTokenizer2.tokenizer('上海市闵行长宁罗阳路莲花河畔景苑19号801室', true)
// println tokens
// Pattern p = Pattern.compile("[a-zA-Z\\d\\-负一二三四五六七八九十零两个百千万亿]+(#|弄|号楼?|[甲乙丙丁]?室|幢|橦|栋|座|口|楼|房|单元|区|层)?")
// Pattern p = Pattern.compile("[a-zA-Z\\d\\-负一二三四五六七八九十零两个百千万亿]+(#|弄|号楼?|[甲乙丙丁]?室|幢|橦|栋|座|口|楼|房|单元|区|层)?")
//
// BufferedReader reader = new File('C:\\Users\\yk\\Desktop\\yongjia\\永嘉上海0730.txt').newReader('UTF-8')
// BufferedReader reader = new File('C:\\Users\\yk\\Desktop\\yongjia\\永嘉上海0730.txt').newReader('UTF-8')
// reader.eachLine { line ->
// if (!line) {
// return
@@ -17,43 +17,43 @@ import com.mongodb.DBObject
import com.mongodb.Mongo
class Yongjia {
def final static QUANJIAO = ['0' : '0',
'1' : '1',
'2' : '2',
'3' : '3',
'4' : '4',
'5' : '5',
'6' : '6',
'7' : '7',
'8' : '8',
'9' : '9',
'A' : 'A',
'B' : 'B',
'C' : 'C',
'D' : 'D',
'E' : 'E',
'F' : 'F',
'G' : 'G',
'H' : 'H',
'I' : 'I',
'J' : 'J',
'K' : 'K',
'L' : 'L',
'M' : 'M',
'N' : 'N',
'O' : 'O',
'P' : 'P',
'Q' : 'Q',
'R' : 'R',
'S' : 'S',
'T' : 'T',
'U' : 'U',
'V' : 'V',
'W' : 'W',
'X' : 'X',
'Y' : 'Y',
'Z' : 'Z',
'#' : '#']
def final static QUANJIAO = ['0' : '0',
'1' : '1',
'2' : '2',
'3' : '3',
'4' : '4',
'5' : '5',
'6' : '6',
'7' : '7',
'8' : '8',
'9' : '9',
'A' : 'A',
'B' : 'B',
'C' : 'C',
'D' : 'D',
'E' : 'E',
'F' : 'F',
'G' : 'G',
'H' : 'H',
'I' : 'I',
'J' : 'J',
'K' : 'K',
'L' : 'L',
'M' : 'M',
'N' : 'N',
'O' : 'O',
'P' : 'P',
'Q' : 'Q',
'R' : 'R',
'S' : 'S',
'T' : 'T',
'U' : 'U',
'V' : 'V',
'W' : 'W',
'X' : 'X',
'Y' : 'Y',
'Z' : 'Z',
'#' : '#']
def final static QUANJIAOKEYS = QUANJIAO.keySet()
@@ -66,56 +66,56 @@ class Yongjia {
final static token = "FF231CF2F7268316D937374DB96AC91ED7478F663E9D233D7389B1F0A809038F59F52B983E627AD8E95FB644D8C43601E5FEE7F107B3BFD8F5AF2D73853591F481A34E26723B657C2DD2D05684DB12F69C37447B272F857EFE2BEC49BADB44A3A2DBDB7CDBE2AA25"
def static c00 = 0 // 两个地址都没匹配
def static c01 = 0 // 家庭地址没匹配,单位地址匹配
def static c10 = 0 // 家庭地址匹配,单位地址没匹配
def static c11 = 0 // 两个地址都匹配
def static c00 = 0 // 两个地址都没匹配
def static c01 = 0 // 家庭地址没匹配,单位地址匹配
def static c10 = 0 // 家庭地址匹配,单位地址没匹配
def static c11 = 0 // 两个地址都匹配
public static void main(args) {
ExcelParser parser = new ExcelParser('C:\\Users\\yk\\Desktop\\yongjia\\永嘉数据-0910.xlsx')
ExcelParser parser = new ExcelParser('C:\\Users\\yk\\Desktop\\yongjia\\永嘉数据-0910.xlsx')
def (headers, rows) = parser.parse()
/*
BufferedWriter writer = new File('C:\\Users\\yk\\Desktop\\yongjia\\test\\上海-0730.txt').newWriter('GBK')
BufferedWriter writer = new File('C:\\Users\\yk\\Desktop\\yongjia\\test\\上海-0730.txt').newWriter('GBK')
for(row in rows){
def map = parser.toMap(headers, row)
def a = replace(map['家庭地址'])
def a = replace(map['家庭地址'])
if (a && a.length() > 0)
writer.writeLine(a)
a = replace(map['单位地址'])
a = replace(map['单位地址'])
if (a && a.length() > 0)
writer.writeLine(a)
}
writer.close()
*/
File file = new File("C:\\Users\\yk\\Desktop\\yongjia\\永嘉地址数据(上海)-done.csv");
File file = new File("C:\\Users\\yk\\Desktop\\yongjia\\永嘉地址数据(上海)-done.csv");
if (file.exists()){
file.delete();
}
file.append("家庭地址,", CHARSET)
file.append("名称,", CHARSET)
file.append("类型,", CHARSET)
file.append("周边,", CHARSET)
file.append("均价,", CHARSET)
file.append("面积,", CHARSET)
file.append("估价,", CHARSET)
file.append("开发商,", CHARSET)
file.append("物业公司,", CHARSET)
file.append("小区户数,", CHARSET)
file.append("家庭地址,", CHARSET)
file.append("名称,", CHARSET)
file.append("类型,", CHARSET)
file.append("周边,", CHARSET)
file.append("均价,", CHARSET)
file.append("面积,", CHARSET)
file.append("估价,", CHARSET)
file.append("开发商,", CHARSET)
file.append("物业公司,", CHARSET)
file.append("小区户数,", CHARSET)
file.append("单位地址,", CHARSET)
file.append("名称,", CHARSET)
file.append("类型,", CHARSET)
file.append("上班距离 KM,", CHARSET)
file.append("单位地址,", CHARSET)
file.append("名称,", CHARSET)
file.append("类型,", CHARSET)
file.append("上班距离 KM,", CHARSET)
file.append(System.getProperty("line.separator"))
for(row in rows){
def map = parser.toMap(headers, row)
process(map, file, '上海')
process(map, file, '上海')
}
println c00
@@ -125,16 +125,16 @@ class Yongjia {
}
def static process(map, csvFile, city) {
def orgAddr1 = map['家庭地址']
def orgAddr1 = map['家庭地址']
println orgAddr1
def orgAddr2 = map['单位地址']
def orgAddr2 = map['单位地址']
def newAddr1 = replace(orgAddr1)
def newAddr2 = replace(orgAddr2)
// 家庭地址
// 家庭地址
csvFile.append("${orgAddr1},", CHARSET)
// 新旧地址都为空
// 新旧地址都为空
if (!newAddr1 && !newAddr2) {
csvFile.append(",", CHARSET)
csvFile.append(",", CHARSET)
@@ -161,14 +161,14 @@ class Yongjia {
def b0 = false
def b1 = false
if (newAddr1) {
// 匹配小区
// 匹配小区
start = System.currentTimeMillis()
parseRet = addressParserService.parseOneDistrict(city, newAddr1, 'district')
println '小区:' + (System.currentTimeMillis() - start)
println '小区:' + (System.currentTimeMillis() - start)
if (parseRet && parseRet.name) {
start = System.currentTimeMillis()
parseRet = getDistrictInfo(parseRet)
println '价格:' + (System.currentTimeMillis() - start)
println '价格:' + (System.currentTimeMillis() - start)
} else {
parseRet = addressParserService.parseOneDistrict(city, newAddr1, 'office')
@@ -188,17 +188,17 @@ class Yongjia {
// 经纬度
// 经纬度
start = System.currentTimeMillis()
latlng = baiduAPIService.geocoder(city, newAddr1)
println '经纬度:' + (System.currentTimeMillis() - start)
println '经纬度:' + (System.currentTimeMillis() - start)
if (latlng) {
// 周边
// 周边
start = System.currentTimeMillis()
def s1 = "小区: ${roundSerach(latlng[0], latlng[1], '小区')}\r\n\r\n"
def s2 = "办公楼: ${roundSerach(latlng[0], latlng[1], '写字楼')}\r\n\r\n"
def s3 = "银行: ${roundSerach(latlng[0], latlng[1], '银行')}"
println '周边:' + (System.currentTimeMillis() - start)
def s1 = "小区: ${roundSerach(latlng[0], latlng[1], '小区')}\r\n\r\n"
def s2 = "办公楼: ${roundSerach(latlng[0], latlng[1], '写字楼')}\r\n\r\n"
def s3 = "银行: ${roundSerach(latlng[0], latlng[1], '银行')}"
println '周边:' + (System.currentTimeMillis() - start)
csvFile.append("\"${s1 + s2 + s3}\",", CHARSET)
} else {
csvFile.append(",", CHARSET)
@@ -209,11 +209,11 @@ class Yongjia {
csvFile.append(",", CHARSET)
}
// 均价
// 均价
if(parseRet && parseRet.avgPrice){
csvFile.append(parseRet.avgPrice+ ",","gbk")
// 估价
// 估价
if (parseRet.area && parseRet.fangjiaTotalPrice) {
csvFile.append("${parseRet.area},","gbk")
csvFile.append("${parseRet.fangjiaTotalPrice},","gbk")
@@ -227,7 +227,7 @@ class Yongjia {
csvFile.append(",","gbk")
}
// 开发商、物业、绿化率、人口
// 开发商、物业、绿化率、人口
if (parseRet && parseRet.precise == 0) {
def district = SEAWEED.findOne(new BasicDBObject('city', parseRet.city).append('region', parseRet.region).append('name', parseRet.name).append('cat', parseRet.qw[0]))
if (district) {
@@ -245,7 +245,7 @@ class Yongjia {
csvFile.append(",", CHARSET)
}
// 单位地址
// 单位地址
csvFile.append("\"${orgAddr2}\",", CHARSET)
if (newAddr2) {
@@ -267,14 +267,14 @@ class Yongjia {
csvFile.append(",", CHARSET)
}
// 经纬度
// 经纬度
latlng2 = baiduAPIService.geocoder(city, newAddr2)
} else {
csvFile.append(",", CHARSET)
csvFile.append(",", CHARSET)
}
// 距离
// 距离
if (latlng && latlng2) {
def d = GeoUtils.calculateDistance(latlng[0], latlng[1], latlng2[0], latlng2[1])
@@ -313,9 +313,9 @@ class Yongjia {
for (s in QUANJIAOKEYS) {
str = str.replaceAll(s, QUANJIAO[s])
}
str = str.replaceAll('无+', '').replaceAll('^0$', '').replaceAll('^-+$', '')
.replaceAll('(上海)+', '上海').replaceAll('^上海$', '')
.replaceAll('【', '').replaceAll('\\*+', '').replaceAll('[\\s ]+', '')
str = str.replaceAll('无+', '').replaceAll('^0$', '').replaceAll('^-+$', '')
.replaceAll('(上海)+', '上海').replaceAll('^上海$', '')
.replaceAll('【', '').replaceAll('\\*+', '').replaceAll('[\\s ]+', '')
return str
}
@@ -344,7 +344,7 @@ class Yongjia {
}
}
//房价网估价
//房价网估价
// m.fangjiaTotalPrice = evaluation(m, token);
// if(!m.fangjiaTotalPrice && m.avgPrice){
// m.fangjiaTotalPrice = (m.avgPrice * m.area).toInteger();
@@ -352,7 +352,7 @@ class Yongjia {
//m.rentPrice = queryRentPriceRange(m.city, m.region, m.name, lastMonth);
//其它网站均价
//其它网站均价
// def otherWebsiteMap = CommonDatautils.queryOtherWebsitePrice(m.city, m.name);
// m.sunfunPrice = otherWebsiteMap.price_soufun
// m.anjukePrice = otherWebsiteMap.price_anjuke
@@ -391,7 +391,7 @@ class Yongjia {
}
//println "周边均价:" + totalPrice +"【${url_str}】"
//println "周边均价:" + totalPrice +"【${url_str}】"
return totalPrice;
}
@@ -407,7 +407,7 @@ class Yongjia {
conn.setRequestMethod("GET");
conn.doOutput = true
inputStream = conn.getInputStream();
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
while((line = buf.readLine())!=null){
sb.append(line);
}
@@ -28,25 +28,25 @@ class Yongjia2 {
final static token = "FF231CF2F7268316D937374DB96AC91ED7478F663E9D233D7389B1F0A809038F59F52B983E627AD8E95FB644D8C43601E5FEE7F107B3BFD8F5AF2D73853591F481A34E26723B657C2DD2D05684DB12F69C37447B272F857EFE2BEC49BADB44A3A2DBDB7CDBE2AA25"
public static void main(args) {
BufferedReader reader = new File('C:\\Users\\yk\\Desktop\\yongjia\\test\\上海-0730.txt').newReader('UTF-8')
BufferedReader reader = new File('C:\\Users\\yk\\Desktop\\yongjia\\test\\上海-0730.txt').newReader('UTF-8')
File file = new File("C:\\Users\\yk\\Desktop\\yongjia\\test\\永嘉地址数据(上海)-done.csv");
File file = new File("C:\\Users\\yk\\Desktop\\yongjia\\test\\永嘉地址数据(上海)-done.csv");
if (file.exists()){
file.delete();
}
file.append("地址,", CHARSET)
file.append("名称,", CHARSET)
file.append("类型,", CHARSET)
file.append("地址,", CHARSET)
file.append("楼号,", CHARSET)
file.append("单元号,", CHARSET)
file.append("楼层,", CHARSET)
file.append("室号,", CHARSET)
file.append("地址,", CHARSET)
file.append("名称,", CHARSET)
file.append("类型,", CHARSET)
file.append("地址,", CHARSET)
file.append("楼号,", CHARSET)
file.append("单元号,", CHARSET)
file.append("楼层,", CHARSET)
file.append("室号,", CHARSET)
file.append(System.getProperty("line.separator"))
reader.eachLine { line ->
process(line, file, '上海')
process(line, file, '上海')
}
}
@@ -58,11 +58,11 @@ class Yongjia2 {
return
}
// 家庭地址
// 家庭地址
csvFile.append("${orgAddr1},", CHARSET)
// 匹配小区
// 匹配小区
def parseRet = addressParserService.parseOneDistrict(city, orgAddr1, 'district')
if (parseRet && parseRet.name) {
@@ -123,7 +123,7 @@ class Yongjia2 {
}
}
//房价网估价
//房价网估价
// m.fangjiaTotalPrice = evaluation(m, token);
// if(!m.fangjiaTotalPrice && m.avgPrice){
// m.fangjiaTotalPrice = (m.avgPrice * m.area).toInteger();
@@ -131,7 +131,7 @@ class Yongjia2 {
//m.rentPrice = queryRentPriceRange(m.city, m.region, m.name, lastMonth);
//其它网站均价
//其它网站均价
// def otherWebsiteMap = CommonDatautils.queryOtherWebsitePrice(m.city, m.name);
// m.sunfunPrice = otherWebsiteMap.price_soufun
// m.anjukePrice = otherWebsiteMap.price_anjuke
@@ -170,7 +170,7 @@ class Yongjia2 {
}
//println "周边均价:" + totalPrice +"【${url_str}】"
//println "周边均价:" + totalPrice +"【${url_str}】"
return totalPrice;
}
@@ -186,7 +186,7 @@ class Yongjia2 {
conn.setRequestMethod("GET");
conn.doOutput = true
inputStream = conn.getInputStream();
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
while((line = buf.readLine())!=null){
sb.append(line);
}
@@ -10,7 +10,7 @@ import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
/**
* 地址分析任务
* 地址分析任务
*/
public class AddressAnalysisJob implements Runnable{
@@ -34,7 +34,7 @@ public class AddressAnalysisJob implements Runnable{
@Override
void run() {
long startTime = System.currentTimeMillis()
logger.info("Job ${taskId} 开始运行")
logger.info("Job ${taskId} 开始运行")
def alreadyAnalyAddressSeq = []
// def limit = ADDRESS_DETAIL.count(new BasicDBObject("task_id", taskId))
def limit = 100
@@ -83,12 +83,12 @@ public class AddressAnalysisJob implements Runnable{
}catch (Exception e){
e.printStackTrace()
}
logger.info("Job ${taskId} 运行结束")
logger.info("Job ${taskId} 运行结束")
long endTime = System.currentTimeMillis()
println("Job ${taskId} 运行结束,分析记录数:${total} 耗时:${endTime - startTime}毫秒")
println("Job ${taskId} 运行结束,分析记录数:${total} 耗时:${endTime - startTime}毫秒")
}
/**
* 清理历史数据
* 清理历史数据
* @param taskId
*/
void clearHistory(String taskId){
@@ -96,7 +96,7 @@ public class AddressAnalysisJob implements Runnable{
ANALYTICS_OVERVIEW.remove(new BasicDBObject("task_id", taskId))
}
/**
* 统计小区区域范围分布
* 统计小区区域范围分布
* @param taskId
*/
void regionDistrictCount(String taskId){
@@ -117,7 +117,7 @@ public class AddressAnalysisJob implements Runnable{
}
}
/**
* 统计匹配的小区数量
* 统计匹配的小区数量
* @param taskId
*/
void countMatchCount(String taskId){
@@ -129,7 +129,7 @@ public class AddressAnalysisJob implements Runnable{
}
/**
* 分析物业类型占比
* 分析物业类型占比
* @param taskId
*/
void analyEstateType(String taskId){
@@ -147,7 +147,7 @@ public class AddressAnalysisJob implements Runnable{
}
/**
* 分析总价分布
* 分析总价分布
* @param taskId
*/
void analyTotalPrice(String taskId){
@@ -196,20 +196,20 @@ public class AddressAnalysisJob implements Runnable{
ANALYTICS_OVERVIEW.update(new BasicDBObject("task_id", taskId), new BasicDBObject('$set', data), true, false)
}
/**
* 计算总价区间标签
* 计算总价区间标签
* @param totalPrice
* @return
*/
def calTotalPriceLabel(long totalPrice){
if(totalPrice > 2000 * 10000){
return "大于2000"
return "大于2000"
}
int offset = 100 * 10000
int multiple = (int)(totalPrice / offset)
return "${multiple * 100}-${(multiple + 1) * 100}"
}
/**
* 产生总价标签
* 产生总价标签
* @return
*/
def genTotalPriceSerial(){
@@ -217,7 +217,7 @@ public class AddressAnalysisJob implements Runnable{
for(int i = 0 ; i < 20; i++){
serial << "${i * 100}-${(i + 1) * 100}"
}
serial << "大于2000"
serial << "大于2000"
}
@@ -15,7 +15,7 @@ import org.apache.log4j.Logger
import java.util.regex.Pattern
/**
* 地址分析器
* 地址分析器
*/
public class AddressAnalyzer implements Runnable{
@@ -28,8 +28,8 @@ public class AddressAnalyzer implements Runnable{
final BasicDBObject addressDetail
final BasicDBObject task
final TASK_TYPE_FENG_KONG = "风控"
final TASK_TYPE_YING_XIAO = "营销"
final TASK_TYPE_FENG_KONG = "风控"
final TASK_TYPE_YING_XIAO = "营销"
public AddressAnalyzer(BasicDBObject addressDetail,BasicDBObject task){
this.addressDetail = addressDetail
@@ -58,10 +58,10 @@ public class AddressAnalyzer implements Runnable{
onFinish()
return
}
if(city.endsWith("市")){
city = city.replaceAll("市\$", "")
if(city.endsWith("市")){
city = city.replaceAll("市\$", "")
}
if(city != "上海"){
if(city != "上海"){
onFinish()
return
}
@@ -87,7 +87,7 @@ public class AddressAnalyzer implements Runnable{
double lng = matchResult.lng
double lat = matchResult.lat
BasicDBObject district = DbUtils.seaweed.findById(matchResult.getString("id"), new BasicDBObject("estate_type2", 1))
/**匹配接口使用缓存,一天内被删除的小区有可能查不到*/
/**匹配接口使用缓存,一天内被删除的小区有可能查不到*/
if(district == null){
onFinish()
return
@@ -135,13 +135,13 @@ public class AddressAnalyzer implements Runnable{
JSONObject newest = totalPrices.get(totalPrices.size() - 1)
def lastMonth = DateUtil.add(new Date(), Calendar.MONTH, -1).format("yyyy-MM")
def thisMonth = new Date().format("yyyy-MM")
if(newest.getString("date") == lastMonth || newest.getString("date") == thisMonth){//防止月初还未计算出均价
if(newest.getString("date") == lastMonth || newest.getString("date") == thisMonth){//防止月初还未计算出均价
result.totalPrice = newest.getLong("totalPrice")
}
}
def now = OpenApiClient.evaluateDistrictPrice(matchCity, matchRegion, name, result.area, estateType == "普通住宅"? 0 : 1, new Date())
def oneYearAge = OpenApiClient.evaluateDistrictPrice(matchCity, matchRegion, name, result.area, estateType == "普通住宅"? 0 : 1, DateUtil.add(new Date(), Calendar.YEAR, -1))
def now = OpenApiClient.evaluateDistrictPrice(matchCity, matchRegion, name, result.area, estateType == "普通住宅"? 0 : 1, new Date())
def oneYearAge = OpenApiClient.evaluateDistrictPrice(matchCity, matchRegion, name, result.area, estateType == "普通住宅"? 0 : 1, DateUtil.add(new Date(), Calendar.YEAR, -1))
if(now && oneYearAge){
def markUp = DataUtil.roundDouble((now.getLong("totalPrice") - oneYearAge.getLong("totalPrice")) * 100/oneYearAge.getLong("totalPrice"), 2)
result.append("mark_up", markUp)
@@ -161,11 +161,11 @@ public class AddressAnalyzer implements Runnable{
e.printStackTrace()
}
long end = System.currentTimeMillis()
// println("消耗时间:" + (end - s) + "毫秒")
// println("消耗时间:" + (end - s) + "毫秒")
}
/**
* 计算房屋面积
* 计算房屋面积
* @param city
* @param region
* @param name
@@ -230,7 +230,7 @@ public class AddressAnalyzer implements Runnable{
}
private String parseBuildingNumber(String buildingNumber){
Pattern pattern = Pattern.compile("\\w+(栋|幢|座)")
Pattern pattern = Pattern.compile("\\w+(栋|幢|座)")
def matcher = pattern.matcher(buildingNumber)
if(matcher.find()){
return matcher.group().substring(0, matcher.group().length() - 1)
@@ -239,7 +239,7 @@ public class AddressAnalyzer implements Runnable{
}
private String parseUnitNumber(String buildingNumber){
Pattern pattern = Pattern.compile("\\w+单元")
Pattern pattern = Pattern.compile("\\w+单元")
def matcher = pattern.matcher(buildingNumber)
if(matcher.find()){
return matcher.group().substring(0, matcher.group().length() - 2)
@@ -9,8 +9,8 @@ import com.mongodb.DBObject
/**
*
* 说明:地址标准化处理脚本
* @author 王文艳
* 说明:地址标准化处理脚本
* @author 王文艳
*
*/
class AddressStandardization {
@@ -22,13 +22,13 @@ class AddressStandardization {
static def handlers = [new ShengShiQuRemoveHandler(),new DuplicateRemovalHandler(),new SymbolRemoveHandler(),new RegionRemoveHandler()];
public static void main(String[] args){
districtAddressStandard("district","舟山");
districtAddressStandard("district","舟山");
}
/**
*
* @param category 建筑物类型
* @param city 城市
* @param category 建筑物类型
* @param city 城市
* @return
*/
static def districtAddressStandard(String category,String city){
@@ -45,9 +45,9 @@ class AddressStandardization {
fields.put("region", 1)
fields.put("name", 1)
fields.put("address", 1)
def file = new File("D:\\data\\logs\\地名修正日志.log");
file.append("开始执行地址标准化,开始时间:"+new Date().format("yyyy年MM月dd日 HH:mm:ss ")+"\n");
file.append("处理条件:"+query+"\n");
def file = new File("D:\\data\\logs\\地名修正日志.log");
file.append("开始执行地址标准化,开始时间:"+new Date().format("yyyy年MM月dd日 HH:mm:ss ")+"\n");
file.append("处理条件:"+query+"\n");
DSCursor cur = ddColl.find(query, fields)
Iterator<DBObject> iter = cur.iterator();
DBObject dbObject;
@@ -72,7 +72,7 @@ class AddressStandardization {
println ""+dbObject.get("_id")+"("+dbObject.get("name")+")"+":"+dbObject.get("address")+"---->" + request.getAddress();
}
}
file.append("地名标准化结束,结束时间:"+new Date().format("yyyy年MM月dd日 HH:mm:ss ")+"总记录:"+totalCount+",标准化记录:"+countRecord+",标准化百分比:"+((int)(countRecord/totalCount)*100))+""+"\n";
file.append("地名标准化结束,结束时间:"+new Date().format("yyyy年MM月dd日 HH:mm:ss ")+"总记录:"+totalCount+",标准化记录:"+countRecord+",标准化百分比:"+((int)(countRecord/totalCount)*100))+""+"\n";
}
}
@@ -43,13 +43,13 @@ class AddressToBaiduLngLat {
def task = new AddressToBaiduLngLat();
//task.run();
def citys = ["红河"]//task.readTxt("H:\\", "citys2.txt");//读文件//["遵义"]//CommonDatautils.findAllCity();//所有城市
def citys = ["红河"]//task.readTxt("H:\\", "citys2.txt");//读文件//["遵义"]//CommonDatautils.findAllCity();//所有城市
def cat = "district";
def index = 0;
for(def city : citys){
index++;
task.districtAddressCover(city, cat);
println "${index} "+city + "完成......."
println "${index} "+city + "完成......."
}
}
@@ -68,7 +68,7 @@ class AddressToBaiduLngLat {
def districtAddressCover(city, cat){
def ret = getAllDistricts(city, cat);
println city + " 共:"+ret.size()
println city + " 共:"+ret.size()
def index = 0;
def total = ret.size();
def row = 0;
@@ -108,7 +108,7 @@ class AddressToBaiduLngLat {
long lat = abcLngLatMap.lat * 10000000000L
index ++;
//println "["+row+"]"+obj.region +" "+obj.district_name+" 新baidu:"+m.baiduLat + " "+m.baiduLng+" 新mapabc:"+lng +" "+lat+" "+distance
//println "["+row+"]"+obj.region +" "+obj.district_name+" 新baidu:"+m.baiduLat + " "+m.baiduLng+" 新mapabc:"+lng +" "+lat+" "+distance
if(index%100==0){
print "."
}
@@ -124,9 +124,9 @@ class AddressToBaiduLngLat {
}
}
println "【${city}】index : "+index;
println "【${city}】noLatLngIndex :"+noLatLngIndex;
println "【${city}】total : "+total
println "【${city}】index : "+index;
println "【${city}】noLatLngIndex :"+noLatLngIndex;
println "【${city}】total : "+total
}
@@ -235,7 +235,7 @@ class AddressToBaiduLngLat {
for(poi in poiList){
def level = poi.level.text()
if (isFirst){
//如果是第一次,则以第一次作为优先选择
//如果是第一次,则以第一次作为优先选择
getDataFromPoi(ret, poi)
isFirst = false
}
@@ -290,380 +290,380 @@ class AddressToBaiduLngLat {
static {
CITY_CODE_MAP.put("北京", "010")
CITY_CODE_MAP.put("广州", "020")
CITY_CODE_MAP.put("重庆", "023")
CITY_CODE_MAP.put("南京", "025")
CITY_CODE_MAP.put("武汉", "027")
CITY_CODE_MAP.put("成都", "028")
CITY_CODE_MAP.put("保定", "0312")
CITY_CODE_MAP.put("承德", "0314")
CITY_CODE_MAP.put("唐山", "0315")
CITY_CODE_MAP.put("廊坊", "0316")
CITY_CODE_MAP.put("秦皇岛", "0335")
CITY_CODE_MAP.put("新乡", "0373")
CITY_CODE_MAP.put("洛阳", "0379")
CITY_CODE_MAP.put("大连", "0411")
CITY_CODE_MAP.put("鞍山", "0412")
CITY_CODE_MAP.put("抚顺", "0413")
CITY_CODE_MAP.put("本溪", "0414")
CITY_CODE_MAP.put("丹东", "0415")
CITY_CODE_MAP.put("辽阳", "0419")
CITY_CODE_MAP.put("葫芦岛", "0429")
CITY_CODE_MAP.put("长春", "0431")
CITY_CODE_MAP.put("哈尔滨", "0451")
CITY_CODE_MAP.put("佳木斯", "0454")
CITY_CODE_MAP.put("包头", "0472")
CITY_CODE_MAP.put("镇江", "0511")
CITY_CODE_MAP.put("苏州", "0512")
CITY_CODE_MAP.put("昆山", "0512")
CITY_CODE_MAP.put("南通", "0513")
CITY_CODE_MAP.put("扬州", "0514")
CITY_CODE_MAP.put("徐州", "0516")
CITY_CODE_MAP.put("常州", "0519")
CITY_CODE_MAP.put("济南", "0531")
CITY_CODE_MAP.put("青岛", "0532")
CITY_CODE_MAP.put("蚌埠", "0552")
CITY_CODE_MAP.put("黄山", "0559")
CITY_CODE_MAP.put("杭州", "0571")
CITY_CODE_MAP.put("湖州", "0572")
CITY_CODE_MAP.put("台州", "0576")
CITY_CODE_MAP.put("温州", "0577")
CITY_CODE_MAP.put("福州", "0591")
CITY_CODE_MAP.put("厦门", "0592")
CITY_CODE_MAP.put("宁德", "0593")
CITY_CODE_MAP.put("莆田", "0594")
CITY_CODE_MAP.put("泉州", "0595")
CITY_CODE_MAP.put("漳州", "0596")
CITY_CODE_MAP.put("威海", "0631")
CITY_CODE_MAP.put("汕尾", "0660")
CITY_CODE_MAP.put("汕尾岛", "0660")
CITY_CODE_MAP.put("阳江", "0662")
CITY_CODE_MAP.put("揭阳", "0663")
CITY_CODE_MAP.put("茂名", "0668")
CITY_CODE_MAP.put("襄阳", "0710")
CITY_CODE_MAP.put("黄石", "0714")
CITY_CODE_MAP.put("宜昌", "0717")
CITY_CODE_MAP.put("十堰", "0719")
CITY_CODE_MAP.put("荆门", "0724")
CITY_CODE_MAP.put("岳阳", "0730")
CITY_CODE_MAP.put("长沙", "0731")
CITY_CODE_MAP.put("湘潭", "0732")
CITY_CODE_MAP.put("株洲", "0733")
CITY_CODE_MAP.put("常德", "0736")
CITY_CODE_MAP.put("江门", "0750")
CITY_CODE_MAP.put("汕头", "0754")
CITY_CODE_MAP.put("深圳", "0755")
CITY_CODE_MAP.put("珠海", "0756")
CITY_CODE_MAP.put("湛江", "0759")
CITY_CODE_MAP.put("中山", "0760")
CITY_CODE_MAP.put("防城港", "0770")
CITY_CODE_MAP.put("南宁", "0771")
CITY_CODE_MAP.put("桂林", "0773")
CITY_CODE_MAP.put("北海", "0779")
CITY_CODE_MAP.put("南昌", "0791")
CITY_CODE_MAP.put("九江", "0792")
CITY_CODE_MAP.put("景德镇", "0798")
CITY_CODE_MAP.put("自贡", "0813")
CITY_CODE_MAP.put("泸州", "0830")
CITY_CODE_MAP.put("宜宾", "0831")
CITY_CODE_MAP.put("内江", "01832")
CITY_CODE_MAP.put("眉山", "0833")
CITY_CODE_MAP.put("贵阳", "0851")
CITY_CODE_MAP.put("三亚", "0899")
CITY_CODE_MAP.put("咸阳", "0910")
CITY_CODE_MAP.put("延安", "0911")
CITY_CODE_MAP.put("延边朝鲜族自治州", "0433")
CITY_CODE_MAP.put("七台河", "0464")
CITY_CODE_MAP.put("三明", "0598")
CITY_CODE_MAP.put("三门峡", "0398")
CITY_CODE_MAP.put("上海", "021")
CITY_CODE_MAP.put("上饶", "0793")
CITY_CODE_MAP.put("东莞", "0769")
CITY_CODE_MAP.put("东营", "0546")
CITY_CODE_MAP.put("中卫", "01953")
CITY_CODE_MAP.put("临夏回族自治州", "0930")
CITY_CODE_MAP.put("临汾", "0357")
CITY_CODE_MAP.put("临沂", "0539")
CITY_CODE_MAP.put("丽水", "0578")
CITY_CODE_MAP.put("乌兰察布", "0474")
CITY_CODE_MAP.put("乌海", "0473")
CITY_CODE_MAP.put("乌鲁木齐", "0991")
CITY_CODE_MAP.put("乐山", "0833")
CITY_CODE_MAP.put("云浮", "0766")
CITY_CODE_MAP.put("亳州", "0558")
CITY_CODE_MAP.put("仙桃", "0728")
CITY_CODE_MAP.put("伊春", "0458")
CITY_CODE_MAP.put("伊犁哈萨克自治州", "0999")
CITY_CODE_MAP.put("佛山", "0757")
CITY_CODE_MAP.put("信阳", "0376")
CITY_CODE_MAP.put("克拉玛依", "0990")
CITY_CODE_MAP.put("六安", "0564")
CITY_CODE_MAP.put("兰州", "0931")
CITY_CODE_MAP.put("兴安盟", "0482")
CITY_CODE_MAP.put("凉山彝族自治州", "0834")
CITY_CODE_MAP.put("南充", "0817")
CITY_CODE_MAP.put("南平", "0599")
CITY_CODE_MAP.put("南阳", "0377")
CITY_CODE_MAP.put("博尔塔拉蒙古自治州", "0909")
CITY_CODE_MAP.put("双鸭山", "0469")
CITY_CODE_MAP.put("合肥", "0551")
CITY_CODE_MAP.put("吉安", "0796")
CITY_CODE_MAP.put("吉林", "0432")
CITY_CODE_MAP.put("吐鲁番地区", "0995")
CITY_CODE_MAP.put("吕梁", "0358")
CITY_CODE_MAP.put("吴忠", "0953")
CITY_CODE_MAP.put("周口", "0394")
CITY_CODE_MAP.put("呼伦贝尔", "0470")
CITY_CODE_MAP.put("呼和浩特", "0471")
CITY_CODE_MAP.put("咸宁", "0715")
CITY_CODE_MAP.put("商丘", "0370")
CITY_CODE_MAP.put("商洛", "0914")
CITY_CODE_MAP.put("嘉兴", "0573")
CITY_CODE_MAP.put("四平", "0434")
CITY_CODE_MAP.put("塔城地区", "0901")
CITY_CODE_MAP.put("大兴安岭地区", "0457")
CITY_CODE_MAP.put("大同", "0352")
CITY_CODE_MAP.put("大庆", "0459")
CITY_CODE_MAP.put("大理白族自治州", "0872")
CITY_CODE_MAP.put("天水", "0938")
CITY_CODE_MAP.put("天津", "022")
CITY_CODE_MAP.put("天门", "01728")
CITY_CODE_MAP.put("太原", "0351")
CITY_CODE_MAP.put("娄底", "0738")
CITY_CODE_MAP.put("孝感", "0712")
CITY_CODE_MAP.put("宁波", "0574")
CITY_CODE_MAP.put("安庆", "0556")
CITY_CODE_MAP.put("安康", "0915")
CITY_CODE_MAP.put("安阳", "0372")
CITY_CODE_MAP.put("安顺", "0853")
CITY_CODE_MAP.put("定西", "0932")
CITY_CODE_MAP.put("宜春", "0795")
CITY_CODE_MAP.put("宝鸡", "0917")
CITY_CODE_MAP.put("宣城", "0563")
CITY_CODE_MAP.put("宿州", "0557")
CITY_CODE_MAP.put("宿迁", "0527")
CITY_CODE_MAP.put("山南地区", "0893")
CITY_CODE_MAP.put("崇左", "01771")
CITY_CODE_MAP.put("巢湖", "0565")
CITY_CODE_MAP.put("巴中", "0827")
CITY_CODE_MAP.put("平凉", "0933")
CITY_CODE_MAP.put("平顶山", "0375")
CITY_CODE_MAP.put("广元", "0839")
CITY_CODE_MAP.put("广安", "0826")
CITY_CODE_MAP.put("庆阳", "0934")
CITY_CODE_MAP.put("开封", "0378")
CITY_CODE_MAP.put("张家口", "0313")
CITY_CODE_MAP.put("张家界", "0744")
CITY_CODE_MAP.put("张掖", "0936")
CITY_CODE_MAP.put("德州", "0534")
CITY_CODE_MAP.put("德阳", "0838")
CITY_CODE_MAP.put("忻州", "0350")
CITY_CODE_MAP.put("怀化", "0745")
CITY_CODE_MAP.put("恩施土家族苗族自治州", "0718")
CITY_CODE_MAP.put("惠州", "0752")
CITY_CODE_MAP.put("抚州", "0794")
CITY_CODE_MAP.put("拉萨", "0891")
CITY_CODE_MAP.put("攀枝花", "0812")
CITY_CODE_MAP.put("新余", "0790")
CITY_CODE_MAP.put("无锡", "0510")
CITY_CODE_MAP.put("日喀则地区", "0892")
CITY_CODE_MAP.put("日照", "0633")
CITY_CODE_MAP.put("昆明", "0871")
CITY_CODE_MAP.put("昌吉回族自治州", "0994")
CITY_CODE_MAP.put("昭通", "0870")
CITY_CODE_MAP.put("晋中", "0354")
CITY_CODE_MAP.put("晋城", "0356")
CITY_CODE_MAP.put("曲靖", "0874")
CITY_CODE_MAP.put("朔州", "0349")
CITY_CODE_MAP.put("朝阳", "0421")
CITY_CODE_MAP.put("来宾", "01772")
CITY_CODE_MAP.put("松原", "0438")
CITY_CODE_MAP.put("枣庄", "0632")
CITY_CODE_MAP.put("柳州", "0772")
CITY_CODE_MAP.put("梅州", "0753")
CITY_CODE_MAP.put("梧州", "0774")
CITY_CODE_MAP.put("楚雄彝族自治州", "0878")
CITY_CODE_MAP.put("榆林", "0912")
CITY_CODE_MAP.put("武威", "01935")
CITY_CODE_MAP.put("毕节地区", "0857")
CITY_CODE_MAP.put("永州", "0746")
CITY_CODE_MAP.put("汉中", "0916")
CITY_CODE_MAP.put("池州", "0566")
CITY_CODE_MAP.put("沈阳", "024")
CITY_CODE_MAP.put("沧州", "0317")
CITY_CODE_MAP.put("河池", "0778")
CITY_CODE_MAP.put("河源", "0762")
CITY_CODE_MAP.put("泰安", "0538")
CITY_CODE_MAP.put("泰州", "0523")
CITY_CODE_MAP.put("济宁", "0537")
CITY_CODE_MAP.put("济源", "01391")
CITY_CODE_MAP.put("海东地区", "0972")
CITY_CODE_MAP.put("海北藏族自治州", "0970")
CITY_CODE_MAP.put("海南藏族自治州", "0974")
CITY_CODE_MAP.put("海口", "0898")
CITY_CODE_MAP.put("淄博", "0533")
CITY_CODE_MAP.put("淮北", "0561")
CITY_CODE_MAP.put("淮南", "0554")
CITY_CODE_MAP.put("淮安", "0517")
CITY_CODE_MAP.put("清远", "0763")
CITY_CODE_MAP.put("渭南", "0913")
CITY_CODE_MAP.put("湘西土家族苗族自治州", "0743")
CITY_CODE_MAP.put("滁州", "0550")
CITY_CODE_MAP.put("滨州", "0543")
CITY_CODE_MAP.put("漯河", "0395")
CITY_CODE_MAP.put("潍坊", "0536")
CITY_CODE_MAP.put("潜江", "02728")
CITY_CODE_MAP.put("潮州", "0768")
CITY_CODE_MAP.put("濮阳", "0393")
CITY_CODE_MAP.put("烟台", "0535")
CITY_CODE_MAP.put("焦作", "0391")
CITY_CODE_MAP.put("牡丹江", "0453")
CITY_CODE_MAP.put("玉林", "0775")
CITY_CODE_MAP.put("玉溪", "0877")
CITY_CODE_MAP.put("甘孜藏族自治州", "0836")
CITY_CODE_MAP.put("白城", "0436")
CITY_CODE_MAP.put("白山", "0439")
CITY_CODE_MAP.put("白银", "0943")
CITY_CODE_MAP.put("益阳", "0737")
CITY_CODE_MAP.put("盐城", "0515")
CITY_CODE_MAP.put("盘锦", "0427")
CITY_CODE_MAP.put("石嘴山", "0952")
CITY_CODE_MAP.put("石家庄", "0311")
CITY_CODE_MAP.put("石河子", "0993")
CITY_CODE_MAP.put("神农架林区", "01719")
CITY_CODE_MAP.put("红河哈尼族彝族自治州", "0873")
CITY_CODE_MAP.put("绍兴", "0575")
CITY_CODE_MAP.put("绥化", "0455")
CITY_CODE_MAP.put("绵阳", "0816")
CITY_CODE_MAP.put("聊城", "0635")
CITY_CODE_MAP.put("肇庆", "0758")
CITY_CODE_MAP.put("舟山", "0580")
CITY_CODE_MAP.put("芜湖", "0553")
CITY_CODE_MAP.put("荆州", "0716")
CITY_CODE_MAP.put("莱芜", "0634")
CITY_CODE_MAP.put("菏泽", "0530")
CITY_CODE_MAP.put("萍乡", "0799")
CITY_CODE_MAP.put("营口", "0417")
CITY_CODE_MAP.put("衡水", "0318")
CITY_CODE_MAP.put("衡阳", "0734")
CITY_CODE_MAP.put("衢州", "0570")
CITY_CODE_MAP.put("西双版纳傣族自治州", "0691")
CITY_CODE_MAP.put("西宁", "0971")
CITY_CODE_MAP.put("西安", "029")
CITY_CODE_MAP.put("许昌", "0374")
CITY_CODE_MAP.put("贵港", "01755")
CITY_CODE_MAP.put("贺州", "01774")
CITY_CODE_MAP.put("资阳", "0832")
CITY_CODE_MAP.put("赣州", "0797")
CITY_CODE_MAP.put("赤峰", "0476")
CITY_CODE_MAP.put("辽源", "0437")
CITY_CODE_MAP.put("达州", "0818")
CITY_CODE_MAP.put("运城", "0359")
CITY_CODE_MAP.put("连云港", "0518")
CITY_CODE_MAP.put("通化", "0435")
CITY_CODE_MAP.put("通辽", "0475")
CITY_CODE_MAP.put("遂宁", "0825")
CITY_CODE_MAP.put("遵义", "0852")
CITY_CODE_MAP.put("邢台", "0319")
CITY_CODE_MAP.put("邯郸", "0310")
CITY_CODE_MAP.put("邵阳", "0739")
CITY_CODE_MAP.put("郑州", "0371")
CITY_CODE_MAP.put("郴州", "0735")
CITY_CODE_MAP.put("鄂尔多斯", "0477")
CITY_CODE_MAP.put("鄂州", "0711")
CITY_CODE_MAP.put("酒泉", "0937")
CITY_CODE_MAP.put("金华", "0579")
CITY_CODE_MAP.put("金昌", "0935")
CITY_CODE_MAP.put("钦州", "0777")
CITY_CODE_MAP.put("铁岭", "0410")
CITY_CODE_MAP.put("铜仁地区", "0856")
CITY_CODE_MAP.put("铜川", "0919")
CITY_CODE_MAP.put("铜陵", "0562")
CITY_CODE_MAP.put("银川", "0951")
CITY_CODE_MAP.put("锦州", "0416")
CITY_CODE_MAP.put("长治", "0355")
CITY_CODE_MAP.put("阜新", "0418")
CITY_CODE_MAP.put("阜阳", "01558")
CITY_CODE_MAP.put("阳泉", "0353")
CITY_CODE_MAP.put("阿坝藏族羌族自治州", "0837")
CITY_CODE_MAP.put("陇南", "02935")
CITY_CODE_MAP.put("随州", "0722")
CITY_CODE_MAP.put("雅安", "0835")
CITY_CODE_MAP.put("韶关", "0751")
CITY_CODE_MAP.put("马鞍山", "0555")
CITY_CODE_MAP.put("驻马店", "0396")
CITY_CODE_MAP.put("鸡西", "0467")
CITY_CODE_MAP.put("鹤壁", "0392")
CITY_CODE_MAP.put("鹤岗", "0468")
CITY_CODE_MAP.put("鹰潭", "0701")
CITY_CODE_MAP.put("黄冈", "0713")
CITY_CODE_MAP.put("黑河", "0456")
CITY_CODE_MAP.put("黔东南苗族侗族自治州", "0855")
CITY_CODE_MAP.put("黔南布依族苗族自治州", "0854")
CITY_CODE_MAP.put("齐齐哈尔", "0452")
CITY_CODE_MAP.put("龙岩", "0597")
CITY_CODE_MAP.put("甘南藏族自治州", "0941")
CITY_CODE_MAP.put("嘉峪关", "1937")
CITY_CODE_MAP.put("百色", "0776")
CITY_CODE_MAP.put("六盘水", "0858")
CITY_CODE_MAP.put("黔西南布依族苗族自治州", "0859")
CITY_CODE_MAP.put("白沙黎族自治县", "0802")
CITY_CODE_MAP.put("保亭黎族苗族自治县", "0801")
CITY_CODE_MAP.put("昌江黎族自治县", "0803")
CITY_CODE_MAP.put("澄迈县", "0804")
CITY_CODE_MAP.put("定安县", "0806")
CITY_CODE_MAP.put("东方", "0807")
CITY_CODE_MAP.put("东沙群岛", "0808")
CITY_CODE_MAP.put("乐东黎族自治县", "2802")
CITY_CODE_MAP.put("临高县", "1896")
CITY_CODE_MAP.put("陵水黎族自治县", "0809")
CITY_CODE_MAP.put("南沙群岛", "1891")
CITY_CODE_MAP.put("琼海", "1894")
CITY_CODE_MAP.put("琼中黎族苗族自治县", "1899")
CITY_CODE_MAP.put("屯昌县", "1892")
CITY_CODE_MAP.put("万宁", "1898")
CITY_CODE_MAP.put("文昌", "1893")
CITY_CODE_MAP.put("五指山", "1897")
CITY_CODE_MAP.put("西沙群岛", "1895")
CITY_CODE_MAP.put("中沙群岛", "2801")
CITY_CODE_MAP.put("儋州", "0805")
CITY_CODE_MAP.put("阿拉善盟", "0483")
CITY_CODE_MAP.put("巴彦淖尔市", "0478")
CITY_CODE_MAP.put("锡林郭勒盟", "0479")
CITY_CODE_MAP.put("固原", "0954")
CITY_CODE_MAP.put("果洛藏族自治州", "0975")
CITY_CODE_MAP.put("海西蒙古族藏族自治州", "0977")
CITY_CODE_MAP.put("黄南藏族自治州", "0973")
CITY_CODE_MAP.put("玉树藏族自治州", "0976")
CITY_CODE_MAP.put("阿里地区", "0897")
CITY_CODE_MAP.put("昌都地区", "0895")
CITY_CODE_MAP.put("林芝地区", "0894")
CITY_CODE_MAP.put("那曲地区", "0896")
CITY_CODE_MAP.put("阿克苏地区", "0997")
CITY_CODE_MAP.put("阿勒泰地区", "0906")
CITY_CODE_MAP.put("巴音郭楞蒙古自治州", "0996")
CITY_CODE_MAP.put("哈密地区", "0902")
CITY_CODE_MAP.put("和田地区", "0903")
CITY_CODE_MAP.put("喀什地区", "0998")
CITY_CODE_MAP.put("克孜勒苏柯尔克孜自治州", "0908")
CITY_CODE_MAP.put("保山", "0875")
CITY_CODE_MAP.put("德宏傣族景颇族自治州", "0692")
CITY_CODE_MAP.put("迪庆藏族自治州", "0887")
CITY_CODE_MAP.put("丽江", "0888")
CITY_CODE_MAP.put("临沧市", "0883")
CITY_CODE_MAP.put("怒江傈僳族自治州", "0886")
CITY_CODE_MAP.put("普洱", "0879")
CITY_CODE_MAP.put("文山壮族苗族自治州", "0876")
CITY_CODE_MAP.put("香港", "1852")
CITY_CODE_MAP.put("澳门", "1853")
CITY_CODE_MAP.put("五家渠市", "652300")
CITY_CODE_MAP.put("阿拉尔市", "652900")
CITY_CODE_MAP.put("图木舒克市", "653000")
CITY_CODE_MAP.put("北京", "010")
CITY_CODE_MAP.put("广州", "020")
CITY_CODE_MAP.put("重庆", "023")
CITY_CODE_MAP.put("南京", "025")
CITY_CODE_MAP.put("武汉", "027")
CITY_CODE_MAP.put("成都", "028")
CITY_CODE_MAP.put("保定", "0312")
CITY_CODE_MAP.put("承德", "0314")
CITY_CODE_MAP.put("唐山", "0315")
CITY_CODE_MAP.put("廊坊", "0316")
CITY_CODE_MAP.put("秦皇岛", "0335")
CITY_CODE_MAP.put("新乡", "0373")
CITY_CODE_MAP.put("洛阳", "0379")
CITY_CODE_MAP.put("大连", "0411")
CITY_CODE_MAP.put("鞍山", "0412")
CITY_CODE_MAP.put("抚顺", "0413")
CITY_CODE_MAP.put("本溪", "0414")
CITY_CODE_MAP.put("丹东", "0415")
CITY_CODE_MAP.put("辽阳", "0419")
CITY_CODE_MAP.put("葫芦岛", "0429")
CITY_CODE_MAP.put("长春", "0431")
CITY_CODE_MAP.put("哈尔滨", "0451")
CITY_CODE_MAP.put("佳木斯", "0454")
CITY_CODE_MAP.put("包头", "0472")
CITY_CODE_MAP.put("镇江", "0511")
CITY_CODE_MAP.put("苏州", "0512")
CITY_CODE_MAP.put("昆山", "0512")
CITY_CODE_MAP.put("南通", "0513")
CITY_CODE_MAP.put("扬州", "0514")
CITY_CODE_MAP.put("徐州", "0516")
CITY_CODE_MAP.put("常州", "0519")
CITY_CODE_MAP.put("济南", "0531")
CITY_CODE_MAP.put("青岛", "0532")
CITY_CODE_MAP.put("蚌埠", "0552")
CITY_CODE_MAP.put("黄山", "0559")
CITY_CODE_MAP.put("杭州", "0571")
CITY_CODE_MAP.put("湖州", "0572")
CITY_CODE_MAP.put("台州", "0576")
CITY_CODE_MAP.put("温州", "0577")
CITY_CODE_MAP.put("福州", "0591")
CITY_CODE_MAP.put("厦门", "0592")
CITY_CODE_MAP.put("宁德", "0593")
CITY_CODE_MAP.put("莆田", "0594")
CITY_CODE_MAP.put("泉州", "0595")
CITY_CODE_MAP.put("漳州", "0596")
CITY_CODE_MAP.put("威海", "0631")
CITY_CODE_MAP.put("汕尾", "0660")
CITY_CODE_MAP.put("汕尾岛", "0660")
CITY_CODE_MAP.put("阳江", "0662")
CITY_CODE_MAP.put("揭阳", "0663")
CITY_CODE_MAP.put("茂名", "0668")
CITY_CODE_MAP.put("襄阳", "0710")
CITY_CODE_MAP.put("黄石", "0714")
CITY_CODE_MAP.put("宜昌", "0717")
CITY_CODE_MAP.put("十堰", "0719")
CITY_CODE_MAP.put("荆门", "0724")
CITY_CODE_MAP.put("岳阳", "0730")
CITY_CODE_MAP.put("长沙", "0731")
CITY_CODE_MAP.put("湘潭", "0732")
CITY_CODE_MAP.put("株洲", "0733")
CITY_CODE_MAP.put("常德", "0736")
CITY_CODE_MAP.put("江门", "0750")
CITY_CODE_MAP.put("汕头", "0754")
CITY_CODE_MAP.put("深圳", "0755")
CITY_CODE_MAP.put("珠海", "0756")
CITY_CODE_MAP.put("湛江", "0759")
CITY_CODE_MAP.put("中山", "0760")
CITY_CODE_MAP.put("防城港", "0770")
CITY_CODE_MAP.put("南宁", "0771")
CITY_CODE_MAP.put("桂林", "0773")
CITY_CODE_MAP.put("北海", "0779")
CITY_CODE_MAP.put("南昌", "0791")
CITY_CODE_MAP.put("九江", "0792")
CITY_CODE_MAP.put("景德镇", "0798")
CITY_CODE_MAP.put("自贡", "0813")
CITY_CODE_MAP.put("泸州", "0830")
CITY_CODE_MAP.put("宜宾", "0831")
CITY_CODE_MAP.put("内江", "01832")
CITY_CODE_MAP.put("眉山", "0833")
CITY_CODE_MAP.put("贵阳", "0851")
CITY_CODE_MAP.put("三亚", "0899")
CITY_CODE_MAP.put("咸阳", "0910")
CITY_CODE_MAP.put("延安", "0911")
CITY_CODE_MAP.put("延边朝鲜族自治州", "0433")
CITY_CODE_MAP.put("七台河", "0464")
CITY_CODE_MAP.put("三明", "0598")
CITY_CODE_MAP.put("三门峡", "0398")
CITY_CODE_MAP.put("上海", "021")
CITY_CODE_MAP.put("上饶", "0793")
CITY_CODE_MAP.put("东莞", "0769")
CITY_CODE_MAP.put("东营", "0546")
CITY_CODE_MAP.put("中卫", "01953")
CITY_CODE_MAP.put("临夏回族自治州", "0930")
CITY_CODE_MAP.put("临汾", "0357")
CITY_CODE_MAP.put("临沂", "0539")
CITY_CODE_MAP.put("丽水", "0578")
CITY_CODE_MAP.put("乌兰察布", "0474")
CITY_CODE_MAP.put("乌海", "0473")
CITY_CODE_MAP.put("乌鲁木齐", "0991")
CITY_CODE_MAP.put("乐山", "0833")
CITY_CODE_MAP.put("云浮", "0766")
CITY_CODE_MAP.put("亳州", "0558")
CITY_CODE_MAP.put("仙桃", "0728")
CITY_CODE_MAP.put("伊春", "0458")
CITY_CODE_MAP.put("伊犁哈萨克自治州", "0999")
CITY_CODE_MAP.put("佛山", "0757")
CITY_CODE_MAP.put("信阳", "0376")
CITY_CODE_MAP.put("克拉玛依", "0990")
CITY_CODE_MAP.put("六安", "0564")
CITY_CODE_MAP.put("兰州", "0931")
CITY_CODE_MAP.put("兴安盟", "0482")
CITY_CODE_MAP.put("凉山彝族自治州", "0834")
CITY_CODE_MAP.put("南充", "0817")
CITY_CODE_MAP.put("南平", "0599")
CITY_CODE_MAP.put("南阳", "0377")
CITY_CODE_MAP.put("博尔塔拉蒙古自治州", "0909")
CITY_CODE_MAP.put("双鸭山", "0469")
CITY_CODE_MAP.put("合肥", "0551")
CITY_CODE_MAP.put("吉安", "0796")
CITY_CODE_MAP.put("吉林", "0432")
CITY_CODE_MAP.put("吐鲁番地区", "0995")
CITY_CODE_MAP.put("吕梁", "0358")
CITY_CODE_MAP.put("吴忠", "0953")
CITY_CODE_MAP.put("周口", "0394")
CITY_CODE_MAP.put("呼伦贝尔", "0470")
CITY_CODE_MAP.put("呼和浩特", "0471")
CITY_CODE_MAP.put("咸宁", "0715")
CITY_CODE_MAP.put("商丘", "0370")
CITY_CODE_MAP.put("商洛", "0914")
CITY_CODE_MAP.put("嘉兴", "0573")
CITY_CODE_MAP.put("四平", "0434")
CITY_CODE_MAP.put("塔城地区", "0901")
CITY_CODE_MAP.put("大兴安岭地区", "0457")
CITY_CODE_MAP.put("大同", "0352")
CITY_CODE_MAP.put("大庆", "0459")
CITY_CODE_MAP.put("大理白族自治州", "0872")
CITY_CODE_MAP.put("天水", "0938")
CITY_CODE_MAP.put("天津", "022")
CITY_CODE_MAP.put("天门", "01728")
CITY_CODE_MAP.put("太原", "0351")
CITY_CODE_MAP.put("娄底", "0738")
CITY_CODE_MAP.put("孝感", "0712")
CITY_CODE_MAP.put("宁波", "0574")
CITY_CODE_MAP.put("安庆", "0556")
CITY_CODE_MAP.put("安康", "0915")
CITY_CODE_MAP.put("安阳", "0372")
CITY_CODE_MAP.put("安顺", "0853")
CITY_CODE_MAP.put("定西", "0932")
CITY_CODE_MAP.put("宜春", "0795")
CITY_CODE_MAP.put("宝鸡", "0917")
CITY_CODE_MAP.put("宣城", "0563")
CITY_CODE_MAP.put("宿州", "0557")
CITY_CODE_MAP.put("宿迁", "0527")
CITY_CODE_MAP.put("山南地区", "0893")
CITY_CODE_MAP.put("崇左", "01771")
CITY_CODE_MAP.put("巢湖", "0565")
CITY_CODE_MAP.put("巴中", "0827")
CITY_CODE_MAP.put("平凉", "0933")
CITY_CODE_MAP.put("平顶山", "0375")
CITY_CODE_MAP.put("广元", "0839")
CITY_CODE_MAP.put("广安", "0826")
CITY_CODE_MAP.put("庆阳", "0934")
CITY_CODE_MAP.put("开封", "0378")
CITY_CODE_MAP.put("张家口", "0313")
CITY_CODE_MAP.put("张家界", "0744")
CITY_CODE_MAP.put("张掖", "0936")
CITY_CODE_MAP.put("德州", "0534")
CITY_CODE_MAP.put("德阳", "0838")
CITY_CODE_MAP.put("忻州", "0350")
CITY_CODE_MAP.put("怀化", "0745")
CITY_CODE_MAP.put("恩施土家族苗族自治州", "0718")
CITY_CODE_MAP.put("惠州", "0752")
CITY_CODE_MAP.put("抚州", "0794")
CITY_CODE_MAP.put("拉萨", "0891")
CITY_CODE_MAP.put("攀枝花", "0812")
CITY_CODE_MAP.put("新余", "0790")
CITY_CODE_MAP.put("无锡", "0510")
CITY_CODE_MAP.put("日喀则地区", "0892")
CITY_CODE_MAP.put("日照", "0633")
CITY_CODE_MAP.put("昆明", "0871")
CITY_CODE_MAP.put("昌吉回族自治州", "0994")
CITY_CODE_MAP.put("昭通", "0870")
CITY_CODE_MAP.put("晋中", "0354")
CITY_CODE_MAP.put("晋城", "0356")
CITY_CODE_MAP.put("曲靖", "0874")
CITY_CODE_MAP.put("朔州", "0349")
CITY_CODE_MAP.put("朝阳", "0421")
CITY_CODE_MAP.put("来宾", "01772")
CITY_CODE_MAP.put("松原", "0438")
CITY_CODE_MAP.put("枣庄", "0632")
CITY_CODE_MAP.put("柳州", "0772")
CITY_CODE_MAP.put("梅州", "0753")
CITY_CODE_MAP.put("梧州", "0774")
CITY_CODE_MAP.put("楚雄彝族自治州", "0878")
CITY_CODE_MAP.put("榆林", "0912")
CITY_CODE_MAP.put("武威", "01935")
CITY_CODE_MAP.put("毕节地区", "0857")
CITY_CODE_MAP.put("永州", "0746")
CITY_CODE_MAP.put("汉中", "0916")
CITY_CODE_MAP.put("池州", "0566")
CITY_CODE_MAP.put("沈阳", "024")
CITY_CODE_MAP.put("沧州", "0317")
CITY_CODE_MAP.put("河池", "0778")
CITY_CODE_MAP.put("河源", "0762")
CITY_CODE_MAP.put("泰安", "0538")
CITY_CODE_MAP.put("泰州", "0523")
CITY_CODE_MAP.put("济宁", "0537")
CITY_CODE_MAP.put("济源", "01391")
CITY_CODE_MAP.put("海东地区", "0972")
CITY_CODE_MAP.put("海北藏族自治州", "0970")
CITY_CODE_MAP.put("海南藏族自治州", "0974")
CITY_CODE_MAP.put("海口", "0898")
CITY_CODE_MAP.put("淄博", "0533")
CITY_CODE_MAP.put("淮北", "0561")
CITY_CODE_MAP.put("淮南", "0554")
CITY_CODE_MAP.put("淮安", "0517")
CITY_CODE_MAP.put("清远", "0763")
CITY_CODE_MAP.put("渭南", "0913")
CITY_CODE_MAP.put("湘西土家族苗族自治州", "0743")
CITY_CODE_MAP.put("滁州", "0550")
CITY_CODE_MAP.put("滨州", "0543")
CITY_CODE_MAP.put("漯河", "0395")
CITY_CODE_MAP.put("潍坊", "0536")
CITY_CODE_MAP.put("潜江", "02728")
CITY_CODE_MAP.put("潮州", "0768")
CITY_CODE_MAP.put("濮阳", "0393")
CITY_CODE_MAP.put("烟台", "0535")
CITY_CODE_MAP.put("焦作", "0391")
CITY_CODE_MAP.put("牡丹江", "0453")
CITY_CODE_MAP.put("玉林", "0775")
CITY_CODE_MAP.put("玉溪", "0877")
CITY_CODE_MAP.put("甘孜藏族自治州", "0836")
CITY_CODE_MAP.put("白城", "0436")
CITY_CODE_MAP.put("白山", "0439")
CITY_CODE_MAP.put("白银", "0943")
CITY_CODE_MAP.put("益阳", "0737")
CITY_CODE_MAP.put("盐城", "0515")
CITY_CODE_MAP.put("盘锦", "0427")
CITY_CODE_MAP.put("石嘴山", "0952")
CITY_CODE_MAP.put("石家庄", "0311")
CITY_CODE_MAP.put("石河子", "0993")
CITY_CODE_MAP.put("神农架林区", "01719")
CITY_CODE_MAP.put("红河哈尼族彝族自治州", "0873")
CITY_CODE_MAP.put("绍兴", "0575")
CITY_CODE_MAP.put("绥化", "0455")
CITY_CODE_MAP.put("绵阳", "0816")
CITY_CODE_MAP.put("聊城", "0635")
CITY_CODE_MAP.put("肇庆", "0758")
CITY_CODE_MAP.put("舟山", "0580")
CITY_CODE_MAP.put("芜湖", "0553")
CITY_CODE_MAP.put("荆州", "0716")
CITY_CODE_MAP.put("莱芜", "0634")
CITY_CODE_MAP.put("菏泽", "0530")
CITY_CODE_MAP.put("萍乡", "0799")
CITY_CODE_MAP.put("营口", "0417")
CITY_CODE_MAP.put("衡水", "0318")
CITY_CODE_MAP.put("衡阳", "0734")
CITY_CODE_MAP.put("衢州", "0570")
CITY_CODE_MAP.put("西双版纳傣族自治州", "0691")
CITY_CODE_MAP.put("西宁", "0971")
CITY_CODE_MAP.put("西安", "029")
CITY_CODE_MAP.put("许昌", "0374")
CITY_CODE_MAP.put("贵港", "01755")
CITY_CODE_MAP.put("贺州", "01774")
CITY_CODE_MAP.put("资阳", "0832")
CITY_CODE_MAP.put("赣州", "0797")
CITY_CODE_MAP.put("赤峰", "0476")
CITY_CODE_MAP.put("辽源", "0437")
CITY_CODE_MAP.put("达州", "0818")
CITY_CODE_MAP.put("运城", "0359")
CITY_CODE_MAP.put("连云港", "0518")
CITY_CODE_MAP.put("通化", "0435")
CITY_CODE_MAP.put("通辽", "0475")
CITY_CODE_MAP.put("遂宁", "0825")
CITY_CODE_MAP.put("遵义", "0852")
CITY_CODE_MAP.put("邢台", "0319")
CITY_CODE_MAP.put("邯郸", "0310")
CITY_CODE_MAP.put("邵阳", "0739")
CITY_CODE_MAP.put("郑州", "0371")
CITY_CODE_MAP.put("郴州", "0735")
CITY_CODE_MAP.put("鄂尔多斯", "0477")
CITY_CODE_MAP.put("鄂州", "0711")
CITY_CODE_MAP.put("酒泉", "0937")
CITY_CODE_MAP.put("金华", "0579")
CITY_CODE_MAP.put("金昌", "0935")
CITY_CODE_MAP.put("钦州", "0777")
CITY_CODE_MAP.put("铁岭", "0410")
CITY_CODE_MAP.put("铜仁地区", "0856")
CITY_CODE_MAP.put("铜川", "0919")
CITY_CODE_MAP.put("铜陵", "0562")
CITY_CODE_MAP.put("银川", "0951")
CITY_CODE_MAP.put("锦州", "0416")
CITY_CODE_MAP.put("长治", "0355")
CITY_CODE_MAP.put("阜新", "0418")
CITY_CODE_MAP.put("阜阳", "01558")
CITY_CODE_MAP.put("阳泉", "0353")
CITY_CODE_MAP.put("阿坝藏族羌族自治州", "0837")
CITY_CODE_MAP.put("陇南", "02935")
CITY_CODE_MAP.put("随州", "0722")
CITY_CODE_MAP.put("雅安", "0835")
CITY_CODE_MAP.put("韶关", "0751")
CITY_CODE_MAP.put("马鞍山", "0555")
CITY_CODE_MAP.put("驻马店", "0396")
CITY_CODE_MAP.put("鸡西", "0467")
CITY_CODE_MAP.put("鹤壁", "0392")
CITY_CODE_MAP.put("鹤岗", "0468")
CITY_CODE_MAP.put("鹰潭", "0701")
CITY_CODE_MAP.put("黄冈", "0713")
CITY_CODE_MAP.put("黑河", "0456")
CITY_CODE_MAP.put("黔东南苗族侗族自治州", "0855")
CITY_CODE_MAP.put("黔南布依族苗族自治州", "0854")
CITY_CODE_MAP.put("齐齐哈尔", "0452")
CITY_CODE_MAP.put("龙岩", "0597")
CITY_CODE_MAP.put("甘南藏族自治州", "0941")
CITY_CODE_MAP.put("嘉峪关", "1937")
CITY_CODE_MAP.put("百色", "0776")
CITY_CODE_MAP.put("六盘水", "0858")
CITY_CODE_MAP.put("黔西南布依族苗族自治州", "0859")
CITY_CODE_MAP.put("白沙黎族自治县", "0802")
CITY_CODE_MAP.put("保亭黎族苗族自治县", "0801")
CITY_CODE_MAP.put("昌江黎族自治县", "0803")
CITY_CODE_MAP.put("澄迈县", "0804")
CITY_CODE_MAP.put("定安县", "0806")
CITY_CODE_MAP.put("东方", "0807")
CITY_CODE_MAP.put("东沙群岛", "0808")
CITY_CODE_MAP.put("乐东黎族自治县", "2802")
CITY_CODE_MAP.put("临高县", "1896")
CITY_CODE_MAP.put("陵水黎族自治县", "0809")
CITY_CODE_MAP.put("南沙群岛", "1891")
CITY_CODE_MAP.put("琼海", "1894")
CITY_CODE_MAP.put("琼中黎族苗族自治县", "1899")
CITY_CODE_MAP.put("屯昌县", "1892")
CITY_CODE_MAP.put("万宁", "1898")
CITY_CODE_MAP.put("文昌", "1893")
CITY_CODE_MAP.put("五指山", "1897")
CITY_CODE_MAP.put("西沙群岛", "1895")
CITY_CODE_MAP.put("中沙群岛", "2801")
CITY_CODE_MAP.put("儋州", "0805")
CITY_CODE_MAP.put("阿拉善盟", "0483")
CITY_CODE_MAP.put("巴彦淖尔市", "0478")
CITY_CODE_MAP.put("锡林郭勒盟", "0479")
CITY_CODE_MAP.put("固原", "0954")
CITY_CODE_MAP.put("果洛藏族自治州", "0975")
CITY_CODE_MAP.put("海西蒙古族藏族自治州", "0977")
CITY_CODE_MAP.put("黄南藏族自治州", "0973")
CITY_CODE_MAP.put("玉树藏族自治州", "0976")
CITY_CODE_MAP.put("阿里地区", "0897")
CITY_CODE_MAP.put("昌都地区", "0895")
CITY_CODE_MAP.put("林芝地区", "0894")
CITY_CODE_MAP.put("那曲地区", "0896")
CITY_CODE_MAP.put("阿克苏地区", "0997")
CITY_CODE_MAP.put("阿勒泰地区", "0906")
CITY_CODE_MAP.put("巴音郭楞蒙古自治州", "0996")
CITY_CODE_MAP.put("哈密地区", "0902")
CITY_CODE_MAP.put("和田地区", "0903")
CITY_CODE_MAP.put("喀什地区", "0998")
CITY_CODE_MAP.put("克孜勒苏柯尔克孜自治州", "0908")
CITY_CODE_MAP.put("保山", "0875")
CITY_CODE_MAP.put("德宏傣族景颇族自治州", "0692")
CITY_CODE_MAP.put("迪庆藏族自治州", "0887")
CITY_CODE_MAP.put("丽江", "0888")
CITY_CODE_MAP.put("临沧市", "0883")
CITY_CODE_MAP.put("怒江傈僳族自治州", "0886")
CITY_CODE_MAP.put("普洱", "0879")
CITY_CODE_MAP.put("文山壮族苗族自治州", "0876")
CITY_CODE_MAP.put("香港", "1852")
CITY_CODE_MAP.put("澳门", "1853")
CITY_CODE_MAP.put("五家渠市", "652300")
CITY_CODE_MAP.put("阿拉尔市", "652900")
CITY_CODE_MAP.put("图木舒克市", "653000")
}
/**
* 从poi中获得需要的数据
* 从poi中获得需要的数据
* @param ret
* @param poi
*/
@@ -688,7 +688,7 @@ class AddressToBaiduLngLat {
m.mabc_lat = o[1];
}else{
index++;
println "["+index+"]"+m.address +" 没有解析出经纬度 "+m.id
println "["+index+"]"+m.address +" 没有解析出经纬度 "+m.id
}
@@ -727,7 +727,7 @@ class AddressToBaiduLngLat {
conn.setRequestMethod("GET");
conn.doOutput = true
inputStream = conn.getInputStream();
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
while((line = buf.readLine())!=null){
sb.append(line);
}
@@ -35,18 +35,18 @@ class AnShouDataTest {
def task = new AnShouDataTest();
def token = task.getToken();
println token
// task.run("E:\\数据测试\\数据\\", "安硕测试数据.txt");
// task.batchTestApi("E:\\数据测试\\数据\\", "安硕测试数据", ".txt");
// task.run("E:\\数据测试\\数据\\", "安硕测试数据.txt");
// task.batchTestApi("E:\\数据测试\\数据\\", "安硕测试数据", ".txt");
}
def batchTestApi(def path, def fileName, def fileType){
def ret = readTxt(path, fileName+fileType);//读文件
def ret = readTxt(path, fileName+fileType);//读文件
def result = [];
def index = 0;
for(def m : ret){
if(m.estateType=="住房"){
if(m.estateType=="住房"){
index ++;
@@ -83,28 +83,28 @@ class AnShouDataTest {
}
}
//writeCSV2("E:\\数据测试\\结果\\", "批量地址测试.csv", result);
writeCSV("E:\\数据测试\\结果\\", fileName+"(新).csv", result);
//writeCSV2("E:\\数据测试\\结果\\", "批量地址测试.csv", result);
writeCSV("E:\\数据测试\\结果\\", fileName+"(新).csv", result);
}
def run(def path, def fileName){
def ret = readTxt(path, fileName);//读文件
def citys = CommonDatautils.findAllCity();//所有城市
def regionMap = CommonDatautils.getAllCityRegions();//所有城市
def provinces = CommonDatautils.findAllCityProvince();//城市所对省
def sroucesData = selectDistrict("住房", ret, citys);//选取住宅类型
println "住宅共:"+sroucesData.size();
def results = getResult(sroucesData, provinces, citys, regionMap);//填充结果
writeCSV("E:\\数据测试\\结果\\", "安硕住宅数据测试结果.csv", results);
//writeCSV3("E:\\数据测试\\结果\\", "安硕住宅数据测试结果(6-13).csv", results);
def ret = readTxt(path, fileName);//读文件
def citys = CommonDatautils.findAllCity();//所有城市
def regionMap = CommonDatautils.getAllCityRegions();//所有城市
def provinces = CommonDatautils.findAllCityProvince();//城市所对省
def sroucesData = selectDistrict("住房", ret, citys);//选取住宅类型
println "住宅共:"+sroucesData.size();
def results = getResult(sroucesData, provinces, citys, regionMap);//填充结果
writeCSV("E:\\数据测试\\结果\\", "安硕住宅数据测试结果.csv", results);
//writeCSV3("E:\\数据测试\\结果\\", "安硕住宅数据测试结果(6-13).csv", results);
}
def getResult(def sroucesData, def provinces, def citys, def regionMap){
def token = getToken();
def index = 0;
def matchIndex = 0;
// def lastMonth = CommonDatautils.findLastMonth("上海");
// def lastMonth = CommonDatautils.findLastMonth("上海");
for(def m : sroucesData){
index++;
@@ -115,9 +115,9 @@ class AnShouDataTest {
// continue;
// }
println "【${index}】 old:"+m.oldAddress
println "【${index}】 old:"+m.oldAddress
m = selectCity(m, citys);
//省
//省
if(m.city){
m.address = AddressParserUtils.filterCity(m.city, m.oldAddress);
m.province = provinces.get(m.city);
@@ -127,9 +127,9 @@ class AnShouDataTest {
def numMap = AddressParserUtils.selectHouseNumAndRoom(m.address?m.address:m.oldAddress);
println "【旧方法】"+numMap
println "【旧方法】"+numMap
def numMap2 = AddressParserUtils.selectHouseNumAndRoom2(m.address?m.address:m.oldAddress);
println "【新方法】"+numMap2
println "【新方法】"+numMap2
m.houseNum = numMap.houseNum;
m.roomNum = numMap.roomNum;
m.unitNum = numMap.unitNum;
@@ -145,12 +145,12 @@ class AnShouDataTest {
m.address1 = numMap2.s1?numMap2.s1:numMap2.s2;
m.distName1 = numMap2.s2;
//地址匹配
//地址匹配
def rett = null;
if(m.address){
rett = getNameByAPI(m.city, m.address, token);
}
//地址没匹配上,如果能提出小区名,用city+name查找
//地址没匹配上,如果能提出小区名,用city+name查找
if(!rett && m.distName){
rett = getNameByAPI(m.city, m.distName, token);
if(!rett){
@@ -176,7 +176,7 @@ class AnShouDataTest {
if(!m.address){
m.address = rett.address
}else{
if((rett.address.indexOf("弄")!=-1||rett.address.indexOf("号")!=-1||rett.address.indexOf("街")!=-1)&&m.address.indexOf("号")==-1&&m.address.indexOf("路")==-1){
if((rett.address.indexOf("弄")!=-1||rett.address.indexOf("号")!=-1||rett.address.indexOf("街")!=-1)&&m.address.indexOf("号")==-1&&m.address.indexOf("路")==-1){
m.address = rett.address
}
}
@@ -195,24 +195,24 @@ class AnShouDataTest {
m = getDistrictInfo(m);
if(m.address){
m.address = m.address.replace("#", "#").replaceAll(",", "、");
m.address = m.address.replace("#", "#").replaceAll(",", "、");
}
if(m.address){
m.address = m.address.replace("#", "#").replaceAll(",", "、");
m.address = m.address.replace("#", "#").replaceAll(",", "、");
}
//if(m.houseNum ||m.roomNum){
println "结果1: 【${m.name}】【${m.houseNum}】【${m.unitNum}】【${m.roomNum}】"
println "结果1: 【${m.name}】【${numMap2.houseNum}】【${numMap2.unitNum}】【${numMap2.roomNum}】"
println "结果2:【${m.address}】【${m.distName}】【${m.houseInfo}】【${m.baiduMatch}】【${m.baiduAddress}】"
println "结果1: 【${m.name}】【${m.houseNum}】【${m.unitNum}】【${m.roomNum}】"
println "结果1: 【${m.name}】【${numMap2.houseNum}】【${numMap2.unitNum}】【${numMap2.roomNum}】"
println "结果2:【${m.address}】【${m.distName}】【${m.houseInfo}】【${m.baiduMatch}】【${m.baiduAddress}】"
println "..................................................."
// }
}
println "能匹配上的有:[ "+matchIndex+" ]";
println "能匹配上的有:[ "+matchIndex+" ]";
return sroucesData;
}
@@ -230,7 +230,7 @@ class AnShouDataTest {
m.avgPrice = CommonDatautils.queryDistrictPrice(m.city, m.region, m.name, weekend, m.cat);
}
//房价网估价
//房价网估价
m.fangjiaTotalPrice = evaluation(m, token);
if(!m.fangjiaTotalPrice && m.avgPrice){
m.fangjiaTotalPrice = (m.avgPrice * m.area).toInteger();
@@ -238,7 +238,7 @@ class AnShouDataTest {
//m.rentPrice = queryRentPriceRange(m.city, m.region, m.name, lastMonth);
//其它网站均价
//其它网站均价
def otherWebsiteMap = CommonDatautils.queryOtherWebsitePrice(m.city, m.name);
m.sunfunPrice = otherWebsiteMap.price_soufun
m.anjukePrice = otherWebsiteMap.price_anjuke
@@ -267,7 +267,7 @@ class AnShouDataTest {
m.city = m.oldCity;
return m;
}else{
def city = m.oldAddress.substring(0, m.oldAddress.indexOf("市"));
def city = m.oldAddress.substring(0, m.oldAddress.indexOf("市"));
if(city != m.oldCity){
m.city = city;
}
@@ -281,8 +281,8 @@ class AnShouDataTest {
}
def selectRegion(def m){
def address = m.oldAddress;
if(address.indexOf("市")!=-1 && address.indexOf("区")!=-1){
m.region = address.substring(address.indexOf("市")+1, address.indexOf("区"));
if(address.indexOf("市")!=-1 && address.indexOf("区")!=-1){
m.region = address.substring(address.indexOf("市")+1, address.indexOf("区"));
}
return m;
}
@@ -291,25 +291,25 @@ class AnShouDataTest {
m.roomNum = filterRoomNum(m.roomNum);
m.unitNum = selectUnitNum(m.address!=null?m.address:m.oldAddress);
//所在楼层
//所在楼层
//m = selectHouseFloor(m);
m = standardAddress(m);
return m;
}
def selectUnitNum(def address){
def ret = findPattern(address, "[幢|栋|楼](\\d+|\\w+|\\S)[单元|座]",0);
def ret = findPattern(address, "[幢|栋|楼](\\d+|\\w+|\\S)[单元|座]",0);
if(!ret.num1){
ret = findPattern(address, "[层](\\d+|\\w+|\\S)[单元|座]",0);
ret = findPattern(address, "[层](\\d+|\\w+|\\S)[单元|座]",0);
}
if(!ret.num1){
ret = findPattern(address, "(\\d+|\\w+|\\S)[单元|座]",0);
ret = findPattern(address, "(\\d+|\\w+|\\S)[单元|座]",0);
}
if(ret.num1){
ret.num1 = ret.num1.replace("一", "1").replace("二", "2").replace("三", "3").replace("四", "4").replace("五", "5").replace("六", "6").replace("七", "7").replace("八", "8").replace("九", "9").replace("十", "10")
ret.num1 = ret.num1.replace("一", "1").replace("二", "2").replace("三", "3").replace("四", "4").replace("五", "5").replace("六", "6").replace("七", "7").replace("八", "8").replace("九", "9").replace("十", "10")
}
return ret.num1;
}
@@ -334,8 +334,8 @@ class AnShouDataTest {
def address = m.address!=null?m.address:m.oldAddress;
if(!m.region && address.indexOf("县")!=-1){
m.region = address.substring(0, address.indexOf("县"));
if(!m.region && address.indexOf("县")!=-1){
m.region = address.substring(0, address.indexOf("县"));
if(m.region){
m.address = address.substring(address.indexOf(m.region)+m.region.length() + 1, address.length());
}
@@ -343,15 +343,15 @@ class AnShouDataTest {
}
if(!m.region && address.indexOf("区")!=-1){
m.region = address.substring(0, address.indexOf("区")+1);
if(!m.region && address.indexOf("区")!=-1){
m.region = address.substring(0, address.indexOf("区")+1);
if(m.region){
m.address = address.substring(address.indexOf(m.region)+m.region.length(), address.length());
def index = m.address.indexOf("区");
def index = m.address.indexOf("区");
if(index!=-1){
def end = m.address.indexOf("街");
def end = m.address.indexOf("街");
if(end==-1){
end = m.address.indexOf("路");
end = m.address.indexOf("路");
}
if(index < end){
@@ -362,22 +362,22 @@ class AnShouDataTest {
m.region = findStandardRegion(m.city, m.region, regionMap);
}
if(!m.region && address.indexOf("市")!=-1){
m.region = address.substring(0, address.indexOf("市"));
if(!m.region && address.indexOf("市")!=-1){
m.region = address.substring(0, address.indexOf("市"));
if(m.region){
m.address = address.substring(address.indexOf(m.region)+m.region.length() + 1, address.length());
}
m.region = findStandardRegion(m.city, m.region, regionMap);
}
m.oldAddress = m.oldAddress.replaceAll(",", "、");
m.address = m.address.replace("#", "#").replaceAll(",", "、");
m.oldAddress = m.oldAddress.replaceAll(",", "、");
m.address = m.address.replace("#", "#").replaceAll(",", "、");
return m;
}
def standardAddress(def m){
if(m.address.indexOf("号")!=-1){
m.address = m.address.substring(0, m.address.indexOf("号")+1);
if(m.address.indexOf("号")!=-1){
m.address = m.address.substring(0, m.address.indexOf("号")+1);
}else if(m.houseNum){
m.address = m.address.substring(0, m.address.indexOf(m.houseNum));
}else if(m.roomNum){
@@ -391,53 +391,53 @@ class AnShouDataTest {
def selectHouseNumAndRoom(def map){
def address = map.address?map.address:map.oldAddress;
def ret = findPattern(address, "[弄|区](\\d+)[号|幢|栋](\\w{2,4})?[室]?",1);
def ret = findPattern(address, "[弄|区](\\d+)[号|幢|栋](\\w{2,4})?[室]?",1);
map.houseNum = ret.num1;
map.roomNum = ret.num2;
if(!map.houseNum){
address = map.address?map.address:map.oldAddress;
ret = findPattern(address, "号(\\d+)[幢|栋](\\d+)?[室|房]", 1);
ret = findPattern(address, "号(\\d+)[幢|栋](\\d+)?[室|房]", 1);
map.houseNum = ret.num1;
map.roomNum = ret.num2;
}
if(!map.houseNum){
address = map.address?map.address:map.oldAddress;
ret = findPattern(address, "(\\d+|\\w+)[幢|栋]", 0);
ret = findPattern(address, "(\\d+|\\w+)[幢|栋]", 0);
map.houseNum = ret.num1;
}
if(!map.houseNum){
address = map.address?map.address:map.oldAddress;
ret = findPattern(address, "(\\d+|\\w+)号楼", 0);
ret = findPattern(address, "(\\d+|\\w+)号楼", 0);
map.houseNum = ret.num1;
}
if(!map.houseNum){
address = map.address?map.address:map.oldAddress;
ret = findPattern(address, "(\\d+)#楼", 0);
ret = findPattern(address, "(\\d+)#楼", 0);
map.houseNum = ret.num1;
}
if(!map.roomNum){
address = map.address?map.address:map.oldAddress;
ret = findPattern(address, "(\\w{1,4}|\\d+)[室|房]",0);
ret = findPattern(address, "(\\w{1,4}|\\d+)[室|房]",0);
map.roomNum = ret.num1;
}
if(!map.roomNum){
address = map.address?map.address:map.oldAddress;
ret = findPattern(address, "单元(\\w{1,4})[号|室]",0);
ret = findPattern(address, "单元(\\w{1,4})[号|室]",0);
map.roomNum = ret.num1;
}
if(!map.houseNum&&!map.roomNum && map.oldAddress.indexOf("路")==-1 && map.oldAddress.indexOf("街")==-1){
ret = findPattern(map.oldAddress, "(\\d+)[号|幢|栋](\\w{2,4})?[室]?",0);
if(!map.houseNum&&!map.roomNum && map.oldAddress.indexOf("路")==-1 && map.oldAddress.indexOf("街")==-1){
ret = findPattern(map.oldAddress, "(\\d+)[号|幢|栋](\\w{2,4})?[室]?",0);
map.houseNum = ret.num1;
map.roomNum = ret.num2;
@@ -446,7 +446,7 @@ class AnShouDataTest {
if(!map.roomNum){
address = map.address?map.address:map.oldAddress;
ret = findPattern(address, "[座|号楼|梯|层](\\d{3,5})",0);
ret = findPattern(address, "[座|号楼|梯|层](\\d{3,5})",0);
map.roomNum = ret.num1;
}
@@ -456,7 +456,7 @@ class AnShouDataTest {
map.houseNum = ret.num1;
map.roomNum = ret.num2;
}
//println map.oldAddress+" 楼栋号结果:["+map.houseNum+"]["+map.roomNum+"]["+map.address+"]";
//println map.oldAddress+" 楼栋号结果:["+map.houseNum+"]["+map.roomNum+"]["+map.address+"]";
return map;
}
@@ -478,7 +478,7 @@ class AnShouDataTest {
}
if(!map.floor){
def address = map.address?map.address:map.oldAddress;
def ret = findPattern(address, "(\\d+)[层|F|f]", 0);
def ret = findPattern(address, "(\\d+)[层|F|f]", 0);
map.floor = ret.num1;
if(ret.address){
map.address = ret.address;
@@ -492,7 +492,7 @@ class AnShouDataTest {
if(!map.floor){
map.floor = null;
}
//println map.oldAddress +" 结果:"+map.floor+" ["+map.address+"]";
//println map.oldAddress +" 结果:"+map.floor+" ["+map.address+"]";
return map;
}
@@ -575,7 +575,7 @@ class AnShouDataTest {
}
//println "周边均价:" + totalPrice +"【${url_str}】"
//println "周边均价:" + totalPrice +"【${url_str}】"
return totalPrice;
}
@@ -592,7 +592,7 @@ class AnShouDataTest {
List vs = vo.get("cla");
for(String[] o : vs){
String s = o[1];
if(s.indexOf("小区")!=-1||s.indexOf("楼盘")!=-1){
if(s.indexOf("小区")!=-1||s.indexOf("楼盘")!=-1){
address = vo.get("addr");
break;
}
@@ -705,7 +705,7 @@ class AnShouDataTest {
conn.setRequestMethod("GET");
conn.doOutput = true
inputStream = conn.getInputStream();
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
while((line = buf.readLine())!=null){
sb.append(line);
}
@@ -743,14 +743,14 @@ class AnShouDataTest {
for(def m : ret){
if(m.estateType == estateType){
result << m;
// if(m.address.indexOf("市")!=-1){
// def city = m.address.substring(0, m.address.lastIndexOf("市"));
// if(city.indexOf("市")!=-1){
// def tempCity = city.substring(city.indexOf("市")+1, city.length());
// if(m.address.indexOf("市")!=-1){
// def city = m.address.substring(0, m.address.lastIndexOf("市"));
// if(city.indexOf("市")!=-1){
// def tempCity = city.substring(city.indexOf("市")+1, city.length());
// if(citys.contains(tempCity)){
// city = tempCity;
// }else{
// city = m.address.substring(0, m.address.indexOf("市"))
// city = m.address.substring(0, m.address.indexOf("市"))
// }
// }
// if(city){
@@ -784,11 +784,11 @@ class AnShouDataTest {
String estateType = arrs[4]
def m = [:];
m.cid = cid;
if(estateType=="住房"){
if(estateType=="住房"){
m.cat = "district";
}else if(estateType=='写字楼'){
}else if(estateType=='写字楼'){
m.cat = "office";
}else if(estateType=="商铺"){
}else if(estateType=="商铺"){
m.cat = "shop";
}else{
m.cat = "other";
@@ -811,16 +811,16 @@ class AnShouDataTest {
}
file.append("城市,")
file.append("地址,")
file.append("楼盘名称,")
file.append("省,")
file.append("楼盘地址,")
file.append("别名,")
file.append("楼号,")
file.append("室号,")
file.append("单元号,")
file.append("状态,")
file.append("城市,")
file.append("地址,")
file.append("楼盘名称,")
file.append("省,")
file.append("楼盘地址,")
file.append("别名,")
file.append("楼号,")
file.append("室号,")
file.append("单元号,")
file.append("状态,")
file.append(System.getProperty("line.separator"));
ret.each{
@@ -864,29 +864,29 @@ class AnShouDataTest {
}
file.append("房产编号,")
file.append("房产地址,")
file.append("城市,")
file.append("建筑面积,")
file.append("抵押物类型,")
file.append("标准化-省,")
file.append("标准化-区,")
file.append("标准化-市,")
file.append("标准化-板块,")
file.append("标准化-楼盘,")
file.append("标准化-楼盘地址,")
file.append("楼栋,")
file.append("单元(号),")
file.append("房号,")
file.append("楼盘单价,")
file.append("批量估值(精准),")
file.append("搜房网楼盘单价,")
file.append("搜房网精确估值,")
file.append("安居客楼盘价格,")
//file.append("匹配状态,")
//file.append("别名,")
file.append("安居客精确估值,")
// file.append("租金,")
file.append("房产编号,")
file.append("房产地址,")
file.append("城市,")
file.append("建筑面积,")
file.append("抵押物类型,")
file.append("标准化-省,")
file.append("标准化-区,")
file.append("标准化-市,")
file.append("标准化-板块,")
file.append("标准化-楼盘,")
file.append("标准化-楼盘地址,")
file.append("楼栋,")
file.append("单元(号),")
file.append("房号,")
file.append("楼盘单价,")
file.append("批量估值(精准),")
file.append("搜房网楼盘单价,")
file.append("搜房网精确估值,")
file.append("安居客楼盘价格,")
//file.append("匹配状态,")
//file.append("别名,")
file.append("安居客精确估值,")
// file.append("租金,")
file.append(System.getProperty("line.separator"));
ret.each{
file.append(it.cid + ",","gbk")
@@ -993,17 +993,17 @@ class AnShouDataTest {
}
file.append("房产地址,")
file.append("标准化-区,")
file.append("标准化-板块,")
file.append("标准化-楼盘,")
file.append("标准化-楼盘地址,")
file.append("楼栋,")
//file.append("楼栋-新,")
file.append("单元(号),")
//file.append("单元(号)-新,")
file.append("房号,")
//file.append("房号-新,")
file.append("房产地址,")
file.append("标准化-区,")
file.append("标准化-板块,")
file.append("标准化-楼盘,")
file.append("标准化-楼盘地址,")
file.append("楼栋,")
//file.append("楼栋-新,")
file.append("单元(号),")
//file.append("单元(号)-新,")
file.append("房号,")
//file.append("房号-新,")
file.append(System.getProperty("line.separator"));
ret.each{
@@ -12,7 +12,7 @@ import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
*批量地址分析脚本
*批量地址分析脚本
*/
@BatchOwner(Watcher.WangWenYan)
public class BulkAddressAnalyBatch extends BaseBatch{
@@ -41,7 +41,7 @@ public class BulkAddressAnalyBatch extends BaseBatch{
}
}
/**
* 恢复被打断的任务
* 恢复被打断的任务
*/
void resumeInterrtuptedTask(){
@@ -57,7 +57,7 @@ public class BulkAddressAnalyBatch extends BaseBatch{
task.status = TaskStatus.TASK_STATUS_DOING
TASK.save(task)
executorService.submit(new AddressAnalysisJob(task, false))
logger.info("提交任务:${task._id.toString()}")
logger.info("提交任务:${task._id.toString()}")
}
}
@@ -69,7 +69,7 @@ public class BulkEvaluationForTest extends BaseBatch{
def findNewTask(MqProvider mqProvider){
ADDRESS_COLL.find(new BasicDBObject("finish_mark",0)).sort(new BasicDBObject("version", -1)).each {BasicDBObject dbo->
ADDRESS_COLL.update(new BasicDBObject("_id", dbo._id), new BasicDBObject('$set', new BasicDBObject("finish_mark", 1).append("send_date", new Date())))
println("${new Date().format("yyyy-MM-dd HH:mm:ss")}Ìá½»ÈÎÎñ${dbo.city} ${dbo.address}")
println("${new Date().format("yyyy-MM-dd HH:mm:ss")}提交任务${dbo.city} ${dbo.address}")
mqProvider.send(dbo.get("_id").toString())
//executorService.submit(new BulkEvaluationTask(dbo))
//new BulkEvaluationTask(dbo);
@@ -53,7 +53,7 @@ public class BulkEvaluationTask implements Runnable {
Integer height = task.height
String build_date = task.build_date
String buildDate = null
String estateType = "普通住宅"
String estateType = "普通住宅"
if(StringUtils.isNotBlank(build_date)){
Pattern pattern = Pattern.compile("^\\d\\d\\d\\d");
def matcher = pattern.matcher(build_date)
@@ -62,15 +62,15 @@ public class BulkEvaluationTask implements Runnable {
}
}
if (StringUtils.isBlank(city) || area == null || area <= 0 || area > 1000) {
task.msg = "参数非法"
task.msg = "参数非法"
finishTask()
return
}
if (city.endsWith("市")) {
city = city.replaceAll("市\$", "")
if (city.endsWith("市")) {
city = city.replaceAll("市\$", "")
}
if((districtName && districtName.contains("别墅")) || (address && address.contains("别墅"))){
estateType = "别墅"
if((districtName && districtName.contains("别墅")) || (address && address.contains("别墅"))){
estateType = "别墅"
}
JSONObject matchResult = null
@@ -117,7 +117,7 @@ public class BulkEvaluationTask implements Runnable {
BasicDBObject district = DbUtils.seaweed.findById(matchResult.getString("id"));
def propertyType = null;
if(district){
if(district.estate_type2.contains("别墅")){
if(district.estate_type2.contains("别墅")){
propertyType = 1;
}else{
propertyType = 0;
@@ -198,7 +198,7 @@ public class BulkEvaluationTask implements Runnable {
task.fjw_price = (task.total_price/task.area).toInteger();
}
//房估估价格
//房估估价格
if(task.match_city && task.match_region && task.match_name){
def priceObj = findFangguguAndYungujiaPrice(task.match_city, task.match_region, task.match_name);
if(priceObj){
@@ -211,7 +211,7 @@ public class BulkEvaluationTask implements Runnable {
}
//其它网站价格
//其它网站价格
def startDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -6, "yyyy-MM");
startDate = startDate.replace("-", "")
@@ -300,7 +300,7 @@ public class BulkEvaluationTask implements Runnable {
} catch (Exception e) {
e.printStackTrace()
}
println("${new Date().format("yyyy-MM-dd HH:mm:ss")}估价完成")
println("${new Date().format("yyyy-MM-dd HH:mm:ss")}估价完成")
}
def findFangguguAndYungujiaPrice(def city, def region, def name){
@@ -514,7 +514,7 @@ public class BulkEvaluationTask implements Runnable {
def values = [];
def fjw_price = fjwPrice;
//找出最小差
//找出最小差
for(def i=0;i<prices.size()-1;i++){
def a = prices[i]
for(def j=i+1;j<prices.size();j++){
@@ -526,7 +526,7 @@ public class BulkEvaluationTask implements Runnable {
}
}
//根据最小差找出结果
//根据最小差找出结果
for(def i=0;i<prices.size()-1;i++){
def a = prices[i]
for(def j=i+1;j<prices.size();j++){
@@ -1,7 +1,7 @@
package com.fangjia.script.address
/**
* 去除相邻重复的字符串(不包括单个字符重复)
* @author 王文艳
* 去除相邻重复的字符串(不包括单个字符重复)
* @author 王文艳
*
*/
class DuplicateRemovalHandler extends AddressStandardizeHandler {
@@ -11,7 +11,7 @@ class DuplicateRemovalHandler extends AddressStandardizeHandler {
try{
request.setAddress(duplicateRemoval(request.getAddress()));
}catch(Exception e){
println "DuplicateRemovalHandler: " + request.getAddress()+" 处理失败,message:"+e.getMessage();
println "DuplicateRemovalHandler: " + request.getAddress()+" 处理失败,message:"+e.getMessage();
}
}
@@ -26,7 +26,7 @@ class DuplicateRemovalHandler extends AddressStandardizeHandler {
for(i=0; i< s.length();i++){
for(int j= i+1; j < s.length() ; j++ ){
foundStr = s.substring(i, j+1);
if(s.indexOf(foundStr,j+1) == j+1 && !(foundStr ==~/\d+/)){//方向相邻重复且非数字的字符串片段
if(s.indexOf(foundStr,j+1) == j+1 && !(foundStr ==~/\d+/)){//方向相邻重复且非数字的字符串片段
s = s.replaceFirst(foundStr, "");
continue outer;
}
@@ -36,7 +36,7 @@ class DuplicateRemovalHandler extends AddressStandardizeHandler {
}
public static void main(String[] args){
Request request = new Request("西南1000米)");
Request request = new Request("西南1000米)");
new DuplicateRemovalHandler().handleRequest(request)
print request.getAddress();
}
@@ -36,15 +36,15 @@ class JinDongDataTest {
static main(args) {
def task = new JinDongDataTest();
//task.batchTestApi("E:\\数据测试\\数据\\", "京东匹配成功数据", ".txt");
task.batchTest1("E:\\数据测试\\数据\\", "商铺-京东", ".txt");
//task.batchTest1("E:\\数据测试\\数据\\", "住宅找到小区-京东", ".txt");
//task.batchTest2("E:\\数据测试\\数据\\", "京东测试数据", ".txt");
//task.batchTestApi("E:\\数据测试\\数据\\", "京东匹配成功数据", ".txt");
task.batchTest1("E:\\数据测试\\数据\\", "商铺-京东", ".txt");
//task.batchTest1("E:\\数据测试\\数据\\", "住宅找到小区-京东", ".txt");
//task.batchTest2("E:\\数据测试\\数据\\", "京东测试数据", ".txt");
}
def batchTestApi(def path, def fileName, def fileType){
def ret = readTxt(path, fileName+fileType);//读文件
def ret = readTxt(path, fileName+fileType);//读文件
def result = [];
def index = 0;
for(def map : ret){
@@ -58,40 +58,40 @@ class JinDongDataTest {
rett = getDistrictBaseInfo(rett);
rett = getBiduLngLat(rett.city, rett.oldAddress, rett);
rett = getDistrictKind(rett);
println "基本信息查询完成....."
println "基本信息查询完成....."
rett = getDistrictInfo(rett)
println "小区均价查询完成....."
println "小区均价查询完成....."
rett = getCityAvgPriceInfo(rett);
println "城市均价查询完成....."
println "城市均价查询完成....."
rett = getRentAvgPriceInfo(rett);
println "租金信息查询完成....."
println "租金信息查询完成....."
rett = getOneYearDistrictPriceInfo(rett);
println "最大最小均价完成....."
println "最大最小均价完成....."
rett = getPricePredict(rett);
println "预测完成....."
println "预测完成....."
rett = getAvgConsumption(rett);
println "人均消费完成....."
println "人均消费完成....."
rett = getOfficeCount(rett);
println "办公人口完成....."
println "办公人口完成....."
rett = getPoiCount(rett);
println "poi完成....."
println "poi完成....."
rett = getRegionInfo(rett);
rett = getBlockInfo(rett);
println "区域板块完成....."
println "区域板块完成....."
rett = getHouseCount(rett);
println "房子数完成....."
println "房子数完成....."
result << rett;
println "["+index+"]"+rett
}
def sysTime = System.currentTimeMillis();
writeCSV("E:\\数据测试\\结果\\", fileName+"("+sysTime+").csv", result);
writeCSV("E:\\数据测试\\结果\\", fileName+"("+sysTime+").csv", result);
}
def batchTest1(def path, def fileName, def fileType){
def ret = readTxt(path, fileName+fileType);//读文件
def ret = readTxt(path, fileName+fileType);//读文件
def result = [];
def index = 0;
for(def rett : ret){
@@ -108,54 +108,54 @@ class JinDongDataTest {
rett.oldName = parseMap.oldName;
rett.db_cat = parseMap.cat
rett.area = 100;
rett.cat = rett.type=='写字楼'?'office':(rett.type=='商铺'?'shop':(rett.type=='住宅'?"district":""));
rett.cat = rett.type=='写字楼'?'office':(rett.type=='商铺'?'shop':(rett.type=='住宅'?"district":""));
rett = getDistrictBaseInfo(rett);
if(!rett.type && !rett.cat){
rett.cat = "district";
}
rett = getBiduLngLat(rett.city, rett.oldAddress, rett);
rett = getDistrictKind(rett);
println "基本信息查询完成....."
println "基本信息查询完成....."
// rett = getDistrictInfo(rett)
// println "小区均价查询完成....."
// println "小区均价查询完成....."
// if(rett.cat=='office'||rett.cat=="shop"){
// rett = getShopOrOfficeCityAvgPriceInfo(rett);
// println "城市均价查询完成....."
// println "城市均价查询完成....."
// rett = getShopOrOfficeRentAvgPriceInfo(rett);
// println "租金信息查询完成....."
// println "租金信息查询完成....."
// }else{
// rett = getCityAvgPriceInfo(rett);
// println "城市均价查询完成....."
// println "城市均价查询完成....."
// rett = getRentAvgPriceInfo(rett);
// println "租金信息查询完成....."
// println "租金信息查询完成....."
// }
// rett = getOneYearDistrictPriceInfo(rett);
// println "最大最小均价完成....."
// println "最大最小均价完成....."
// rett = getPricePredict(rett);
// println "预测完成....."
// println "预测完成....."
// rett = getAvgConsumption(rett);
// println "人均消费完成....."
// println "人均消费完成....."
// rett = getOfficeCount(rett);
// println "写字楼完成....."
// println "写字楼完成....."
// rett = getPoiCount(rett);
// println "poi完成....."
// println "poi完成....."
// rett = getRegionInfo(rett);
// rett = getBlockInfo(rett);
// println "区域板块完成....."
// println "区域板块完成....."
// rett = getHouseCount(rett);
// println "房子数完成....."
// println "房子数完成....."
result << rett;
println "["+index+"]"+rett
}
def sysTime = System.currentTimeMillis();
writeCSV("E:\\数据测试\\结果\\", fileName+"("+sysTime+").csv", result);
writeCSV("E:\\数据测试\\结果\\", fileName+"("+sysTime+").csv", result);
}
def batchTest2(def path, def fileName, def fileType){
def ret = readTxt(path, fileName+fileType);//读文件
def ret = readTxt(path, fileName+fileType);//读文件
def index = 1;
def result = [];
for(def rett : ret){
@@ -172,7 +172,7 @@ class JinDongDataTest {
}
writeCSV2("E:\\数据测试\\结果\\", fileName+"(周边人均消费).csv", result);
writeCSV2("E:\\数据测试\\结果\\", fileName+"(周边人均消费).csv", result);
}
def getDistrictInfo(m){
if(m.name){
@@ -200,7 +200,7 @@ class JinDongDataTest {
}
//周边均价
//周边均价
if(!m.avg_price && m.oldAddress && m.area && m.city){
@@ -219,7 +219,7 @@ class JinDongDataTest {
if(city){
def nearCityPrice = CommonDatautils.findCityPrice(city, null)
//半年前最近的城市均价
//半年前最近的城市均价
def weekend = DateUtil.formatByPattern(DateUtil.getFirstDayOfWeek(new Date()), "yyyy-MM-dd");
def monthAgo = DateUtils.add(weekend, -30);
def monthAgoPrice = CommonDatautils.findCityPrice(city, monthAgo);
@@ -237,7 +237,7 @@ class JinDongDataTest {
if(city){
def nearCityPrice = CommonDatautils.findShopOrOfficeCityPrice(city, rett.cat, null)
//半年前最近的城市均价
//半年前最近的城市均价
def weekend = DateUtil.formatByPattern(DateUtil.getFirstDayOfWeek(new Date()), "yyyy-MM-dd");
def monthAgo = DateUtils.add(weekend, -30);
def monthAgoPrice = CommonDatautils.findShopOrOfficeCityPrice(city, rett.cat, monthAgo);
@@ -280,7 +280,7 @@ class JinDongDataTest {
public static def getOneYearDistrictPriceInfo(def rett){
if(rett.city && rett.region && rett.name){
def weekend = DateUtil.formatByPattern(DateUtil.getFirstDayOfWeek(new Date()), "yyyy-MM-dd");
//一前内小区均价
//一前内小区均价
def oneYearAgo = DateUtils.add(weekend, -365);
def oneYearPriceList = CommonDatautils.queryDistrictPrices(rett.city, rett.region, rett.name, rett.kind, oneYearAgo, weekend);
if(oneYearPriceList.size()>0){
@@ -345,11 +345,11 @@ class JinDongDataTest {
def getPoiCount(def rett){
if(rett.lng2 && rett.lat2){
rett.hspt_count = CommonDatautils.findPoiCount("医院", rett.lat2, rett.lng2)
rett.bank_count = CommonDatautils.findPoiCount("银行", rett.lat2, rett.lng2)
rett.school_count = CommonDatautils.findPoiCount("学校", rett.lat2, rett.lng2)
rett.bus_count = CommonDatautils.findPoiCount("公交", rett.lat2, rett.lng2)
rett.subway_count = CommonDatautils.findPoiCount("地铁", rett.lat2, rett.lng2)
rett.hspt_count = CommonDatautils.findPoiCount("医院", rett.lat2, rett.lng2)
rett.bank_count = CommonDatautils.findPoiCount("银行", rett.lat2, rett.lng2)
rett.school_count = CommonDatautils.findPoiCount("学校", rett.lat2, rett.lng2)
rett.bus_count = CommonDatautils.findPoiCount("公交", rett.lat2, rett.lng2)
rett.subway_count = CommonDatautils.findPoiCount("地铁", rett.lat2, rett.lng2)
}
return rett;
@@ -425,21 +425,21 @@ class JinDongDataTest {
def getDistrictKind(def rett){
if(rett.type){
if(rett.type.indexOf("写字楼")!=-1){
rett.kind = "写字楼";
}else if(rett.type.indexOf("商铺")!=-1){
rett.kind = "商铺";
if(rett.type.indexOf("写字楼")!=-1){
rett.kind = "写字楼";
}else if(rett.type.indexOf("商铺")!=-1){
rett.kind = "商铺";
}else{
if(rett.estateType){
if(rett.estateType.indexOf("别墅")!=-1){
rett.kind = "别墅";
if(rett.estateType.indexOf("别墅")!=-1){
rett.kind = "别墅";
}
}
}
}
if(!rett.kind){
rett.kind = "小区";
rett.kind = "小区";
}
return rett;
}
@@ -487,7 +487,7 @@ class JinDongDataTest {
}
//println "周边均价:" + totalPrice +"【${url_str}】"
//println "周边均价:" + totalPrice +"【${url_str}】"
return totalPrice;
}
@@ -504,7 +504,7 @@ class JinDongDataTest {
conn.setRequestMethod("GET");
conn.doOutput = true
inputStream = conn.getInputStream();
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
while((line = buf.readLine())!=null){
sb.append(line);
}
@@ -531,11 +531,11 @@ class JinDongDataTest {
if (file.exists()){
file.delete();
}
file.append("序号,")
file.append("地址,")
file.append("经度,")
file.append("纬度,")
file.append("周边餐饮的人均消费,")
file.append("序号,")
file.append("地址,")
file.append("经度,")
file.append("纬度,")
file.append("周边餐饮的人均消费,")
file.append(System.getProperty("line.separator"));
ret.each{
file.append(it.id + ",","gbk")
@@ -570,47 +570,47 @@ class JinDongDataTest {
}
file.append("序号,")
file.append("地址,")
file.append("地址属性,")
file.append("标准化-省,")
file.append("城市,")
file.append("区域,")
file.append("板块,")
file.append("数据库类型,")
file.append("楼盘名称,")
file.append("匹配楼盘名称,")
file.append("地址中楼盘名称,")
file.append("物业地址,")
file.append("物业类型,")
file.append("经度,")
file.append("纬度,")
file.append("户数,")
file.append("产权性质,")
file.append("竣工时间,")
file.append("开发商,")
file.append("物业公司,")
file.append("物业费,")
file.append("建筑面积,")
file.append("楼盘单价,")
file.append("城市均价,")
file.append("月城市均价涨跌幅,")
file.append("租房均价,")
file.append("小区月均价涨跌幅,")
file.append("一年内小区历史最高价,")
file.append("一年内小区历史最低价,")
file.append("未来一年房价预测,")
file.append("周边餐饮的人均消费,")
file.append("板块均价排名,")
file.append("板块楼盘个数,")
file.append("区域均价排名,")
file.append("区域楼盘个数,")
file.append("写字楼数量,")
file.append("医院数量,")
file.append("银行数量,")
file.append("学校数量,")
file.append("公交车站数量,")
file.append("地铁数量,")
file.append("序号,")
file.append("地址,")
file.append("地址属性,")
file.append("标准化-省,")
file.append("城市,")
file.append("区域,")
file.append("板块,")
file.append("数据库类型,")
file.append("楼盘名称,")
file.append("匹配楼盘名称,")
file.append("地址中楼盘名称,")
file.append("物业地址,")
file.append("物业类型,")
file.append("经度,")
file.append("纬度,")
file.append("户数,")
file.append("产权性质,")
file.append("竣工时间,")
file.append("开发商,")
file.append("物业公司,")
file.append("物业费,")
file.append("建筑面积,")
file.append("楼盘单价,")
file.append("城市均价,")
file.append("月城市均价涨跌幅,")
file.append("租房均价,")
file.append("小区月均价涨跌幅,")
file.append("一年内小区历史最高价,")
file.append("一年内小区历史最低价,")
file.append("未来一年房价预测,")
file.append("周边餐饮的人均消费,")
file.append("板块均价排名,")
file.append("板块楼盘个数,")
file.append("区域均价排名,")
file.append("区域楼盘个数,")
file.append("写字楼数量,")
file.append("医院数量,")
file.append("银行数量,")
file.append("学校数量,")
file.append("公交车站数量,")
file.append("地铁数量,")
file.append(System.getProperty("line.separator"));
ret.each{
file.append(it.id + ",","gbk")
@@ -888,7 +888,7 @@ class JinDongDataTest {
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("address", address);
params.put("businessMatch", "1");//没有找到,进行商铺写字楼匹配
params.put("businessMatch", "1");//没有找到,进行商铺写字楼匹配
params.put("token", "FF231CF2F7268316D937374DB96AC91ED7478F663E9D233D7389B1F0A809038F59F52B983E627AD8E95FB644D8C43601E5FEE7F107B3BFD8F5AF2D73853591F481A34E26723B657C2DD2D05684DB12F69C37447B272F857EFE2BEC49BADB44A3A2DBDB7CDBE2AA25");
HttpRequest http = new HttpRequest();
Map<String, Object> map = null;
@@ -14,14 +14,14 @@ class OfficeLivePrecent {
static main(args) {
def task = new OfficeLivePrecent();
task.run("上海", "E:\\数据测试\\数据\\", "写字楼.txt");
task.run("上海", "E:\\数据测试\\数据\\", "写字楼.txt");
}
def run(def city, def path, def fileName){
def ret = readTxt(path, fileName);
def offices = findOffice(city);
def results = matchOffice(ret, offices);
writeCSV("E:\\数据测试\\结果\\", "写字楼.csv", results);
writeCSV("E:\\数据测试\\结果\\", "写字楼.csv", results);
}
def matchOffice(def ret, def offices){
@@ -67,8 +67,8 @@ class OfficeLivePrecent {
if (file.exists()){
file.delete();
}
file.append("名称,")
file.append("入住率,")
file.append("名称,")
file.append("入住率,")
file.append(System.getProperty("line.separator"));
list.each{m->
@@ -21,8 +21,8 @@ public class RegionRemoveHandler extends AddressStandardizeHandler {
}
public static void main(String[] args){
Request request = new Request("팹顧팹顧쨀소領돎땜켓沽쨀베쨌宅쨀깝켓쨌슥썹뇹");
request.setRegion("팹顧");
Request request = new Request("普陀普陀鲁家峙岛东南涂鲁海路与鲁滨南路交界处");
request.setRegion("普陀");
new RegionRemoveHandler().handleRequest(request);
println request.getAddress();
}
@@ -21,7 +21,7 @@ class ShopAddressBatch {
def run(){
def data = readExcel("E:\\数据导出\\商铺\\201412.xlsx");
def data = readExcel("E:\\数据导出\\商铺\\201412.xlsx");
data = getLatLng(data);
saveData(data,"2014-12");
}
@@ -30,7 +30,7 @@ class ShopAddressBatch {
for(def row : data){
DBObject obj = new BasicDBObject()
obj.city = "上海";
obj.city = "上海";
obj.region = row[0];
obj.block = row[1];
obj.name = row[2];
@@ -81,17 +81,17 @@ class ShopAddressBatch {
def rowCount = workbook.getSheetAt(0).getLastRowNum();
def sheet1=workbook.getSheetAt(0);
def rett = [];
//循环行
//循环行
sheet1.eachWithIndex {row, index->
if(index>0){
def cells = row.physicalNumberOfCells;//取得列数
def cells = row.physicalNumberOfCells;//取得列数
if(cells>1){
def ret = [];
for(def i=0;i<10;i++){
def value = row.getCell(i);
def name = value.toString();
if(name.indexOf("E")!=-1&&!name.contains("路")){
if(name.indexOf("E")!=-1&&!name.contains("路")){
BigDecimal db = new BigDecimal(name);
name = db.toPlainString()+".0"
}
@@ -135,7 +135,7 @@ class ShopAddressBatch {
for(def row : data){
def address = row[3];
def map = getLonLngV2("上海市"+address);
def map = getLonLngV2("上海市"+address);
if(map){
row << map.lat
@@ -175,7 +175,7 @@ class ShopAddressBatch {
conn.setRequestMethod("GET");
conn.doOutput = true
inputStream = conn.getInputStream();
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
while((line = buf.readLine())!=null){
sb.append(line);
}
@@ -4,14 +4,14 @@ class SymbolRemoveHandler extends AddressStandardizeHandler {
String[] SYMBOLS = ["...", "?", ",", ".", ",", "。", ";", "-", "?"];
String[] SYMBOLS = ["...", "?", ",", ".", ",", "。", ";", "-", "?"];
@Override
public void handleRequest(Request request) {
try{
request.setAddress(removeEllipsis(request.getAddress()));
}catch(Exception e){
println "SymbolRemoveHandler: " + request.getAddress()+" 处理失败,message:"+e.getMessage();
println "SymbolRemoveHandler: " + request.getAddress()+" 处理失败,message:"+e.getMessage();
}
}
@@ -33,6 +33,6 @@ class SymbolRemoveHandler extends AddressStandardizeHandler {
}
public static void main(String[] arhs){
print new SymbolRemoveHandler().removeEllipsis("...?ssss........?,,");
print new SymbolRemoveHandler().removeEllipsis("...?ssss........?,,");
}
}
@@ -8,14 +8,14 @@ import com.fangjia.core.util.DateUtil;
class Test {
private static final Pattern shihaoPattern = Pattern.compile("(\\w+)[幢|栋|号楼|号|单元|楼|座](\\w+||\\d{3,4})室?");
private static final Pattern shihaoPattern = Pattern.compile("(\\w+)[幢|栋|号楼|号|单元|楼|座](\\w+||\\d{3,4})室?");
static main(args) {
// println CosineSimilarAlgorithm.getSimilarity("沪太路1700弄","沪太路1700弄小区")
// println CosineSimilarAlgorithm.getSimilarity("沪太路1700弄","沪太路631弄", )
// println CosineSimilarAlgorithm.getSimilarity("沪太路1700弄","沪太路951号")
// println CosineSimilarAlgorithm.getSimilarity("证大立方大厦","富庆国定大厦")
// println CosineSimilarAlgorithm.getSimilarity("紫光大厦","元福大厦")
// println CosineSimilarAlgorithm.getSimilarity("沪太路1700弄","沪太路1700弄小区")
// println CosineSimilarAlgorithm.getSimilarity("沪太路1700弄","沪太路631弄", )
// println CosineSimilarAlgorithm.getSimilarity("沪太路1700弄","沪太路951号")
// println CosineSimilarAlgorithm.getSimilarity("证大立方大厦","富庆国定大厦")
// println CosineSimilarAlgorithm.getSimilarity("紫光大厦","元福大厦")
// def month = 360;
@@ -26,12 +26,12 @@ class Test {
// println getPayMonth1(9360000, month, rate, 0.5);
// println getPayMonth1(9360000, month, rate, 0.6);
// def map = parseLouDouInfo("新泾三村A栋1501");
// def map = parseLouDouInfo("新泾三村A栋1501");
// println map
def obj = "发布时间:2016/4/9"
obj = obj.substring(obj.indexOf("发布时间"));
obj = obj.replace("发布时间:", "").replace("发布时间", "").replace("发布", "").replace("/","-")
def obj = "发布时间:2016/4/9"
obj = obj.substring(obj.indexOf("发布时间"));
obj = obj.replace("发布时间:", "").replace("发布时间", "").replace("发布", "").replace("/","-")
def pattern = ~/\d{4}-\d{1,2}-\d{1,2}/
def matcher = pattern.matcher(obj)
if (matcher.find()) {
@@ -31,20 +31,20 @@ class YongjiaDataTest {
static main(args) {
def task = new YongjiaDataTest()
task.batchTestApi("上海", "C:\\Users\\yk\\Desktop\\yongjia\\上海-0730.txt")
task.batchTestApi("上海", "C:\\Users\\yk\\Desktop\\yongjia\\上海-0730.txt")
}
def batchTestApi(def city, def fileName){
def result = []
def index = 0
File file = new File("C:\\Users\\yk\\Desktop\\yongjia\\上海-0730.csv")
File file = new File("C:\\Users\\yk\\Desktop\\yongjia\\上海-0730.csv")
if (file.exists()){
file.delete();
}
file.append("房产地址,")
file.append("标准化-楼盘,")
file.append("楼盘单价,")
file.append("房产地址,")
file.append("标准化-楼盘,")
file.append("楼盘单价,")
file.append(System.getProperty("line.separator"));
BufferedReader reader = new File(fileName).newReader('utf-8')
@@ -90,7 +90,7 @@ class YongjiaDataTest {
m.avgPrice = CommonDatautils.queryDistrictPrice(m.city, m.region, m.name, weekend, m.cat);
}
//房价网估价
//房价网估价
// m.fangjiaTotalPrice = evaluation(m, token);
// if(!m.fangjiaTotalPrice && m.avgPrice){
// m.fangjiaTotalPrice = (m.avgPrice * m.area).toInteger();
@@ -98,7 +98,7 @@ class YongjiaDataTest {
//m.rentPrice = queryRentPriceRange(m.city, m.region, m.name, lastMonth);
//其它网站均价
//其它网站均价
// def otherWebsiteMap = CommonDatautils.queryOtherWebsitePrice(m.city, m.name);
// m.sunfunPrice = otherWebsiteMap.price_soufun
// m.anjukePrice = otherWebsiteMap.price_anjuke
@@ -137,7 +137,7 @@ class YongjiaDataTest {
}
//println "周边均价:" + totalPrice +"【${url_str}】"
//println "周边均价:" + totalPrice +"【${url_str}】"
return totalPrice;
}
@@ -153,7 +153,7 @@ class YongjiaDataTest {
conn.setRequestMethod("GET");
conn.doOutput = true
inputStream = conn.getInputStream();
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
while((line = buf.readLine())!=null){
sb.append(line);
}
@@ -29,7 +29,7 @@ class HouseLoudongAnaBatch extends BaseBatch {
}
/**
* 从楼栋表分析出小区楼号、室号信息
* 从楼栋表分析出小区楼号、室号信息
*/
private anaHouseLoudong() {
def index = 0;
@@ -44,7 +44,7 @@ class HouseLoudongAnaBatch extends BaseBatch {
query.put('region', house.region);
query.put('name', house.name);
// 处理小区内楼号
// 处理小区内楼号
DBObject districtHouse = DB_2.findOne(query);
if (districtHouse) {
BasicDBList houseNums = districtHouse.house_nums;
@@ -73,7 +73,7 @@ class HouseLoudongAnaBatch extends BaseBatch {
DB_2.save(obj);
}
// 处理同一楼号内的室号
// 处理同一楼号内的室号
query.put('house_num', houseNumStr);
DBObject houseRoom = DB_3.findOne(query);
if (houseRoom) {
@@ -23,211 +23,211 @@ class SortDistrictPics {
def final static DEL_HEIGHT = 82
static main(args) {
// process('shop', '上海')
process('office', '上海')
// process('shop', '上海')
process('office', '上海')
// process('district', '上海')
// process('district', '北京')
// process('district', '广州')
// process('district', '深圳')
// process('district', '贵阳')
// process('district', '乌鲁木齐')
// process('district', '泰州')
// process('district', '江门')
// process('district', '秦皇岛')
// process('district', '温州')
// process('district', '台州')
// process('district', '大庆')
// process('district', '无锡')
// process('district', '清远')
// process('district', '珠海')
// process('district', '兰州')
// process('district', '佛山')
// process('district', '柳州')
// process('district', '西安')
// process('district', '湛江')
// process('district', '镇江')
// process('district', '武汉')
// process('district', '湖州')
// process('district', '南通')
// process('district', '赣州')
// process('district', '丹东')
// process('district', '咸阳')
// process('district', '丽江')
// process('district', '张家界')
// process('district', '银川')
// process('district', '大理')
// process('district', '宿迁')
// process('district', '襄阳')
// process('district', '唐山')
// process('district', '大连')
// process('district', '芜湖')
// process('district', '马鞍山')
// process('district', '惠州')
// process('district', '玉林')
// process('district', '潍坊')
// process('district', '南昌')
// process('district', '哈尔滨')
// process('district', '连云港')
// process('district', '太原')
// process('district', '延安')
// process('district', '宁波')
// process('district', '九江')
// process('district', '三亚')
// process('district', '南京')
// process('district', '海口')
// process('district', '济南')
// process('district', '宜昌')
// process('district', '烟台')
// process('district', '东莞')
// process('district', '天津')
// process('district', '常州')
// process('district', '威海')
// process('district', '六盘水')
// process('district', '沈阳')
// process('district', '吉林')
// process('district', '徐州')
// process('district', '鞍山')
// process('district', '保定')
// process('district', '宜宾')
// process('district', '莆田')
// process('district', '金华')
// process('district', '西宁')
// process('district', '郑州')
// process('district', '盐城')
// process('district', '自贡')
// process('district', '石家庄')
// process('district', '绍兴')
// process('district', '昆明')
// process('district', '淄博')
// process('district', '攀枝花')
// process('district', '南宁')
// process('district', '衢州')
// process('district', '扬州')
// process('district', '泉州')
// process('district', '长春')
// process('district', '苏州')
// process('district', '邯郸')
// process('district', '北海')
// process('district', '合肥')
// process('district', '青岛')
// process('district', '廊坊')
// process('district', '洛阳')
// process('district', '呼和浩特')
// process('district', '嘉兴')
// process('district', '中山')
// process('district', '桂林')
// process('district', '长沙')
// process('district', '株洲')
// process('district', '湘潭')
// process('district', '成都')
// process('district', '重庆')
// process('district', '杭州')
// process('district', '厦门')
// process('district', '福州')
// process('district', '绵阳')
// process('district', '上海')
// process('district', '北京')
// process('district', '广州')
// process('district', '深圳')
// process('district', '贵阳')
// process('district', '乌鲁木齐')
// process('district', '泰州')
// process('district', '江门')
// process('district', '秦皇岛')
// process('district', '温州')
// process('district', '台州')
// process('district', '大庆')
// process('district', '无锡')
// process('district', '清远')
// process('district', '珠海')
// process('district', '兰州')
// process('district', '佛山')
// process('district', '柳州')
// process('district', '西安')
// process('district', '湛江')
// process('district', '镇江')
// process('district', '武汉')
// process('district', '湖州')
// process('district', '南通')
// process('district', '赣州')
// process('district', '丹东')
// process('district', '咸阳')
// process('district', '丽江')
// process('district', '张家界')
// process('district', '银川')
// process('district', '大理')
// process('district', '宿迁')
// process('district', '襄阳')
// process('district', '唐山')
// process('district', '大连')
// process('district', '芜湖')
// process('district', '马鞍山')
// process('district', '惠州')
// process('district', '玉林')
// process('district', '潍坊')
// process('district', '南昌')
// process('district', '哈尔滨')
// process('district', '连云港')
// process('district', '太原')
// process('district', '延安')
// process('district', '宁波')
// process('district', '九江')
// process('district', '三亚')
// process('district', '南京')
// process('district', '海口')
// process('district', '济南')
// process('district', '宜昌')
// process('district', '烟台')
// process('district', '东莞')
// process('district', '天津')
// process('district', '常州')
// process('district', '威海')
// process('district', '六盘水')
// process('district', '沈阳')
// process('district', '吉林')
// process('district', '徐州')
// process('district', '鞍山')
// process('district', '保定')
// process('district', '宜宾')
// process('district', '莆田')
// process('district', '金华')
// process('district', '西宁')
// process('district', '郑州')
// process('district', '盐城')
// process('district', '自贡')
// process('district', '石家庄')
// process('district', '绍兴')
// process('district', '昆明')
// process('district', '淄博')
// process('district', '攀枝花')
// process('district', '南宁')
// process('district', '衢州')
// process('district', '扬州')
// process('district', '泉州')
// process('district', '长春')
// process('district', '苏州')
// process('district', '邯郸')
// process('district', '北海')
// process('district', '合肥')
// process('district', '青岛')
// process('district', '廊坊')
// process('district', '洛阳')
// process('district', '呼和浩特')
// process('district', '嘉兴')
// process('district', '中山')
// process('district', '桂林')
// process('district', '长沙')
// process('district', '株洲')
// process('district', '湘潭')
// process('district', '成都')
// process('district', '重庆')
// process('district', '杭州')
// process('district', '厦门')
// process('district', '福州')
// process('district', '绵阳')
//
//
// process('new_district', '上海')
// process('new_district', '北京')
// process('new_district', '广州')
// process('new_district', '深圳')
// process('new_district', '贵阳')
// process('new_district', '乌鲁木齐')
// process('new_district', '泰州')
// process('new_district', '江门')
// process('new_district', '秦皇岛')
// process('new_district', '温州')
// process('new_district', '台州')
// process('new_district', '大庆')
// process('new_district', '无锡')
// process('new_district', '清远')
// process('new_district', '珠海')
// process('new_district', '兰州')
// process('new_district', '佛山')
// process('new_district', '柳州')
// process('new_district', '西安')
// process('new_district', '湛江')
// process('new_district', '镇江')
// process('new_district', '武汉')
// process('new_district', '湖州')
// process('new_district', '南通')
// process('new_district', '赣州')
// process('new_district', '丹东')
// process('new_district', '咸阳')
// process('new_district', '丽江')
// process('new_district', '张家界')
// process('new_district', '银川')
// process('new_district', '大理')
// process('new_district', '宿迁')
// process('new_district', '襄阳')
// process('new_district', '唐山')
// process('new_district', '大连')
// process('new_district', '芜湖')
// process('new_district', '马鞍山')
// process('new_district', '惠州')
// process('new_district', '玉林')
// process('new_district', '潍坊')
// process('new_district', '南昌')
// process('new_district', '哈尔滨')
// process('new_district', '连云港')
// process('new_district', '太原')
// process('new_district', '延安')
// process('new_district', '宁波')
// process('new_district', '九江')
// process('new_district', '三亚')
// process('new_district', '南京')
// process('new_district', '海口')
// process('new_district', '济南')
// process('new_district', '宜昌')
// process('new_district', '烟台')
// process('new_district', '东莞')
// process('new_district', '天津')
// process('new_district', '常州')
// process('new_district', '威海')
// process('new_district', '六盘水')
// process('new_district', '沈阳')
// process('new_district', '吉林')
// process('new_district', '徐州')
// process('new_district', '鞍山')
// process('new_district', '保定')
// process('new_district', '宜宾')
// process('new_district', '莆田')
// process('new_district', '金华')
// process('new_district', '西宁')
// process('new_district', '郑州')
// process('new_district', '盐城')
// process('new_district', '自贡')
// process('new_district', '石家庄')
// process('new_district', '绍兴')
// process('new_district', '昆明')
// process('new_district', '淄博')
// process('new_district', '攀枝花')
// process('new_district', '南宁')
// process('new_district', '衢州')
// process('new_district', '扬州')
// process('new_district', '泉州')
// process('new_district', '长春')
// process('new_district', '苏州')
// process('new_district', '邯郸')
// process('new_district', '北海')
// process('new_district', '合肥')
// process('new_district', '青岛')
// process('new_district', '廊坊')
// process('new_district', '洛阳')
// process('new_district', '呼和浩特')
// process('new_district', '嘉兴')
// process('new_district', '中山')
// process('new_district', '桂林')
// process('new_district', '长沙')
// process('new_district', '株洲')
// process('new_district', '湘潭')
// process('new_district', '成都')
// process('new_district', '重庆')
// process('new_district', '杭州')
// process('new_district', '厦门')
// process('new_district', '福州')
// process('new_district', '绵阳')
// process('new_district', '上海')
// process('new_district', '北京')
// process('new_district', '广州')
// process('new_district', '深圳')
// process('new_district', '贵阳')
// process('new_district', '乌鲁木齐')
// process('new_district', '泰州')
// process('new_district', '江门')
// process('new_district', '秦皇岛')
// process('new_district', '温州')
// process('new_district', '台州')
// process('new_district', '大庆')
// process('new_district', '无锡')
// process('new_district', '清远')
// process('new_district', '珠海')
// process('new_district', '兰州')
// process('new_district', '佛山')
// process('new_district', '柳州')
// process('new_district', '西安')
// process('new_district', '湛江')
// process('new_district', '镇江')
// process('new_district', '武汉')
// process('new_district', '湖州')
// process('new_district', '南通')
// process('new_district', '赣州')
// process('new_district', '丹东')
// process('new_district', '咸阳')
// process('new_district', '丽江')
// process('new_district', '张家界')
// process('new_district', '银川')
// process('new_district', '大理')
// process('new_district', '宿迁')
// process('new_district', '襄阳')
// process('new_district', '唐山')
// process('new_district', '大连')
// process('new_district', '芜湖')
// process('new_district', '马鞍山')
// process('new_district', '惠州')
// process('new_district', '玉林')
// process('new_district', '潍坊')
// process('new_district', '南昌')
// process('new_district', '哈尔滨')
// process('new_district', '连云港')
// process('new_district', '太原')
// process('new_district', '延安')
// process('new_district', '宁波')
// process('new_district', '九江')
// process('new_district', '三亚')
// process('new_district', '南京')
// process('new_district', '海口')
// process('new_district', '济南')
// process('new_district', '宜昌')
// process('new_district', '烟台')
// process('new_district', '东莞')
// process('new_district', '天津')
// process('new_district', '常州')
// process('new_district', '威海')
// process('new_district', '六盘水')
// process('new_district', '沈阳')
// process('new_district', '吉林')
// process('new_district', '徐州')
// process('new_district', '鞍山')
// process('new_district', '保定')
// process('new_district', '宜宾')
// process('new_district', '莆田')
// process('new_district', '金华')
// process('new_district', '西宁')
// process('new_district', '郑州')
// process('new_district', '盐城')
// process('new_district', '自贡')
// process('new_district', '石家庄')
// process('new_district', '绍兴')
// process('new_district', '昆明')
// process('new_district', '淄博')
// process('new_district', '攀枝花')
// process('new_district', '南宁')
// process('new_district', '衢州')
// process('new_district', '扬州')
// process('new_district', '泉州')
// process('new_district', '长春')
// process('new_district', '苏州')
// process('new_district', '邯郸')
// process('new_district', '北海')
// process('new_district', '合肥')
// process('new_district', '青岛')
// process('new_district', '廊坊')
// process('new_district', '洛阳')
// process('new_district', '呼和浩特')
// process('new_district', '嘉兴')
// process('new_district', '中山')
// process('new_district', '桂林')
// process('new_district', '长沙')
// process('new_district', '株洲')
// process('new_district', '湘潭')
// process('new_district', '成都')
// process('new_district', '重庆')
// process('new_district', '杭州')
// process('new_district', '厦门')
// process('new_district', '福州')
// process('new_district', '绵阳')
// def floors = ['http://img1.soufunimg.com/agents1/2011_05/23/17/53/94/sh/houseinfo/404243101600.jpg',
@@ -423,7 +423,7 @@ class SortDistrictPics {
DBObject query = new BasicDBObject('_id', district._id);
DBObject data = new BasicDBObject();
// 图片总数超过上限,则取top
// 图片总数超过上限,则取top
if (retFloors.size() + retOuts.size() > maxSize) {
if (retFloors.size() > (maxSize / 2)) {
data.put('floorplan_pics', getTop(retFloors, maxSize / 2));
@@ -433,7 +433,7 @@ class SortDistrictPics {
data.put('out_pics', getTop(retOuts, maxSize - data.floorplan_pics.size()));
} else {
// 没超过上限,全部保留
// 没超过上限,全部保留
data.put('floorplan_pics', retFloors);
data.put('out_pics', retOuts);
}
@@ -587,7 +587,7 @@ class SortDistrictPics {
}
/**
* 取得fs中不存在的图片URL
* 取得fs中不存在的图片URL
*/
static remove404Pics(picInfos) {
def ret = [];
@@ -603,7 +603,7 @@ class SortDistrictPics {
static doSort(picInfos) {
// Collections.sort(picInfos, new Comparator<PicInfo>() {
// public int compare(PicInfo info1, PicInfo info2) {
// // 图片不存在或者没正确读取信息,往后排
// // 图片不存在或者没正确读取信息,往后排
// if (info1.is404 == true || info1.isEmpty == true || info1.isSmall == true) {
// return 1;
// }
@@ -611,7 +611,7 @@ class SortDistrictPics {
// return -1;
// }
//
// // 图片长宽比较大,往后排
// // 图片长宽比较大,往后排
// if (info1.hdw < 0.5 || info1.hdw > 2) {
// return 1;
// }
@@ -626,7 +626,7 @@ class SortDistrictPics {
// });
picInfos.sort { info1, info2 ->
// 图片不存在或者没正确读取信息,往后排
// 图片不存在或者没正确读取信息,往后排
if (info1.isF0f() || info1.isSmall()) {
return 1;
}
@@ -634,7 +634,7 @@ class SortDistrictPics {
return -1;
}
// 图片长宽比较大,往后排
// 图片长宽比较大,往后排
if (info1.getHdw() < 0.5 || info1.getHdw() > 2) {
return 1;
}
@@ -663,7 +663,7 @@ class SortDistrictPics {
}
/**
* 房型图列表转换为string列表
* 房型图列表转换为string列表
*
* @param srcs
* @return
@@ -709,19 +709,19 @@ class SortDistrictPics {
// if(directories.hasNext()){
// Directory directory = (Directory) directories.next();
//
// // 如果 imageInfo中无法读取,从exif中读取dpi
// // 如果 imageInfo中无法读取,从exif中读取dpi
// if(imageInfo.getPhysicalWidthDpi() <= 0 || imageInfo.getPhysicalHeightDpi() <= 0){
// dipx = new BigDecimal(directory.getString(ExifDirectory.TAG_X_RESOLUTION));
// dipy = new BigDecimal(directory.getString(ExifDirectory.TAG_Y_RESOLUTION));
// if(dipx.equals(new BigDecimal(72)) || dipy.equals(new BigDecimal(72))) {
// // 如果dpi为72的话,抛出错误,因为暂时无法确定从exif中读出的数据是对的
// // 如果dpi为72的话,抛出错误,因为暂时无法确定从exif中读出的数据是对的
// } else{
// dipx = new BigDecimal(imageInfo.getPhysicalWidthDpi());
// dipy = new BigDecimal(imageInfo.getPhysicalHeightDpi());
// }
// }
// }
// } catch (Exception e) { // 处理出错
// } catch (Exception e) { // 处理出错
//// e.printStackTrace();
// }
@@ -732,12 +732,12 @@ class SortDistrictPics {
public static class PicInfo {
private String imgUrl;
private int hxw; // 图片高度*宽度(像素)
private double hdw; // 图片高度÷宽度(像素)
private double dpi; // 图片DPI
private boolean f0f; // fs库中不存在
private boolean small; // 是否小于120*82
private boolean vertical; // 是否竖向图片
private int hxw; // 图片高度*宽度(像素)
private double hdw; // 图片高度÷宽度(像素)
private double dpi; // 图片DPI
private boolean f0f; // fs库中不存在
private boolean small; // 是否小于120*82
private boolean vertical; // 是否竖向图片
public double score;
public String getImgUrl() {
@@ -30,8 +30,8 @@ import java.util.concurrent.atomic.AtomicInteger
import java.util.regex.Pattern
/**
* 对解析好的网页进行分析
* 获取小区解析的网站数量
* 对解析好的网页进行分析
* 获取小区解析的网站数量
* @author gaojuan
*
*/
@@ -46,23 +46,23 @@ class AnalyDistrictLog extends BaseBatch {
public static final DBCollection VERIFY_RULE_COLL = MongoUtils.getCollectionByName("coredata", "verify_rule", "coredata.mongo.db.fangjia.com")
public static final DSCollection SEABIRD = new DataStore("datastore.db.fangjia.com").getCollection("fangjia.seabird")
public static final DSCollection seawater = new DataStore("192.168.0.65", 27777).getCollection("fangjia.seawater")
/**人工删除*/
/**人工删除*/
private static final int MANUAL_DELETE = 1;
/**区域错误删除*/
/**区域错误删除*/
private static final int REGION_ERROR_DELETE = 2;
/**别名关联删除*/
/**别名关联删除*/
private static final int ALIAS_RELATE_DELETE = 3;
/**名称非法删除*/
/**名称非法删除*/
private static final int NAME_ILLEGAL_DELETE = 4;
/**通过百度API判定类型不合法删除*/
/**通过百度API判定类型不合法删除*/
private static final int BAIDU_JUDGE_DELETE = 5;
/**CPDB已经删除*/
/**CPDB已经删除*/
private static final int CPDB_DELETE = 6;
/**正常数据*/
/**正常数据*/
private static final int NORMAL_DATA = 0;
private static final int MACHINE_DETELE = 8
/**数据抓取失败标示*/
/**数据抓取失败标示*/
private static final int ANALY_ROAD_STATUS_ERROR = 2;
private Date now = new Date()
private int THREE_MONTHS_AGO = DateUtil.add(now, Calendar.MONTH, -6).format("yyyyMMdd").toInteger()
@@ -73,14 +73,14 @@ class AnalyDistrictLog extends BaseBatch {
private static List districtUnContainsWords = []
static List<Pattern> illegalDistirctNamePatterns = [Pattern.compile(".*地图定位.*"), Pattern.compile(".*地块.*"), Pattern.compile(".*售罄.*"),
Pattern.compile(".*-\$"), Pattern.compile(".*….*"), Pattern.compile(".*\\d#.*")
, /.*(东|南|西|北|一|二|三|四|五|正|前|后)+门.*/, /.*出售.*/, /.*(\d|一|二|三|四|五)号楼$/]
static List<Pattern> illegalDistirctNamePatterns = [Pattern.compile(".*地图定位.*"), Pattern.compile(".*地块.*"), Pattern.compile(".*售罄.*"),
Pattern.compile(".*-\$"), Pattern.compile(".*….*"), Pattern.compile(".*\\d#.*")
, /.*(东|南|西|北|一|二|三|四|五|正|前|后)+门.*/, /.*出售.*/, /.*(\d|一|二|三|四|五)号楼$/]
static main(args) {
def s = System.currentTimeMillis()
runBatch(new AnalyDistrictLog(), args)
println("运行时间:${(int) ((System.currentTimeMillis() - s)) / (1000 * 60)}分钟")
println("运行时间:${(int) ((System.currentTimeMillis() - s)) / (1000 * 60)}分钟")
}
@Override
@@ -90,8 +90,8 @@ class AnalyDistrictLog extends BaseBatch {
districtUnContainsWords << dbo.getString("value")
}
def allCity = CommonUtils.findAllCity()
if (SystemUtils.IS_OS_WINDOWS) {//本地手动跑特定城市数据
allCity = ["南京"]
if (SystemUtils.IS_OS_WINDOWS) {//本地手动跑特定城市数据
allCity = ["南京"]
}else{
updateTempData()
}
@@ -102,19 +102,19 @@ class AnalyDistrictLog extends BaseBatch {
def index = 1;
for (def city : allCity) {
def s = System.currentTimeMillis()
println "[${index}]开始分析 ${city} ${new Date().format('yyyy/MM/dd HH:mm:ss')}"
println "[${index}]开始分析 ${city} ${new Date().format('yyyy/MM/dd HH:mm:ss')}"
for (def dataType : dataTypes) {
process(city, dataType);
// if(allOfflineCity.contains(city) && dataType == "district"){//统计线下城市房源
// if(allOfflineCity.contains(city) && dataType == "district"){//统计线下城市房源
// calcHouseCount(city)
// }
}
index++;
println "[${index}]分析结束 ${city} ${new Date().format('yyyy/MM/dd HH:mm:ss')} 耗时:${(int) ((System.currentTimeMillis() - s) / (1000 * 60))}分钟"
println "[${index}]分析结束 ${city} ${new Date().format('yyyy/MM/dd HH:mm:ss')} 耗时:${(int) ((System.currentTimeMillis() - s) / (1000 * 60))}分钟"
}
}
/*统计线下城市房源的数量*/
/*统计线下城市房源的数量*/
def calcHouseCount(def city){
def startDate = DateUtil.add(new Date(), Calendar.MONTH, -6).format("yyyyMMdd").toInteger()
@@ -132,10 +132,10 @@ class AnalyDistrictLog extends BaseBatch {
long count = (long) seawater.count(houseQuery)
ANALY_SITE_COLL.update(new BasicDBObject("_id",dbo._id), new BasicDBObject('$set',new BasicDBObject('house_num',count)))
if(idx % 100 == 0){
println("${city}统计进度:${DataUtil.roundDouble((idx *100)/total,2)}%")
println("${city}统计进度:${DataUtil.roundDouble((idx *100)/total,2)}%")
}
}
println("${city}分析结束")
println("${city}分析结束")
}
/**
@@ -152,7 +152,7 @@ class AnalyDistrictLog extends BaseBatch {
query.append("version", new BasicDBObject('$lt', version))
DBCursor cursor = ANALY_SITE_COLL.find(query, new BasicDBObject("city", 1).append("region", 1).append("district_name", 1).append("cat", 1)).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(300)
while (cursor.hasNext()) {
println("删除:${cursor.next()}")
println("删除:${cursor.next()}")
}
ANALY_SITE_COLL.remove(query)
query.removeField("version")
@@ -182,13 +182,13 @@ class AnalyDistrictLog extends BaseBatch {
updater.put("region", formateRegion)
}
}
String key = city + JOIN_SEPARATOR + formateRegion + JOIN_SEPARATOR + name.toLowerCase().replaceAll(' ', " ").replaceAll(" ", " ").replaceAll('(^\\s+)|(\\s+$)', '');
String key = city + JOIN_SEPARATOR + formateRegion + JOIN_SEPARATOR + name.toLowerCase().replaceAll(' ', " ").replaceAll(" ", " ").replaceAll('(^\\s+)|(\\s+$)', '');
int audit_status = analySite.getInt("audit_status", 0)//审阅状态 0:未审核 1:已审核 4:已审阅
int audit_status = analySite.getInt("audit_status", 0)//审阅状态 0:未审核 1:已审核 4:已审阅
int data_status = analySite.getInt("data_status", NORMAL_DATA)
int exists
def district = allRightData.get(key);
//判断小区是否在CPDB库中
//判断小区是否在CPDB库中
if (district != null) {
exists = 1
} else {
@@ -215,7 +215,7 @@ class AnalyDistrictLog extends BaseBatch {
} else if (deletedData.containsKey(key)) {
data_status = CPDB_DELETE
audit_status = 1
} else if (name.endsWith("宿舍") && analySite.containsField("map_compare_result") && analySite.getString("map_compare_result") == "not_exist") {
} else if (name.endsWith("宿舍") && analySite.containsField("map_compare_result") && analySite.getString("map_compare_result") == "not_exist") {
data_status = NAME_ILLEGAL_DELETE
audit_status = 1
} else {
@@ -240,7 +240,7 @@ class AnalyDistrictLog extends BaseBatch {
ANALY_SITE_COLL.update(new BasicDBObject("_id", analySite._id), new BasicDBObject('$set', updater))
}
if (progress % 1000 == 0) {
println("状态进度:${DataUtil.roundDouble((progress / count) * 100, 1)}%")
println("状态进度:${DataUtil.roundDouble((progress / count) * 100, 1)}%")
}
}
}
@@ -277,10 +277,10 @@ class AnalyDistrictLog extends BaseBatch {
*/
public def process(def city, def cat) {
def analyRoads = findAnalyRoads(city, cat);//查找抓取数据
def analyRoads = findAnalyRoads(city, cat);//查找抓取数据
if (analyRoads.size() > 0) {
def districts = getAllDistricts(city, cat);
//按网页合并
//按网页合并
def List<BasicDBObject> mergedRoad = analyzeAnalyRoads(analyRoads);
saveAnalySite(mergedRoad, cat, city);
updateOldVersionAnalySiteDataStatus(city, cat, districts)
@@ -288,7 +288,7 @@ class AnalyDistrictLog extends BaseBatch {
}
/**
* 通过城市区域小区名对Road数据进行分组
* 通过城市区域小区名对Road数据进行分组
* @param analyRoads
* @return
*/
@@ -314,7 +314,7 @@ class AnalyDistrictLog extends BaseBatch {
item.get("site_names") << analyRoad.site_name
}
if (progress % 10000 == 0) {
println("分组进度:${DataUtil.roundDouble((progress / analyRoads.size()) * 100, 0)}%")
println("分组进度:${DataUtil.roundDouble((progress / analyRoads.size()) * 100, 0)}%")
}
}
@@ -326,7 +326,7 @@ class AnalyDistrictLog extends BaseBatch {
dbo.put("site_count", siteNames.size())
result << dbo
}
println "[analyzeAnalyRoads]耗时:${System.currentTimeMillis() - start}"
println "[analyzeAnalyRoads]耗时:${System.currentTimeMillis() - start}"
return result;
}
@@ -344,7 +344,7 @@ class AnalyDistrictLog extends BaseBatch {
def item = new BasicDBObject()
if(!obj.primary_map){
ANALY_ROAD_TEMP_COLL.remove(new BasicDBObject("_id", obj._id))
println("移除TEMP:" + obj.toString())
println("移除TEMP:" + obj.toString())
continue
}
BasicDBObject primaryMap = obj.get("primary_map")
@@ -397,7 +397,7 @@ class AnalyDistrictLog extends BaseBatch {
def name = obj.getString("name");
List alias = obj.get("alias");
def status = obj.getInt("status", 0);
String key = city + JOIN_SEPARATOR + region + JOIN_SEPARATOR + name.toLowerCase().replaceAll(' ', " ").replaceAll(" ", " ").replaceAll('(^\\s+)|(\\s+$)', '');
String key = city + JOIN_SEPARATOR + region + JOIN_SEPARATOR + name.toLowerCase().replaceAll(' ', " ").replaceAll(" ", " ").replaceAll('(^\\s+)|(\\s+$)', '');
if (0 != status) {
deletedData.put(key, obj)
} else {
@@ -406,7 +406,7 @@ class AnalyDistrictLog extends BaseBatch {
if (alias) {
for (String aliaName : alias) {
if (StringUtils.isNotBlank(aliaName) && aliaName != name) {
key = city + JOIN_SEPARATOR + region + JOIN_SEPARATOR + aliaName.toLowerCase().replaceAll(' ', " ").replaceAll(" ", " ").replaceAll('(^\\s+)|(\\s+$)', '');
key = city + JOIN_SEPARATOR + region + JOIN_SEPARATOR + aliaName.toLowerCase().replaceAll(' ', " ").replaceAll(" ", " ").replaceAll('(^\\s+)|(\\s+$)', '');
if (0 != status) {
deletedData.put(key, obj);
} else {
@@ -417,7 +417,7 @@ class AnalyDistrictLog extends BaseBatch {
}
}
}
//移除拥有和正常数据一样KEY的被删除数据
//移除拥有和正常数据一样KEY的被删除数据
for (String k : allRightData.keySet()) {
deletedData.remove(k)
}
@@ -495,7 +495,7 @@ class AnalyDistrictLog extends BaseBatch {
}
progress.getAndIncrement()
if (progress.get() % 1000 == 0) {
println("入库进度:${DataUtil.roundDouble((progress.get() / mergedRoads.size()) * 100, 1)}%")
println("入库进度:${DataUtil.roundDouble((progress.get() / mergedRoads.size()) * 100, 1)}%")
}
}
})
@@ -520,7 +520,7 @@ class AnalyDistrictLog extends BaseBatch {
if (StringUtils.isBlank(region)) {
return false;
}
if (region.contains("周边") || region.contains("其他")) {
if (region.contains("周边") || region.contains("其他")) {
return false;
}
return RegionData.isRegion(city, region)
@@ -528,7 +528,7 @@ class AnalyDistrictLog extends BaseBatch {
def updateTempData() {
println("开始更新临时数据")
println("开始更新临时数据")
DBObject query = new BasicDBObject();
query.append("status", new BasicDBObject('$ne', ANALY_ROAD_STATUS_ERROR))
query.append("c_date", new BasicDBObject('$gt', DateUtil.add(now, Calendar.DATE, -2)))
@@ -572,10 +572,10 @@ class AnalyDistrictLog extends BaseBatch {
}
}
if(sb.length() > 0 && sb.length() < 10000){
sendEmail("zhoutongyun@fangjia.com", "疑似解析错误小区", sb.toString())
sendEmail("wangwenyan@fangjia.com", "疑似解析错误小区", sb.toString())
sendEmail("zhoutongyun@fangjia.com", "疑似解析错误小区", sb.toString())
sendEmail("wangwenyan@fangjia.com", "疑似解析错误小区", sb.toString())
}
println("临时数据更新结束")
println("临时数据更新结束")
}
def assembleObj(BasicDBObject srcDbo) {
@@ -27,8 +27,8 @@ import java.util.concurrent.atomic.AtomicInteger
import java.util.regex.Pattern
/**
* 对解析好的网页进行分析
* 获取小区解析的网站数量
* 对解析好的网页进行分析
* 获取小区解析的网站数量
* @author gaojuan
*
*/
@@ -42,21 +42,21 @@ class AnalyDistrictLogTest extends BaseBatch{
public static final DBCollection REGION_BLOCK_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com")
public static final DBCollection VERIFY_RULE_COLL = MongoUtils.getCollectionByName("coredata", "verify_rule", "coredata.mongo.db.fangjia.com")
public static DSCollection seaweed = new DataStore("192.168.0.251", 27007).getCollection("fangjia.seaweed")
/**人工删除*/
/**人工删除*/
private static final int MANUAL_DELETE = 1;
/**区域错误删除*/
/**区域错误删除*/
private static final int REGION_ERROR_DELETE = 2;
/**别名关联删除*/
/**别名关联删除*/
private static final int ALIAS_RELATE_DELETE = 3;
/**名称非法删除*/
/**名称非法删除*/
private static final int NAME_ILLEGAL_DELETE = 4;
/**通过百度API判定类型不合法删除*/
/**通过百度API判定类型不合法删除*/
private static final int BAIDU_JUDGE_DELETE = 5;
/**CPDB已经删除*/
/**CPDB已经删除*/
private static final int CPDB_DELETE = 6;
/**正常数据*/
/**正常数据*/
private static final int NORMAL_DATA = 0;
/**数据抓取失败标示*/
/**数据抓取失败标示*/
private static final int ANALY_ROAD_STATUS_ERROR = 2;
private Date now = new Date()
@@ -66,14 +66,14 @@ class AnalyDistrictLogTest extends BaseBatch{
private static List districtUnContainsWords = []
static List<Pattern> illegalDistirctNamePatterns = [Pattern.compile(".*地图定位.*"),Pattern.compile(".*地块.*"),Pattern.compile(".*售罄.*"),
Pattern.compile(".*-\$"),Pattern.compile(".*….*"),Pattern.compile(".*\\d#.*")
,/.*(东|南|西|北|一|二|三|四|五|正|前|后)+门.*/,/.*出售.*/,/.*(\d|一|二|三|四|五)号楼$/]
static List<Pattern> illegalDistirctNamePatterns = [Pattern.compile(".*地图定位.*"),Pattern.compile(".*地块.*"),Pattern.compile(".*售罄.*"),
Pattern.compile(".*-\$"),Pattern.compile(".*….*"),Pattern.compile(".*\\d#.*")
,/.*(东|南|西|北|一|二|三|四|五|正|前|后)+门.*/,/.*出售.*/,/.*(\d|一|二|三|四|五)号楼$/]
static main(args) {
def s = System.currentTimeMillis()
runBatch(new AnalyDistrictLogTest(), args)
println("运行时间:${(int)((System.currentTimeMillis() - s))/(1000*60)}分钟")
println("运行时间:${(int)((System.currentTimeMillis() - s))/(1000*60)}分钟")
}
@Override
@@ -84,19 +84,19 @@ class AnalyDistrictLogTest extends BaseBatch{
}
def allCity = CommonUtils.findAllCity()
updateTempData()
if(SystemUtils.IS_OS_WINDOWS){//本地手动跑特定城市数据
allCity = ["石家庄", "沈阳", "上海", "杭州", "济南", "青岛", "广州", "重庆", "成都", "西安"]
if(SystemUtils.IS_OS_WINDOWS){//本地手动跑特定城市数据
allCity = ["石家庄", "沈阳", "上海", "杭州", "济南", "青岛", "广州", "重庆", "成都", "西安"]
}
def dataTypes = ["district","office"];
def index = 1;
for(def city : allCity){
def s = System.currentTimeMillis()
println "[${index}]开始分析 ${city} ${new Date().format('yyyy/MM/dd HH:mm:ss')}"
println "[${index}]开始分析 ${city} ${new Date().format('yyyy/MM/dd HH:mm:ss')}"
for(def dataType : dataTypes){
process(city, dataType);
}
index ++;
println "[${index}]分析结束 ${city} ${new Date().format('yyyy/MM/dd HH:mm:ss')} 耗时:${(int)((System.currentTimeMillis() - s)/(1000 * 60))}分钟"
println "[${index}]分析结束 ${city} ${new Date().format('yyyy/MM/dd HH:mm:ss')} 耗时:${(int)((System.currentTimeMillis() - s)/(1000 * 60))}分钟"
}
}
@@ -115,7 +115,7 @@ class AnalyDistrictLogTest extends BaseBatch{
query.append("version",new BasicDBObject('$lt',version))
DBCursor cursor = ANALY_SITE_COLL.find(query,new BasicDBObject("city", 1).append("region",1).append("district_name",1).append("cat",1)).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(300)
while(cursor.hasNext()){
println("删除:${cursor.next()}")
println("删除:${cursor.next()}")
}
ANALY_SITE_COLL.remove(query)
query.removeField("version")
@@ -145,13 +145,13 @@ class AnalyDistrictLogTest extends BaseBatch{
updater.put("region",formateRegion)
}
}
String key = city + JOIN_SEPARATOR + formateRegion + JOIN_SEPARATOR + name.toLowerCase().replaceAll(' '," ").replaceAll(" "," ").replaceAll('(^\\s+)|(\\s+$)','');
String key = city + JOIN_SEPARATOR + formateRegion + JOIN_SEPARATOR + name.toLowerCase().replaceAll(' '," ").replaceAll(" "," ").replaceAll('(^\\s+)|(\\s+$)','');
int audit_status = analySite.getInt("audit_status",0)//审阅状态 0:未审核 1:已审核 4:已审阅
int audit_status = analySite.getInt("audit_status",0)//审阅状态 0:未审核 1:已审核 4:已审阅
int data_status = analySite.getInt("data_status",NORMAL_DATA)
int exists
def district = allRightData.get(key);
//判断小区是否在CPDB库中
//判断小区是否在CPDB库中
if(district != null){
exists = 1
}else{
@@ -200,7 +200,7 @@ class AnalyDistrictLogTest extends BaseBatch{
ANALY_SITE_COLL.update(new BasicDBObject("_id",analySite._id), new BasicDBObject('$set',updater))
}
if(progress % 1000 == 0){
println("状态进度:${DataUtil.roundDouble((progress/count)*100,1)}%")
println("状态进度:${DataUtil.roundDouble((progress/count)*100,1)}%")
}
}
}
@@ -212,10 +212,10 @@ class AnalyDistrictLogTest extends BaseBatch{
*/
public def process(def city, def cat){
def analyRoads = findAnalyRoads(city, cat);//查找抓取数据
def analyRoads = findAnalyRoads(city, cat);//查找抓取数据
if(analyRoads.size() > 0){
def districts = getAllDistricts(city, cat);
//按网页合并
//按网页合并
def List<BasicDBObject> mergedRoad = analyzeAnalyRoads(analyRoads);
saveAnalySite(mergedRoad,cat);
updateOldVersionAnalySiteDataStatus(city,cat,districts)
@@ -223,7 +223,7 @@ class AnalyDistrictLogTest extends BaseBatch{
}
/**
* 通过城市区域小区名对Road数据进行分组
* 通过城市区域小区名对Road数据进行分组
* @param analyRoads
* @return
*/
@@ -249,7 +249,7 @@ class AnalyDistrictLogTest extends BaseBatch{
item.get("site_names") << analyRoad.site_name
}
if(progress % 10000 == 0){
println("分组进度:${DataUtil.roundDouble((progress/analyRoads.size())*100,0)}%")
println("分组进度:${DataUtil.roundDouble((progress/analyRoads.size())*100,0)}%")
}
}
@@ -261,7 +261,7 @@ class AnalyDistrictLogTest extends BaseBatch{
dbo.put("site_count",siteNames.size())
result << dbo
}
println "[analyzeAnalyRoads]耗时:${System.currentTimeMillis() - start}"
println "[analyzeAnalyRoads]耗时:${System.currentTimeMillis() - start}"
return result;
}
@@ -327,7 +327,7 @@ class AnalyDistrictLogTest extends BaseBatch{
def name = obj.getString("name");
List alias = obj.get("alias");
def status = obj.getInt("status", 0);
String key = city + JOIN_SEPARATOR + region + JOIN_SEPARATOR + name.toLowerCase().replaceAll(' '," ").replaceAll(" "," ").replaceAll('(^\\s+)|(\\s+$)','');
String key = city + JOIN_SEPARATOR + region + JOIN_SEPARATOR + name.toLowerCase().replaceAll(' '," ").replaceAll(" "," ").replaceAll('(^\\s+)|(\\s+$)','');
if(0 != status){
deletedData.put(key,obj)
}else{
@@ -336,7 +336,7 @@ class AnalyDistrictLogTest extends BaseBatch{
if(alias){
for(String aliaName : alias){
if(StringUtils.isNotBlank(aliaName) && aliaName != name){
key = city + JOIN_SEPARATOR +region+ JOIN_SEPARATOR + aliaName.toLowerCase().replaceAll(' '," ").replaceAll(" "," ").replaceAll('(^\\s+)|(\\s+$)','');
key = city + JOIN_SEPARATOR +region+ JOIN_SEPARATOR + aliaName.toLowerCase().replaceAll(' '," ").replaceAll(" "," ").replaceAll('(^\\s+)|(\\s+$)','');
if(0 != status){
deletedData.put(key,obj);
}else{
@@ -347,7 +347,7 @@ class AnalyDistrictLogTest extends BaseBatch{
}
}
}
//移除拥有和正常数据一样KEY的被删除数据
//移除拥有和正常数据一样KEY的被删除数据
for(String k : allRightData.keySet()){
deletedData.remove(k)
}
@@ -415,7 +415,7 @@ class AnalyDistrictLogTest extends BaseBatch{
}
progress.getAndIncrement()
if(progress.get() % 1000 == 0){
println("入库进度:${DataUtil.roundDouble((progress.get()/mergedRoads.size())*100,1)}%")
println("入库进度:${DataUtil.roundDouble((progress.get()/mergedRoads.size())*100,1)}%")
}
}
})
@@ -438,7 +438,7 @@ class AnalyDistrictLogTest extends BaseBatch{
if(StringUtils.isBlank(region)){
return false;
}
if(region.contains("周边")||region.contains("其他")){
if(region.contains("周边")||region.contains("其他")){
return false;
}
if(regions.containsKey(region)){
@@ -455,7 +455,7 @@ class AnalyDistrictLogTest extends BaseBatch{
def updateTempData(){
println("开始更新临时数据")
println("开始更新临时数据")
DBObject query = new BasicDBObject();
query.append("status",new BasicDBObject('$ne',ANALY_ROAD_STATUS_ERROR))
query.append("c_date", new BasicDBObject('$gt',DateUtil.add(now, Calendar.DATE, -2)))
@@ -488,7 +488,7 @@ class AnalyDistrictLogTest extends BaseBatch{
println(count)
}
}
println("临时数据更新结束")
println("临时数据更新结束")
}
def assembleObj(BasicDBObject srcDbo){
@@ -39,12 +39,12 @@ class AutoAddDistrict {
}
def run(){
def ret = findNeedAddDistrict("南京","玄武");
println "玄武:" + ret.size();
def ret = findNeedAddDistrict("南京","玄武");
println "玄武:" + ret.size();
def distRet = findAnalyRoads(ret);
def ret1 = findNeedAddDistrict("南京","建邺");
println "建邺:" + ret1.size();
def ret1 = findNeedAddDistrict("南京","建邺");
println "建邺:" + ret1.size();
def distRet1 = findAnalyRoads(ret1);
//updateDistrict(distRet);
}
@@ -96,8 +96,8 @@ class AutoAddDistrict {
for(def m : rett){
def siteName = m.key;
def values = m.values;
if(siteName=='网上房地产'){
println "网上房地产"
if(siteName=='网上房地产'){
println "网上房地产"
}else{
for(def field : fields){
@@ -17,9 +17,9 @@ public class AutoAddSouFangDistrict{
DBCollection analySiteColl = MongoUtils.getCollectionByName("coredata","analy_site", "coredata.mongo.db.fangjia.com",27017)
def static citys = ["温州","金华","衢州","湖州","绍兴","嘉兴","舟山","泰州","徐州","镇江","淮安","连云港","宿迁","扬州","盐城","南通","无锡","丽水","重庆","佛山","江门","珠海","郑州","洛阳","许昌","新乡","开封","安阳","大连","鞍山","营口","丹东","西安","榆林","渭南","宝鸡","太原","忻州","运城","朔州","长治","晋中","哈尔滨","齐齐哈尔","大庆","南宁","柳州","桂林","呼和浩特","鄂尔多斯","呼伦贝尔","包头"]
def static citys = ["温州","金华","衢州","湖州","绍兴","嘉兴","舟山","泰州","徐州","镇江","淮安","连云港","宿迁","扬州","盐城","南通","无锡","丽水","重庆","佛山","江门","珠海","郑州","洛阳","许昌","新乡","开封","安阳","大连","鞍山","营口","丹东","西安","榆林","渭南","宝鸡","太原","忻州","运城","朔州","长治","晋中","哈尔滨","齐齐哈尔","大庆","南宁","柳州","桂林","呼和浩特","鄂尔多斯","呼伦贝尔","包头"]
def static final DBCollection ANALY_ROAD_COLL = MongoUtils.getCollectionByName("coredata", "analy_road", "coredata.mongo.db.fangjia.com")
def districtNameNoise = [".","·","。","#","@"," ","?"]
def districtNameNoise = [".","·","。","#","@"," ","?"]
def static main(args){
def obj = new AutoAddSouFangDistrict()
citys.each {
@@ -34,7 +34,7 @@ public class AutoAddSouFangDistrict{
},new BasicDBObject("city",1).append("region",1).append("name",1))
def keys = new BasicDBObject("city",1).append("region",1).append("district_name",1).append("site_names",1).append("site_count",1)
DBCursor cursor = analySiteColl.find(new BasicDBObject("city",city).append("cat","district").append("exists",0).append("audit_status",0).append("data_status",0),keys)//append("site_names",Pattern.compile(".*搜房.*")
DBCursor cursor = analySiteColl.find(new BasicDBObject("city",city).append("cat","district").append("exists",0).append("audit_status",0).append("data_status",0),keys)//append("site_names",Pattern.compile(".*搜房.*")
cursor.batchSize(10000)
cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT)
cursor.each {BasicDBObject dbo ->
@@ -43,10 +43,10 @@ public class AutoAddSouFangDistrict{
def name = formatName(orgName)
def validSite = isSelectedSites(dbo);
if(!validSite){
println("网站不合法:" + dbo)
println("网站不合法:" + dbo)
}
if(validSite && !isEndsWith(name) && !isContainsChars(name)){
name = name.replaceAll("\\(|(","#").replaceAll("\\)|)","@")
name = name.replaceAll("\\(|(","#").replaceAll("\\)|)","@")
if(name.indexOf("#") > 0 && name.indexOf("@") > 0){
name = name.substring(0,name.indexOf("#"))
}
@@ -68,8 +68,8 @@ public class AutoAddSouFangDistrict{
seaweed.append("status",0)
def info = findDistrictInfo(city,region,name)
def baiDuEstateType = getBaiDuEstateType(city,name)
if(info[0] == "写字楼"|| "写字楼" == baiDuEstateType || "商铺" == baiDuEstateType){
println("写字楼/商铺:" + seaweed)
if(info[0] == "写字楼"|| "写字楼" == baiDuEstateType || "商铺" == baiDuEstateType){
println("写字楼/商铺:" + seaweed)
}else{
seaweed.append("estate_type2",info[0])
if(StringUtils.isNotBlank(info[1])){
@@ -92,10 +92,10 @@ public class AutoAddSouFangDistrict{
if(dbo.getInt("site_count") > 2){
return true;
}
if(dbo.getInt("site_count") == 1 &&(dbo.getString("site_names").contains("图吧") || dbo.getString("site_names").contains("千万间"))){
if(dbo.getInt("site_count") == 1 &&(dbo.getString("site_names").contains("图吧") || dbo.getString("site_names").contains("千万间"))){
return false
}
if(dbo.getInt("site_count") == 2 && dbo.getString("site_names").contains("图吧") && dbo.getString("site_names").contains("千万间")){
if(dbo.getInt("site_count") == 2 && dbo.getString("site_names").contains("图吧") && dbo.getString("site_names").contains("千万间")){
return false
}
return true;
@@ -120,9 +120,9 @@ public class AutoAddSouFangDistrict{
if(StringUtils.isNotBlank(estate_type)){
estateTypes << estate_type
}
if(dbo.getString("site_name") == "搜房" && address == null){
if(dbo.getString("site_name") == "搜房" && address == null){
address = result.get("address")
}else if(otherAddress == null ||(result.get("address") && result.getString("address") ==~ /.*\d+号.*/)){
}else if(otherAddress == null ||(result.get("address") && result.getString("address") ==~ /.*\d+号.*/)){
otherAddress = result.get("address")
}
@@ -133,16 +133,16 @@ public class AutoAddSouFangDistrict{
int liangYongCount = 0
int zhuZhaiCount = 0
estateTypes.each {
if(it.contains("商铺") || it.contains("写字楼") || it.contains("商业") || it.contains("办公")){
if(it.contains("商铺") || it.contains("写字楼") || it.contains("商业") || it.contains("办公")){
officeCount ++
}
if(it.contains("别墅")){
if(it.contains("别墅")){
bieshuCount ++
}
if(it.contains("商住两用")){
if(it.contains("商住两用")){
liangYongCount ++
}
if(it.contains("住宅")){
if(it.contains("住宅")){
zhuZhaiCount ++
}
}
@@ -151,29 +151,29 @@ public class AutoAddSouFangDistrict{
}
if(zhuZhaiCount > 0){
if(bieshuCount > 0){
return ["别墅",address]
return ["别墅",address]
}else if(liangYongCount > 0){
return ["商住两用",address]
return ["商住两用",address]
}
return ["普通住宅",address]
return ["普通住宅",address]
}else if(officeCount > zhuZhaiCount){
return ["写字楼",address]
return ["写字楼",address]
}else if(bieshuCount > 0){
return ["别墅",address]
return ["别墅",address]
}else if(liangYongCount > 0){
return ["商住两用",address]
return ["商住两用",address]
}else{
return ["普通住宅",address]
return ["普通住宅",address]
}
}
def formateAddress(String city,String address){
if(address.endsWith("交汇")){
if(address.endsWith("交汇")){
return address
}
String orgAddress = address
if(address ==~ /.*\d+$/){
address += "号"
address += "号"
}
JSONObject result = OpenApiClient.addressStandardize(city, address)
String formateAddr;
@@ -183,7 +183,7 @@ public class AutoAddSouFangDistrict{
formateAddr += result.getString("streetNumber")
}
}
if(formateAddr && (formateAddr.contains("市") || formateAddr.contains("县"))){
if(formateAddr && (formateAddr.contains("市") || formateAddr.contains("县"))){
result = OpenApiClient.addressStandardize(city, formateAddr)
if(result.containsKey("street")){
formateAddr = result.getString("street")
@@ -198,7 +198,7 @@ public class AutoAddSouFangDistrict{
formateAddr = addrs[0] + addrs[1]
return formateAddr
}else if(addrs.size() == 3){
formateAddr = addrs[0] + "与" + addrs[1].replaceFirst("^(与|和)","") + "交汇"
formateAddr = addrs[0] + "与" + addrs[1].replaceFirst("^(与|和)","") + "交汇"
return formateAddr
}
}
@@ -214,12 +214,12 @@ public class AutoAddSouFangDistrict{
if(item.containsKey("detail_info") && item.getString("name") == name){
String tag = item.getJSONObject("detail_info")?.get("tag")?.toString();
if (StringUtils.isNotBlank(tag)) {
if(tag.contains("写字楼")){
return "写字楼"
}else if(tag.contains("商场") || tag.contains("购物") || tag.contains("商铺")){
return "商铺"
}else if(tag.contains("住宅区")){
return "普通住宅"
if(tag.contains("写字楼")){
return "写字楼"
}else if(tag.contains("商场") || tag.contains("购物") || tag.contains("商铺")){
return "商铺"
}else if(tag.contains("住宅区")){
return "普通住宅"
}
}
}
@@ -233,22 +233,22 @@ public class AutoAddSouFangDistrict{
districtNameNoise.each {
name = name.replace(it,"")
}
if(name ==~ /.*(村|小区)\d+号$/){
name = name.replaceAll("\\d+号\$","")
if(name ==~ /.*(村|小区)\d+号$/){
name = name.replaceAll("\\d+号\$","")
}
if(name ==~ /.*路\d+号$/){
name = name + "小区"
if(name ==~ /.*路\d+号$/){
name = name + "小区"
}
name = name.replaceAll("\\d+幢\$","")
name = name.replaceAll("\\d+幢\$","")
return name;
}
def isEndsWith(def s){
def endChars = ["路","街","店","贸区","巷","村","设计院","社区","地块","交易中心","综合大楼","附近","对面","安置房"];
def endChars = ["路","街","店","贸区","巷","村","设计院","社区","地块","交易中心","综合大楼","附近","对面","安置房"];
def flag = false;
for(def key : endChars){
if(s.endsWith(key)){
if(key=="村"||key=="巷"){
if(key=="村"||key=="巷"){
if(s.length()<=3){
flag = true;
}
@@ -263,7 +263,7 @@ public class AutoAddSouFangDistrict{
}
def isContainsChars(s) {
def chars = ["世贸中心", "商务中心", "家具中心", "商业中心", "建材市场", "生活广场", "购物广场", "商城", "电器城"]
def chars = ["世贸中心", "商务中心", "家具中心", "商业中心", "建材市场", "生活广场", "购物广场", "商城", "电器城"]
def flag = false;
for (def key : chars) {
if (s.indexOf(key) != -1) {
@@ -276,7 +276,7 @@ public class AutoAddSouFangDistrict{
def List<String> splitAddress(String addr){
List<String> addres = []
def keys = ["街道","大道","国道","道路","路","街","线"]
def keys = ["街道","大道","国道","道路","路","街","线"]
String remain = addr;
while(true){
int p = Integer.MAX_VALUE
@@ -25,7 +25,7 @@ class AutoDistrict {
public static void main(String[] args){
def task = new AutoDistrict();
task.run(["西安", "成都"]);//"北京",
task.run(["西安", "成都"]);//"北京",
}
@@ -154,7 +154,7 @@ class AutoDistrict {
s = s + " "+ estateType;
}
}
if(s.indexOf("写字楼")!=-1||s.indexOf("办字楼")!=-1||s.indexOf("商铺")!=-1||s.indexOf("商住楼")!=-1){
if(s.indexOf("写字楼")!=-1||s.indexOf("办字楼")!=-1||s.indexOf("商铺")!=-1||s.indexOf("商住楼")!=-1){
return false;
}
return true;
@@ -233,7 +233,7 @@ class AutoDistrict {
}
def writeCSV(city, results, fields){
File file = new File("e:\\自动入库("+city+").csv");
File file = new File("e:\\自动入库("+city+").csv");
if (file.exists()){
file.delete();
}
@@ -13,7 +13,7 @@ class CityAnaDataBatch extends BaseBatch {
DBCollection CITY_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "city_sell_stat", "mongo.db.fangjia.com")
DBCollection CITY_ANA_DATA = MongoUtils.getCollectionByName("fangjia_stat", "city_ana_data", "mongo.db.fangjia.com")
List CENTER_CITY = ["石家庄","太原","呼和浩特","沈阳","长春","哈尔滨","南京","杭州","合肥","福州","南昌","济南","郑州","武汉","长沙","广州","南宁","海口","成都","贵阳","昆明","拉萨","西安","兰州","西宁","银川","乌鲁木齐","台北"]
List CENTER_CITY = ["石家庄","太原","呼和浩特","沈阳","长春","哈尔滨","南京","杭州","合肥","福州","南昌","济南","郑州","武汉","长沙","广州","南宁","海口","成都","贵阳","昆明","拉萨","西安","兰州","西宁","银川","乌鲁木齐","台北"]
static void main(String[] args) {
runBatch(new CityAnaDataBatch(), args)
@@ -21,7 +21,7 @@ class CityAnaDataBatch extends BaseBatch {
@Override
void process(Object args) {
Integer yearMonth = CITY_SELL_STAT.find(new BasicDBObject("city", "上海")).sort(new BasicDBObject("date", -1)).limit(1).toList()[0].date
Integer yearMonth = CITY_SELL_STAT.find(new BasicDBObject("city", "上海")).sort(new BasicDBObject("date", -1)).limit(1).toList()[0].date
List orders = []
CITY_SELL_STAT.find(new BasicDBObject("date", yearMonth)).each {BasicDBObject dbo->
orders << new CityOrderItem(dbo.city, dbo.avg_price)
@@ -31,11 +31,11 @@ class CityAnaDataBatch extends BaseBatch {
CityOrderItem item = orders.get(i - 1)
def data = new BasicDBObject('city_rank', i).append("city_total_rank", orders.size()).append("avg_price", item.avgPrice)
if(ProvinceData.isGovernmentCity(item.city)){
data.put("level", "直辖市")
data.put("level", "直辖市")
}else if(CENTER_CITY.contains(item.city)){
data.put("level", "省会城市")
data.put("level", "省会城市")
}else{
data.put("level", "地址市")
data.put("level", "地址市")
}
CITY_ANA_DATA.update(new BasicDBObject("city", item.city), new BasicDBObject('$set', data), true,true)
}
@@ -11,7 +11,7 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 城市区域人均消费统计脚本
* 城市区域人均消费统计脚本
* @author
*/
@BatchOwner(Watcher.WangWenYan)
@@ -38,7 +38,7 @@ public class CityAndRegionAverageConsume extends BaseBatch{
def analyRegionAverageConsume(def city, def region){
List<Integer> numbers = new ArrayList<Integer>();
BUSINESS_SHOP_COLL.find(new BasicDBObject("business_type", "美食").append("city", city).append("region", region), new BasicDBObject("consumption", 1)).each {BasicDBObject dbo->
BUSINESS_SHOP_COLL.find(new BasicDBObject("business_type", "美食").append("city", city).append("region", region), new BasicDBObject("consumption", 1)).each {BasicDBObject dbo->
if(dbo.consumption && dbo.consumption > 0){
numbers << dbo.consumption
}
@@ -59,7 +59,7 @@ public class CityAndRegionAverageConsume extends BaseBatch{
def analyCityAverageConsume(def city){
List<Integer> numbers = new ArrayList<Integer>();
BUSINESS_SHOP_COLL.find(new BasicDBObject("business_type", "美食").append("city", city), new BasicDBObject("consumption", 1)).each {BasicDBObject dbo->
BUSINESS_SHOP_COLL.find(new BasicDBObject("business_type", "美食").append("city", city), new BasicDBObject("consumption", 1)).each {BasicDBObject dbo->
if(dbo.consumption && dbo.consumption > 0){
numbers << dbo.consumption
}
@@ -15,8 +15,8 @@ import java.sql.Connection
import java.sql.DriverManager
/**
* 小区均价涨跌幅纠正脚本
* 功能说明:
* 小区均价涨跌幅纠正脚本
* 功能说明:
*/
@BatchOwner(Watcher.WangWenYan)
class ClickHouseDistrictStatMarkupCalcBatch extends BaseBatch {
@@ -39,10 +39,10 @@ class ClickHouseDistrictStatMarkupCalcBatch extends BaseBatch {
query.city = district.city
query.region = district.region
query.district_name = district.name
if(district.estate_type2 && "别墅" == district.estate_type2){
query.estate_type = "别墅"
if(district.estate_type2 && "别墅" == district.estate_type2){
query.estate_type = "别墅"
}else{
query.estate_type = "小区"
query.estate_type = "小区"
}
List<BasicDBObject> items = DISTRICT_STAT.find(query, new BasicDBObject("avg_price", 1).append("weekend_markup", 1).append("weekend", 1)).batchSize(2000).addOption(Bytes.QUERYOPTION_SLAVEOK).sort(new BasicDBObject("weekend", 1)).toList()
if(items.size() < 2){
@@ -11,7 +11,7 @@ import com.mongodb.DBCollection
/**
* Created by wenyan on 2016/1/22.
*/
def cityList = ["南昌","包头","安庆","泸州","遵义","常德","南充","牡丹江","锦州","岳阳","韶关","济宁","平顶山"]
def cityList = ["南昌","包头","安庆","泸州","遵义","常德","南充","牡丹江","锦州","岳阳","韶关","济宁","平顶山"]
DBCollection ANALY_SITE_COLL = MongoUtils.getCollectionByName("coredata", "analy_site", "coredata.mongo.db.fangjia.com")
DSCollection seawater = new DataStore("192.168.0.65", 27777).getCollection("fangjia.seawater")
def startDate = DateUtil.add(new Date(), Calendar.MONTH, -6).format("yyyyMMdd").toInteger()
@@ -30,9 +30,9 @@ cityList.each {def city->
long count = (long) seawater.count(houseQuery)
ANALY_SITE_COLL.update(new BasicDBObject("_id",dbo._id), new BasicDBObject('$set',new BasicDBObject('house_num',count)))
if(idx % 100 == 0){
println("${city}统计进度:${DataUtil.roundDouble((idx *100)/total,2)}%")
println("${city}统计进度:${DataUtil.roundDouble((idx *100)/total,2)}%")
}
}
println("${city}分析结束")
println("${city}分析结束")
}
@@ -18,8 +18,8 @@ class CoverPrecent {
public static final DBCollection OFFICE_STAT_COLL = MongoUtils.getCollectionByName("fangjia", "office_stat", "mongo.db.fangjia.com")
static main(args) {
//def citys = ["天津","重庆","成都","大连","沈阳","哈尔滨","牡丹江","大庆","齐齐哈尔","佳木斯","伊春","鸡西","鹤岗","双鸭山","七台河","绥化","黑河"];
def citys = ["上海"]
//def citys = ["天津","重庆","成都","大连","沈阳","哈尔滨","牡丹江","大庆","齐齐哈尔","佳木斯","伊春","鸡西","鹤岗","双鸭山","七台河","绥化","黑河"];
def citys = ["上海"]
def cat = "office";
def weekend = "2014-11-16";
@@ -41,7 +41,7 @@ class CoverPrecent {
result << ret;
}
task.writeCSV3("E:\\数据测试\\结果\\商铺写字楼覆盖率\\","覆盖率-"+cat+".csv", result)
task.writeCSV3("E:\\数据测试\\结果\\商铺写字楼覆盖率\\","覆盖率-"+cat+".csv", result)
}
@@ -52,18 +52,18 @@ class CoverPrecent {
}
file.append("城市,")
file.append("楼盘数量,")
file.append("售价数量,")
file.append("租金数量,")
file.append("售价占比,")
file.append("租金占比,")
file.append("租房房源数,")
file.append("租房房源小区量,")
file.append("出售房源数,")
file.append("出售房源小区量,")
file.append("出租房源小区占比,")
file.append("出售房源小区占比,")
file.append("城市,")
file.append("楼盘数量,")
file.append("售价数量,")
file.append("租金数量,")
file.append("售价占比,")
file.append("租金占比,")
file.append("租房房源数,")
file.append("租房房源小区量,")
file.append("出售房源数,")
file.append("出售房源小区量,")
file.append("出租房源小区占比,")
file.append("出售房源小区占比,")
file.append(System.getProperty("line.separator"));
ret.each{ map->
file.append(map.city+ ",","gbk")
@@ -87,9 +87,9 @@ class CoverPrecent {
def totalCount = findTotalCount(city, cat);
def kind = "";
if(cat == "shop"){
kind = "商铺";
kind = "商铺";
}else if(cat == "office"){
kind = "写字楼";
kind = "写字楼";
}
def sellCount = findSellPriceCountByCat(city, kind, weekend);
def rentCount = findRentPriceCountByCat(city, kind, weekend);
@@ -107,7 +107,7 @@ class CoverPrecent {
ret.rentCount = rentCount;
ret.sellCoverPecent = DataUtil.getRound(2, sellCoverPecent) ;
ret.rentCoverPecent = DataUtil.getRound(2, rentCoverPecent) ;
println "【${city}】:"+totalCount+" sellCount:"+sellCount+" rentCount:"+rentCount+" sellCoverPecent:"+sellCoverPecent+" rentCoverPecent:"+rentCoverPecent
println "【${city}】:"+totalCount+" sellCount:"+sellCount+" rentCount:"+rentCount+" sellCoverPecent:"+sellCoverPecent+" rentCoverPecent:"+rentCoverPecent
return ret;
}
@@ -134,7 +134,7 @@ class CoverPrecent {
DBObject q = new BasicDBObject();
q.put("city", city);
q.put('weekend', DateUtils.parseFromTightYYYYMMDD(weekend));
if(kind == '商铺'){
if(kind == '商铺'){
return SHOP_STAT_COLL.count(q);
}else{
return OFFICE_STAT_COLL.count(q);
@@ -13,7 +13,7 @@ import com.mongodb.Mongo
import com.mongodb.MongoURI
/**
* 功能说明:监控数据库数据量
* 功能说明:监控数据库数据量
*/
@BatchOwner(Watcher.WangWenYan)
class DatabaseStatWatcherBatch extends BaseBatch {
@@ -16,7 +16,7 @@ import com.mongodb.DBObject
import java.util.concurrent.ConcurrentHashMap
/**
* 小区周边数据统计脚本
* 小区周边数据统计脚本
*/
@BatchOwner(Watcher.WangWenYan)
public class DistrictAroundDataAnaly extends BaseBatch {
@@ -58,7 +58,7 @@ public class DistrictAroundDataAnaly extends BaseBatch {
DbUtils.multiThreadingExecuteQueryOnSeaweed(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), { BasicDBObject dbo ->
def data = new BasicDBObject()
if (dbo.lng && polygonMap.size() > 0) {
def economics = ["中心城区", "普通城区", "郊区"]
def economics = ["中心城区", "普通城区", "郊区"]
for (def economic : economics) {
def latlngs = polygonMap.get(economic)
if (latlngs != null) {
@@ -75,10 +75,10 @@ public class DistrictAroundDataAnaly extends BaseBatch {
}
}
if (!data.containsField("economic")) {
data.put("economic", "郊外")
data.put("economic", "郊外")
}
}
def types = ["商场": "060101", "麦当劳": "050302", "肯德基": "050301", "星巴克": "050501", "地铁站": "150500", "小学": "141202", "中学": "141203", "市中心": "190500"]
def types = ["商场": "060101", "麦当劳": "050302", "肯德基": "050301", "星巴克": "050501", "地铁站": "150500", "小学": "141202", "中学": "141203", "市中心": "190500"]
if (dbo.lng) {
def nearest_classic_pois = []
types.each { def type, def code ->
@@ -98,7 +98,7 @@ public class DistrictAroundDataAnaly extends BaseBatch {
AMAP_POI_COLL.find(query, new BasicDBObject("name", 1).append("lat", 1).append("lng", 1)).limit(1).each { BasicDBObject poi ->
def classicPoi = new BasicDBObject()
classicPoi.code = code
classicPoi.name = poi.name.toString().replaceAll("((|\\().*()|\\))", "")
classicPoi.name = poi.name.toString().replaceAll("((|\\().*()|\\))", "")
classicPoi.lng = poi.lng
classicPoi.lat = poi.lat
classicPoi.distance = GeoUtils.calculateDistance(dbo.lng / 10000000000d, dbo.lat / 10000000000d, classicPoi.lng, classicPoi.lat)
@@ -186,7 +186,7 @@ public class DistrictAroundDataAnaly extends BaseBatch {
def query = new BasicDBObject()
query.append('lat', new BasicDBObject('$gte', arr[0]).append('$lte', arr[1]))
query.append('lng', new BasicDBObject('$gte', arr[2]).append('$lte', arr[3]))
query.append("business_type", "美食")
query.append("business_type", "美食")
def keys = new BasicDBObject("consumption", 1)
List<Integer> numbers = new ArrayList<Integer>();
BUSINESS_SHOP_COLL.find(query, keys).each { BasicDBObject dbo ->
@@ -31,14 +31,14 @@ class DistrictBackupReset {
def startDate = "2014-12-07";
def index = 0;
for(def city : citys){
if(city!="上海"){
if(city!="上海"){
index ++;
def backUpMap = findBackUp(city);
println "【${city}】备份数据:"+backUpMap.size();
println "【${city}】备份数据:"+backUpMap.size();
def districts = findUpdateDistrictData(city, startDate)
println "【${city}】可能修改的小区数据有:"+districts.size();
println "【${city}】可能修改的小区数据有:"+districts.size();
updateDistrict(districts, backUpMap);
println "${index} 更新完成..........................."
println "${index} 更新完成..........................."
}
}
@@ -58,8 +58,8 @@ class DistrictBackupReset {
}else{
index ++;
println "[${index}]当前:"+obj;
println "[${index}]备份:"+map;
println "[${index}]当前:"+obj;
println "[${index}]备份:"+map;
DBObject udata = new BasicDBObject();
udata.put("m_date", new Date());
@@ -134,7 +134,7 @@ class DistrictBackupReset {
if(cursor){
for(def dbo : cursor){
def city = dbo.get("display");
if(city!='全国'){
if(city!='全国'){
citys << city;
}
}
@@ -11,7 +11,7 @@ import com.fangjia.script.util.MongoUtils
import com.fangjia.core.util.DateUtil;
import com.fangjia.core.util.DataUtil
/**
* 分期竣工时间覆盖率
* 分期竣工时间覆盖率
* @author gaojuan
*
*/
@@ -25,7 +25,7 @@ class DistrictFinishDateCover {
}
def run(){
def citys = ["上海"];//findAllCity()
def citys = ["上海"];//findAllCity()
for(def city : citys){
def districts = getAllDistricts(city, "district");
findFenQiData(districts);
@@ -43,24 +43,24 @@ class DistrictFinishDateCover {
def findFenQiData(districts){
def fenqiDatas = [];
//查找分期数据
//查找分期数据
if(districts && districts.size() > 0){
for(def map : districts){
def keys = map.key
def dbo = map.value;
def name = dbo.name;
if(name.indexOf("期")!=-1){
if(name.indexOf("期")!=-1){
fenqiDatas << dbo;
}
}
}
//查找总小区
//查找总小区
def dists = [];
for(def dbo : fenqiDatas){
def name = dbo.name
def region = dbo.region
name = name.replaceAll("[一|二|三|四|五|六|七|八|九|十]期", "")
name = name.replaceAll("[一|二|三|四|五|六|七|八|九|十]期", "")
def obj = districts.get(region + "@"+name);
if(obj){
if(!dists.contains(obj)){
@@ -103,8 +103,8 @@ class DistrictFinishDateCover {
def rett1 = calDateCover(dists);
//def rett2 = calDateCover(rets);
writeCSV("E:\\数据测试\\结果\\","上海分期(总).csv", rett1)
//writeCSV("E:\\数据测试\\结果\\","上海分期.csv", rett2)
writeCSV("E:\\数据测试\\结果\\","上海分期(总).csv", rett1)
//writeCSV("E:\\数据测试\\结果\\","上海分期.csv", rett2)
}
def calDateCover(dists){
@@ -160,8 +160,8 @@ class DistrictFinishDateCover {
def count = haveTimeDists.size();
def precent = count * 100/total;
precent = DataUtil.getRound(2,precent);
println "覆盖率:" + precent
println "共:" + total
println "覆盖率:" + precent
println "共:" + total
return haveTimeDists;
}
@@ -172,9 +172,9 @@ class DistrictFinishDateCover {
}
file.append("区域,")
file.append("名称,")
file.append("时间,")
file.append("区域,")
file.append("名称,")
file.append("时间,")
file.append(System.getProperty("line.separator"));
ret.each{ map->
@@ -217,7 +217,7 @@ class DistrictFinishDateCover {
if(cursor){
for(def dbo : cursor){
def city = dbo.get("display");
if(city!='全国'){
if(city!='全国'){
citys << city;
}
}
@@ -58,7 +58,7 @@ class DistrictPriceRatingBatch extends BaseBatch {
if(item.maxArea && item.maxArea < 40){
continue
}
item.tag = "봉빽遼酪"
item.tag = "豪华住宅"
haoHuaList << item
}
list.removeAll(haoHuaList)
@@ -78,19 +78,19 @@ class DistrictPriceRatingBatch extends BaseBatch {
if(item.maxArea && item.maxArea < 60){
continue
}
item.tag = "멕도遼酪"
item.tag = "高档住宅"
gaoDangList << item
}
for(int i = regionList.size() * 0.75 ; i <= regionList.size(); i++){
DistrictPriceOrderItem item = regionList.get(i - 1)
item.tag = "됴도遼酪"
item.tag = "低档住宅"
diDangList << item
}
}
list.removeAll(gaoDangList)
list.removeAll(diDangList)
list.each {DistrictPriceOrderItem item->
item.tag = "팹繫遼酪"
item.tag = "普通住宅"
}
haoHuaList.each {
DISTRICT_ANALY_DATA_COLL.update(new BasicDBObject("city", city).append("region", it.region).append("district_name", it.name), new BasicDBObject('$set', new BasicDBObject('tag', it.tag)), true, true)
@@ -12,8 +12,8 @@ import com.mongodb.Bytes
import com.mongodb.DBCollection
/**
* 小区均价涨跌幅纠正脚本
* 功能说明:
* 小区均价涨跌幅纠正脚本
* 功能说明:
*/
@BatchOwner(Watcher.WangWenYan)
class DistrictStatMarkupCalcBatch extends BaseBatch {
@@ -29,19 +29,19 @@ class DistrictStatMarkupCalcBatch extends BaseBatch {
@Override
void process(Object args) {
def cityList = CommonUtils.findAllCity()
cityList =["北海"]
cityList =["北海"]
int total = cityList.size()
int count = 1
cityList.each {def city ->
DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city,"海城","东旭华庭"),{BasicDBObject district->
DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city,"海城","东旭华庭"),{BasicDBObject district->
def query = new BasicDBObject()
query.city = district.city
query.region = district.region
query.district_name = district.name
if(district.estate_type2 && "别墅" == district.estate_type2){
query.estate_type = "别墅"
if(district.estate_type2 && "别墅" == district.estate_type2){
query.estate_type = "别墅"
}else{
query.estate_type = "小区"
query.estate_type = "小区"
}
List<BasicDBObject> items = DISTRICT_STAT.find(query, new BasicDBObject("avg_price", 1).append("weekend_markup", 1).append("weekend", 1)).batchSize(2000).addOption(Bytes.QUERYOPTION_SLAVEOK).sort(new BasicDBObject("weekend", 1)).toList()
if(items.size() < 2){
@@ -37,7 +37,7 @@ class DistrictStatRankBatch extends BaseBatch {
public void process(Object args) {
def cities = ["深圳"]//ServerAreaMap.findAllCities()
def cities = ["深圳"]//ServerAreaMap.findAllCities()
for(String city : cities) {
// DISTRICT_STAT_MAP.clear()
// Map<String, List<String>> regions = RegionData.getAllRegions(city)
@@ -187,7 +187,7 @@ class DistrictStatRankBatch extends BaseBatch {
String city = district.city
String region = district.region
String name = district.name
String estateType = (district.estate_type2 != null && district.estate_type2.indexOf('别墅') > -1 ? '别墅' : '小区')
String estateType = (district.estate_type2 != null && district.estate_type2.indexOf('别墅') > -1 ? '别墅' : '小区')
Date momWeekend = DateUtil.getThisSunday(DateUtil.getBeforeDay(new Date(), 30))
Date sosWeekend = DateUtil.getThisSunday(DateUtil.getBeforeDay(new Date(), 90))
@@ -339,9 +339,9 @@ class DistrictStatRankBatch extends BaseBatch {
}
public static int calculatePercentile(def values, def targetValue) {
// 找出数字在List中的排名
// 找出数字在List中的排名
def rank = values.indexOf(targetValue) + 1
// 计算百分位数
// 计算百分位数
def percentile = (int)((rank / values.size()) * 100)
return percentile;
}
@@ -25,7 +25,7 @@ class EstateTypeChange {
def offices = getAllDistricts(city, "office");
updateEstateType("office", offices);
println "【${city}】 完成......"
println "【${city}】 完成......"
}
}
@@ -38,21 +38,21 @@ class EstateTypeChange {
def new_estate_type2 = "";
if(cat == "shop"){
if(estate_type2 == "市场类商铺"||estate_type2 == "专项卖场"){
new_estate_type2 = "专业市场";
}else if(estate_type2 == "百货购物中心商铺" || estate_type2 == "商业综合体" || estate_type2 == "商务区商铺"){
new_estate_type2 = "商场商铺";
}else if(estate_type2 == "社区内铺"){
new_estate_type2 = "社区商铺";
if(estate_type2 == "市场类商铺"||estate_type2 == "专项卖场"){
new_estate_type2 = "专业市场";
}else if(estate_type2 == "百货购物中心商铺" || estate_type2 == "商业综合体" || estate_type2 == "商务区商铺"){
new_estate_type2 = "商场商铺";
}else if(estate_type2 == "社区内铺"){
new_estate_type2 = "社区商铺";
}
}else if(cat == "office"){
if(estate_type2 == "园区写字楼"){
new_estate_type2 = "创意园区";
}else if(estate_type2 == "酒店商务楼" || estate_type2 == "商业综合楼"){
new_estate_type2 = "标准写字楼";
}else if(estate_type2 == "商住两用楼"){
new_estate_type2 = "商住两用";
if(estate_type2 == "园区写字楼"){
new_estate_type2 = "创意园区";
}else if(estate_type2 == "酒店商务楼" || estate_type2 == "商业综合楼"){
new_estate_type2 = "标准写字楼";
}else if(estate_type2 == "商住两用楼"){
new_estate_type2 = "商住两用";
}
}
@@ -69,10 +69,10 @@ class EstateTypeChange {
}
// else{
// if(estate_type2!="社区商铺" && estate_type2!="商务区商铺" && estate_type2!="商业街商铺"){
// if(estate_type2!="社区商铺" && estate_type2!="商务区商铺" && estate_type2!="商业街商铺"){
// println "[ ${index} ]"+ estate_type2+" "
// }
// if(estate_type2!="标准写字楼"){
// if(estate_type2!="标准写字楼"){
// println "[ ${index} ]"+ estate_type2+" "
// }
//
@@ -10,12 +10,12 @@ import com.mongodb.BasicDBObject
import org.apache.commons.lang.StringUtils
import java.util.regex.Pattern
// 分期分区与主小区映射关系表
// 分期分区与主小区映射关系表
@BatchOwner(Watcher.WangWenYan)
class FindStageDistrictBatch extends BaseBatch {
Pattern STAGE_PATTERN = Pattern.compile("(([一二三四五六七八九十百千\\d]+)期?[,、至到]?)?([一二三四五六七八九十百千\\d]+)期\$");
Pattern PARTITION_PATTERN = Pattern.compile("(\\w|东|南|北|中|西|甲|乙|丙|丁|一|二|三|四|五|六|七|八|九|十)区\$");
Pattern STAGE_PATTERN = Pattern.compile("(([一二三四五六七八九十百千\\d]+)期?[,、至到]?)?([一二三四五六七八九十百千\\d]+)期\$");
Pattern PARTITION_PATTERN = Pattern.compile("(\\w|东|南|北|中|西|甲|乙|丙|丁|一|二|三|四|五|六|七|八|九|十)区\$");
static void main(String[] args) {
@@ -27,14 +27,14 @@ class FindStageDistrictBatch extends BaseBatch {
CommonUtils.findAllCity().each { def city ->
CommonDatautils.SEAWEED_COLL.find(DbUtils.buildSeaweedQuery2(city), new BasicDBObject("region", 1).append("name", 1).append("master_district_name", 1).append("city", 1)).each { BasicDBObject dbo ->
if (dbo.name =~ STAGE_PATTERN && StringUtils.isBlank(dbo.master_district_name)) {
def mainName = dbo.name.toString().replaceAll("([一二三四五六七八九十百千\\d]+)期\$", "")
def mainName = dbo.name.toString().replaceAll("([一二三四五六七八九十百千\\d]+)期\$", "")
def query = DbUtils.buildSeaweedQuery(city, dbo.region, mainName)
if (DbUtils.seaweed.count(query) != 0) {
DbUtils.executeUpdateOnSeaweed(new BasicDBObject("_id", dbo._id), new BasicDBObject("master_district_name", mainName))
println(dbo)
}
} else if (dbo.name =~ PARTITION_PATTERN && StringUtils.isBlank(dbo.master_district_name)) {
def mainName = dbo.name.toString().replaceAll("(\\w|东|南|北|中|西|甲|乙|丙|丁|一|二|三|四|五|六|七|八|九|十)区\$", "")
def mainName = dbo.name.toString().replaceAll("(\\w|东|南|北|中|西|甲|乙|丙|丁|一|二|三|四|五|六|七|八|九|十)区\$", "")
def query = DbUtils.buildSeaweedQuery(city, dbo.region, mainName)
if (DbUtils.seaweed.count(query) != 0) {
DbUtils.executeUpdateOnSeaweed(new BasicDBObject("_id", dbo._id), new BasicDBObject("master_district_name", mainName))
@@ -59,13 +59,13 @@ public class ManualOperationDataStat extends BaseBatch{
aliasAddQuery.append("cat", "district")
aliasAddQuery.append("c_date", new BasicDBObject('$gte', startDate).append('$lt', endDate))
aliasAddQuery.append("city", city)
aliasAddQuery.append("opt", "Ìí¼Ó")
aliasAddQuery.append("opt", "添加")
def aliasRemoveQuery = new BasicDBObject()
aliasRemoveQuery.append("cat", "district")
aliasRemoveQuery.append("c_date", new BasicDBObject('$gte', startDate).append('$lt', endDate))
aliasRemoveQuery.append("city", city)
aliasRemoveQuery.append("opt", "ɾ³ý")
aliasRemoveQuery.append("opt", "删除")
data << [city, DbUtils.seaweed.count(createQuery), DbUtils.seaweed.count(removeQuery), ALIAS_UPDATE_OPERATION.count(aliasAddQuery), ALIAS_UPDATE_OPERATION.count(aliasRemoveQuery)]
println([city, DbUtils.seaweed.count(createQuery), DbUtils.seaweed.count(removeQuery), ALIAS_UPDATE_OPERATION.count(aliasAddQuery), ALIAS_UPDATE_OPERATION.count(aliasRemoveQuery)])
@@ -122,15 +122,15 @@ public class ManualOperationDataStat extends BaseBatch{
def content = sd.getContent(map)
println(content)
sendEmail(Watcher.WangWenYan.email, "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("wangchencen@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
// sendEmail("caoyuyan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
// sendEmail("shenyuan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
// sendEmail("xuminxia@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
// sendEmail("huangcunliang@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
// sendEmail("yaojingmeng@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
// sendEmail("muliangshu@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
// sendEmail("sunjian@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail(Watcher.WangWenYan.email, "后台统计量", content)
sendEmail("wangchencen@fangjia.com", "后台统计量", content)
// sendEmail("caoyuyan@fangjia.com", "后台统计量", content)
// sendEmail("shenyuan@fangjia.com", "后台统计量", content)
// sendEmail("xuminxia@fangjia.com", "后台统计量", content)
// sendEmail("huangcunliang@fangjia.com", "后台统计量", content)
// sendEmail("yaojingmeng@fangjia.com", "后台统计量", content)
// sendEmail("muliangshu@fangjia.com", "后台统计量", content)
// sendEmail("sunjian@fangjia.com", "后台统计量", content)
}
}
@@ -13,7 +13,7 @@ import com.mongodb.DBCollection
import java.util.regex.Pattern
/**
*MongoDB慢日志分析脚本
*MongoDB慢日志分析脚本
*/
@BatchOwner(Watcher.WangWenYan)
class MongoAccessLogAnalyzer extends BaseBatch {
@@ -49,7 +49,7 @@ class MongoAccessLogAnalyzer extends BaseBatch {
MONGO_SLOW_LOG.save(dbo)
}
}catch (Exception e){
println(line + "\t\t发生异常")
println(line + "\t\t发生异常")
e.printStackTrace()
}
@@ -129,9 +129,9 @@ class MongoAccessLogAnalyzer extends BaseBatch {
map.put('totalCount', totalCount)
map.put('avgTime', totalCountTime/totalCount)
def content = sd.getContent(map)
sendEmail(Watcher.WangWenYan.email, DateUtil.add(new Date(), Calendar.DATE, -1).format("yyyy年MM月dd日")+ "超时日志报表", content)
sendEmail(Watcher.DuXiaoYu.email, DateUtil.add(new Date(), Calendar.DATE, -1).format("yyyy年MM月dd日")+ "超时日志报表", content)
sendEmail("joe@fangjia.com", DateUtil.add(new Date(), Calendar.DATE, -1).format("yyyy年MM月dd日")+ "超时日志报表", content)
sendEmail(Watcher.WangWenYan.email, DateUtil.add(new Date(), Calendar.DATE, -1).format("yyyy年MM月dd日")+ "超时日志报表", content)
sendEmail(Watcher.DuXiaoYu.email, DateUtil.add(new Date(), Calendar.DATE, -1).format("yyyy年MM月dd日")+ "超时日志报表", content)
sendEmail("joe@fangjia.com", DateUtil.add(new Date(), Calendar.DATE, -1).format("yyyy年MM月dd日")+ "超时日志报表", content)
}
static final COUNTERS_PATTERNS = ['ntoreturn': Pattern.compile('ntoreturn:\\d+'), 'ntoskip': Pattern.compile('ntoskip:\\d+'),
@@ -26,14 +26,14 @@ class PriceStat {
static main(args) {
def task = new PriceStat();
//task.run("南京", "shop");
//task.houseCountStat("北京","写字楼");
def citys = ["天津","重庆","成都","大连","沈阳","哈尔滨","牡丹江","大庆","齐齐哈尔","佳木斯","伊春","鸡西","鹤岗","双鸭山","七台河","绥化","黑河"];
//task.run("南京", "shop");
//task.houseCountStat("北京","写字楼");
def citys = ["天津","重庆","成都","大连","沈阳","哈尔滨","牡丹江","大庆","齐齐哈尔","佳木斯","伊春","鸡西","鹤岗","双鸭山","七台河","绥化","黑河"];
// task.run("上海", "district");
// task.run("上海", "district");
// def citys = ["重庆","大连","哈尔滨","沈阳","大庆","牡丹江","齐齐哈尔"];
// def citys = ["重庆","大连","哈尔滨","沈阳","大庆","牡丹江","齐齐哈尔"];
// for(def city : citys){
// task.backupStat(city, "district");
// }
@@ -55,7 +55,7 @@ class PriceStat {
def backupStat(def city, def cat, def prefix){
def resultMap = getAllDistricts(city, cat);
def ret = resultMap.ret;
println "[${city}] "+cat + " 共"+ret.size();
println "[${city}] "+cat + " 共"+ret.size();
//def detail = houseBackupStat(city);
def detail = houseStat(city, cat +"_"+ prefix);
def res = [];
@@ -69,7 +69,7 @@ class PriceStat {
def result = [:];
result.put(prefix+"HouseCount", ret.size());
result.put(prefix+"HouseDistCount", res.size());
println "[${city}]"+" 共:"+ret.size()+" 有房源的[${cat}]:"+res.size();
println "[${city}]"+" 共:"+ret.size()+" 有房源的[${cat}]:"+res.size();
return result;
}
@@ -169,15 +169,15 @@ class PriceStat {
}
// println "共"+index;
// println "有售价共:"+sellIndex
// println "有租价共:"+rentIndex
// println "竣工时间共:"+finishDateIndex
// println "建筑时间共:"+buildDateIndex
// println "居住时间共:"+liveDateIndex
// println "有时间共:"+dateIndex
// println "经纬度共:"+latlngIndex
// println "图片共:"+picIndex
// println "共"+index;
// println "有售价共:"+sellIndex
// println "有租价共:"+rentIndex
// println "竣工时间共:"+finishDateIndex
// println "建筑时间共:"+buildDateIndex
// println "居住时间共:"+liveDateIndex
// println "有时间共:"+dateIndex
// println "经纬度共:"+latlngIndex
// println "图片共:"+picIndex
resultMap.sellPriceOffices = sellPriceOffices;
resultMap.ret = ret;
@@ -199,7 +199,7 @@ class PriceStat {
detail << key;
}
println "参与计算的共:"+sum+"房源!"
println "参与计算的共:"+sum+"房源!"
return detail;
}
@@ -22,7 +22,7 @@ class RegionAnaDataBatch extends BaseBatch {
@Override
void process(Object args) {
Integer yearMonth = REGION_SELL_STAT.find(new BasicDBObject("city", "上海").append("region", "长宁")).sort(new BasicDBObject("date", -1)).limit(1).toList()[0].date
Integer yearMonth = REGION_SELL_STAT.find(new BasicDBObject("city", "上海").append("region", "长宁")).sort(new BasicDBObject("date", -1)).limit(1).toList()[0].date
CommonUtils.findAllCity().each {def city->
List orders = []
for(def region : RegionData.getAllRegions(city).keySet()){
@@ -37,7 +37,7 @@ public boolean isLegalRegion(String city,String region){
if(StringUtils.isBlank(region)){
return false;
}
if(region.contains("Öܱß")||region.contains("ÆäËû")){
if(region.contains("周边")||region.contains("其他")){
return false;
}
if(regions.containsKey(region)){
@@ -14,9 +14,9 @@ import org.bson.types.ObjectId
class SeaweedFieldStat {
DSCollection dataStoreConnection = null;
public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
def distTypes = ["多层","小高层","超高层","高层","花园洋房","老洋房","四合院"];
def villaTypes = ["独栋","双拼","联排","叠加"];
def shopTypes = ["多层","小高层","超高层","高层","独栋","双拼","联排","叠加"];
def distTypes = ["多层","小高层","超高层","高层","花园洋房","老洋房","四合院"];
def villaTypes = ["独栋","双拼","联排","叠加"];
def shopTypes = ["多层","小高层","超高层","高层","独栋","双拼","联排","叠加"];
static main(args) {
@@ -25,7 +25,7 @@ class SeaweedFieldStat {
}
def run(){
def citys = ["上海"]//findAllCity();
def citys = ["上海"]//findAllCity();
for(def city : citys){
def districts = getAllDistricts(city, "district");
//processEstateType(districts);
@@ -46,7 +46,7 @@ class SeaweedFieldStat {
public def statBlock(districts){
println "小区共:"+districts.size();
println "小区共:"+districts.size();
def index=0;
for(def district : districts){
@@ -55,7 +55,7 @@ class SeaweedFieldStat {
index++;
}
}
println "无block:"+index;
println "无block:"+index;
}
public def getAllDistricts(def city, def cat){
DBObject q = new BasicDBObject();
@@ -91,35 +91,35 @@ class SeaweedFieldStat {
for(def map : districts){
def estate_type2 = map.estate_type;
if(estate_type2){
if(!estate_type2.equals("普通住宅")&&!estate_type2.equals("别墅")&&!estate_type2.equals("商住")){
if(!estate_type2.equals("普通住宅")&&!estate_type2.equals("别墅")&&!estate_type2.equals("商住")){
DBObject udata = new BasicDBObject();
udata.put("m_date", new Date());
udata.put("estate_type2", "普通住宅");
udata.put("estate_type2", "普通住宅");
DBObject q = new BasicDBObject();
q.put("_id", map.id);
println "update ${map.id} "+estate_type2
getDataStoreConnection().update(q, new BasicDBObject('$set', udata))
}
}else{//物业类弄为空
}else{//物业类弄为空
def name = map.name;
if(name.indexOf("别墅")!=-1){
if(name.indexOf("别墅")!=-1){
DBObject udata = new BasicDBObject();
udata.put("m_date", new Date());
udata.put("estate_type2", "别墅");
udata.put("estate_type2", "别墅");
DBObject q = new BasicDBObject();
q.put("_id", map.id);
println "update ${map.id} 改为:别墅"
println "update ${map.id} 改为:别墅"
getDataStoreConnection().update(q, new BasicDBObject('$set', udata))
}else{
DBObject udata = new BasicDBObject();
udata.put("m_date", new Date());
udata.put("estate_type2", "普通住宅");
udata.put("estate_type2", "普通住宅");
DBObject q = new BasicDBObject();
q.put("_id", map.id);
println "update ${map.id} 无物业类型"
println "update ${map.id} 无物业类型"
getDataStoreConnection().update(q, new BasicDBObject('$set', udata))
}
@@ -132,15 +132,15 @@ class SeaweedFieldStat {
for(def map : districts){
def build_type = map.build_type;
def estate_type2 = map.estate_type;
if(map.name == "佘山东郡"){
if(map.name == "佘山东郡"){
println "ssssss"
}
def result ;
if(estate_type2 == "普通住宅"){
if(estate_type2 == "普通住宅"){
result = changeValue(estate_type2, build_type, distTypes);
}else if(estate_type2 == "别墅"){
}else if(estate_type2 == "别墅"){
result = changeValue(estate_type2, build_type, villaTypes);
}else if(estate_type2 == "商住"){
}else if(estate_type2 == "商住"){
result = changeValue(estate_type2, build_type, shopTypes);
}
if(build_type != result){
@@ -159,50 +159,50 @@ class SeaweedFieldStat {
if(buildType && buildType!='null'){
def tempBuildTypes = [];
println "(${estate_type2})处理前:"+buildType;
println "(${estate_type2})处理前:"+buildType;
if(buildType.indexOf("别墅")!=-1){
buildType = buildType.replace("别墅","");
if(buildType.indexOf("别墅")!=-1){
buildType = buildType.replace("别墅","");
}
if(buildType.indexOf("叠拼")!=-1){
buildType = buildType.replace("叠拼","");
if(buildType.indexOf("叠拼")!=-1){
buildType = buildType.replace("叠拼","");
}
if(buildType.indexOf("(")!=-1){
def startIndex = buildType.indexOf("(");
def endIndex = buildType.indexOf(")")+1;
if(buildType.indexOf("(")!=-1){
def startIndex = buildType.indexOf("(");
def endIndex = buildType.indexOf(")")+1;
def s = buildType.substring(startIndex, endIndex)
buildType = buildType.replace(s, "");
}
if(buildType.indexOf(",")!=-1||buildType.indexOf(",")!=-1||buildType.indexOf(";")!=-1||buildType.indexOf(";")!=-1||buildType.indexOf(" ")!=-1||buildType.indexOf("、")!=-1||buildType.indexOf("|")!=-1){
tempBuildTypes = buildType.split(",|;| |、|,|;||")
if(buildType.indexOf(",")!=-1||buildType.indexOf(",")!=-1||buildType.indexOf(";")!=-1||buildType.indexOf(";")!=-1||buildType.indexOf(" ")!=-1||buildType.indexOf("、")!=-1||buildType.indexOf("|")!=-1){
tempBuildTypes = buildType.split(",|;| |、|,|;||")
}else{
tempBuildTypes << buildType
}
if(tempBuildTypes.size()==1 && buildType.length() > 5){
println "可能含有其它分隔符......"
println "可能含有其它分隔符......"
}
if(tempBuildTypes.size() > 0){
buildType = standardType(tempBuildTypes, types);
println "处理后:"+buildType;
println "处理后:"+buildType;
if(!buildType){
println "不合要求"
println "不合要求"
}
}
}else{
if(estate_type2 == "普通住宅"){
buildType = "小高层";
println "默认 "
}else if(estate_type2 == "别墅"){
buildType = "联排";
println "默认 "
}else if(estate_type2 == "商住"){
buildType = "小高层";
println "默认 "
if(estate_type2 == "普通住宅"){
buildType = "小高层";
println "默认 "
}else if(estate_type2 == "别墅"){
buildType = "联排";
println "默认 "
}else if(estate_type2 == "商住"){
buildType = "小高层";
println "默认 "
}
}
return buildType;
@@ -228,7 +228,7 @@ class SeaweedFieldStat {
if(cursor){
for(def dbo : cursor){
def city = dbo.get("display");
if(city!='全国'){
if(city!='全国'){
citys << city;
}
}
@@ -24,7 +24,7 @@ class SeedsScore {
private def run(){
// def ids = findWebsitByScope("全国类");
// def ids = findWebsitByScope("全国类");
// updateSeedsScoreByParentId(ids, 0);
// def ret = readText("anjuke.txt");
// def ret = readText("focus.txt");
@@ -74,12 +74,12 @@ class SeedsScore {
private def readText(def fileName){
File file = new File('h:\\'+fileName)
def ret = [];
//对文件内容的每一行进行分割处理,比较常用在处理csv文件
//对文件内容的每一行进行分割处理,比较常用在处理csv文件
file.splitEachLine(" ") {
def temp = it
ret << temp;
}
//转为Writable对象,可重定向输出
//转为Writable对象,可重定向输出
file.asWritable()
return ret;
}
@@ -12,8 +12,8 @@ import com.mongodb.DBCollection
/**
* @author 文艳
* 短息使用量监控脚本
* @author 文艳
* 短息使用量监控脚本
*/
@BatchOwner(Watcher.WangWenYan)
public class SmsUsageMonitorBatch extends BaseBatch{
@@ -34,21 +34,21 @@ public class SmsUsageMonitorBatch extends BaseBatch{
for(int i = 0; i < 7; i++){
def date = calendar.getTime();
def item = [:]
item.day = date.format("yyyy年MM月dd日")
item.day = date.format("yyyy年MM月dd日")
item.count = countUsageByDay(date)
weekUsage << item
calendar.add(Calendar.DATE, 1);
}
/**获取剩余短信数量*/
/**获取剩余短信数量*/
int remain = (int)(SingletonClient.getClient().balance * 10)
EmailTemplate sd = new EmailTemplate("smsUsage.ftl")
Map map = new HashMap()
map.put('weekUsage', weekUsage)
map.put('remain', remain)
def content = sd.getContent(map)
sendEmail(Watcher.WangWenYan.email, "短信近期使用情况", content)
sendEmail(Watcher.DuXiaoYu.email, "短信近期使用情况", content)
sendEmail("yangqixing@fangjia.com", "短信近期使用情况", content)
sendEmail(Watcher.WangWenYan.email, "短信近期使用情况", content)
sendEmail(Watcher.DuXiaoYu.email, "短信近期使用情况", content)
sendEmail("yangqixing@fangjia.com", "短信近期使用情况", content)
}
def countUsageByDay(Date date){
@@ -33,7 +33,7 @@ class SplitWebsiteTable {
def task = new SplitWebsiteTable();
// task.run();
String s="周围商场:超市、便利商店等一应俱全,国美电器、易初莲花休闲生活:中小学:新城小学侨光中学,川沙中学华夏西校 商场:超市、便利商店等一应俱全,国美电器、易初莲花 医院:安平医院、川沙镇中心医院 邮局:中国邮政唐镇邮政所(浦东新区川沙分局) 银行:交通银行、工商银行、建设银行、农业银行 其他:六团邮政所、唐镇邮政所、避风塘茶楼、川沙体育场羽毛球馆餐饮设施:暂无资料周围学校:暂无资料医院:安平医院、川沙镇中心医院"
String s="周围商场:超市、便利商店等一应俱全,国美电器、易初莲花休闲生活:中小学:新城小学侨光中学,川沙中学华夏西校 商场:超市、便利商店等一应俱全,国美电器、易初莲花 医院:安平医院、川沙镇中心医院 邮局:中国邮政唐镇邮政所(浦东新区川沙分局) 银行:交通银行、工商银行、建设银行、农业银行 其他:六团邮政所、唐镇邮政所、避风塘茶楼、川沙体育场羽毛球馆餐饮设施:暂无资料周围学校:暂无资料医院:安平医院、川沙镇中心医院"
def ss = task.test2(s);
println ss ;
}
@@ -41,27 +41,27 @@ class SplitWebsiteTable {
def test2(obj){
String s = obj.toString();
String ret = "";
if(s=='周围商场:休闲生活:餐饮设施:周围学校:医院:'){
if(s=='周围商场:休闲生活:餐饮设施:周围学校:医院:'){
return "";
}else{
String s1 = s.substring(0, s.indexOf("休闲生活:"));
String s2 = s.substring(s.indexOf("休闲生活:"), s.indexOf("餐饮设施:"));
String s3 = s.substring(s.indexOf("餐饮设施:"), s.indexOf("周围学校:"));
String s4 = s.substring(s.indexOf("周围学校:"), s.indexOf("医院:"));
String s5 = s.substring(s.indexOf("医院:"), s.length());
if(s1!="周围商场:"){
String s1 = s.substring(0, s.indexOf("休闲生活:"));
String s2 = s.substring(s.indexOf("休闲生活:"), s.indexOf("餐饮设施:"));
String s3 = s.substring(s.indexOf("餐饮设施:"), s.indexOf("周围学校:"));
String s4 = s.substring(s.indexOf("周围学校:"), s.indexOf("医院:"));
String s5 = s.substring(s.indexOf("医院:"), s.length());
if(s1!="周围商场:"){
ret = s1+"<br/>";
}
if(s2!="休闲生活:"){
if(s2!="休闲生活:"){
ret = ret + s2 +"<br/>";
}
if(s3!="餐饮设施:"){
if(s3!="餐饮设施:"){
ret = ret + s3 +"<br/>";
}
if(s4!="周围学校:"){
if(s4!="周围学校:"){
ret = ret + s4 +"<br/>";
}
if(s5!="医院:"){
if(s5!="医院:"){
ret = ret + s5 +"<br/>";
}
}
@@ -73,19 +73,19 @@ class SplitWebsiteTable {
def test(obj){
String s = obj.toString();
String ret = "";
String[] traffices = s.split("周围商场:");
String[] traffices = s.split("周围商场:");
String traffice = traffices[0];
if(traffice=='公交线路:地铁线路:'){
if(traffice=='公交线路:地铁线路:'){
return "";
}else{
String[] arr = traffice.split("地铁线路:");
if(arr[0]!='公交线路:'){
String[] arr = traffice.split("地铁线路:");
if(arr[0]!='公交线路:'){
ret = arr[0]+"<br/>";
}
if(arr.length==2){
arr[1] = arr[1].trim();
if(arr[1].length()>2){
ret = ret +"地铁线路:"+arr[1];
ret = ret +"地铁线路:"+arr[1];
}
}
@@ -126,9 +126,9 @@ class SplitWebsiteTable {
private List<Map<String, Object>> relatedJobs;
def siteScores = ["搜房网":9,"新浪乐居":9,"安居客":8.5,"搜狐焦点":8.5,"网上房地产":10,"网易":8,
"链家在线":8,"带看":8,"腾讯":7.5,"推推99房产网":7.5,"淘房屋":7,"我来房产":6.5,"星空地产网":6,"百度乐居":6,"房之窗":5,"房产之门":5,"上海热线":5,"吉屋网":4,"楼盘网":4,"聊宅网":3.5,"爱易网":3.5,"淘房":3,
"房王":3,"赶集":2,"58同城":2,"猎房网":1];
def siteScores = ["搜房网":9,"新浪乐居":9,"安居客":8.5,"搜狐焦点":8.5,"网上房地产":10,"网易":8,
"链家在线":8,"带看":8,"腾讯":7.5,"推推99房产网":7.5,"淘房屋":7,"我来房产":6.5,"星空地产网":6,"百度乐居":6,"房之窗":5,"房产之门":5,"上海热线":5,"吉屋网":4,"楼盘网":4,"聊宅网":3.5,"爱易网":3.5,"淘房":3,
"房王":3,"赶集":2,"58同城":2,"猎房网":1];
public def saveNewData(website){
for(def obj : website){
DBObject udata = new BasicDBObject();
@@ -39,8 +39,8 @@ public class Test {
public static void main(String[] ars){
def test = new Test();
// test.test();
// //test.run("district", "上海");
// test.analyDistrictPk("district", "天津");//小区统计与小区的对比
// //test.run("district", "上海");
// test.analyDistrictPk("district", "天津");//小区统计与小区的对比
// println new Date(new ObjectId('5469bb4cda29ee73010fa577').getTime())
// test.addTestData();
test.findRepeatGuidingPrice()
@@ -62,16 +62,16 @@ public class Test {
}
def datas = []
datas << ["id", "城市","区域","名称","添加人","指导价","日期"]
datas << ["id", "城市","区域","名称","添加人","指导价","日期"]
map.each{key,list->
if(list.size()>1){
def userNames = []
for(def ret : list){
userNames << ret.guiding_person
}
if(userNames.contains("付贵生")){
if(userNames.contains("付贵生")){
for(def r : list){
if(r.guiding_person!="付贵生"){
if(r.guiding_person!="付贵生"){
GUIDING_PRICE_COLL.remove(new BasicDBObject("_id", r._id))
}
}
@@ -80,15 +80,15 @@ public class Test {
}
}
// ExcelUtils.write("d:\\数据\\福州\\指导价.xlsx",datas, "datas")
// ExcelUtils.write("d:\\数据\\福州\\指导价.xlsx",datas, "datas")
}
public void addTestData(){
DBObject obj = new BasicDBObject();
obj.put("city", "北京");
obj.put("region", "朝阳");
obj.put("name", "朝阳广场");
obj.put("city", "北京");
obj.put("region", "朝阳");
obj.put("name", "朝阳广场");
obj.put("guiding_date", new Date());
obj.put("guiding_value", 60000);
obj.put("category", "office");
@@ -99,7 +99,7 @@ public class Test {
}
public void test(){
// 所有城市
// 所有城市
def allCitys = [:];
SERVER_AREA_CONN.find(new BasicDBObject('status', new BasicDBObject('$in', [0, 2]))).each { allCitys.put(it.display, it); }
@@ -108,7 +108,7 @@ public class Test {
citys << k.toString();
}
// 当期周末
// 当期周末
Date weekend = DateUtil.getFirstDayOfWeek(new Date());
DBObject q = new BasicDBObject();
q.put("weekend", weekend);
@@ -132,8 +132,8 @@ public class Test {
public analyDistrictPk(def cat, def city){
def seaweedDists = findDistrictIds(cat, city);
def districtInfos = findDistrictInfo(city, "2014-10-26");
println "小区数:"+seaweedDists.size();
println "小区分析表数:"+districtInfos.size();
println "小区数:"+seaweedDists.size();
println "小区分析表数:"+districtInfos.size();
def index = 0;
for(def dist : districtInfos){
if(!seaweedDists.contains(dist)){
@@ -191,13 +191,13 @@ public class Test {
}
public String run(def cat, def city) {
def analyDists = findAnalySiteDistrict(cat, city);//分析表
def analyDists = findAnalySiteDistrict(cat, city);//分析表
def surplusDists = findSurplusDistrict(cat, city);//不存在小区表
def surplusDists = findSurplusDistrict(cat, city);//不存在小区表
analyDists = analyDists + surplusDists;
def seaweedDists = CommonDatautils.getSimpleDistricts(cat, city);//总小区表
def seaweedDists = CommonDatautils.getSimpleDistricts(cat, city);//总小区表
println analyDists.size()
println seaweedDists.size();
def index = 0;
@@ -205,14 +205,14 @@ public class Test {
if(!seaweedDists.contains(site)){
index++;
println "[${index}] 不在小区表:"+site;
println "[${index}] 不在小区表:"+site;
}
}
for(def dist : seaweedDists){
if(!analyDists.contains(dist)){
index++;
println "[${index}] 小区不在分析表:"+dist;
println "[${index}] 小区不在分析表:"+dist;
}
}
}
@@ -39,7 +39,7 @@ class UpdateCityLevel {
def arr = line.split("\t");
def map = [:];
map.city = arr[0];
map.level = arr[1].replace("级", "").replace("一", "1").replace("二", "2").replace("三", "3").replace("四", "4").replace("五", "5").replace("六", "6")
map.level = arr[1].replace("级", "").replace("一", "1").replace("二", "2").replace("三", "3").replace("四", "4").replace("五", "5").replace("六", "6")
ret << map;
}
@@ -53,7 +53,7 @@ public class UserAgentAnalysisBatch extends BaseBatch{
int times = 3
while(!logFile.exists() && times > 0){
println("${new Date().format("yyyy-MM-dd HH:mm:ss")}\t${logFile.getAbsolutePath()} 未找到!!!!!")
println("${new Date().format("yyyy-MM-dd HH:mm:ss")}\t${logFile.getAbsolutePath()} 未找到!!!!!")
Thread.sleep(1000 * 60 * 60)
times --
}
@@ -18,7 +18,7 @@ import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
/**
* 微众银行房源统计脚本
* 微众银行房源统计脚本
*/
@BatchOwner(Watcher.WangWenYan)
class WeBankHouseAnalyBatch extends BaseBatch {
@@ -20,8 +20,8 @@ class AnalyCommonUtils {
public static final DBCollection ANALY_TASK_COLL = MongoUtils.getCollectionByName("coredata", "analy_task", "coredata.mongo.db.fangjia.com")
public static final DBCollection ANALY_ROAD_COLL = MongoUtils.getCollectionByName("coredata", "analy_road", "coredata.mongo.db.fangjia.com")
static def villas = ["独栋","联排","双拼","叠加"]
static def districts = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院"];
static def villas = ["独栋","联排","双拼","叠加"]
static def districts = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院"];
static def standardRegion(def regionMap,def region){
def lastRegion = "";
@@ -154,7 +154,7 @@ class AnalyCommonUtils {
if(taskId!="null"&&taskId){
saveAnalyRoad(city, ret, taskId, site_name, page_name, cat);
}else{
println "没有相对应该的task_id";
println "没有相对应该的task_id";
}
return taskId;
}
@@ -247,7 +247,7 @@ class AnalyCommonUtils {
def newEstateType = "";
def newBuildType = "";
if(build_type){
def buildTypes = build_type.split(",|、|;")
def buildTypes = build_type.split(",|、|;")
def distTypes = []
def villaTypes = []
for(def type : buildTypes){
@@ -260,35 +260,35 @@ class AnalyCommonUtils {
}
}
if(estate_type2.indexOf("别墅")!=-1 && estate_type2.indexOf("普通住宅")!=-1){
if(estate_type2.indexOf("别墅")!=-1 && estate_type2.indexOf("普通住宅")!=-1){
if(villaTypes.size() > 0){
newEstateType = "别墅";
newEstateType = "别墅";
newBuildType = villaTypes.unique().join(",");
}else{
newEstateType = "普通住宅";
newEstateType = "普通住宅";
if(distTypes.size() > 0){
newBuildType = distTypes.unique().join(",");
}
}
}else if(estate_type2.indexOf("别墅")==-1 && estate_type2.indexOf("普通住宅")!=-1){
newEstateType = "普通住宅";
}else if(estate_type2.indexOf("别墅")==-1 && estate_type2.indexOf("普通住宅")!=-1){
newEstateType = "普通住宅";
if(distTypes.size() > 0){
newBuildType = distTypes.unique().join(",");
}
}else if(estate_type2.indexOf("别墅")!=-1 && estate_type2.indexOf("普通住宅")==-1){
newEstateType = "别墅";
}else if(estate_type2.indexOf("别墅")!=-1 && estate_type2.indexOf("普通住宅")==-1){
newEstateType = "别墅";
if(villaTypes.size()>0){
newBuildType = villaTypes.unique().join(",");
}
}else{
newEstateType = "普通住宅";
newEstateType = "普通住宅";
}
}else{
if(estate_type2.indexOf("别墅")!=-1){
newEstateType = "别墅";
if(estate_type2.indexOf("别墅")!=-1){
newEstateType = "别墅";
}else{
newEstateType = "普通住宅";
newEstateType = "普通住宅";
}
}
@@ -301,7 +301,7 @@ class AnalyCommonUtils {
def index = 1;
for(def map : results){
updateDistrict(map, city);
println "[ "+city+" ]"+index +" 修改完成....."
println "[ "+city+" ]"+index +" 修改完成....."
index++;
}
}
@@ -327,7 +327,7 @@ class AnalyCommonUtils {
if (!dbValue ) {
udata.put(k, v);
}else{
if(dbValue.toString().indexOf("暂无")!=-1){
if(dbValue.toString().indexOf("暂无")!=-1){
udata.put(k, v);
}
}
@@ -343,7 +343,7 @@ class AnalyCommonUtils {
udata.put("estate_type2", typeMap.estate_type);
udata.put("build_type", typeMap.build_type);
}
println id.toString() + " " + obj.get("region") + " " + obj.get("name") + " 修改的值 :" + udata
println id.toString() + " " + obj.get("region") + " " + obj.get("name") + " 修改的值 :" + udata
udata.put("m_date", new Date());
getDataStoreConnection().update(q, new BasicDBObject('$set', udata))
}
@@ -36,7 +36,7 @@ class BkoffActionSelectAnaly{
char[] chars = new char[1];
chars[0] = str.charAt(0);
String temp = new String(chars);
if (chars[0] >= 'A' && chars[0] <= 'Z') {//当为字母时,则转换为小写
if (chars[0] >= 'A' && chars[0] <= 'Z') {//当为字母时,则转换为小写
return str.replaceFirst(temp, temp.toLowerCase());
}
return str;
@@ -91,7 +91,7 @@ class BkoffActionSelectAnaly{
files << tempList[i];
}
// if (tempList[i].isDirectory()) {
// System.out.println("文件夹:"+tempList[i]);
// System.out.println("文件夹:"+tempList[i]);
// }
}
return files;
@@ -26,7 +26,7 @@ import static com.mongodb.client.model.Projections.include
import static com.mongodb.client.model.Updates.combine
import static com.mongodb.client.model.Updates.set
@BatchOwner(value = Watcher.WangWenYan, describe = "分析小区特征")
@BatchOwner(value = Watcher.WangWenYan, describe = "分析小区特征")
class DistrictFeatureAnalyBatch extends BaseBatch {
@@ -51,7 +51,7 @@ class DistrictFeatureAnalyBatch extends BaseBatch {
keys.finish_building_date = 1
UpdateOptions options = new UpdateOptions()
options.upsert(true)
["佛山", "济南", "北京", "广州", "深圳"].each { def city ->
["佛山", "济南", "北京", "广州", "深圳"].each { def city ->
println(city)
def statMap = new ConcurrentHashMap<>()
def query = and(eq("city", city), eq("weekend", DateUtil.getThisSunday(new Date())))
@@ -108,8 +108,8 @@ class DistrictFeatureAnalyBatch extends BaseBatch {
sqlBuffer.append("SELECT count(*) as count from esf_deal_price_lansi WHERE 1=1")
sqlBuffer.append(" and fjCity = '${city}'")
sqlBuffer.append(" and fjRegion ='${region}'")
sqlBuffer.append(" and (districtType != '写字楼' and districtType != '商铺' and districtType != '办公' and districtType !='工厂' and districtType != '其它' " +
" and houseType != '写字楼' and houseType != '商铺' and houseType != '办公' and houseType !='工厂' and houseType != '其它')")
sqlBuffer.append(" and (districtType != '写字楼' and districtType != '商铺' and districtType != '办公' and districtType !='工厂' and districtType != '其它' " +
" and houseType != '写字楼' and houseType != '商铺' and houseType != '办公' and houseType !='工厂' and houseType != '其它')")
List<String> namePars = new ArrayList<String>()
for (String alia : names) {
if (StringUtils.isNotBlank(alia) && alia.trim().length() > 1) {
@@ -5,7 +5,7 @@ import com.mongodb.DBCollection;
import com.mongodb.DBObject
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.DateUtils
//估价数据分析
//估价数据分析
class EvaluationAnaly{
public static final DBCollection EVALUATION_COLL = MongoUtils.getCollectionByName("fangjia", "evaluation", "mongo.db.fangjia.com")
@@ -32,7 +32,7 @@ class EvaluationAnaly{
map.put(ip, detailMap)
}
//ip估价次数统计
//ip估价次数统计
def totalCount = detailMap.get("total_count");
if(!totalCount){
detailMap.put("total_count", 1)
@@ -8,8 +8,8 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 房价网首页
* 估价记录和API总量统计
* 房价网首页
* 估价记录和API总量统计
*
*/
@BatchOwner(Watcher.WangWenYan)
@@ -19,7 +19,7 @@ import org.jsoup.select.Elements;
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
/**
* 解析克而瑞小区概况页(二手小区)
* 解析克而瑞小区概况页(二手小区)
* @author gaojuan
*
*/
@@ -39,25 +39,25 @@ class NewKerEsfCrawlTask extends BaseBatch{
@Override
public void process(Object args) {
run(["沈阳","大连","天津","成都","重庆"]);
run(["沈阳","大连","天津","成都","重庆"]);
}
def run(def citys){
def site_name = "克而瑞房价";
def page_name = "首页";
def site_name = "克而瑞房价";
def page_name = "首页";
def cat = "district"
for(def city : citys){
def webpages = findWebpages(site_name, page_name, city, cat);
println "【${city}】共找到搜房 "+webpages.size()+"个网页";
println "【${city}】共找到搜房 "+webpages.size()+"个网页";
def regions = AnalyCommonUtils.findRegion(city);
def blocks = AnalyCommonUtils.findRegionBlock(city);
def ret = analyWebpages(city, webpages, regions, blocks);
AnalyCommonUtils.addAnalyRoad(site_name, "概况", city, ret, cat);
AnalyCommonUtils.addAnalyRoad(site_name, "概况", city, ret, cat);
}
@@ -78,22 +78,22 @@ class NewKerEsfCrawlTask extends BaseBatch{
def doc = Jsoup.parse(s);
resultMap.city = city;
def alias = doc.select(".esgrid_left p:contains(别名:) em").text().trim()
def alias = doc.select(".esgrid_left p:contains(别名:) em").text().trim()
def ele = doc.select(".descr_cont_info")
resultMap.name = ele.select("li:contains(小区名称) a").text().trim()
def addressInfo = ele.select("li:contains(小区地址) cite").text().trim();
resultMap.build_area = AnalyCommonUtils.getNumeric(ele.select("li:contains(建筑面积) cite").text().replace("-", "").replace(",", "").trim(), "(\\d+)");
resultMap.developer = ele.select("li:contains(开发商) cite").text().replace("-", "").trim();
resultMap.estate_company = ele.select("li:contains(物业公司) cite").text().replace("-", "").trim();
resultMap.virescence_percent = AnalyCommonUtils.getNumeric(ele.select("li:contains(绿化率) cite").text().replace("-", "").replace("%", "").trim(), "(\\d+[\\.]?[\\d+]?(\\S+)(.*)?)");
resultMap.cubage_percent = AnalyCommonUtils.getNumeric(ele.select("li:contains(容积率) cite").text().replace("-", "").trim(), "(\\d+[\\.]?[\\d+]?(\\S+)(.*)?)");
resultMap.name = ele.select("li:contains(小区名称) a").text().trim()
def addressInfo = ele.select("li:contains(小区地址) cite").text().trim();
resultMap.build_area = AnalyCommonUtils.getNumeric(ele.select("li:contains(建筑面积) cite").text().replace("-", "").replace(",", "").trim(), "(\\d+)");
resultMap.developer = ele.select("li:contains(开发商) cite").text().replace("-", "").trim();
resultMap.estate_company = ele.select("li:contains(物业公司) cite").text().replace("-", "").trim();
resultMap.virescence_percent = AnalyCommonUtils.getNumeric(ele.select("li:contains(绿化率) cite").text().replace("-", "").replace("%", "").trim(), "(\\d+[\\.]?[\\d+]?(\\S+)(.*)?)");
resultMap.cubage_percent = AnalyCommonUtils.getNumeric(ele.select("li:contains(容积率) cite").text().replace("-", "").trim(), "(\\d+[\\.]?[\\d+]?(\\S+)(.*)?)");
if(alias){
def aliasList = [];
aliasList << alias;
resultMap.alias = aliasList;
}
//区域及地址解析
//区域及地址解析
if(addressInfo){
def region = addressInfo.substring(addressInfo.indexOf("[")+1, addressInfo.indexOf("]"));
resultMap.region = AnalyCommonUtils.standardRegion(regions, region);
@@ -108,8 +108,8 @@ class NewKerEsfCrawlTask extends BaseBatch{
resultMap.address = addressInfo.substring(addressInfo.lastIndexOf("]")+1, addressInfo.size());
}
//物业费解析
def estate_charge = ele.select("li:contains(物业费) cite").text().replace("元/㎡/月", "").trim();
//物业费解析
def estate_charge = ele.select("li:contains(物业费) cite").text().replace("元/㎡/月", "").trim();
if(estate_charge){
if(estate_charge.indexOf("-")==-1){
resultMap.estate_charge = estate_charge;
@@ -125,8 +125,8 @@ class NewKerEsfCrawlTask extends BaseBatch{
}
//建造时间解析
def houseAge = ele.select("li:contains(小区房龄) cite").text().replace("年", "").trim();
//建造时间解析
def houseAge = ele.select("li:contains(小区房龄) cite").text().replace("年", "").trim();
if(houseAge && AnalyCommonUtils.isNumeric(houseAge)){
def age = houseAge.toInteger();
if(age > 1000){
@@ -139,18 +139,18 @@ class NewKerEsfCrawlTask extends BaseBatch{
}
//户数解析
def houseInfo = ele.select("li:contains(小区房源) cite").text().replace("-", "").trim();
//户数解析
def houseInfo = ele.select("li:contains(小区房源) cite").text().replace("-", "").trim();
if(houseInfo){
def household_count = AnalyCommonUtils.getNumeric(houseInfo, "共(\\d+)套")
def household_count = AnalyCommonUtils.getNumeric(houseInfo, "共(\\d+)套")
resultMap.household_count = household_count;
}
//页面异步加载,拿不到相关信息
//交通
//页面异步加载,拿不到相关信息
//交通
// resultMap.traffic = doc.select("#data_list_station")
//
// //周边配套
// //周边配套
// def facilities = "";
// def facilities_edu = doc.select("#data_list_edu cite")
// def facilities_life = doc.select("#data_list_life cite")
@@ -175,24 +175,24 @@ class NewKerEsfCrawlTask extends BaseBatch{
resultMap.url = url+"_des";
resultMap = AnalyCommonUtils.filerValueIsNull(resultMap);
println "【 ${index} 】"+resultMap;
println "【 ${index} 】"+resultMap;
if(resultMap.name){
sucCount++;
results << resultMap;
}else{
failCount++;
println "小区名称没解析出来"
println "小区名称没解析出来"
}
}else{
failCount++;
println "没有网页内容"
println "没有网页内容"
}
}
println "失败:"+failCount
println "成功:"+sucCount
println "失败:"+failCount
println "成功:"+sucCount
return results;
}
@@ -17,7 +17,7 @@ import org.jsoup.nodes.Document
import org.jsoup.select.Elements;
/**
* 克而瑞新房抓取,解析,空值替换
* 克而瑞新房抓取,解析,空值替换
* @author gaojuan
*
*/
@@ -34,11 +34,11 @@ class NewKerXfCrawlTask {
static main(args) {
def task = new NewKerXfCrawlTask();
def ret = [
"沈阳":"http://www.fangjiadp.com/shenyang/advanced/search",
"大连":"http://www.fangjiadp.com/dalian/advanced/search",
"天津":"http://www.fangjiadp.com/tianjin/advanced/search",
"成都":"http://www.fangjiadp.com/chengdu/advanced/search",
"重庆":"http://www.fangjiadp.com/chongqing/advanced/search"
"沈阳":"http://www.fangjiadp.com/shenyang/advanced/search",
"大连":"http://www.fangjiadp.com/dalian/advanced/search",
"天津":"http://www.fangjiadp.com/tianjin/advanced/search",
"成都":"http://www.fangjiadp.com/chengdu/advanced/search",
"重庆":"http://www.fangjiadp.com/chongqing/advanced/search"
];
task.run(ret, "district");
}
@@ -61,14 +61,14 @@ class NewKerXfCrawlTask {
def blocks = AnalyCommonUtils.findRegionBlock(city);
def ret = analyWebpages(city, detailUrls, regions, blocks);
AnalyCommonUtils.addAnalyRoad("克而瑞房价", "详情", city, ret, cat);
AnalyCommonUtils.addAnalyRoad("克而瑞房价", "详情", city, ret, cat);
}
def getNextPageLink(def url, def urls){
String s = http.get(url, [:], "utf-8");
if(s){
def doc = Jsoup.parse(s);
def nextPageElement = doc.select("a:contains(下一页)");
def nextPageElement = doc.select("a:contains(下一页)");
if(nextPageElement){
def nextPageUrl = nextPageElement.attr("href");
def newUrl = "http://www.fangjiadp.com/"+nextPageUrl;
@@ -108,7 +108,7 @@ class NewKerXfCrawlTask {
String s = http.get(url, [:], "utf-8");
if(s){
def doc = Jsoup.parse(s);
def detailUrl = doc.select("a:contains(楼盘详情)");
def detailUrl = doc.select("a:contains(楼盘详情)");
if(detailUrl){
detailUrl = detailUrl.attr("href");
allUrls << "http://www.fangjiadp.com" + detailUrl;
@@ -140,30 +140,30 @@ class NewKerXfCrawlTask {
def doc = Jsoup.parse(s);
resultMap.city = city;
resultMap.name = doc.select("li:contains(楼盘名称:) cite,.a_t").text().trim();
resultMap.address = doc.select(".property_m>dl:contains(地址:)>dd").text().replace(";", "").trim();
resultMap.estate_type2 = doc.select(".property_f>ul>li:contains(物业类型:)>span").text().replace(";", "").trim();
resultMap.name = doc.select("li:contains(楼盘名称:) cite,.a_t").text().trim();
resultMap.address = doc.select(".property_m>dl:contains(地址:)>dd").text().replace(";", "").trim();
resultMap.estate_type2 = doc.select(".property_f>ul>li:contains(物业类型:)>span").text().replace(";", "").trim();
resultMap.live_date = doc.select(".property_f>ul>li:contains(交房日期:)>span").text().trim();
resultMap.developer = doc.select(".property_f>ul>li:contains(开发商:)>span").text().replace("-", "").trim();
resultMap.build_area = AnalyCommonUtils.getNumeric(doc.select(".property_f>ul>li:contains(建筑面积:)>span").text().replace("-", "").replace(",", "").trim(), "(\\d+)");
resultMap.build_type = doc.select(".property_f>ul>li:contains(建筑形态:)>span").text().trim();
resultMap.household_count = AnalyCommonUtils.getNumeric(doc.select(".property_f>ul>li:contains(规划户数:)>span").text().trim(), "(\\d+)");
resultMap.property_life = doc.select("li:contains(产权信息:) cite").text().replace(";", "").trim();
resultMap.fitment = doc.select(".property_f>ul>li:contains(装修情况:)>span").text().replace(";", "").trim();
resultMap.fitment_level = doc.select(".property_f>ul>li:contains(装修标准:)>span").text().replace("-元/㎡", "").trim();
resultMap.virescence_percent = AnalyCommonUtils.getNumeric(doc.select(".property_f>ul>li:contains(绿化率:)>span").text().replace("-", "").replace("%", "").trim(), "(\\d+[\\.]?[\\d+]?(\\S+)(.*)?)");
resultMap.cubage_percent = AnalyCommonUtils.getNumeric(doc.select(".property_f>ul>li:contains(容积率:)>span").text().replace("-", "").trim(), "(\\d+[\\.]?[\\d+]?(\\S+)(.*)?)");
resultMap.estate_company = doc.select(".property_f>ul>li:contains(物业公司:)>span").text().trim();
resultMap.live_date = doc.select(".property_f>ul>li:contains(交房日期:)>span").text().trim();
resultMap.developer = doc.select(".property_f>ul>li:contains(开发商:)>span").text().replace("-", "").trim();
resultMap.build_area = AnalyCommonUtils.getNumeric(doc.select(".property_f>ul>li:contains(建筑面积:)>span").text().replace("-", "").replace(",", "").trim(), "(\\d+)");
resultMap.build_type = doc.select(".property_f>ul>li:contains(建筑形态:)>span").text().trim();
resultMap.household_count = AnalyCommonUtils.getNumeric(doc.select(".property_f>ul>li:contains(规划户数:)>span").text().trim(), "(\\d+)");
resultMap.property_life = doc.select("li:contains(产权信息:) cite").text().replace(";", "").trim();
resultMap.fitment = doc.select(".property_f>ul>li:contains(装修情况:)>span").text().replace(";", "").trim();
resultMap.fitment_level = doc.select(".property_f>ul>li:contains(装修标准:)>span").text().replace("-元/㎡", "").trim();
resultMap.virescence_percent = AnalyCommonUtils.getNumeric(doc.select(".property_f>ul>li:contains(绿化率:)>span").text().replace("-", "").replace("%", "").trim(), "(\\d+[\\.]?[\\d+]?(\\S+)(.*)?)");
resultMap.cubage_percent = AnalyCommonUtils.getNumeric(doc.select(".property_f>ul>li:contains(容积率:)>span").text().replace("-", "").trim(), "(\\d+[\\.]?[\\d+]?(\\S+)(.*)?)");
resultMap.estate_company = doc.select(".property_f>ul>li:contains(物业公司:)>span").text().trim();
resultMap.car_site = doc.select(".property_f>ul>li:contains(车位数:)>span").text().trim();
resultMap.sell_address = doc.select(".property_f>ul>li:contains(售楼处:)>span").text().trim();
resultMap.mobile = doc.select(".property_f>ul>li:contains(售楼电话:)>span").text().trim();
resultMap.car_site = doc.select(".property_f>ul>li:contains(车位数:)>span").text().trim();
resultMap.sell_address = doc.select(".property_f>ul>li:contains(售楼处:)>span").text().trim();
resultMap.mobile = doc.select(".property_f>ul>li:contains(售楼电话:)>span").text().trim();
def regionHtml = doc.select(".property_m>dl:contains(区域:)>dd").html();
def regionHtml = doc.select(".property_m>dl:contains(区域:)>dd").html();
//区域板块及环线
//区域板块及环线
if(regionHtml){
def array = regionHtml.split("</em><em>")
if(array.size()>=1){
@@ -185,8 +185,8 @@ class NewKerXfCrawlTask {
}
}
//物业费解析
def estate_charge = doc.select(".property_f>ul>li:contains(物业费用:)>span").text().replace("元/㎡/月", "").trim();
//物业费解析
def estate_charge = doc.select(".property_f>ul>li:contains(物业费用:)>span").text().replace("元/㎡/月", "").trim();
if(estate_charge){
if(estate_charge.indexOf("-")==-1){
resultMap.estate_charge = estate_charge;
@@ -202,30 +202,30 @@ class NewKerXfCrawlTask {
}
//开盘时间
def open_date_str = doc.select(".property_f>ul>li:contains(开盘日期:)>span").text().trim();
//开盘时间
def open_date_str = doc.select(".property_f>ul>li:contains(开盘日期:)>span").text().trim();
if(open_date_str){
resultMap.open_date = DateUtil.parseStringToDate(open_date_str)
}
resultMap.url = url;
resultMap = AnalyCommonUtils.filerValueIsNull(resultMap);
println "【 ${index} 】"+resultMap;
println "【 ${index} 】"+resultMap;
if(resultMap.name){
sucCount++;
results << resultMap;
}else{
failCount++;
println "小区名称没解析出来"
println "小区名称没解析出来"
}
}else{
failCount++;
println "没有网页内容"
println "没有网页内容"
}
}
println "失败:"+failCount
println "成功:"+sucCount
println "失败:"+failCount
println "成功:"+sucCount
return results;
}
}
@@ -15,7 +15,7 @@ class Test {
def task = new Test();
def obj = "23平米"
def obj = "23平米"
Pattern pa = Pattern.compile("(\\d+(.\\d+)?)")
def ma=pa.matcher(obj)
def value = null;
@@ -27,10 +27,10 @@ class Test {
println value
// def names = task.findDistricts("shop", "北京", "房山");
// println "库中:"+ names.size();
// def names = task.findDistricts("shop", "北京", "房山");
// println "库中:"+ names.size();
// def ret = task.readTxt("H:\\", "shop.txt");
// println "Excel:"+ ret.size();
// println "Excel:"+ ret.size();
//
// def index = 0;
// for(def exclName : ret){
@@ -87,13 +87,13 @@ return;
//// }
//
// String h = new String(s.getBytes("iso-8859-1"),"UTF-8");
// if(StringUtils.contains(h, "opt:更新")){
// if(StringUtils.contains(h, "opt:更新")){
// count_update++;
// } else if(StringUtils.contains(h, "opt:放弃")){
// } else if(StringUtils.contains(h, "opt:放弃")){
// count_abort++;
// } else if(StringUtils.contains(h, "opt:保存")){
// } else if(StringUtils.contains(h, "opt:保存")){
// count_save++;
// } else if(StringUtils.contains(h, "opt:过滤")){
// } else if(StringUtils.contains(h, "opt:过滤")){
// count_filter++;
// }
//
@@ -42,14 +42,14 @@ class WebAnalyLogBatch extends BaseBatch {
if (file.exists()) {
def retMap = analyFile(filePath);
println fileName + " 文件分析完成......"
println fileName + " 文件分析完成......"
saveToDB(retMap, yesterDay);
//移除10天前的文件
//移除10天前的文件
removeFile(path);
} else {
println fileName + " 没有找到文件!";
println fileName + " 没有找到文件!";
}
}
@@ -63,9 +63,9 @@ class WebAnalyLogBatch extends BaseBatch {
if (file.exists()) {
file.delete();
println "删除文件 ......" + fileName;
println "删除文件 ......" + fileName;
} else {
println "要删的文件不存在 ......" + fileName;
println "要删的文件不存在 ......" + fileName;
}
}
@@ -92,29 +92,29 @@ class WebAnalyLogBatch extends BaseBatch {
obj.put("city", keys[1]);
obj.put("site_name", keys[2]);
obj.put("stat_date", DateUtils.parseFromTightYYYYMMDD(yesterDay));
if (v.getAt("总量")) {
obj.put("total_count", v.getAt("总量"));
if (v.getAt("总量")) {
obj.put("total_count", v.getAt("总量"));
}
if (v.getAt("保存")) {
obj.put("add_count", v.getAt("保存"));
if (v.getAt("保存")) {
obj.put("add_count", v.getAt("保存"));
}
if (v.getAt("更新")) {
obj.put("update_count", v.getAt("更新"));
if (v.getAt("更新")) {
obj.put("update_count", v.getAt("更新"));
}
if (v.getAt("过滤")) {
obj.put("filter_count", v.getAt("过滤"));
if (v.getAt("过滤")) {
obj.put("filter_count", v.getAt("过滤"));
}
if (v.getAt("异常")) {
obj.put("error_count", v.getAt("异常"));
if (v.getAt("异常")) {
obj.put("error_count", v.getAt("异常"));
}
if (v.getAt("放弃")) {
obj.put("lost_count", v.getAt("放弃"));
if (v.getAt("放弃")) {
obj.put("lost_count", v.getAt("放弃"));
}
obj.put("c_date", new Date());
@@ -132,7 +132,7 @@ class WebAnalyLogBatch extends BaseBatch {
}
def writeTxt(def s, def day) {
FileUtils.writeStringToFile(new File("E:\\数据\\logs\\结果\\" + day + ".txt"), s, "GBK")
FileUtils.writeStringToFile(new File("E:\\数据\\logs\\结果\\" + day + ".txt"), s, "GBK")
}
def printlnBaseInfo(retMap) {
@@ -143,7 +143,7 @@ class WebAnalyLogBatch extends BaseBatch {
def readTxt1(def file) {
//方式一
//方式一
def retMap = [:];
FileInputStream fis = new FileInputStream(file);
RandomAccessFile raf = new RandomAccessFile(new File(file), "r");
@@ -173,11 +173,11 @@ class WebAnalyLogBatch extends BaseBatch {
def s = line.substring(line.indexOf("-") + 2, line.length());
def infoMap = splitToMap(s)
// def array = s.split("\\s+");
//房源类型
//房源类型
def cat = infoMap['cat'];
//城市详情
//城市详情
def city = infoMap['city'];
//解析操作
//解析操作
def optType = infoMap['opt'];
/*if(array[5].toString().endsWith('"city":"')){
array[5] = array[5] + array[6];
@@ -191,14 +191,14 @@ class WebAnalyLogBatch extends BaseBatch {
detailMap = [:]
retMap.putAt(key, detailMap);
}
//房源类型总量
def totalCount = detailMap["总量"];
//房源类型总量
def totalCount = detailMap["总量"];
if (!totalCount) {
totalCount = 0;
}
detailMap.put("总量", totalCount + 1);
detailMap.put("总量", totalCount + 1);
//各种操作类型统计
//各种操作类型统计
def count = detailMap.get(optType);
if (!count) {
count = 0;
@@ -244,13 +244,13 @@ class WebAnalyLogBatch extends BaseBatch {
webDetailMap.put(key, infoMap);
}
def totalCount = infoMap.get("总量");
def totalCount = infoMap.get("总量");
if (!totalCount) {
totalCount = 0;
}
infoMap.put("总量", totalCount + 1);
infoMap.put("总量", totalCount + 1);
//解析操作
//解析操作
def optType = getOptionType(array[0]);
def count = infoMap.get(optType);
if (!count) {
@@ -288,7 +288,7 @@ class WebAnalyLogBatch extends BaseBatch {
def readTxt3(def file) {
Charset charset = Charset.forName("UTF-8");
//Java.nio.charset.Charset处理了字符转换问题。它通过构造CharsetEncoder和CharsetDecoder将字符序列转换成字节和逆转换。
//Java.nio.charset.Charset处理了字符转换问题。它通过构造CharsetEncoder和CharsetDecoder将字符序列转换成字节和逆转换。
CharsetDecoder decoder = charset.newDecoder();
FileInputStream fis = null;
def retMap = [:];
@@ -319,7 +319,7 @@ class WebAnalyLogBatch extends BaseBatch {
println byteBuffer.toString();
if (charBuffer.toString().contains("[1009 10:39:22 711 INFO ] [exe1] - opt:异")) {
if (charBuffer.toString().contains("[1009 10:39:22 711 INFO ] [exe1] - opt:异")) {
println "sss"
}
while (true) {
@@ -429,7 +429,7 @@ class WebAnalyLogBatch extends BaseBatch {
def readTxt4(def file) {
// try {
// BufferedInputStream bis = new BufferedInputStream(new FileInputStream(new File(file)));
// BufferedReader ino = new BufferedReader(new InputStreamReader(bis, "utf-8"), 10 * 1024 * 1024);//10M缓存
// BufferedReader ino = new BufferedReader(new InputStreamReader(bis, "utf-8"), 10 * 1024 * 1024);//10M缓存
// int index = 0;
// while(ino.ready()) {
// String line = ino.readLine();
@@ -446,7 +446,7 @@ class WebAnalyLogBatch extends BaseBatch {
// ex.printStackTrace();
// }
//
//非一行一行读取
//非一行一行读取
// File inFile = new File(file);
// FileChannel inFileChannel = new FileInputStream(inFile).getChannel();
// byte[] bs = new byte[1024];
@@ -460,7 +460,7 @@ class WebAnalyLogBatch extends BaseBatch {
// buffer.flip();
// def s = charset.decode(buffer);
// sb.append(s);
// if(s.contains("[1009 10:39:22 711 INFO ] [exe1] - opt:异")){
// if(s.contains("[1009 10:39:22 711 INFO ] [exe1] - opt:异")){
// println "sss"
// }
// println "[${index}]"+s;
@@ -36,10 +36,10 @@ class XingyeDataBatch {
public static void main(String[] args) {
List<String> cities = ServerAreaMap.findAllCities()
List<List<Object>> rows = ExcelUtils.read("D:/房产测试合并数据21.xls", 0, 0)
//新序号 序号 地区 楼宇名称 房产地址 建筑面积 单价 总价值 城市 名称 地址 经度 纬度
rows.get(0).add('计算类型')
rows.get(0).add('可信度')
List<List<Object>> rows = ExcelUtils.read("D:/房产测试合并数据21.xls", 0, 0)
//新序号 序号 地区 楼宇名称 房产地址 建筑面积 单价 总价值 城市 名称 地址 经度 纬度
rows.get(0).add('计算类型')
rows.get(0).add('可信度')
for (int i = 1; i < rows.size(); i++) {
println rows.get(i).get(0)
List<Object> cloumns = rows.get(i)
@@ -57,15 +57,15 @@ class XingyeDataBatch {
String realCity = cloumns.get(11).toString().trim()
realCity = formatCity(realCity, cities)
//todo 过滤掉不是小区的数据
if('其他' == cat || StringUtils.isBlank(realCity) || !cities.contains(realCity)) {
//todo 过滤掉不是小区的数据
if('其他' == cat || StringUtils.isBlank(realCity) || !cities.contains(realCity)) {
rows.get(i).add('')
rows.get(i).add('')
}else {
Res res = null;
if ('商铺' == cat) {
if ('商铺' == cat) {
res = getShopPriceResult(realCity, realName, address, lng, lat)
} else if ('写字楼' == cat) {
} else if ('写字楼' == cat) {
res = getOfficePriceResult(realCity, realName, address, lng, lat)
} else {
res = getDistrictPriceResult(realCity, realName, address, lng, lat)
@@ -85,7 +85,7 @@ class XingyeDataBatch {
}
}
ExcelUtils.write("D:/房产测试合并数据21-output.xls", rows, "数据")
ExcelUtils.write("D:/房产测试合并数据21-output.xls", rows, "数据")
}
private static String formatCity(String city, List<String> cities) {
@@ -93,8 +93,8 @@ class XingyeDataBatch {
return city
}
if(city.endsWith('市')) {
city = StringUtils.substringBeforeLast(city, "市")
if(city.endsWith('市')) {
city = StringUtils.substringBeforeLast(city, "市")
if(cities.contains(city)) {
return city
}
@@ -121,13 +121,13 @@ class XingyeDataBatch {
}
}
/**推测真正的正式名*/
/**推测真正的正式名*/
static String speculateCityNameByRegion(String province, String orgName){
List<String> foundCityList = new ArrayList<String>();
/**根据区域推测所在的城市*/
/**根据区域推测所在的城市*/
String temp;
for(Map.Entry<String,Set<String>> entry : CITY_REGION_MAP.entrySet()){
temp = orgName.replaceFirst("^" + entry.getKey(),"");//排除无"市"标示的城市信息干扰 如:北京海定区
temp = orgName.replaceFirst("^" + entry.getKey(),"");//排除无"市"标示的城市信息干扰 如:北京海定区
if(entry.getValue().contains(temp)){
if(province == null){
foundCityList.add(entry.getKey());
@@ -183,10 +183,10 @@ class XingyeDataBatch {
private static Res getDistrictPriceResult(String realCity, String realName, String realAddr, Double lng, Double lat) {
Res res = null;
DBObject district = chooseDistrict(realCity, realName, realAddr, 'district')
if(district != null) {//利用计算的均价
if(district != null) {//利用计算的均价
String proRegion = district.get('region')
String proName = district.get('name')
String proEstateType = (district.get('estate_type2') != null && district.get('estate_type2').indexOf('别墅') > -1 ? '别墅' : '小区');
String proEstateType = (district.get('estate_type2') != null && district.get('estate_type2').indexOf('别墅') > -1 ? '别墅' : '小区');
DBObject statq = new BasicDBObject()
statq.put('city', realCity)
statq.put('region', proRegion)
@@ -200,7 +200,7 @@ class XingyeDataBatch {
}
}
//利用样本房源进行计算
//利用样本房源进行计算
if(res == null) {
List<DBObject> houses = fetchHousesInSeawater(realCity, realName, 1000)
if(houses != null && houses.size() > 10) {
@@ -212,7 +212,7 @@ class XingyeDataBatch {
}
}
//利用周边计算
//利用周边计算
if(res == null && lat != null && lng != null) {
String js = HTTP.invoke('/evaluation/lngLatEvaluate', [city : realCity, lng:lng, lat:lat, lngLatType:1], "UTF-8")
if(js != null) {
@@ -241,7 +241,7 @@ class XingyeDataBatch {
}
/**
* 在seaweed中选择一个最优的小区
* 在seaweed中选择一个最优的小区
* @param city
* @param name
* @param address
@@ -341,7 +341,7 @@ class XingyeDataBatch {
/**
* 数量得分(最高得分0.1)
* 数量得分(最高得分0.1)
*
* @param points
* @return
@@ -350,14 +350,14 @@ class XingyeDataBatch {
if (points.size() == 0) {
return 0;
}
// 每10个样本=0.01
// 每10个样本=0.01
double c1 = (double) points.size() / 10.0 / 100.0;
c1 = (c1 > 0.1 ? 0.1 : c1);
return c1;
}
/**
* 质量得分(最高得分0.1)
* 质量得分(最高得分0.1)
*
* @param points
* @return
@@ -373,21 +373,21 @@ class XingyeDataBatch {
for (DBObject p : points) {
Date d = p.get('publish_date')
long diff = calcDate.getTime() - d.getTime();
if (diff <= 30 * 24 * 3600 * 1000) {// 一个月内
if (diff <= 30 * 24 * 3600 * 1000) {// 一个月内
c2 = c2 + perc * 1.00d;
} else if (diff <= 60 * 24 * 3600 * 1000) {// 两个月内
} else if (diff <= 60 * 24 * 3600 * 1000) {// 两个月内
c2 = c2 + perc * 0.90d;
} else if (diff <= 90 * 24 * 3600 * 1000) {// 三个月内
} else if (diff <= 90 * 24 * 3600 * 1000) {// 三个月内
c2 = c2 + perc * 0.85d;
} else if (diff <= 180 * 24 * 3600 * 1000) {// 半年内
} else if (diff <= 180 * 24 * 3600 * 1000) {// 半年内
c2 = c2 + perc * 0.80d;
} else if (diff <= 365 * 24 * 3600 * 1000) {// 一年内
} else if (diff <= 365 * 24 * 3600 * 1000) {// 一年内
c2 = c2 + perc * 0.70d;
} else if (diff <= 540 * 24 * 3600 * 1000) {// 一年半内
} else if (diff <= 540 * 24 * 3600 * 1000) {// 一年半内
c2 = c2 + perc * 0.60d;
} else if (diff <= 720 * 24 * 3600 * 1000) {// 两年内
} else if (diff <= 720 * 24 * 3600 * 1000) {// 两年内
c2 = c2 + perc * 0.40d;
} else {// 超过两年
} else {// 超过两年
c2 = c2 + perc * 0.10d;
}
}
@@ -24,7 +24,7 @@ class XingyeDataPrice {
public static HttpHelper HTTP = new HttpHelper("192.168.10.66:7180");
public static void main(String[] args) {
String fileName = "介入数据跑价格补城市价格";
String fileName = "介入数据跑价格补城市价格";
List<String> cities = ServerAreaMap.findAllCities()
List<List<Object>> rows = ExcelUtils.read("D:/${fileName}.xls", 0, 0)
for (int i = 1; i < rows.size(); i++) {
@@ -78,14 +78,14 @@ class XingyeDataPrice {
cloumns.set(16, "0.01")
}
ExcelUtils.write("D:/${fileName}-final.xls", rows, "数据")
ExcelUtils.write("D:/${fileName}-final.xls", rows, "数据")
}
public static void maindd3(String[] args) {
List<String> idList = FileUtils.readLines(new File("D:/介入ID.txt"))
List<String> idList = FileUtils.readLines(new File("D:/介入ID.txt"))
String fileName = "房价调查完成合并数据0324终-final16";
String fileName = "房价调查完成合并数据0324终-final16";
List<String> cities = ServerAreaMap.findAllCities()
List<List<Object>> rows = ExcelUtils.read("D:/${fileName}.xls", 0, 0)
List<List<Object>> newRows = [rows.get(0)];
@@ -161,10 +161,10 @@ class XingyeDataPrice {
}
ExcelUtils.write("D:/${fileName}-final.xls", newRows, "数据")
ExcelUtils.write("D:/${fileName}-final.xls", newRows, "数据")
}
public static void maind(String[] args) {
String fileName = "其他类";
String fileName = "其他类";
List<String> cities = ServerAreaMap.findAllCities()
List<List<Object>> rows = ExcelUtils.read("D:/${fileName}.xlsx", 0, 0)
def eschars = [] as String[]
@@ -243,7 +243,7 @@ class XingyeDataPrice {
cloumns.set(16, "0.01")
}
ExcelUtils.write("D:/${fileName}-final.xlsx", rows, "数据")
ExcelUtils.write("D:/${fileName}-final.xlsx", rows, "数据")
}
@@ -252,8 +252,8 @@ class XingyeDataPrice {
return city
}
if(city.endsWith('市')) {
city = StringUtils.substringBeforeLast(city, "市")
if(city.endsWith('市')) {
city = StringUtils.substringBeforeLast(city, "市")
if(cities.contains(city)) {
return city
}
@@ -279,13 +279,13 @@ class XingyeDataPrice {
}
}
/**推测真正的正式名*/
/**推测真正的正式名*/
static String speculateCityNameByRegion(String province, String orgName){
List<String> foundCityList = new ArrayList<String>();
/**根据区域推测所在的城市*/
/**根据区域推测所在的城市*/
String temp;
for(Map.Entry<String,Set<String>> entry : CITY_REGION_MAP.entrySet()){
temp = orgName.replaceFirst("^" + entry.getKey(),"");//排除无"市"标示的城市信息干扰 如:北京海定区
temp = orgName.replaceFirst("^" + entry.getKey(),"");//排除无"市"标示的城市信息干扰 如:北京海定区
if(entry.getValue().contains(temp)){
if(province == null){
foundCityList.add(entry.getKey());
@@ -28,95 +28,95 @@ import com.mongodb.QueryOperators
class DistrictExportBatch {
public static final FILELD_MAP = [
"_id":"楼盘编号",
"city":"所在城市",
"region":"所在地区",
"block":"所在板块",
"name":"楼盘名称",
"estate_type2":"物业类型",
"build_type":"建筑类型",
"live_date":"竣工时间(建成年份)",
"lng2":"经度",
"lat2":"纬度",
"developer":"开发商名称",
"estate_company":"物业公司",
"estate_charge":"物业费",
"licence":"商品房销售许可证号",
"address":"楼盘地址",
"area":"面积",
"avg_price":"楼盘单价",
"totalPrice":"评估总价",
"city_price":"城市均价",
"city_price_markup":"近半年城市均价涨跌幅",
"rent_price":"租房均价",
"year_markup":"均价涨跌幅",
"month_markup":"环比上月",
"max_price":"历史最高单价",
"min_price":"历史最低单价",
"futrue_price":"未来一年房价预测",
"futrue_markup":"未来一年房价涨跌幅预测",
"last_month_house_count":"上个月挂牌量",
"half_year_house_count":"半年挂牌量",
"attention_level":"近三个月买家关注度",
"block_rank":"板块排名",
"block_count":"板块个数",
"region_rank":"区域排名",
"region_count":"区域个数",
"consumption":"周边餐饮的人均消费",
"consumption_level":"综合消费水平高于所属城市的楼盘数",
"_id":"楼盘编号",
"city":"所在城市",
"region":"所在地区",
"block":"所在板块",
"name":"楼盘名称",
"estate_type2":"物业类型",
"build_type":"建筑类型",
"live_date":"竣工时间(建成年份)",
"lng2":"经度",
"lat2":"纬度",
"developer":"开发商名称",
"estate_company":"物业公司",
"estate_charge":"物业费",
"licence":"商品房销售许可证号",
"address":"楼盘地址",
"area":"面积",
"avg_price":"楼盘单价",
"totalPrice":"评估总价",
"city_price":"城市均价",
"city_price_markup":"近半年城市均价涨跌幅",
"rent_price":"租房均价",
"year_markup":"均价涨跌幅",
"month_markup":"环比上月",
"max_price":"历史最高单价",
"min_price":"历史最低单价",
"futrue_price":"未来一年房价预测",
"futrue_markup":"未来一年房价涨跌幅预测",
"last_month_house_count":"上个月挂牌量",
"half_year_house_count":"半年挂牌量",
"attention_level":"近三个月买家关注度",
"block_rank":"板块排名",
"block_count":"板块个数",
"region_rank":"区域排名",
"region_count":"区域个数",
"consumption":"周边餐饮的人均消费",
"consumption_level":"综合消费水平高于所属城市的楼盘数",
"KTV_count":"KTV数量",
"cinema_count":"电影院数量",
"jianshen_count":"健身中心数量",
"zhonghesc_count":"综合商场数量",
"spa_count":"美容院数量",
"hotel_count":"星级酒店数量",
"bigmarket_count":"大型超市数量",
"gef_count":"高尔夫球场数量",
"slschool_count":"私立学校数量",
"zrzx_count":"整容整形数量",
"coffee_count":"咖啡厅数量",
"bath_count":"洗浴数量",
"office_count":"写字楼数量",
"hspt_count":"医院数量",
"bank_count":"银行数量",
"atm_count":"ATM数量",
"park_count":"公园绿地数量",
"comm_count":"商圈数量",
"bus_count":"公交车站数量",
"subway_count":"地铁数量",
"school_count":"学校数量",
"air_count":"机场数量",
"market_count":"超市数量",
"wb_count":"网吧数量",
"ywss_count":"厌恶设施数量",
"all_area":"楼盘占地总面积",
"build_area":"楼盘建筑总面积",
"household_count":"总套数",
// "":"可售套数",
// "":"所在省份",
// "alias1":"楼盘别名1",
// "alias2":"楼盘别名2",
// "alias3":"楼盘别名3",
"kind":"小区类型",
"virescence_percent":"绿化率",
"cubage_percent":"容积率",
// "facilities":"配套信息",
"traffic":"交通情况",
"household_count":"户数",
"car_site":"车位情况",
// "":"具体楼盘号",
"property_life":"产权性质",
"build_structure":"房屋结构",
"open_date":"开盘时间",
"sell_address":"售楼地址",
"discount":"折扣信息",
// "fitment_materials":"建材装修",
"floor_info":"楼层状况",
"description":"描述",
// "c_date":"登记日期",
// "m_date":"更新日期",
// "out_pics":"外景图",
"KTV_count":"KTV数量",
"cinema_count":"电影院数量",
"jianshen_count":"健身中心数量",
"zhonghesc_count":"综合商场数量",
"spa_count":"美容院数量",
"hotel_count":"星级酒店数量",
"bigmarket_count":"大型超市数量",
"gef_count":"高尔夫球场数量",
"slschool_count":"私立学校数量",
"zrzx_count":"整容整形数量",
"coffee_count":"咖啡厅数量",
"bath_count":"洗浴数量",
"office_count":"写字楼数量",
"hspt_count":"医院数量",
"bank_count":"银行数量",
"atm_count":"ATM数量",
"park_count":"公园绿地数量",
"comm_count":"商圈数量",
"bus_count":"公交车站数量",
"subway_count":"地铁数量",
"school_count":"学校数量",
"air_count":"机场数量",
"market_count":"超市数量",
"wb_count":"网吧数量",
"ywss_count":"厌恶设施数量",
"all_area":"楼盘占地总面积",
"build_area":"楼盘建筑总面积",
"household_count":"总套数",
// "":"可售套数",
// "":"所在省份",
// "alias1":"楼盘别名1",
// "alias2":"楼盘别名2",
// "alias3":"楼盘别名3",
"kind":"小区类型",
"virescence_percent":"绿化率",
"cubage_percent":"容积率",
// "facilities":"配套信息",
"traffic":"交通情况",
"household_count":"户数",
"car_site":"车位情况",
// "":"具体楼盘号",
"property_life":"产权性质",
"build_structure":"房屋结构",
"open_date":"开盘时间",
"sell_address":"售楼地址",
"discount":"折扣信息",
// "fitment_materials":"建材装修",
"floor_info":"楼层状况",
"description":"描述",
// "c_date":"登记日期",
// "m_date":"更新日期",
// "out_pics":"外景图",
]
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
public static final DBCollection DIST_STAT_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "district_stat_rent", "mongo.db.fangjia.com")
@@ -134,7 +134,7 @@ class DistrictExportBatch {
public static def radius = 1000;
public static void main(String[] args) {
for(String city : ["上海"]) {//"北京",,
for(String city : ["上海"]) {//"北京",,
areas = [];
StringBuilder builder = new StringBuilder()
FILELD_MAP.each() {column, display->
@@ -160,7 +160,7 @@ class DistrictExportBatch {
c ++
def kind = dbo.get("kind").toString();
if(kind.indexOf("别墅")!=-1){
if(kind.indexOf("别墅")!=-1){
continue;
}
if(!dbo.get("lng2") || !dbo.get("lat2")){
@@ -170,7 +170,7 @@ class DistrictExportBatch {
//if(!regions.contains(dbo.get("region"))){
// regions << dbo.get("region").toString()
//房子估价
//房子估价
def params = findLoudongInfo(city, dbo.get("region"), dbo.get("name"));
params.city = city;
params.region = dbo.get("region")
@@ -180,33 +180,33 @@ class DistrictExportBatch {
continue;
}
//最近城市均价
//最近城市均价
def nearCityPrice = findCityPrice(city, null);
//半年前最近的城市均价
//半年前最近的城市均价
def weekend = DateUtil.formatByPattern(DateUtil.getFirstDayOfWeek(new Date()), "yyyy-MM-dd");
def halfYearAgo = DateUtils.add(weekend, -180);
def halfYearAgoPrice = findCityPrice(city, halfYearAgo);
//租金价格
//租金价格
def endMonthDate = DateUtils.add(DateUtils.formatToYYYYMMDD(DateUtils.getFirstDayOfMonth(new Date())),-1);
def houseTypePriceMap = findDistrictRentStat(city, dbo.get("region"), dbo.get("name"),endMonthDate);
//最近期小区均价
//最近期小区均价
def nearDistAvgPrice = queryDistrictPrice(city, dbo.get("region"), dbo.get("name"), null, 1);
//一前内小区均价
//一前内小区均价
def oneYearAgo = DateUtils.add(weekend, -365);
def oneYearPriceList = queryDistrictPrices(city, dbo.get("region"), dbo.get("name"), oneYearAgo, weekend);
//小区在所在区域或板块中的点击量排名
//小区在所在区域或板块中的点击量排名
def clickRank = findRegionBlockRank(city, dbo.get("region"), dbo.get("name"));
//某板块中的小区量
//某板块中的小区量
def blockCount = findRegionBlockDistrictCount(city, dbo.get("region"), dbo.get("block"));
//某区域中的小区量
//某区域中的小区量
def regionCount = findRegionBlockDistrictCount(city, dbo.get("region"), null);
//一年后的预测价格
//一年后的预测价格
def futurePrice = findPricePredict(city, dbo.get("region"), dbo.get("name"));
//人均消费水平
//人均消费水平
def avgConsumption = 0;
FILELD_MAP.each() {column, display->
@@ -216,15 +216,15 @@ class DistrictExportBatch {
val = ''
}else if(column == 'address'){
def address = dbo.get("address");
if(address.indexOf("号")!=-1){
val = address + params.houseNumber +"栋"+params.roomNumber+"室";
if(address.indexOf("号")!=-1){
val = address + params.houseNumber +"栋"+params.roomNumber+"室";
}else{
val = address + params.houseNumber +"号"+params.roomNumber+"室";
val = address + params.houseNumber +"号"+params.roomNumber+"室";
}
}else if(column == 'area'){
val = params.area
}else if(column == 'totalPrice'){
val = DataUtil.getRound(2, totalPrice*1.0/10000) +"万元";
val = DataUtil.getRound(2, totalPrice*1.0/10000) +"万元";
}else if(column.startsWith('alias')) {
if(dbo.get('alias')) {
List aliList = (List)dbo.get('alias');
@@ -247,7 +247,7 @@ class DistrictExportBatch {
}else if(column == 'estate_charge'){
def s = dbo.get(column);
if(s){
val = s.toString().replaceAll("元/㎡/月","").replaceAll("元/月/平米","")
val = s.toString().replaceAll("元/㎡/月","").replaceAll("元/月/平米","")
}else{
val = "";
}
@@ -309,20 +309,20 @@ class DistrictExportBatch {
def three_room = houseTypePriceMap.get("3");
val = "";
if(one_room){
val = "1室:"+one_room+"元";
val = "1室:"+one_room+"元";
}
if(two_room){
if(val){
val = val + " ";
}
val = val + "2室:"+two_room+"元";
val = val + "2室:"+two_room+"元";
}
if(three_room){
if(val){
val = val + " ";
}
val = val + "3室:"+three_room+"元";
val = val + "3室:"+three_room+"元";
}
@@ -330,11 +330,11 @@ class DistrictExportBatch {
if(clickRank){
def attent = clickRank.get("region_click_rank")/regionCount * 1.0
if(attent<=0.3){
val = "高";
val = "高";
}else if(attent>0.3 && attent<=0.7){
val = "一般"
val = "一般"
}else{
val = "较低";
val = "较低";
}
}else{
val = "";
@@ -353,11 +353,11 @@ class DistrictExportBatch {
val = "";
}
}else if(column.indexOf("_count")!=-1 && display.indexOf("数量")!=-1){
}else if(column.indexOf("_count")!=-1 && display.indexOf("数量")!=-1){
if(dbo.get("lng2") && dbo.get("lat2")){
def lat = (double)dbo.get("lat2") / 10000000000.0
def lng = (double)dbo.get("lng2") / 10000000000.0
val = findPoiCount(display.toString(), lat.toString(), lng.toString()) +"个";
val = findPoiCount(display.toString(), lat.toString(), lng.toString()) +"个";
}else{
val = "";
}
@@ -380,7 +380,7 @@ class DistrictExportBatch {
}
}else if(column == "month_markup"){
//一个月以前最近的小区均价
//一个月以前最近的小区均价
def monthAgo = DateUtils.add(weekend, -30);
def monthAgoAvgPrice = queryDistrictPrice(city, dbo.get("region"), dbo.get("name"), monthAgo, 1);
if(oneYearPriceList.size()>0 && monthAgoAvgPrice>0){
@@ -551,7 +551,7 @@ class DistrictExportBatch {
public static def findPoiCount(display, lat, lng){
display = display.toString().replaceAll("数量","");
display = display.toString().replaceAll("数量","");
def ret = 0;
def page_num = 0;
@@ -591,7 +591,7 @@ class DistrictExportBatch {
conn.setRequestMethod("GET");
conn.doOutput = true
inputStream = conn.getInputStream();
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
while((line = buf.readLine())!=null){
sb.append(line);
}
@@ -703,7 +703,7 @@ class DistrictExportBatch {
public static def findBusinessShop(city, maxLat, minLat, maxLng, minLng){
DBObject q = new BasicDBObject()
q.put("city", city);
q.put("business_type", "餐厅");
q.put("business_type", "餐厅");
q.put("consumption", new BasicDBObject(QueryOperators.GT, 0));
q.put('lat', new BasicDBObject(QueryOperators.LTE, maxLat).append(QueryOperators.GTE, minLat));
q.put('lng', new BasicDBObject(QueryOperators.LTE, maxLng).append(QueryOperators.GTE, minLng));
@@ -57,12 +57,12 @@ class HouseLoudongExportBatch {
static final String XML_FILE_NAME = "fangzi"
static def FOCUS_FIELDS = [
"city":"城市", "region":"区域", "name":"小区名", "house_area":"面积", "house_num":"房号", "room_num":"室号", "house_num_unit":"单元",
"house_height":"楼高", "house_floor":"所在楼层", "room":"几室", "hall":"几厅", "toilet":"几卫", "direction":"朝向", "build_type":"建筑类型",
"garden_area":"花园面积", "outer_area":"露台面积", "attic_area":"阁楼面积", "worry_facility":"厌恶设施",
"noise":"噪音情况", "sight":"景观情况", "inside_pos":"小区位置", "house_property":"产权性质",
"fitment_cost":"装修花费", "fitment_age":"装修年限", "fitment_rank":"装修级别", "lighting_ventilation":"采光通风"
//"basement_area":"地下室面积", "sideset":"边套设施","car_port":"",
"city":"城市", "region":"区域", "name":"小区名", "house_area":"面积", "house_num":"房号", "room_num":"室号", "house_num_unit":"单元",
"house_height":"楼高", "house_floor":"所在楼层", "room":"几室", "hall":"几厅", "toilet":"几卫", "direction":"朝向", "build_type":"建筑类型",
"garden_area":"花园面积", "outer_area":"露台面积", "attic_area":"阁楼面积", "worry_facility":"厌恶设施",
"noise":"噪音情况", "sight":"景观情况", "inside_pos":"小区位置", "house_property":"产权性质",
"fitment_cost":"装修花费", "fitment_age":"装修年限", "fitment_rank":"装修级别", "lighting_ventilation":"采光通风"
//"basement_area":"地下室面积", "sideset":"边套设施","car_port":"",
]
private static final String API_HOST = "open.fangjia.com"//"sha-66.qa.fangjia.com/fangjia-open";
@@ -76,7 +76,7 @@ class HouseLoudongExportBatch {
static HttpClient client = null
public static void main(String[] args) {
def cities = ['北京']
def cities = ['北京']
for(String city : cities) {
exportLoudong(city)
}
@@ -142,7 +142,7 @@ class HouseLoudongExportBatch {
List<Element> elementList = new ArrayList<Element>()
int inx = 1
String fileName = "D:/安硕数据/${XML_FILE_NAME}_${city}(${inx}).xml"
String fileName = "D:/安硕数据/${XML_FILE_NAME}_${city}(${inx}).xml"
while(cur.hasNext()) {
cc++
@@ -186,7 +186,7 @@ class HouseLoudongExportBatch {
String kind = getDistrictKind(city, region, districtName)
if(!kind) {
kind = "小区"
kind = "小区"
}
Date currentWeekend = DateUtil.getThisSunday(new Date())
@@ -251,7 +251,7 @@ class HouseLoudongExportBatch {
if(c % 300000 == 0) {
inx ++
fileName = "D:/安硕数据/${XML_FILE_NAME}_${city}(${inx}).xml"
fileName = "D:/安硕数据/${XML_FILE_NAME}_${city}(${inx}).xml"
}
}
@@ -267,9 +267,9 @@ class HouseLoudongExportBatch {
}
private static def getFromEvaluation(String city, String region, String districtName, double area, String houseNum, String roomNum, String kind) {
String buildType = "公寓"
if(kind == '小区') {
buildType = "公寓"
String buildType = "公寓"
if(kind == '小区') {
buildType = "公寓"
}else {
buildType = kind
}
@@ -9,7 +9,7 @@ import groovy.sql.Sql
import net.sf.json.JSONObject
/**
* 功能说明:
* 功能说明:
*/
@BatchOwner(Watcher.WangWenYan)
class ApiUserDeadTimeMonitorBatch extends BaseBatch {
@@ -24,11 +24,11 @@ class ApiUserDeadTimeMonitorBatch extends BaseBatch {
void process(Object args) {
def sql = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia", "sa", "G00jia", "com.mysql.jdbc.Driver")
def tenDayAgo = DateUtil.add(new Date(), Calendar.DATE, 10).format("yyyy-MM-dd")
String body = "API用户到期通知:\n\n"
String body = "API用户到期通知:\n\n"
int count = 0
sql.eachRow("select ID,COMPANY,DEAD_TIME from fangjia.api_user where status = 0 and DEAD_TIME = '${tenDayAgo}'",{row->
println(row)
String msg = "通知:用户ID为${row['ID']},用户名为:${row['COMPANY']}的API账户将于${((Date)row['DEAD_TIME']).format("yyyy年MM月dd日")}到期!"
String msg = "通知:用户ID为${row['ID']},用户名为:${row['COMPANY']}的API账户将于${((Date)row['DEAD_TIME']).format("yyyy年MM月dd日")}到期!"
body = body + msg + "\n\n"
println(msg)
count++
@@ -37,7 +37,7 @@ class ApiUserDeadTimeMonitorBatch extends BaseBatch {
JSONObject requestBody = new JSONObject()
requestBody.put("msgtype", "markdown")
JSONObject markdown = new JSONObject()
markdown.put("title", "API监控")
markdown.put("title", "API监控")
markdown.put("text", body)
requestBody.put("markdown", markdown)
println(requestBody.toString())
@@ -12,7 +12,7 @@ import groovy.sql.Sql
import net.sf.json.JSONObject
/**
* 功能说明:API使用状态监控脚本
* 功能说明:API使用状态监控脚本
*/
@BatchOwner(Watcher.WangWenYan)
class OpenApiUseStatMonitorBatch extends BaseBatch {
@@ -55,8 +55,8 @@ class OpenApiUseStatMonitorBatch extends BaseBatch {
}
}
def title = "API性能监控"
String body = """失败过多警报[${startDate.format("HH:mm")}-${endDate.format("HH:mm")}]\n"""
def title = "API性能监控"
String body = """失败过多警报[${startDate.format("HH:mm")}-${endDate.format("HH:mm")}]\n"""
def hasWarn = false
def sql = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia", "sa", "G00jia", "com.mysql.jdbc.Driver")
statMap.each {String userId,List<BasicDBObject> logs->
@@ -84,9 +84,9 @@ class OpenApiUseStatMonitorBatch extends BaseBatch {
if(!name){
name = userId
}
item = "> - 用户ID:${userId},用户名:${name}, 访问异常,访问次数:${sucCount + errCount},失败次数:${errCount},失败率:${errRate}\n\n"
item = "> - 用户ID:${userId},用户名:${name}, 访问异常,访问次数:${sucCount + errCount},失败次数:${errCount},失败率:${errRate}\n\n"
}else{
item = "> - 用户ID:${userId},访问异常,访问次数:${sucCount + errCount},失败次数:${errCount},失败率:${errRate}\n\n"
item = "> - 用户ID:${userId},访问异常,访问次数:${sucCount + errCount},失败次数:${errCount},失败率:${errRate}\n\n"
}
body = body + item
hasWarn = true
@@ -11,7 +11,7 @@ import com.fangjia.core.RegionData
import com.fangjia.script.util.DbUtils
/**
* 填充小区的楼栋总数
* 填充小区的楼栋总数
* @author gaojuan
*
*/
@@ -28,10 +28,10 @@ class AddDistrictLouDongCount {
def allCity = CommonUtils.findAllCity();
def cat = "district";
def field = ["city","region","name", "loudong_count", "alias"]
def siteNames = ["链家在线","搜狐焦点"]
def siteNames = ["链家在线","搜狐焦点"]
for(def city : allCity){
def ret = CommonDatautils.findAllDistricts(city, "district", field);
println "[ ${city} ] : "+ret.count();
println "[ ${city} ] : "+ret.count();
def distIndex = 0;
for(def m : ret){
distIndex++;
@@ -187,9 +187,9 @@ class AddDistrictLouDongCount {
}
def getNewValue(result){
def values = result.get("链家在线");
def values = result.get("链家在线");
if(!values){
values = result.get("搜狐焦点");
values = result.get("搜狐焦点");
}
return values[0];
}
@@ -10,7 +10,7 @@ import com.mongodb.DBCursor
import com.mongodb.DBObject
import org.bson.types.ObjectId
/**
* api调用量每半个月自动添加一次
* api调用量每半个月自动添加一次
* @author gaojuan
*
*/
@@ -28,11 +28,11 @@ class AutoAddApiCountBatch extends BaseBatch{
println "running date:"+new Date();
//查询相关接口自增配置
//查询相关接口自增配置
def addMap = getApiAddPrecent();
//查询所有接口调用量
//查询所有接口调用量
def allRows = findApiCount();
//获取新调用量
//获取新调用量
allRows = autoAddCalCount(allRows, addMap);
saveResult(allRows);
@@ -69,7 +69,7 @@ class AutoAddApiCountBatch extends BaseBatch{
println retMap.service_id +" " +addCount+" "
}else{
println "没找到相关自增配置:"+retMap.service_id
println "没找到相关自增配置:"+retMap.service_id
}
}

Some files were not shown because too many files have changed in this diff Show More