修改代码字符编码
This commit is contained in:
@@ -7,7 +7,7 @@
|
||||
<version>0.0.1-SNAPSHOT</version>
|
||||
<name>fangjia-script</name>
|
||||
<properties>
|
||||
<project.build.sourceEncoding>GBK</project.build.sourceEncoding>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
<build>
|
||||
<finalName>fangjia-script</finalName>
|
||||
@@ -19,7 +19,7 @@
|
||||
<configuration>
|
||||
<source>1.8</source>
|
||||
<target>1.8</target>
|
||||
<encoding>GBK</encoding>
|
||||
<encoding>UTF-8</encoding>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
|
||||
@@ -10,17 +10,17 @@ import org.bson.types.ObjectId
|
||||
import org.springframework.core.annotation.AnnotationUtils
|
||||
|
||||
/**
|
||||
* 继承这个类,实现process(args)方法
|
||||
* 写main方法
|
||||
* 继承这个类,实现process(args)方法
|
||||
* 写main方法
|
||||
* static void main(args){
|
||||
* runBatch(new MyBatch(), args)
|
||||
* }
|
||||
* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
* !!!!!任何由process方法抛出的异常都会被监测到,并发出错误警报!!!!!
|
||||
* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
* !!!!!任何由process方法抛出的异常都会被监测到,并发出错误警报!!!!!
|
||||
* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
*
|
||||
* 也可以手动调用jms.monitor(name, message, exception)方法发出错误警报
|
||||
* 在启动时添加 -DuseRollingPrintLog=true 的java参数可以生成滚动日志
|
||||
* 也可以手动调用jms.monitor(name, message, exception)方法发出错误警报
|
||||
* 在启动时添加 -DuseRollingPrintLog=true 的java参数可以生成滚动日志
|
||||
* @author velna
|
||||
*
|
||||
*/
|
||||
@@ -29,11 +29,11 @@ abstract class BaseBatch {
|
||||
private static final Jms __jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
|
||||
//private static final NacosProperties properties = NacosProperties.getInstance()
|
||||
|
||||
private static final BATCH_CONFIG = [singleProcess : false, //是否只可单进程运行,默认true
|
||||
reprocessOnException : false, //在发生异常后是否重新执行,默认false
|
||||
exceptionDelay : 10000, //在发生异常后延迟多少毫秒重新执行,默认10000
|
||||
warnIfProcessNotExit : false, //对singleProcess的程序,如果发现有一个进程还在运行,则发出一个警报
|
||||
isSystemExit : true //运行之后是否执行System.exit();
|
||||
private static final BATCH_CONFIG = [singleProcess : false, //是否只可单进程运行,默认true
|
||||
reprocessOnException : false, //在发生异常后是否重新执行,默认false
|
||||
exceptionDelay : 10000, //在发生异常后延迟多少毫秒重新执行,默认10000
|
||||
warnIfProcessNotExit : false, //对singleProcess的程序,如果发现有一个进程还在运行,则发出一个警报
|
||||
isSystemExit : true //运行之后是否执行System.exit();
|
||||
]
|
||||
|
||||
def logger = new PrintLogger(this.getClass())
|
||||
@@ -41,16 +41,16 @@ abstract class BaseBatch {
|
||||
private static final DBCollection TASK = MongoUtils.getCollectionByName("fangjia_batch", "task", "log.mongo.db.fangjia.com", 52002)
|
||||
|
||||
/**
|
||||
* 执行batch
|
||||
* @param args main方法的参数
|
||||
* 执行batch
|
||||
* @param args main方法的参数
|
||||
*/
|
||||
abstract void process(args);
|
||||
|
||||
/**
|
||||
* 运行batch
|
||||
* @param batch 要运行的batch
|
||||
* @param args 传给process方法的参数,通常为main方法的参数
|
||||
* @param batchConfig 见BATCH_CONFIG定义
|
||||
* 运行batch
|
||||
* @param batch 要运行的batch
|
||||
* @param args 传给process方法的参数,通常为main方法的参数
|
||||
* @param batchConfig 见BATCH_CONFIG定义
|
||||
*/
|
||||
static def runBatch(BaseBatch batch, args, batchConfig = BATCH_CONFIG){
|
||||
def myBatchConfig = [:]
|
||||
@@ -74,7 +74,7 @@ abstract class BaseBatch {
|
||||
}
|
||||
}
|
||||
task.describe = getBatchDescribe(batch)
|
||||
task.status = "运行中"
|
||||
task.status = "运行中"
|
||||
TASK.save(task)
|
||||
checkUseRollingPrintLog(batch.getClass().getSimpleName())
|
||||
if(myBatchConfig.singleProcess){
|
||||
@@ -96,7 +96,7 @@ abstract class BaseBatch {
|
||||
exception = true
|
||||
e.printStackTrace()
|
||||
__jms.monitor(batchName, '', extraData, e)
|
||||
def msg = "脚本发生异常错误 BatchName:${batchName}\t"
|
||||
def msg = "脚本发生异常错误 BatchName:${batchName}\t"
|
||||
def batchOwner = getBatchOwner(batch)
|
||||
if(batchOwner && batchOwner.size() > 0){
|
||||
def owner = null
|
||||
@@ -134,9 +134,9 @@ abstract class BaseBatch {
|
||||
sleep(myBatchConfig.exceptionDelay)
|
||||
}
|
||||
if(exception){
|
||||
task.status = "发生异常"
|
||||
task.status = "发生异常"
|
||||
}else{
|
||||
task.status = "成功"
|
||||
task.status = "成功"
|
||||
}
|
||||
task.finish_date = new Date()
|
||||
TASK.save(task)
|
||||
@@ -167,10 +167,10 @@ abstract class BaseBatch {
|
||||
ret.put(it.name(), it.toMap())
|
||||
}
|
||||
}else{
|
||||
bb.logWarn('!!!请使用'+BatchOwner.class.getName()+'注解标明本Batch的拥有者!!!')
|
||||
bb.logWarn('!!!请使用'+BatchOwner.class.getName()+'注解标明本Batch的拥有者!!!')
|
||||
}
|
||||
}catch(Exception e) {
|
||||
bb.logWarn('!!!请使用'+BatchOwner.class.getName()+'注解标明本Batch的拥有者!!!', e)
|
||||
bb.logWarn('!!!请使用'+BatchOwner.class.getName()+'注解标明本Batch的拥有者!!!', e)
|
||||
}
|
||||
return ret
|
||||
}
|
||||
@@ -187,10 +187,10 @@ abstract class BaseBatch {
|
||||
}
|
||||
}
|
||||
} else {
|
||||
bb.logWarn('!!!请使用' + BatchOwner.class.getName() + '注解标明本Batch的拥有者!!!')
|
||||
bb.logWarn('!!!请使用' + BatchOwner.class.getName() + '注解标明本Batch的拥有者!!!')
|
||||
}
|
||||
} catch (Exception e) {
|
||||
bb.logWarn('!!!解析BatchOwner失败!!!', e)
|
||||
bb.logWarn('!!!解析BatchOwner失败!!!', e)
|
||||
}
|
||||
return phones[0] ? phones[0] : ""
|
||||
}
|
||||
@@ -203,7 +203,7 @@ abstract class BaseBatch {
|
||||
return annotation.describe()
|
||||
}
|
||||
}catch(Exception e) {
|
||||
bb.logWarn('!!!请使用'+BatchOwner.class.getName()+'注解标明本Batch的拥有者!!!', e)
|
||||
bb.logWarn('!!!请使用'+BatchOwner.class.getName()+'注解标明本Batch的拥有者!!!', e)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
+6
-6
@@ -111,7 +111,7 @@ import org.apache.poi.hssf.usermodel.HSSFSheet
|
||||
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
|
||||
import org.apache.poi.ss.usermodel.Row
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
//导出开盘时间需要调查的数据
|
||||
//导出开盘时间需要调查的数据
|
||||
class ExportDistrictData {
|
||||
|
||||
private final static String datastoreDomain = "seaweed.db.fangjia.com"
|
||||
@@ -167,13 +167,13 @@ class ExportDistrictData {
|
||||
}
|
||||
|
||||
if(site.size() > 50){
|
||||
sbu.append("总量"+"\t");
|
||||
sbu.append("总量"+"\t");
|
||||
sbu.append(total+"\t");
|
||||
sbu.append("开盘时间不存在数量"+"\t");
|
||||
sbu.append("开盘时间不存在数量"+"\t");
|
||||
sbu.append(site.size()+"\n");
|
||||
sbu2.append("总量"+"\t");
|
||||
sbu2.append("总量"+"\t");
|
||||
sbu2.append(total+"\t");
|
||||
sbu2.append("开盘时间不存在数量"+"\t");
|
||||
sbu2.append("开盘时间不存在数量"+"\t");
|
||||
sbu2.append(site.size()+"\n");
|
||||
for(int i = 0; i<50 ; i++){
|
||||
int max = site.size()-1;
|
||||
@@ -213,7 +213,7 @@ class ExportDistrictData {
|
||||
public static def getAllCitiesObject() {
|
||||
DBObject q = new BasicDBObject()
|
||||
q.put('status', new BasicDBObject('$lte', 2))
|
||||
q.put('display', "荆州")
|
||||
q.put('display', "荆州")
|
||||
DBObject sq = new BasicDBObject()
|
||||
sq.put('city_grade', 1)
|
||||
sq.put('sequence', 1)
|
||||
|
||||
@@ -76,10 +76,10 @@ class FccsNewHouse {
|
||||
|
||||
BasicDBObject query = new BasicDBObject()
|
||||
query.put("url", detailUrl)
|
||||
query.put("city", "银川")
|
||||
query.put("city", "银川")
|
||||
query.put("category", "district")
|
||||
query.put("site_name", "中国房产超市网")
|
||||
query.put("page_name", "简介")
|
||||
query.put("site_name", "中国房产超市网")
|
||||
query.put("page_name", "简介")
|
||||
DBObject entity = webpage.findOne(query)
|
||||
if (entity != null){
|
||||
DBObject data = new BasicDBObject()
|
||||
@@ -90,10 +90,10 @@ class FccsNewHouse {
|
||||
} else {
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("url", detailUrl)
|
||||
data.put("city", "银川")
|
||||
data.put("city", "银川")
|
||||
data.put("category", "district")
|
||||
data.put("site_name", "中国房产超市网")
|
||||
data.put("page_name", "简介")
|
||||
data.put("site_name", "中国房产超市网")
|
||||
data.put("page_name", "简介")
|
||||
data.put("content", DataUtil.zip(detailContent))
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
@@ -107,6 +107,6 @@ class FccsNewHouse {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
println "总共新增${countSave}总共更新${countUpdate}"
|
||||
println "总共新增${countSave}总共更新${countUpdate}"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,10 +61,10 @@ class FccsSell {
|
||||
// file << detailContent
|
||||
BasicDBObject query = new BasicDBObject()
|
||||
query.put("url", detailUrl)
|
||||
query.put("city", "银川")
|
||||
query.put("city", "银川")
|
||||
query.put("category", "district")
|
||||
query.put("site_name", "中国房产超市网")
|
||||
query.put("page_name", "概况")
|
||||
query.put("site_name", "中国房产超市网")
|
||||
query.put("page_name", "概况")
|
||||
DBObject entity = webpage.findOne(query)
|
||||
if (entity != null){
|
||||
DBObject data = new BasicDBObject()
|
||||
@@ -75,10 +75,10 @@ class FccsSell {
|
||||
} else {
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("url", detailUrl)
|
||||
data.put("city", "银川")
|
||||
data.put("city", "银川")
|
||||
data.put("category", "district")
|
||||
data.put("site_name", "中国房产超市网")
|
||||
data.put("page_name", "概况")
|
||||
data.put("site_name", "中国房产超市网")
|
||||
data.put("page_name", "概况")
|
||||
data.put("content", DataUtil.zip(detailContent))
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
@@ -92,7 +92,7 @@ class FccsSell {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
println "总共新增${countSave}总共更新${countUpdate}"
|
||||
println "总共新增${countSave}总共更新${countUpdate}"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -16,23 +16,23 @@ class Enlight {
|
||||
private static final DBCollection COMMAND_COLL = MongoUtils.getCollectionByName("fangjia_log", "crawl_command", "log.mongo.db.fangjia.com");
|
||||
/*
|
||||
public static final def JOBS = [
|
||||
'5588f2f7e4b0a7ef70e98edb' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'赶集', cat:'rent'],//赶集
|
||||
'5588ec3be4b0ffa7d5e23dad' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'58同城', cat:'rent'],//58同城
|
||||
'5588fbfce4b0a7ef70e98ef3' : [isEncodingUrl:true, encoding:'GBK', enable:true, jobName:'搜房', cat:'rent'],//搜房
|
||||
'5589057ee4b0a7ef70e98f48' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'安居客', cat:'rent'],//安居客
|
||||
'55890362e4b0a7ef70e98f36' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'我爱我家', cat:'rent'],//我爱我家
|
||||
'55890f74e4b0a7ef70e98f87' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'515房产网', cat:'rent'],//515房产网
|
||||
'55892150e4b0a7ef70e98fc4' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'带看网', cat:'rent'],//带看网
|
||||
'558923f5e4b0a7ef70e98fdb' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'购房者', cat:'rent'],//购房者
|
||||
'5589256de4b0a7ef70e98ff3' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'城市房产网', cat:'rent'],//城市房产网
|
||||
'558925bce4b0a7ef70e9903b' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'百姓网', cat:'rent'],//百姓网
|
||||
'563c202ee4b0126639288c3f' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'新浪', cat:'rent'],//新浪
|
||||
'5588f2f7e4b0a7ef70e98edb' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'赶集', cat:'rent'],//赶集
|
||||
'5588ec3be4b0ffa7d5e23dad' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'58同城', cat:'rent'],//58同城
|
||||
'5588fbfce4b0a7ef70e98ef3' : [isEncodingUrl:true, encoding:'GBK', enable:true, jobName:'搜房', cat:'rent'],//搜房
|
||||
'5589057ee4b0a7ef70e98f48' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'安居客', cat:'rent'],//安居客
|
||||
'55890362e4b0a7ef70e98f36' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'我爱我家', cat:'rent'],//我爱我家
|
||||
'55890f74e4b0a7ef70e98f87' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'515房产网', cat:'rent'],//515房产网
|
||||
'55892150e4b0a7ef70e98fc4' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'带看网', cat:'rent'],//带看网
|
||||
'558923f5e4b0a7ef70e98fdb' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'购房者', cat:'rent'],//购房者
|
||||
'5589256de4b0a7ef70e98ff3' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'城市房产网', cat:'rent'],//城市房产网
|
||||
'558925bce4b0a7ef70e9903b' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'百姓网', cat:'rent'],//百姓网
|
||||
'563c202ee4b0126639288c3f' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'新浪', cat:'rent'],//新浪
|
||||
]
|
||||
*/
|
||||
|
||||
static main(args) {
|
||||
/*
|
||||
File f = new File("D:/乌鲁木齐.txt")
|
||||
File f = new File("D:/乌鲁木齐.txt")
|
||||
JOBS.each {jobId, conf->
|
||||
if(!conf.get('enable')) {
|
||||
return false;
|
||||
@@ -49,29 +49,29 @@ class Enlight {
|
||||
vars = "keyword:${line}";
|
||||
}
|
||||
|
||||
String conditions = "city:乌鲁木齐";
|
||||
String conditions = "city:乌鲁木齐";
|
||||
JSONObject res = addTemplateJobUrl(jobId, vars, conditions)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
*/
|
||||
def list = ['昆重小区',
|
||||
'邮政小区',
|
||||
'中央华冠',
|
||||
'瑞源棕榈泉',
|
||||
'卧云仙居',
|
||||
'隆格兰花园',
|
||||
'和风颐园',
|
||||
'半山七哩溪',
|
||||
'培星东岸',
|
||||
'小吉坡小区',
|
||||
'水电设计院小区',
|
||||
def list = ['昆重小区',
|
||||
'邮政小区',
|
||||
'中央华冠',
|
||||
'瑞源棕榈泉',
|
||||
'卧云仙居',
|
||||
'隆格兰花园',
|
||||
'和风颐园',
|
||||
'半山七哩溪',
|
||||
'培星东岸',
|
||||
'小吉坡小区',
|
||||
'水电设计院小区',
|
||||
|
||||
]
|
||||
list.each {item->
|
||||
def encoded = URLEncoder.encode(item, "UTF-8")
|
||||
addTemplateJobUrl("5588ec3be4b0ffa7d5e23dad","keyword:${encoded}","city:昆明")
|
||||
addTemplateJobUrl("5588ec3be4b0ffa7d5e23dad","keyword:${encoded}","city:昆明")
|
||||
}
|
||||
|
||||
|
||||
@@ -79,11 +79,11 @@ class Enlight {
|
||||
|
||||
|
||||
/**
|
||||
* 向抓取系统添加抓取任务
|
||||
* @param jobId 任务号
|
||||
* @param vars 变量
|
||||
* @param conditions 查询条件
|
||||
* @return 200:成功 100:任务已停止 >400任务号错误或不存在 -1:其他错误
|
||||
* 向抓取系统添加抓取任务
|
||||
* @param jobId 任务号
|
||||
* @param vars 变量
|
||||
* @param conditions 查询条件
|
||||
* @return 200:成功 100:任务已停止 >400任务号错误或不存在 -1:其他错误
|
||||
*/
|
||||
protected static JSONObject addTemplateJobUrl(String jobId, String vars, String conditions) {
|
||||
def result = ATTEMIS_HTTP.invoke("/api/addTemplateJobUrl", [jobId : jobId, vars: vars, conditions:conditions], 'UTF-8')
|
||||
|
||||
@@ -16,8 +16,8 @@ class HeExtends {
|
||||
static main(args) {
|
||||
DBCollection COMMAND_COLL = MongoUtils.getCollectionByName("fangjia_log", "crawl_command_monitor", "log.mongo.db.fangjia.com");
|
||||
DBObject fields = new BasicDBObject()
|
||||
fields.put("city","³¤É³")
|
||||
fields.put("keyword", "ÁúÌÁÐ¡Çø")
|
||||
fields.put("city","长沙")
|
||||
fields.put("keyword", "龙塘小区")
|
||||
fields.put("job_id", "559f237de4b0e6f33e2bbb68")
|
||||
fields.put("session_id", "56726578e4b07da06f2efd6b")
|
||||
DBCursor cursor = COMMAND_COLL.find(fields)
|
||||
|
||||
@@ -12,34 +12,34 @@ class HeTests {
|
||||
|
||||
public static final def JOBS = [
|
||||
|
||||
'559f237de4b0e6f33e2bbb68' : [isEncodingUrl:true, encoding:'GBK', enable:true, jobName:'搜房', cat:'sell'],//搜房
|
||||
'559f277ee4b0133c041de396' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'58同城', cat:'sell'],//58同城
|
||||
'559f288ee4b0133c041de3b3' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'赶集网', cat:'sell'],//赶集网
|
||||
'559f2a7be4b0133c041de3d6' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'安居客', cat:'sell'],//安居客
|
||||
'559f2dc6e4b0133c041de42a' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'搜狐焦点', cat:'sell'],//搜狐焦点
|
||||
'559f309ce4b0133c041dea84' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'链家在线', cat:'sell'],//链家在线
|
||||
'559f31c8e4b0133c041df42a' : [isEncodingUrl:true, encoding:'GBK', enable:true, jobName:'新浪乐居', cat:'sell'],//新浪乐居
|
||||
'559f32bbe4b0133c041dfaf7' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'吉屋网', cat:'sell'],//吉屋网
|
||||
'559f37b6e4b0133c041e0e2d' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'515房产网', cat:'sell'],//515房产网
|
||||
'559f3918e4b0133c041e1521' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'我爱我家', cat:'sell'],//我爱我家
|
||||
'559f3a7de4b0133c041e1c41' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'楼盘网', cat:'sell'],//楼盘网
|
||||
'559f3b52e4b0133c041e1f0f' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'好房在线', cat:'sell'],//好房在线
|
||||
'559f3ca8e4b0133c041e247f' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'百度乐居', cat:'sell'],//百度乐居
|
||||
'559f3d91e4b0133c041e27ae' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'带看网', cat:'sell'],//带看网
|
||||
'559f3e6ce4b0133c041e29f4' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'百姓网', cat:'sell'],//百姓网
|
||||
'559f3f61e4b0133c041e2cce' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'城市房产网', cat:'sell'],//城市房产网
|
||||
'559f237de4b0e6f33e2bbb68' : [isEncodingUrl:true, encoding:'GBK', enable:true, jobName:'搜房', cat:'sell'],//搜房
|
||||
'559f277ee4b0133c041de396' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'58同城', cat:'sell'],//58同城
|
||||
'559f288ee4b0133c041de3b3' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'赶集网', cat:'sell'],//赶集网
|
||||
'559f2a7be4b0133c041de3d6' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'安居客', cat:'sell'],//安居客
|
||||
'559f2dc6e4b0133c041de42a' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'搜狐焦点', cat:'sell'],//搜狐焦点
|
||||
'559f309ce4b0133c041dea84' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'链家在线', cat:'sell'],//链家在线
|
||||
'559f31c8e4b0133c041df42a' : [isEncodingUrl:true, encoding:'GBK', enable:true, jobName:'新浪乐居', cat:'sell'],//新浪乐居
|
||||
'559f32bbe4b0133c041dfaf7' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'吉屋网', cat:'sell'],//吉屋网
|
||||
'559f37b6e4b0133c041e0e2d' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'515房产网', cat:'sell'],//515房产网
|
||||
'559f3918e4b0133c041e1521' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'我爱我家', cat:'sell'],//我爱我家
|
||||
'559f3a7de4b0133c041e1c41' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'楼盘网', cat:'sell'],//楼盘网
|
||||
'559f3b52e4b0133c041e1f0f' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'好房在线', cat:'sell'],//好房在线
|
||||
'559f3ca8e4b0133c041e247f' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'百度乐居', cat:'sell'],//百度乐居
|
||||
'559f3d91e4b0133c041e27ae' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'带看网', cat:'sell'],//带看网
|
||||
'559f3e6ce4b0133c041e29f4' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'百姓网', cat:'sell'],//百姓网
|
||||
'559f3f61e4b0133c041e2cce' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'城市房产网', cat:'sell'],//城市房产网
|
||||
|
||||
'5588f2f7e4b0a7ef70e98edb' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'赶集', cat:'rent'],//赶集
|
||||
'5588ec3be4b0ffa7d5e23dad' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'58同城', cat:'rent'],//58同城
|
||||
'5588fbfce4b0a7ef70e98ef3' : [isEncodingUrl:true, encoding:'GBK', enable:true, jobName:'搜房', cat:'rent'],//搜房
|
||||
'5589057ee4b0a7ef70e98f48' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'安居客', cat:'rent'],//安居客
|
||||
'55890362e4b0a7ef70e98f36' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'我爱我家', cat:'rent'],//我爱我家
|
||||
'55890f74e4b0a7ef70e98f87' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'515房产网', cat:'rent'],//515房产网
|
||||
'55892150e4b0a7ef70e98fc4' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'带看网', cat:'rent'],//带看网
|
||||
'558923f5e4b0a7ef70e98fdb' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'购房者', cat:'rent'],//购房者
|
||||
'5589256de4b0a7ef70e98ff3' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'城市房产网', cat:'rent'],//城市房产网
|
||||
'558925bce4b0a7ef70e9903b' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'百姓网', cat:'rent'],//百姓网
|
||||
'563c202ee4b0126639288c3f' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'新浪', cat:'rent'],//新浪
|
||||
'5588f2f7e4b0a7ef70e98edb' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'赶集', cat:'rent'],//赶集
|
||||
'5588ec3be4b0ffa7d5e23dad' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'58同城', cat:'rent'],//58同城
|
||||
'5588fbfce4b0a7ef70e98ef3' : [isEncodingUrl:true, encoding:'GBK', enable:true, jobName:'搜房', cat:'rent'],//搜房
|
||||
'5589057ee4b0a7ef70e98f48' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'安居客', cat:'rent'],//安居客
|
||||
'55890362e4b0a7ef70e98f36' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'我爱我家', cat:'rent'],//我爱我家
|
||||
'55890f74e4b0a7ef70e98f87' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'515房产网', cat:'rent'],//515房产网
|
||||
'55892150e4b0a7ef70e98fc4' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'带看网', cat:'rent'],//带看网
|
||||
'558923f5e4b0a7ef70e98fdb' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'购房者', cat:'rent'],//购房者
|
||||
'5589256de4b0a7ef70e98ff3' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'城市房产网', cat:'rent'],//城市房产网
|
||||
'558925bce4b0a7ef70e9903b' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'百姓网', cat:'rent'],//百姓网
|
||||
'563c202ee4b0126639288c3f' : [isEncodingUrl:true, encoding:'UTF-8', enable:true, jobName:'新浪', cat:'rent'],//新浪
|
||||
]
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
@@ -19,7 +19,7 @@ class IntroTest {
|
||||
// }
|
||||
// newList.each { item -> println item }
|
||||
|
||||
Table table = DatabaseBuilder.open(new File("d:\\profile.accdb")).getTable("»ù±¾²éѯ");
|
||||
Table table = DatabaseBuilder.open(new File("d:\\profile.accdb")).getTable("基本查询");
|
||||
for(Row row : table) {
|
||||
|
||||
String name = row.get("name").toString();
|
||||
|
||||
@@ -20,7 +20,7 @@ class LatitudeTest {
|
||||
def suspectedListNames = []
|
||||
def filePath = "d:\\suspected.xls"
|
||||
field.append("name",1)
|
||||
query.append("city","ÉϺ£")
|
||||
query.append("city","上海")
|
||||
field.append("region", 1)
|
||||
field.append("value_type", 5)
|
||||
// seaweed.find(query, field).limit(10).each {item ->
|
||||
|
||||
@@ -167,7 +167,7 @@ class LoudongOnce {
|
||||
Iterator<Element> iter = elements.iterator()
|
||||
while(iter.hasNext()) {
|
||||
Element ele = iter.next()
|
||||
if(ele.html().indexOf("ÏêÇé") > -1) {
|
||||
if(ele.html().indexOf("详情") > -1) {
|
||||
continue;
|
||||
}
|
||||
String a = ele.getElementsByAttribute("href").toString();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -125,23 +125,23 @@ public class Polygon {
|
||||
int nCross = 0;
|
||||
for (int i = 0; i < verticesCount; ++ i) {
|
||||
Mpoint p1 = vertices.get(i);
|
||||
Mpoint p2 = vertices.get((i + 1) % verticesCount);// 求解 y=p.y 与 p1 p2 的交点
|
||||
if ( p1.getMapLoationY() == p2.getMapLoationY() ) { // p1p2 与 y=p0.y平行
|
||||
Mpoint p2 = vertices.get((i + 1) % verticesCount);// 求解 y=p.y 与 p1 p2 的交点
|
||||
if ( p1.getMapLoationY() == p2.getMapLoationY() ) { // p1p2 与 y=p0.y平行
|
||||
continue;
|
||||
}
|
||||
if ( lat < Math.min(p1.getY(), p2.getY()) ) { // 交点在p1p2延长线上
|
||||
if ( lat < Math.min(p1.getY(), p2.getY()) ) { // 交点在p1p2延长线上
|
||||
continue;
|
||||
}
|
||||
if ( lat >= Math.max(p1.getY(), p2.getY()) ) { // 交点在p1p2延长线上
|
||||
if ( lat >= Math.max(p1.getY(), p2.getY()) ) { // 交点在p1p2延长线上
|
||||
continue;
|
||||
}
|
||||
// 求交点的 X 坐标
|
||||
// 求交点的 X 坐标
|
||||
double x = (lat - p1.getMapLoationY()) * (p2.getMapLoationX() - p1.getMapLoationX()) / (p2.getMapLoationY() - p1.getMapLoationY()) + p1.getMapLoationX();
|
||||
if ( x > lng) { // 只统计单边交点
|
||||
if ( x > lng) { // 只统计单边交点
|
||||
nCross++;
|
||||
}
|
||||
}
|
||||
// 单边交点为偶数,点在多边形之外
|
||||
// 单边交点为偶数,点在多边形之外
|
||||
return (nCross%2==1);
|
||||
}
|
||||
|
||||
|
||||
@@ -13,25 +13,25 @@ class PoseidonOnce {
|
||||
static def poseidonConn = Sql.newInstance("jdbc:mysql://192.168.0.132:4306/poseidon?useUnicode=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver")
|
||||
|
||||
static def soufunRegionMap = [
|
||||
"030" : "宝山",
|
||||
"020" : "长宁",
|
||||
"0996" : "崇明",
|
||||
"032" : "奉贤",
|
||||
"023" : "虹口",
|
||||
"024" : "黄埔",
|
||||
"029" : "嘉定",
|
||||
"035" : "金山",
|
||||
"021" : "静安",
|
||||
"022" : "卢湾",
|
||||
"018" : "闵行",
|
||||
"031" : "青浦",
|
||||
"0586" : "松江",
|
||||
"019" : "徐汇",
|
||||
"026" : "杨浦",
|
||||
"027" : "闸北",
|
||||
"01046" : "上海周边",
|
||||
"028" : "普陀",
|
||||
"025" : "浦东"
|
||||
"030" : "宝山",
|
||||
"020" : "长宁",
|
||||
"0996" : "崇明",
|
||||
"032" : "奉贤",
|
||||
"023" : "虹口",
|
||||
"024" : "黄埔",
|
||||
"029" : "嘉定",
|
||||
"035" : "金山",
|
||||
"021" : "静安",
|
||||
"022" : "卢湾",
|
||||
"018" : "闵行",
|
||||
"031" : "青浦",
|
||||
"0586" : "松江",
|
||||
"019" : "徐汇",
|
||||
"026" : "杨浦",
|
||||
"027" : "闸北",
|
||||
"01046" : "上海周边",
|
||||
"028" : "普陀",
|
||||
"025" : "浦东"
|
||||
]
|
||||
|
||||
public static void main(String[] args) {
|
||||
@@ -111,7 +111,7 @@ class PoseidonOnce {
|
||||
}
|
||||
|
||||
/**
|
||||
* 从crawl_request移除数据
|
||||
* 从crawl_request移除数据
|
||||
* @return
|
||||
*/
|
||||
public static def removeDataFromCrawlRequest() {
|
||||
|
||||
@@ -19,16 +19,16 @@ class SaveToWebpage {
|
||||
println url
|
||||
saveToWebpage(url,fileContents)
|
||||
}
|
||||
println "总共新增${countSave}总共更新${countUpdate}"
|
||||
println "总共新增${countSave}总共更新${countUpdate}"
|
||||
}
|
||||
|
||||
static void saveToWebpage(String detailUrl, String detailContent){
|
||||
BasicDBObject query = new BasicDBObject()
|
||||
query.put("url", detailUrl)
|
||||
query.put("city", "嘉兴")
|
||||
query.put("city", "嘉兴")
|
||||
query.put("category", "district")
|
||||
query.put("site_name", "中国房产超市网")
|
||||
query.put("page_name", "概况")
|
||||
query.put("site_name", "中国房产超市网")
|
||||
query.put("page_name", "概况")
|
||||
DBObject entity = webpage.findOne(query)
|
||||
if (entity != null){
|
||||
DBObject data = new BasicDBObject()
|
||||
@@ -39,10 +39,10 @@ class SaveToWebpage {
|
||||
} else {
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("url", detailUrl)
|
||||
data.put("city", "嘉兴")
|
||||
data.put("city", "嘉兴")
|
||||
data.put("category", "district")
|
||||
data.put("site_name", "中国房产超市网")
|
||||
data.put("page_name", "概况")
|
||||
data.put("site_name", "中国房产超市网")
|
||||
data.put("page_name", "概况")
|
||||
data.put("content", DataUtil.zip(detailContent))
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
|
||||
+4
-4
@@ -11,7 +11,7 @@ class SellDataAddSighterBatch extends BaseBatch{
|
||||
private static final DBCollection HOUSE_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
private static final DBCollection TOBEGRABBED = MongoUtils.getCollectionByName("fangjia_log", "artemis_message", "mongo.db.fangjia.com")
|
||||
|
||||
//假设在周一时运行
|
||||
//假设在周一时运行
|
||||
Date d = new Date() - 6;
|
||||
|
||||
|
||||
@@ -26,12 +26,12 @@ class SellDataAddSighterBatch extends BaseBatch{
|
||||
// String[] attrs = line.split("\t")
|
||||
// println attrs[0] + ":" + attrs[1]
|
||||
// DBObject addDbo = new BasicDBObject()
|
||||
// addDbo.put("city","绍兴")
|
||||
// addDbo.put("city","绍兴")
|
||||
// addDbo.put("c_date", new Date())
|
||||
// addDbo.put("name", attrs[1])
|
||||
// addDbo.put("region",attrs[0])
|
||||
// addDbo.put("status", 0)
|
||||
// addDbo.put("user_name", "机器")
|
||||
// addDbo.put("user_name", "机器")
|
||||
// addDbo.put("category", "sell")
|
||||
// TOBEGRABBED.save(addDbo)
|
||||
// }
|
||||
@@ -57,7 +57,7 @@ class SellDataAddSighterBatch extends BaseBatch{
|
||||
addDbo.put("name", dbo.get('district_name'))
|
||||
addDbo.put("region",dbo.get('region'))
|
||||
addDbo.put("status", 0)
|
||||
addDbo.put("user_name", "贺绿源")
|
||||
addDbo.put("user_name", "贺绿源")
|
||||
addDbo.put("category", "sell")
|
||||
TOBEGRABBED.save(addDbo)
|
||||
i++
|
||||
|
||||
@@ -21,7 +21,7 @@ class TransferTest {
|
||||
static main(args) {
|
||||
int c = 0
|
||||
DBObject query = new BasicDBObject()
|
||||
query.put("city","ÉîÛÚ")
|
||||
query.put("city","深圳")
|
||||
DBCursor dbCursor=LOUDONG_CONN_TEST.find(query)
|
||||
dbCursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
while (dbCursor.hasNext()) {
|
||||
@@ -29,7 +29,7 @@ class TransferTest {
|
||||
c++
|
||||
DBObject obj = dbCursor.next()
|
||||
DBObject queryOnline = new BasicDBObject()
|
||||
queryOnline.put("city","ÉîÛÚ")
|
||||
queryOnline.put("city","深圳")
|
||||
queryOnline.put("region", obj.get("region"))
|
||||
queryOnline.put("name", obj.get("name"))
|
||||
queryOnline.put("house_num", obj.get("house_num"))
|
||||
@@ -60,7 +60,7 @@ class TransferTest {
|
||||
}else
|
||||
{
|
||||
DBObject insertObj = new BasicDBObject()
|
||||
insertObj.put("city", 'ÉîÛÚ')
|
||||
insertObj.put("city", '深圳')
|
||||
insertObj.put("name", obj.get("name"))
|
||||
insertObj.put("region", obj.get("region"))
|
||||
insertObj.put("c_date", obj.get("c_date"))
|
||||
|
||||
@@ -9,20 +9,20 @@ import com.mongodb.DBObject
|
||||
class UpdateData {
|
||||
|
||||
public static final String[] estateType2s = [
|
||||
"商业街商铺",
|
||||
"市场类商铺",
|
||||
"社区内铺",
|
||||
"住宅底商",
|
||||
"百货购物中心商铺",
|
||||
"商业综合体",
|
||||
"商务区商铺",
|
||||
"交通设施商铺"
|
||||
"商业街商铺",
|
||||
"市场类商铺",
|
||||
"社区内铺",
|
||||
"住宅底商",
|
||||
"百货购物中心商铺",
|
||||
"商业综合体",
|
||||
"商务区商铺",
|
||||
"交通设施商铺"
|
||||
];
|
||||
|
||||
static main(args) {
|
||||
DSCollection ddColl = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
|
||||
DBObject query = new BasicDBObject()
|
||||
// query.put('city', '上海')
|
||||
// query.put('city', '上海')
|
||||
query.put('cat', 'shop')
|
||||
query.put('visible', 0)
|
||||
|
||||
@@ -33,8 +33,8 @@ class UpdateData {
|
||||
DBObject dbo = iter.next()
|
||||
String et2 = dbo.get('estate_type2')
|
||||
println dbo.get('name')
|
||||
// if(et2.equals('内铺')) {
|
||||
// et2 = '社区内铺'
|
||||
// if(et2.equals('内铺')) {
|
||||
// et2 = '社区内铺'
|
||||
// }
|
||||
if(estateType2Compare(et2)) {
|
||||
DBObject uq = new BasicDBObject()
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package com.fangjia.script
|
||||
|
||||
public enum Watcher {
|
||||
//ÀëְͬÊÂ
|
||||
//离职同事
|
||||
LiuPeiPei('13621626685', 'wangwenyan@fangjia.com'),
|
||||
YanZhou('13621626685','wangwenyan@fangjia.com'),
|
||||
HuangXiaoHu('13621626685', 'wangwenyan@fangjia.com'),
|
||||
@@ -18,7 +18,7 @@ public enum Watcher {
|
||||
FangRui('17717680589','wangwenyan@fangjia.com'),
|
||||
DuXiaoYu('13621626685', 'wangwenyan@fangjia.com'),
|
||||
|
||||
//ÔÚְͬÊÂ
|
||||
//在职同事
|
||||
LiXue('13311649646','wangwenyan@fangjia.com'),
|
||||
WangWenYan('13916076378','wangwenyan@fangjia.com'),
|
||||
GaoJuan('15021384995','gaojuan@fangjia.com'),
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ class AddMapData {
|
||||
static main(args) {
|
||||
DBObject obj = new BasicDBObject()
|
||||
obj.put("cityCode",21);
|
||||
obj.put("name", "Èýɳ");
|
||||
obj.put("name", "三沙");
|
||||
obj.put("lat",16.838987d);
|
||||
obj.put("lng",112.340047d);
|
||||
obj.put("zoom", 15);
|
||||
|
||||
+10
-10
@@ -43,16 +43,16 @@ class AddressParserService {
|
||||
ret.cat = djson.cat
|
||||
|
||||
if (djson.precise == 0) {
|
||||
// 匹配到小区
|
||||
// 匹配到小区
|
||||
ret.cla = djson.cat
|
||||
ret.qw = djson.cat
|
||||
|
||||
if (ret.cat[0] == 'district') {
|
||||
ret.catStr = '小区'
|
||||
ret.catStr = '小区'
|
||||
} else if (ret.cat[0] == 'office') {
|
||||
ret.catStr = '写字楼'
|
||||
ret.catStr = '写字楼'
|
||||
} else if (ret.cat[0] == 'shop') {
|
||||
ret.catStr = '商铺'
|
||||
ret.catStr = '商铺'
|
||||
}
|
||||
} else if (djson.precise == 1) {
|
||||
ret.cla = djson.cla
|
||||
@@ -60,13 +60,13 @@ class AddressParserService {
|
||||
|
||||
for (c in ret.cla) {
|
||||
if (isDistrict(c)) {
|
||||
ret.catStr = '小区'
|
||||
ret.catStr = '小区'
|
||||
break
|
||||
} else if (isOffice(c)) {
|
||||
ret.catStr = '写字楼'
|
||||
ret.catStr = '写字楼'
|
||||
break
|
||||
} else if (isShop(c)) {
|
||||
ret.catStr = '商铺'
|
||||
ret.catStr = '商铺'
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -80,9 +80,9 @@ class AddressParserService {
|
||||
return ret
|
||||
}
|
||||
|
||||
private final static List<String> DISTRICT_CLAS = Arrays.asList("地产小区", "小区/楼盘", "别墅");
|
||||
private final static List<String> OFFICE_CLAS = Arrays.asList("商务大厦", "办公大厦");
|
||||
private final static List<String> SHOP_CLAS = Arrays.asList("综合商场/购物中心", "购物");
|
||||
private final static List<String> DISTRICT_CLAS = Arrays.asList("地产小区", "小区/楼盘", "别墅");
|
||||
private final static List<String> OFFICE_CLAS = Arrays.asList("商务大厦", "办公大厦");
|
||||
private final static List<String> SHOP_CLAS = Arrays.asList("综合商场/购物中心", "购物");
|
||||
|
||||
private boolean isDistrict(String cla) {
|
||||
for (String c : DISTRICT_CLAS) {
|
||||
|
||||
+2
-2
@@ -129,10 +129,10 @@ class BaiduAPIService {
|
||||
|
||||
static main(args) {
|
||||
BaiduAPIService s = new BaiduAPIService()
|
||||
// println s.geocoder('上海', '一二八纪念路933弄')
|
||||
// println s.geocoder('上海', '一二八纪念路933弄')
|
||||
|
||||
// println s.geoconv6To5('13510836.5,3661517.3')
|
||||
|
||||
println s.roundSearch(121.48175256075, 31.25314063543, 300, '学校', 10, 0, 1)
|
||||
println s.roundSearch(121.48175256075, 31.25314063543, 300, '学校', 10, 0, 1)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,13 +11,13 @@ class Bianmen {
|
||||
static DSCollection ADDREPO = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.addrepo");
|
||||
|
||||
static main(args) {
|
||||
// 处理边门
|
||||
def p0 = Pattern.compile('.*[0-9一二三四五六七八九]号门[))]?$')
|
||||
def p1 = Pattern.compile('.*[东南西北]+[1-9一二三四五六七八九]?[大小]?门[))]?$')
|
||||
def p2 = Pattern.compile('.*[前后正边侧大小路]门[))]?$')
|
||||
def p = Pattern.compile('.*[东南西北]+[1-9一二三四五六七八九]?[大小]?[前后]?(号)?(边)?门$')
|
||||
println p.matcher("XXXX南一门").matches()
|
||||
// DBObject d = new BasicDBObject('$set', new BasicDBObject('std_tag', '出入口'))
|
||||
// 处理边门
|
||||
def p0 = Pattern.compile('.*[0-9一二三四五六七八九]号门[))]?$')
|
||||
def p1 = Pattern.compile('.*[东南西北]+[1-9一二三四五六七八九]?[大小]?门[))]?$')
|
||||
def p2 = Pattern.compile('.*[前后正边侧大小路]门[))]?$')
|
||||
def p = Pattern.compile('.*[东南西北]+[1-9一二三四五六七八九]?[大小]?[前后]?(号)?(边)?门$')
|
||||
println p.matcher("XXXX南一门").matches()
|
||||
// DBObject d = new BasicDBObject('$set', new BasicDBObject('std_tag', '出入口'))
|
||||
//
|
||||
// def index = 0
|
||||
// ADDREPO.find().batchSize(100).bulk(false).each {
|
||||
|
||||
+2
-2
@@ -18,7 +18,7 @@ class CnTokenizer2 extends CnTokenizer {
|
||||
private static Logger LOGGER = LoggerFactory.getLogger(CnTokenizer2.class);
|
||||
|
||||
/**
|
||||
* 分词结果
|
||||
* 分词结果
|
||||
*
|
||||
* @param k
|
||||
* @return
|
||||
@@ -83,7 +83,7 @@ class CnTokenizer2 extends CnTokenizer {
|
||||
return Collections.EMPTY_LIST;
|
||||
}
|
||||
|
||||
private final static Pattern ALPHA_PATTERN = Pattern.compile("[a-zA-Z\\d\\-负一二三四五六七八九十零两个百千万亿]+(#|弄|号楼?|[甲乙丙丁]?室|幢|橦|栋|座|口|楼|房|单元|区|层)?")
|
||||
private final static Pattern ALPHA_PATTERN = Pattern.compile("[a-zA-Z\\d\\-负一二三四五六七八九十零两个百千万亿]+(#|弄|号楼?|[甲乙丙丁]?室|幢|橦|栋|座|口|楼|房|单元|区|层)?")
|
||||
private static String fetchAlpha(List<Token> ret, String t) {
|
||||
Matcher m = ALPHA_PATTERN.matcher(t);
|
||||
def value = ""
|
||||
|
||||
+4
-4
@@ -7,7 +7,7 @@ import org.apache.poi.xssf.usermodel.XSSFFormulaEvaluator
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook
|
||||
|
||||
/**
|
||||
* apache poi excel解析器
|
||||
* apache poi excel解析器
|
||||
* Created by yk on 2014/4/24.
|
||||
*/
|
||||
class ExcelParser {
|
||||
@@ -89,12 +89,12 @@ class ExcelParser {
|
||||
try {
|
||||
CellValue cellValue;
|
||||
cellValue = evaluator.evaluate(cell);
|
||||
switch (cellValue.getCellType()) { //判断公式类型
|
||||
switch (cellValue.getCellType()) { //判断公式类型
|
||||
case Cell.CELL_TYPE_BOOLEAN:
|
||||
value = cellValue.getBooleanValue();
|
||||
break;
|
||||
case Cell.CELL_TYPE_NUMERIC:
|
||||
// 处理日期
|
||||
// 处理日期
|
||||
if (DateUtil.isCellDateFormatted(cell)) {
|
||||
value = cell.getDateCellValue();
|
||||
} else {
|
||||
@@ -136,7 +136,7 @@ class ExcelParser {
|
||||
}
|
||||
|
||||
public static void main(String[]args) {
|
||||
def filename = 'C:\\Users\\yk\\Desktop\\yongjia\\永嘉地址数据(上海)_V1.xlsx'
|
||||
def filename = 'C:\\Users\\yk\\Desktop\\yongjia\\永嘉地址数据(上海)_V1.xlsx'
|
||||
ExcelParser parser = new ExcelParser(filename)
|
||||
def (headers, rows) = parser.parse()
|
||||
println 'Headers'
|
||||
|
||||
+4
-4
@@ -23,8 +23,8 @@ class ExportBaiDuMapCrawlarResult {
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = readTxt("e:\\数据导出\\城市列表\\citys.txt")
|
||||
def keyword = "住宅区,小区,公寓,别墅"
|
||||
def citys = readTxt("e:\\数据导出\\城市列表\\citys.txt")
|
||||
def keyword = "住宅区,小区,公寓,别墅"
|
||||
def rets = [];
|
||||
def index = 0;
|
||||
for(def city : citys){
|
||||
@@ -39,7 +39,7 @@ class ExportBaiDuMapCrawlarResult {
|
||||
rets << ret
|
||||
println index +" " + ret
|
||||
}
|
||||
export(rets, "e:\\数据导出\\百度抓取\\百度抓取结果.xlsx")
|
||||
export(rets, "e:\\数据导出\\百度抓取\\百度抓取结果.xlsx")
|
||||
|
||||
}
|
||||
|
||||
@@ -87,7 +87,7 @@ class ExportBaiDuMapCrawlarResult {
|
||||
def WORK_BOOK = null;
|
||||
try {
|
||||
if (file.exists()) {
|
||||
FileInputStream fs=new FileInputStream(path); //获取d://test.xls
|
||||
FileInputStream fs=new FileInputStream(path); //获取d://test.xls
|
||||
WORK_BOOK = WorkbookFactory.create(fs)
|
||||
WORK_BOOK.setForceFormulaRecalculation(true)
|
||||
sheet = WORK_BOOK.getSheetAt(0);
|
||||
|
||||
+3
-3
@@ -21,12 +21,12 @@ class InvokeHttpRequest {
|
||||
private final static int reCount = 10
|
||||
|
||||
private final static List PROXY_HOSTS = [
|
||||
// 物理机
|
||||
// 物理机
|
||||
new HttpHost("192.168.0.99", 3234),
|
||||
new HttpHost("192.168.0.98", 3234),
|
||||
new HttpHost("192.168.0.96", 3234)
|
||||
|
||||
// 虚拟机
|
||||
// 虚拟机
|
||||
// "192.168.0.102",
|
||||
// "192.168.0.101",
|
||||
// "192.168.0.103",
|
||||
@@ -44,7 +44,7 @@ class InvokeHttpRequest {
|
||||
private static synchronized HttpHost changeProxyHost() {
|
||||
def now = new Date().getTime()
|
||||
if (null != lastChangeTime && now - lastChangeTime < 120000) {
|
||||
// 间隔小于2分钟的不处理
|
||||
// 间隔小于2分钟的不处理
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ class Publish {
|
||||
|
||||
static main(args) {
|
||||
def index = 0
|
||||
// 按城市上线 289是城市代码 上海
|
||||
// 按城市上线 289是城市代码 上海
|
||||
DBCursor cursor = DB_3.find(new BasicDBObject('cityCode', 131)).batchSize(100)
|
||||
cursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
cursor.each {
|
||||
@@ -25,7 +25,7 @@ class Publish {
|
||||
return
|
||||
}
|
||||
|
||||
it.city = '北京'
|
||||
it.city = '北京'
|
||||
def orgRegion = it.region
|
||||
it.region = RegionData.formatRegion(it.city, orgRegion)
|
||||
|
||||
|
||||
@@ -8,14 +8,14 @@ class Test {
|
||||
|
||||
static main(args) {
|
||||
// CnDictionary.init()
|
||||
// List<Token> tokens = CnTokenizer2.tokenizer('上海市闵行长宁罗阳路莲花河畔景苑19号801室', true)
|
||||
// List<Token> tokens = CnTokenizer2.tokenizer('上海市闵行长宁罗阳路莲花河畔景苑19号801室', true)
|
||||
// println tokens
|
||||
|
||||
|
||||
|
||||
// Pattern p = Pattern.compile("[a-zA-Z\\d\\-负一二三四五六七八九十零两个百千万亿]+(#|弄|号楼?|[甲乙丙丁]?室|幢|橦|栋|座|口|楼|房|单元|区|层)?")
|
||||
// Pattern p = Pattern.compile("[a-zA-Z\\d\\-负一二三四五六七八九十零两个百千万亿]+(#|弄|号楼?|[甲乙丙丁]?室|幢|橦|栋|座|口|楼|房|单元|区|层)?")
|
||||
//
|
||||
// BufferedReader reader = new File('C:\\Users\\yk\\Desktop\\yongjia\\永嘉上海0730.txt').newReader('UTF-8')
|
||||
// BufferedReader reader = new File('C:\\Users\\yk\\Desktop\\yongjia\\永嘉上海0730.txt').newReader('UTF-8')
|
||||
// reader.eachLine { line ->
|
||||
// if (!line) {
|
||||
// return
|
||||
|
||||
@@ -17,43 +17,43 @@ import com.mongodb.DBObject
|
||||
import com.mongodb.Mongo
|
||||
|
||||
class Yongjia {
|
||||
def final static QUANJIAO = ['0' : '0',
|
||||
'1' : '1',
|
||||
'2' : '2',
|
||||
'3' : '3',
|
||||
'4' : '4',
|
||||
'5' : '5',
|
||||
'6' : '6',
|
||||
'7' : '7',
|
||||
'8' : '8',
|
||||
'9' : '9',
|
||||
'A' : 'A',
|
||||
'B' : 'B',
|
||||
'C' : 'C',
|
||||
'D' : 'D',
|
||||
'E' : 'E',
|
||||
'F' : 'F',
|
||||
'G' : 'G',
|
||||
'H' : 'H',
|
||||
'I' : 'I',
|
||||
'J' : 'J',
|
||||
'K' : 'K',
|
||||
'L' : 'L',
|
||||
'M' : 'M',
|
||||
'N' : 'N',
|
||||
'O' : 'O',
|
||||
'P' : 'P',
|
||||
'Q' : 'Q',
|
||||
'R' : 'R',
|
||||
'S' : 'S',
|
||||
'T' : 'T',
|
||||
'U' : 'U',
|
||||
'V' : 'V',
|
||||
'W' : 'W',
|
||||
'X' : 'X',
|
||||
'Y' : 'Y',
|
||||
'Z' : 'Z',
|
||||
'#' : '#']
|
||||
def final static QUANJIAO = ['0' : '0',
|
||||
'1' : '1',
|
||||
'2' : '2',
|
||||
'3' : '3',
|
||||
'4' : '4',
|
||||
'5' : '5',
|
||||
'6' : '6',
|
||||
'7' : '7',
|
||||
'8' : '8',
|
||||
'9' : '9',
|
||||
'A' : 'A',
|
||||
'B' : 'B',
|
||||
'C' : 'C',
|
||||
'D' : 'D',
|
||||
'E' : 'E',
|
||||
'F' : 'F',
|
||||
'G' : 'G',
|
||||
'H' : 'H',
|
||||
'I' : 'I',
|
||||
'J' : 'J',
|
||||
'K' : 'K',
|
||||
'L' : 'L',
|
||||
'M' : 'M',
|
||||
'N' : 'N',
|
||||
'O' : 'O',
|
||||
'P' : 'P',
|
||||
'Q' : 'Q',
|
||||
'R' : 'R',
|
||||
'S' : 'S',
|
||||
'T' : 'T',
|
||||
'U' : 'U',
|
||||
'V' : 'V',
|
||||
'W' : 'W',
|
||||
'X' : 'X',
|
||||
'Y' : 'Y',
|
||||
'Z' : 'Z',
|
||||
'#' : '#']
|
||||
|
||||
def final static QUANJIAOKEYS = QUANJIAO.keySet()
|
||||
|
||||
@@ -66,56 +66,56 @@ class Yongjia {
|
||||
|
||||
final static token = "FF231CF2F7268316D937374DB96AC91ED7478F663E9D233D7389B1F0A809038F59F52B983E627AD8E95FB644D8C43601E5FEE7F107B3BFD8F5AF2D73853591F481A34E26723B657C2DD2D05684DB12F69C37447B272F857EFE2BEC49BADB44A3A2DBDB7CDBE2AA25"
|
||||
|
||||
def static c00 = 0 // 两个地址都没匹配
|
||||
def static c01 = 0 // 家庭地址没匹配,单位地址匹配
|
||||
def static c10 = 0 // 家庭地址匹配,单位地址没匹配
|
||||
def static c11 = 0 // 两个地址都匹配
|
||||
def static c00 = 0 // 两个地址都没匹配
|
||||
def static c01 = 0 // 家庭地址没匹配,单位地址匹配
|
||||
def static c10 = 0 // 家庭地址匹配,单位地址没匹配
|
||||
def static c11 = 0 // 两个地址都匹配
|
||||
|
||||
public static void main(args) {
|
||||
ExcelParser parser = new ExcelParser('C:\\Users\\yk\\Desktop\\yongjia\\永嘉数据-0910.xlsx')
|
||||
ExcelParser parser = new ExcelParser('C:\\Users\\yk\\Desktop\\yongjia\\永嘉数据-0910.xlsx')
|
||||
def (headers, rows) = parser.parse()
|
||||
|
||||
/*
|
||||
BufferedWriter writer = new File('C:\\Users\\yk\\Desktop\\yongjia\\test\\上海-0730.txt').newWriter('GBK')
|
||||
BufferedWriter writer = new File('C:\\Users\\yk\\Desktop\\yongjia\\test\\上海-0730.txt').newWriter('GBK')
|
||||
for(row in rows){
|
||||
def map = parser.toMap(headers, row)
|
||||
def a = replace(map['家庭地址'])
|
||||
def a = replace(map['家庭地址'])
|
||||
if (a && a.length() > 0)
|
||||
writer.writeLine(a)
|
||||
|
||||
a = replace(map['单位地址'])
|
||||
a = replace(map['单位地址'])
|
||||
if (a && a.length() > 0)
|
||||
writer.writeLine(a)
|
||||
}
|
||||
writer.close()
|
||||
*/
|
||||
|
||||
File file = new File("C:\\Users\\yk\\Desktop\\yongjia\\永嘉地址数据(上海)-done.csv");
|
||||
File file = new File("C:\\Users\\yk\\Desktop\\yongjia\\永嘉地址数据(上海)-done.csv");
|
||||
if (file.exists()){
|
||||
file.delete();
|
||||
}
|
||||
|
||||
file.append("家庭地址,", CHARSET)
|
||||
file.append("名称,", CHARSET)
|
||||
file.append("类型,", CHARSET)
|
||||
file.append("周边,", CHARSET)
|
||||
file.append("均价,", CHARSET)
|
||||
file.append("面积,", CHARSET)
|
||||
file.append("估价,", CHARSET)
|
||||
file.append("开发商,", CHARSET)
|
||||
file.append("物业公司,", CHARSET)
|
||||
file.append("小区户数,", CHARSET)
|
||||
file.append("家庭地址,", CHARSET)
|
||||
file.append("名称,", CHARSET)
|
||||
file.append("类型,", CHARSET)
|
||||
file.append("周边,", CHARSET)
|
||||
file.append("均价,", CHARSET)
|
||||
file.append("面积,", CHARSET)
|
||||
file.append("估价,", CHARSET)
|
||||
file.append("开发商,", CHARSET)
|
||||
file.append("物业公司,", CHARSET)
|
||||
file.append("小区户数,", CHARSET)
|
||||
|
||||
file.append("单位地址,", CHARSET)
|
||||
file.append("名称,", CHARSET)
|
||||
file.append("类型,", CHARSET)
|
||||
file.append("上班距离 KM,", CHARSET)
|
||||
file.append("单位地址,", CHARSET)
|
||||
file.append("名称,", CHARSET)
|
||||
file.append("类型,", CHARSET)
|
||||
file.append("上班距离 KM,", CHARSET)
|
||||
|
||||
file.append(System.getProperty("line.separator"))
|
||||
|
||||
for(row in rows){
|
||||
def map = parser.toMap(headers, row)
|
||||
process(map, file, '上海')
|
||||
process(map, file, '上海')
|
||||
}
|
||||
|
||||
println c00
|
||||
@@ -125,16 +125,16 @@ class Yongjia {
|
||||
}
|
||||
|
||||
def static process(map, csvFile, city) {
|
||||
def orgAddr1 = map['家庭地址']
|
||||
def orgAddr1 = map['家庭地址']
|
||||
println orgAddr1
|
||||
def orgAddr2 = map['单位地址']
|
||||
def orgAddr2 = map['单位地址']
|
||||
def newAddr1 = replace(orgAddr1)
|
||||
def newAddr2 = replace(orgAddr2)
|
||||
|
||||
// 家庭地址
|
||||
// 家庭地址
|
||||
csvFile.append("${orgAddr1},", CHARSET)
|
||||
|
||||
// 新旧地址都为空
|
||||
// 新旧地址都为空
|
||||
if (!newAddr1 && !newAddr2) {
|
||||
csvFile.append(",", CHARSET)
|
||||
csvFile.append(",", CHARSET)
|
||||
@@ -161,14 +161,14 @@ class Yongjia {
|
||||
def b0 = false
|
||||
def b1 = false
|
||||
if (newAddr1) {
|
||||
// 匹配小区
|
||||
// 匹配小区
|
||||
start = System.currentTimeMillis()
|
||||
parseRet = addressParserService.parseOneDistrict(city, newAddr1, 'district')
|
||||
println '小区:' + (System.currentTimeMillis() - start)
|
||||
println '小区:' + (System.currentTimeMillis() - start)
|
||||
if (parseRet && parseRet.name) {
|
||||
start = System.currentTimeMillis()
|
||||
parseRet = getDistrictInfo(parseRet)
|
||||
println '价格:' + (System.currentTimeMillis() - start)
|
||||
println '价格:' + (System.currentTimeMillis() - start)
|
||||
} else {
|
||||
parseRet = addressParserService.parseOneDistrict(city, newAddr1, 'office')
|
||||
|
||||
@@ -188,17 +188,17 @@ class Yongjia {
|
||||
|
||||
|
||||
|
||||
// 经纬度
|
||||
// 经纬度
|
||||
start = System.currentTimeMillis()
|
||||
latlng = baiduAPIService.geocoder(city, newAddr1)
|
||||
println '经纬度:' + (System.currentTimeMillis() - start)
|
||||
println '经纬度:' + (System.currentTimeMillis() - start)
|
||||
if (latlng) {
|
||||
// 周边
|
||||
// 周边
|
||||
start = System.currentTimeMillis()
|
||||
def s1 = "小区: ${roundSerach(latlng[0], latlng[1], '小区')}\r\n\r\n"
|
||||
def s2 = "办公楼: ${roundSerach(latlng[0], latlng[1], '写字楼')}\r\n\r\n"
|
||||
def s3 = "银行: ${roundSerach(latlng[0], latlng[1], '银行')}"
|
||||
println '周边:' + (System.currentTimeMillis() - start)
|
||||
def s1 = "小区: ${roundSerach(latlng[0], latlng[1], '小区')}\r\n\r\n"
|
||||
def s2 = "办公楼: ${roundSerach(latlng[0], latlng[1], '写字楼')}\r\n\r\n"
|
||||
def s3 = "银行: ${roundSerach(latlng[0], latlng[1], '银行')}"
|
||||
println '周边:' + (System.currentTimeMillis() - start)
|
||||
csvFile.append("\"${s1 + s2 + s3}\",", CHARSET)
|
||||
} else {
|
||||
csvFile.append(",", CHARSET)
|
||||
@@ -209,11 +209,11 @@ class Yongjia {
|
||||
csvFile.append(",", CHARSET)
|
||||
}
|
||||
|
||||
// 均价
|
||||
// 均价
|
||||
if(parseRet && parseRet.avgPrice){
|
||||
csvFile.append(parseRet.avgPrice+ ",","gbk")
|
||||
|
||||
// 估价
|
||||
// 估价
|
||||
if (parseRet.area && parseRet.fangjiaTotalPrice) {
|
||||
csvFile.append("${parseRet.area},","gbk")
|
||||
csvFile.append("${parseRet.fangjiaTotalPrice},","gbk")
|
||||
@@ -227,7 +227,7 @@ class Yongjia {
|
||||
csvFile.append(",","gbk")
|
||||
}
|
||||
|
||||
// 开发商、物业、绿化率、人口
|
||||
// 开发商、物业、绿化率、人口
|
||||
if (parseRet && parseRet.precise == 0) {
|
||||
def district = SEAWEED.findOne(new BasicDBObject('city', parseRet.city).append('region', parseRet.region).append('name', parseRet.name).append('cat', parseRet.qw[0]))
|
||||
if (district) {
|
||||
@@ -245,7 +245,7 @@ class Yongjia {
|
||||
csvFile.append(",", CHARSET)
|
||||
}
|
||||
|
||||
// 单位地址
|
||||
// 单位地址
|
||||
csvFile.append("\"${orgAddr2}\",", CHARSET)
|
||||
|
||||
if (newAddr2) {
|
||||
@@ -267,14 +267,14 @@ class Yongjia {
|
||||
csvFile.append(",", CHARSET)
|
||||
}
|
||||
|
||||
// 经纬度
|
||||
// 经纬度
|
||||
latlng2 = baiduAPIService.geocoder(city, newAddr2)
|
||||
} else {
|
||||
csvFile.append(",", CHARSET)
|
||||
csvFile.append(",", CHARSET)
|
||||
}
|
||||
|
||||
// 距离
|
||||
// 距离
|
||||
if (latlng && latlng2) {
|
||||
def d = GeoUtils.calculateDistance(latlng[0], latlng[1], latlng2[0], latlng2[1])
|
||||
|
||||
@@ -313,9 +313,9 @@ class Yongjia {
|
||||
for (s in QUANJIAOKEYS) {
|
||||
str = str.replaceAll(s, QUANJIAO[s])
|
||||
}
|
||||
str = str.replaceAll('无+', '').replaceAll('^0$', '').replaceAll('^-+$', '')
|
||||
.replaceAll('(上海)+', '上海').replaceAll('^上海$', '')
|
||||
.replaceAll('【', '').replaceAll('\\*+', '').replaceAll('[\\s ]+', '')
|
||||
str = str.replaceAll('无+', '').replaceAll('^0$', '').replaceAll('^-+$', '')
|
||||
.replaceAll('(上海)+', '上海').replaceAll('^上海$', '')
|
||||
.replaceAll('【', '').replaceAll('\\*+', '').replaceAll('[\\s ]+', '')
|
||||
return str
|
||||
}
|
||||
|
||||
@@ -344,7 +344,7 @@ class Yongjia {
|
||||
}
|
||||
}
|
||||
|
||||
//房价网估价
|
||||
//房价网估价
|
||||
// m.fangjiaTotalPrice = evaluation(m, token);
|
||||
// if(!m.fangjiaTotalPrice && m.avgPrice){
|
||||
// m.fangjiaTotalPrice = (m.avgPrice * m.area).toInteger();
|
||||
@@ -352,7 +352,7 @@ class Yongjia {
|
||||
|
||||
//m.rentPrice = queryRentPriceRange(m.city, m.region, m.name, lastMonth);
|
||||
|
||||
//其它网站均价
|
||||
//其它网站均价
|
||||
// def otherWebsiteMap = CommonDatautils.queryOtherWebsitePrice(m.city, m.name);
|
||||
// m.sunfunPrice = otherWebsiteMap.price_soufun
|
||||
// m.anjukePrice = otherWebsiteMap.price_anjuke
|
||||
@@ -391,7 +391,7 @@ class Yongjia {
|
||||
|
||||
}
|
||||
|
||||
//println "周边均价:" + totalPrice +"【${url_str}】"
|
||||
//println "周边均价:" + totalPrice +"【${url_str}】"
|
||||
return totalPrice;
|
||||
}
|
||||
|
||||
@@ -407,7 +407,7 @@ class Yongjia {
|
||||
conn.setRequestMethod("GET");
|
||||
conn.doOutput = true
|
||||
inputStream = conn.getInputStream();
|
||||
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
|
||||
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
|
||||
while((line = buf.readLine())!=null){
|
||||
sb.append(line);
|
||||
}
|
||||
|
||||
+17
-17
@@ -28,25 +28,25 @@ class Yongjia2 {
|
||||
final static token = "FF231CF2F7268316D937374DB96AC91ED7478F663E9D233D7389B1F0A809038F59F52B983E627AD8E95FB644D8C43601E5FEE7F107B3BFD8F5AF2D73853591F481A34E26723B657C2DD2D05684DB12F69C37447B272F857EFE2BEC49BADB44A3A2DBDB7CDBE2AA25"
|
||||
|
||||
public static void main(args) {
|
||||
BufferedReader reader = new File('C:\\Users\\yk\\Desktop\\yongjia\\test\\上海-0730.txt').newReader('UTF-8')
|
||||
BufferedReader reader = new File('C:\\Users\\yk\\Desktop\\yongjia\\test\\上海-0730.txt').newReader('UTF-8')
|
||||
|
||||
File file = new File("C:\\Users\\yk\\Desktop\\yongjia\\test\\永嘉地址数据(上海)-done.csv");
|
||||
File file = new File("C:\\Users\\yk\\Desktop\\yongjia\\test\\永嘉地址数据(上海)-done.csv");
|
||||
if (file.exists()){
|
||||
file.delete();
|
||||
}
|
||||
|
||||
file.append("地址,", CHARSET)
|
||||
file.append("名称,", CHARSET)
|
||||
file.append("类型,", CHARSET)
|
||||
file.append("地址,", CHARSET)
|
||||
file.append("楼号,", CHARSET)
|
||||
file.append("单元号,", CHARSET)
|
||||
file.append("楼层,", CHARSET)
|
||||
file.append("室号,", CHARSET)
|
||||
file.append("地址,", CHARSET)
|
||||
file.append("名称,", CHARSET)
|
||||
file.append("类型,", CHARSET)
|
||||
file.append("地址,", CHARSET)
|
||||
file.append("楼号,", CHARSET)
|
||||
file.append("单元号,", CHARSET)
|
||||
file.append("楼层,", CHARSET)
|
||||
file.append("室号,", CHARSET)
|
||||
file.append(System.getProperty("line.separator"))
|
||||
|
||||
reader.eachLine { line ->
|
||||
process(line, file, '上海')
|
||||
process(line, file, '上海')
|
||||
}
|
||||
|
||||
}
|
||||
@@ -58,11 +58,11 @@ class Yongjia2 {
|
||||
return
|
||||
}
|
||||
|
||||
// 家庭地址
|
||||
// 家庭地址
|
||||
csvFile.append("${orgAddr1},", CHARSET)
|
||||
|
||||
|
||||
// 匹配小区
|
||||
// 匹配小区
|
||||
def parseRet = addressParserService.parseOneDistrict(city, orgAddr1, 'district')
|
||||
|
||||
if (parseRet && parseRet.name) {
|
||||
@@ -123,7 +123,7 @@ class Yongjia2 {
|
||||
}
|
||||
}
|
||||
|
||||
//房价网估价
|
||||
//房价网估价
|
||||
// m.fangjiaTotalPrice = evaluation(m, token);
|
||||
// if(!m.fangjiaTotalPrice && m.avgPrice){
|
||||
// m.fangjiaTotalPrice = (m.avgPrice * m.area).toInteger();
|
||||
@@ -131,7 +131,7 @@ class Yongjia2 {
|
||||
|
||||
//m.rentPrice = queryRentPriceRange(m.city, m.region, m.name, lastMonth);
|
||||
|
||||
//其它网站均价
|
||||
//其它网站均价
|
||||
// def otherWebsiteMap = CommonDatautils.queryOtherWebsitePrice(m.city, m.name);
|
||||
// m.sunfunPrice = otherWebsiteMap.price_soufun
|
||||
// m.anjukePrice = otherWebsiteMap.price_anjuke
|
||||
@@ -170,7 +170,7 @@ class Yongjia2 {
|
||||
|
||||
}
|
||||
|
||||
//println "周边均价:" + totalPrice +"【${url_str}】"
|
||||
//println "周边均价:" + totalPrice +"【${url_str}】"
|
||||
return totalPrice;
|
||||
}
|
||||
|
||||
@@ -186,7 +186,7 @@ class Yongjia2 {
|
||||
conn.setRequestMethod("GET");
|
||||
conn.doOutput = true
|
||||
inputStream = conn.getInputStream();
|
||||
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
|
||||
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
|
||||
while((line = buf.readLine())!=null){
|
||||
sb.append(line);
|
||||
}
|
||||
|
||||
+13
-13
@@ -10,7 +10,7 @@ import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
/**
|
||||
* 地址分析任务
|
||||
* 地址分析任务
|
||||
*/
|
||||
public class AddressAnalysisJob implements Runnable{
|
||||
|
||||
@@ -34,7 +34,7 @@ public class AddressAnalysisJob implements Runnable{
|
||||
@Override
|
||||
void run() {
|
||||
long startTime = System.currentTimeMillis()
|
||||
logger.info("Job ${taskId} 开始运行")
|
||||
logger.info("Job ${taskId} 开始运行")
|
||||
def alreadyAnalyAddressSeq = []
|
||||
// def limit = ADDRESS_DETAIL.count(new BasicDBObject("task_id", taskId))
|
||||
def limit = 100
|
||||
@@ -83,12 +83,12 @@ public class AddressAnalysisJob implements Runnable{
|
||||
}catch (Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
logger.info("Job ${taskId} 运行结束")
|
||||
logger.info("Job ${taskId} 运行结束")
|
||||
long endTime = System.currentTimeMillis()
|
||||
println("Job ${taskId} 运行结束,分析记录数:${total} 耗时:${endTime - startTime}毫秒")
|
||||
println("Job ${taskId} 运行结束,分析记录数:${total} 耗时:${endTime - startTime}毫秒")
|
||||
}
|
||||
/**
|
||||
* 清理历史数据
|
||||
* 清理历史数据
|
||||
* @param taskId
|
||||
*/
|
||||
void clearHistory(String taskId){
|
||||
@@ -96,7 +96,7 @@ public class AddressAnalysisJob implements Runnable{
|
||||
ANALYTICS_OVERVIEW.remove(new BasicDBObject("task_id", taskId))
|
||||
}
|
||||
/**
|
||||
* 统计小区区域范围分布
|
||||
* 统计小区区域范围分布
|
||||
* @param taskId
|
||||
*/
|
||||
void regionDistrictCount(String taskId){
|
||||
@@ -117,7 +117,7 @@ public class AddressAnalysisJob implements Runnable{
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 统计匹配的小区数量
|
||||
* 统计匹配的小区数量
|
||||
* @param taskId
|
||||
*/
|
||||
void countMatchCount(String taskId){
|
||||
@@ -129,7 +129,7 @@ public class AddressAnalysisJob implements Runnable{
|
||||
|
||||
}
|
||||
/**
|
||||
* 分析物业类型占比
|
||||
* 分析物业类型占比
|
||||
* @param taskId
|
||||
*/
|
||||
void analyEstateType(String taskId){
|
||||
@@ -147,7 +147,7 @@ public class AddressAnalysisJob implements Runnable{
|
||||
}
|
||||
|
||||
/**
|
||||
* 分析总价分布
|
||||
* 分析总价分布
|
||||
* @param taskId
|
||||
*/
|
||||
void analyTotalPrice(String taskId){
|
||||
@@ -196,20 +196,20 @@ public class AddressAnalysisJob implements Runnable{
|
||||
ANALYTICS_OVERVIEW.update(new BasicDBObject("task_id", taskId), new BasicDBObject('$set', data), true, false)
|
||||
}
|
||||
/**
|
||||
* 计算总价区间标签
|
||||
* 计算总价区间标签
|
||||
* @param totalPrice
|
||||
* @return
|
||||
*/
|
||||
def calTotalPriceLabel(long totalPrice){
|
||||
if(totalPrice > 2000 * 10000){
|
||||
return "大于2000"
|
||||
return "大于2000"
|
||||
}
|
||||
int offset = 100 * 10000
|
||||
int multiple = (int)(totalPrice / offset)
|
||||
return "${multiple * 100}-${(multiple + 1) * 100}"
|
||||
}
|
||||
/**
|
||||
* 产生总价标签
|
||||
* 产生总价标签
|
||||
* @return
|
||||
*/
|
||||
def genTotalPriceSerial(){
|
||||
@@ -217,7 +217,7 @@ public class AddressAnalysisJob implements Runnable{
|
||||
for(int i = 0 ; i < 20; i++){
|
||||
serial << "${i * 100}-${(i + 1) * 100}"
|
||||
}
|
||||
serial << "大于2000"
|
||||
serial << "大于2000"
|
||||
}
|
||||
|
||||
|
||||
|
||||
+14
-14
@@ -15,7 +15,7 @@ import org.apache.log4j.Logger
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 地址分析器
|
||||
* 地址分析器
|
||||
*/
|
||||
public class AddressAnalyzer implements Runnable{
|
||||
|
||||
@@ -28,8 +28,8 @@ public class AddressAnalyzer implements Runnable{
|
||||
|
||||
final BasicDBObject addressDetail
|
||||
final BasicDBObject task
|
||||
final TASK_TYPE_FENG_KONG = "风控"
|
||||
final TASK_TYPE_YING_XIAO = "营销"
|
||||
final TASK_TYPE_FENG_KONG = "风控"
|
||||
final TASK_TYPE_YING_XIAO = "营销"
|
||||
|
||||
public AddressAnalyzer(BasicDBObject addressDetail,BasicDBObject task){
|
||||
this.addressDetail = addressDetail
|
||||
@@ -58,10 +58,10 @@ public class AddressAnalyzer implements Runnable{
|
||||
onFinish()
|
||||
return
|
||||
}
|
||||
if(city.endsWith("市")){
|
||||
city = city.replaceAll("市\$", "")
|
||||
if(city.endsWith("市")){
|
||||
city = city.replaceAll("市\$", "")
|
||||
}
|
||||
if(city != "上海"){
|
||||
if(city != "上海"){
|
||||
onFinish()
|
||||
return
|
||||
}
|
||||
@@ -87,7 +87,7 @@ public class AddressAnalyzer implements Runnable{
|
||||
double lng = matchResult.lng
|
||||
double lat = matchResult.lat
|
||||
BasicDBObject district = DbUtils.seaweed.findById(matchResult.getString("id"), new BasicDBObject("estate_type2", 1))
|
||||
/**匹配接口使用缓存,一天内被删除的小区有可能查不到*/
|
||||
/**匹配接口使用缓存,一天内被删除的小区有可能查不到*/
|
||||
if(district == null){
|
||||
onFinish()
|
||||
return
|
||||
@@ -135,13 +135,13 @@ public class AddressAnalyzer implements Runnable{
|
||||
JSONObject newest = totalPrices.get(totalPrices.size() - 1)
|
||||
def lastMonth = DateUtil.add(new Date(), Calendar.MONTH, -1).format("yyyy-MM")
|
||||
def thisMonth = new Date().format("yyyy-MM")
|
||||
if(newest.getString("date") == lastMonth || newest.getString("date") == thisMonth){//防止月初还未计算出均价
|
||||
if(newest.getString("date") == lastMonth || newest.getString("date") == thisMonth){//防止月初还未计算出均价
|
||||
result.totalPrice = newest.getLong("totalPrice")
|
||||
}
|
||||
}
|
||||
|
||||
def now = OpenApiClient.evaluateDistrictPrice(matchCity, matchRegion, name, result.area, estateType == "普通住宅"? 0 : 1, new Date())
|
||||
def oneYearAge = OpenApiClient.evaluateDistrictPrice(matchCity, matchRegion, name, result.area, estateType == "普通住宅"? 0 : 1, DateUtil.add(new Date(), Calendar.YEAR, -1))
|
||||
def now = OpenApiClient.evaluateDistrictPrice(matchCity, matchRegion, name, result.area, estateType == "普通住宅"? 0 : 1, new Date())
|
||||
def oneYearAge = OpenApiClient.evaluateDistrictPrice(matchCity, matchRegion, name, result.area, estateType == "普通住宅"? 0 : 1, DateUtil.add(new Date(), Calendar.YEAR, -1))
|
||||
if(now && oneYearAge){
|
||||
def markUp = DataUtil.roundDouble((now.getLong("totalPrice") - oneYearAge.getLong("totalPrice")) * 100/oneYearAge.getLong("totalPrice"), 2)
|
||||
result.append("mark_up", markUp)
|
||||
@@ -161,11 +161,11 @@ public class AddressAnalyzer implements Runnable{
|
||||
e.printStackTrace()
|
||||
}
|
||||
long end = System.currentTimeMillis()
|
||||
// println("消耗时间:" + (end - s) + "毫秒")
|
||||
// println("消耗时间:" + (end - s) + "毫秒")
|
||||
|
||||
}
|
||||
/**
|
||||
* 计算房屋面积
|
||||
* 计算房屋面积
|
||||
* @param city
|
||||
* @param region
|
||||
* @param name
|
||||
@@ -230,7 +230,7 @@ public class AddressAnalyzer implements Runnable{
|
||||
}
|
||||
|
||||
private String parseBuildingNumber(String buildingNumber){
|
||||
Pattern pattern = Pattern.compile("\\w+(栋|幢|座)")
|
||||
Pattern pattern = Pattern.compile("\\w+(栋|幢|座)")
|
||||
def matcher = pattern.matcher(buildingNumber)
|
||||
if(matcher.find()){
|
||||
return matcher.group().substring(0, matcher.group().length() - 1)
|
||||
@@ -239,7 +239,7 @@ public class AddressAnalyzer implements Runnable{
|
||||
}
|
||||
|
||||
private String parseUnitNumber(String buildingNumber){
|
||||
Pattern pattern = Pattern.compile("\\w+单元")
|
||||
Pattern pattern = Pattern.compile("\\w+单元")
|
||||
def matcher = pattern.matcher(buildingNumber)
|
||||
if(matcher.find()){
|
||||
return matcher.group().substring(0, matcher.group().length() - 2)
|
||||
|
||||
+9
-9
@@ -9,8 +9,8 @@ import com.mongodb.DBObject
|
||||
|
||||
/**
|
||||
*
|
||||
* 说明:地址标准化处理脚本
|
||||
* @author 王文艳
|
||||
* 说明:地址标准化处理脚本
|
||||
* @author 王文艳
|
||||
*
|
||||
*/
|
||||
class AddressStandardization {
|
||||
@@ -22,13 +22,13 @@ class AddressStandardization {
|
||||
static def handlers = [new ShengShiQuRemoveHandler(),new DuplicateRemovalHandler(),new SymbolRemoveHandler(),new RegionRemoveHandler()];
|
||||
|
||||
public static void main(String[] args){
|
||||
districtAddressStandard("district","舟山");
|
||||
districtAddressStandard("district","舟山");
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param category 建筑物类型
|
||||
* @param city 城市
|
||||
* @param category 建筑物类型
|
||||
* @param city 城市
|
||||
* @return
|
||||
*/
|
||||
static def districtAddressStandard(String category,String city){
|
||||
@@ -45,9 +45,9 @@ class AddressStandardization {
|
||||
fields.put("region", 1)
|
||||
fields.put("name", 1)
|
||||
fields.put("address", 1)
|
||||
def file = new File("D:\\data\\logs\\地名修正日志.log");
|
||||
file.append("开始执行地址标准化,开始时间:"+new Date().format("yyyy年MM月dd日 HH:mm:ss ")+"\n");
|
||||
file.append("处理条件:"+query+"\n");
|
||||
def file = new File("D:\\data\\logs\\地名修正日志.log");
|
||||
file.append("开始执行地址标准化,开始时间:"+new Date().format("yyyy年MM月dd日 HH:mm:ss ")+"\n");
|
||||
file.append("处理条件:"+query+"\n");
|
||||
DSCursor cur = ddColl.find(query, fields)
|
||||
Iterator<DBObject> iter = cur.iterator();
|
||||
DBObject dbObject;
|
||||
@@ -72,7 +72,7 @@ class AddressStandardization {
|
||||
println ""+dbObject.get("_id")+"("+dbObject.get("name")+")"+":"+dbObject.get("address")+"---->" + request.getAddress();
|
||||
}
|
||||
}
|
||||
file.append("地名标准化结束,结束时间:"+new Date().format("yyyy年MM月dd日 HH:mm:ss ")+"总记录:"+totalCount+",标准化记录:"+countRecord+",标准化百分比:"+((int)(countRecord/totalCount)*100))+""+"\n";
|
||||
file.append("地名标准化结束,结束时间:"+new Date().format("yyyy年MM月dd日 HH:mm:ss ")+"总记录:"+totalCount+",标准化记录:"+countRecord+",标准化百分比:"+((int)(countRecord/totalCount)*100))+""+"\n";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+381
-381
@@ -43,13 +43,13 @@ class AddressToBaiduLngLat {
|
||||
def task = new AddressToBaiduLngLat();
|
||||
//task.run();
|
||||
|
||||
def citys = ["红河"]//task.readTxt("H:\\", "citys2.txt");//读文件//["遵义"]//CommonDatautils.findAllCity();//所有城市
|
||||
def citys = ["红河"]//task.readTxt("H:\\", "citys2.txt");//读文件//["遵义"]//CommonDatautils.findAllCity();//所有城市
|
||||
def cat = "district";
|
||||
def index = 0;
|
||||
for(def city : citys){
|
||||
index++;
|
||||
task.districtAddressCover(city, cat);
|
||||
println "${index} "+city + "完成......."
|
||||
println "${index} "+city + "完成......."
|
||||
}
|
||||
|
||||
}
|
||||
@@ -68,7 +68,7 @@ class AddressToBaiduLngLat {
|
||||
|
||||
def districtAddressCover(city, cat){
|
||||
def ret = getAllDistricts(city, cat);
|
||||
println city + " 共:"+ret.size()
|
||||
println city + " 共:"+ret.size()
|
||||
def index = 0;
|
||||
def total = ret.size();
|
||||
def row = 0;
|
||||
@@ -108,7 +108,7 @@ class AddressToBaiduLngLat {
|
||||
long lat = abcLngLatMap.lat * 10000000000L
|
||||
|
||||
index ++;
|
||||
//println "["+row+"]"+obj.region +" "+obj.district_name+" 新baidu:"+m.baiduLat + " "+m.baiduLng+" 新mapabc:"+lng +" "+lat+" "+distance
|
||||
//println "["+row+"]"+obj.region +" "+obj.district_name+" 新baidu:"+m.baiduLat + " "+m.baiduLng+" 新mapabc:"+lng +" "+lat+" "+distance
|
||||
if(index%100==0){
|
||||
print "."
|
||||
}
|
||||
@@ -124,9 +124,9 @@ class AddressToBaiduLngLat {
|
||||
}
|
||||
|
||||
}
|
||||
println "【${city}】index : "+index;
|
||||
println "【${city}】noLatLngIndex :"+noLatLngIndex;
|
||||
println "【${city}】total : "+total
|
||||
println "【${city}】index : "+index;
|
||||
println "【${city}】noLatLngIndex :"+noLatLngIndex;
|
||||
println "【${city}】total : "+total
|
||||
|
||||
}
|
||||
|
||||
@@ -235,7 +235,7 @@ class AddressToBaiduLngLat {
|
||||
for(poi in poiList){
|
||||
def level = poi.level.text()
|
||||
if (isFirst){
|
||||
//如果是第一次,则以第一次作为优先选择
|
||||
//如果是第一次,则以第一次作为优先选择
|
||||
getDataFromPoi(ret, poi)
|
||||
isFirst = false
|
||||
}
|
||||
@@ -290,380 +290,380 @@ class AddressToBaiduLngLat {
|
||||
|
||||
|
||||
static {
|
||||
CITY_CODE_MAP.put("北京", "010")
|
||||
CITY_CODE_MAP.put("广州", "020")
|
||||
CITY_CODE_MAP.put("重庆", "023")
|
||||
CITY_CODE_MAP.put("南京", "025")
|
||||
CITY_CODE_MAP.put("武汉", "027")
|
||||
CITY_CODE_MAP.put("成都", "028")
|
||||
CITY_CODE_MAP.put("保定", "0312")
|
||||
CITY_CODE_MAP.put("承德", "0314")
|
||||
CITY_CODE_MAP.put("唐山", "0315")
|
||||
CITY_CODE_MAP.put("廊坊", "0316")
|
||||
CITY_CODE_MAP.put("秦皇岛", "0335")
|
||||
CITY_CODE_MAP.put("新乡", "0373")
|
||||
CITY_CODE_MAP.put("洛阳", "0379")
|
||||
CITY_CODE_MAP.put("大连", "0411")
|
||||
CITY_CODE_MAP.put("鞍山", "0412")
|
||||
CITY_CODE_MAP.put("抚顺", "0413")
|
||||
CITY_CODE_MAP.put("本溪", "0414")
|
||||
CITY_CODE_MAP.put("丹东", "0415")
|
||||
CITY_CODE_MAP.put("辽阳", "0419")
|
||||
CITY_CODE_MAP.put("葫芦岛", "0429")
|
||||
CITY_CODE_MAP.put("长春", "0431")
|
||||
CITY_CODE_MAP.put("哈尔滨", "0451")
|
||||
CITY_CODE_MAP.put("佳木斯", "0454")
|
||||
CITY_CODE_MAP.put("包头", "0472")
|
||||
CITY_CODE_MAP.put("镇江", "0511")
|
||||
CITY_CODE_MAP.put("苏州", "0512")
|
||||
CITY_CODE_MAP.put("昆山", "0512")
|
||||
CITY_CODE_MAP.put("南通", "0513")
|
||||
CITY_CODE_MAP.put("扬州", "0514")
|
||||
CITY_CODE_MAP.put("徐州", "0516")
|
||||
CITY_CODE_MAP.put("常州", "0519")
|
||||
CITY_CODE_MAP.put("济南", "0531")
|
||||
CITY_CODE_MAP.put("青岛", "0532")
|
||||
CITY_CODE_MAP.put("蚌埠", "0552")
|
||||
CITY_CODE_MAP.put("黄山", "0559")
|
||||
CITY_CODE_MAP.put("杭州", "0571")
|
||||
CITY_CODE_MAP.put("湖州", "0572")
|
||||
CITY_CODE_MAP.put("台州", "0576")
|
||||
CITY_CODE_MAP.put("温州", "0577")
|
||||
CITY_CODE_MAP.put("福州", "0591")
|
||||
CITY_CODE_MAP.put("厦门", "0592")
|
||||
CITY_CODE_MAP.put("宁德", "0593")
|
||||
CITY_CODE_MAP.put("莆田", "0594")
|
||||
CITY_CODE_MAP.put("泉州", "0595")
|
||||
CITY_CODE_MAP.put("漳州", "0596")
|
||||
CITY_CODE_MAP.put("威海", "0631")
|
||||
CITY_CODE_MAP.put("汕尾", "0660")
|
||||
CITY_CODE_MAP.put("汕尾岛", "0660")
|
||||
CITY_CODE_MAP.put("阳江", "0662")
|
||||
CITY_CODE_MAP.put("揭阳", "0663")
|
||||
CITY_CODE_MAP.put("茂名", "0668")
|
||||
CITY_CODE_MAP.put("襄阳", "0710")
|
||||
CITY_CODE_MAP.put("黄石", "0714")
|
||||
CITY_CODE_MAP.put("宜昌", "0717")
|
||||
CITY_CODE_MAP.put("十堰", "0719")
|
||||
CITY_CODE_MAP.put("荆门", "0724")
|
||||
CITY_CODE_MAP.put("岳阳", "0730")
|
||||
CITY_CODE_MAP.put("长沙", "0731")
|
||||
CITY_CODE_MAP.put("湘潭", "0732")
|
||||
CITY_CODE_MAP.put("株洲", "0733")
|
||||
CITY_CODE_MAP.put("常德", "0736")
|
||||
CITY_CODE_MAP.put("江门", "0750")
|
||||
CITY_CODE_MAP.put("汕头", "0754")
|
||||
CITY_CODE_MAP.put("深圳", "0755")
|
||||
CITY_CODE_MAP.put("珠海", "0756")
|
||||
CITY_CODE_MAP.put("湛江", "0759")
|
||||
CITY_CODE_MAP.put("中山", "0760")
|
||||
CITY_CODE_MAP.put("防城港", "0770")
|
||||
CITY_CODE_MAP.put("南宁", "0771")
|
||||
CITY_CODE_MAP.put("桂林", "0773")
|
||||
CITY_CODE_MAP.put("北海", "0779")
|
||||
CITY_CODE_MAP.put("南昌", "0791")
|
||||
CITY_CODE_MAP.put("九江", "0792")
|
||||
CITY_CODE_MAP.put("景德镇", "0798")
|
||||
CITY_CODE_MAP.put("自贡", "0813")
|
||||
CITY_CODE_MAP.put("泸州", "0830")
|
||||
CITY_CODE_MAP.put("宜宾", "0831")
|
||||
CITY_CODE_MAP.put("内江", "01832")
|
||||
CITY_CODE_MAP.put("眉山", "0833")
|
||||
CITY_CODE_MAP.put("贵阳", "0851")
|
||||
CITY_CODE_MAP.put("三亚", "0899")
|
||||
CITY_CODE_MAP.put("咸阳", "0910")
|
||||
CITY_CODE_MAP.put("延安", "0911")
|
||||
CITY_CODE_MAP.put("延边朝鲜族自治州", "0433")
|
||||
CITY_CODE_MAP.put("七台河", "0464")
|
||||
CITY_CODE_MAP.put("三明", "0598")
|
||||
CITY_CODE_MAP.put("三门峡", "0398")
|
||||
CITY_CODE_MAP.put("上海", "021")
|
||||
CITY_CODE_MAP.put("上饶", "0793")
|
||||
CITY_CODE_MAP.put("东莞", "0769")
|
||||
CITY_CODE_MAP.put("东营", "0546")
|
||||
CITY_CODE_MAP.put("中卫", "01953")
|
||||
CITY_CODE_MAP.put("临夏回族自治州", "0930")
|
||||
CITY_CODE_MAP.put("临汾", "0357")
|
||||
CITY_CODE_MAP.put("临沂", "0539")
|
||||
CITY_CODE_MAP.put("丽水", "0578")
|
||||
CITY_CODE_MAP.put("乌兰察布", "0474")
|
||||
CITY_CODE_MAP.put("乌海", "0473")
|
||||
CITY_CODE_MAP.put("乌鲁木齐", "0991")
|
||||
CITY_CODE_MAP.put("乐山", "0833")
|
||||
CITY_CODE_MAP.put("云浮", "0766")
|
||||
CITY_CODE_MAP.put("亳州", "0558")
|
||||
CITY_CODE_MAP.put("仙桃", "0728")
|
||||
CITY_CODE_MAP.put("伊春", "0458")
|
||||
CITY_CODE_MAP.put("伊犁哈萨克自治州", "0999")
|
||||
CITY_CODE_MAP.put("佛山", "0757")
|
||||
CITY_CODE_MAP.put("信阳", "0376")
|
||||
CITY_CODE_MAP.put("克拉玛依", "0990")
|
||||
CITY_CODE_MAP.put("六安", "0564")
|
||||
CITY_CODE_MAP.put("兰州", "0931")
|
||||
CITY_CODE_MAP.put("兴安盟", "0482")
|
||||
CITY_CODE_MAP.put("凉山彝族自治州", "0834")
|
||||
CITY_CODE_MAP.put("南充", "0817")
|
||||
CITY_CODE_MAP.put("南平", "0599")
|
||||
CITY_CODE_MAP.put("南阳", "0377")
|
||||
CITY_CODE_MAP.put("博尔塔拉蒙古自治州", "0909")
|
||||
CITY_CODE_MAP.put("双鸭山", "0469")
|
||||
CITY_CODE_MAP.put("合肥", "0551")
|
||||
CITY_CODE_MAP.put("吉安", "0796")
|
||||
CITY_CODE_MAP.put("吉林", "0432")
|
||||
CITY_CODE_MAP.put("吐鲁番地区", "0995")
|
||||
CITY_CODE_MAP.put("吕梁", "0358")
|
||||
CITY_CODE_MAP.put("吴忠", "0953")
|
||||
CITY_CODE_MAP.put("周口", "0394")
|
||||
CITY_CODE_MAP.put("呼伦贝尔", "0470")
|
||||
CITY_CODE_MAP.put("呼和浩特", "0471")
|
||||
CITY_CODE_MAP.put("咸宁", "0715")
|
||||
CITY_CODE_MAP.put("商丘", "0370")
|
||||
CITY_CODE_MAP.put("商洛", "0914")
|
||||
CITY_CODE_MAP.put("嘉兴", "0573")
|
||||
CITY_CODE_MAP.put("四平", "0434")
|
||||
CITY_CODE_MAP.put("塔城地区", "0901")
|
||||
CITY_CODE_MAP.put("大兴安岭地区", "0457")
|
||||
CITY_CODE_MAP.put("大同", "0352")
|
||||
CITY_CODE_MAP.put("大庆", "0459")
|
||||
CITY_CODE_MAP.put("大理白族自治州", "0872")
|
||||
CITY_CODE_MAP.put("天水", "0938")
|
||||
CITY_CODE_MAP.put("天津", "022")
|
||||
CITY_CODE_MAP.put("天门", "01728")
|
||||
CITY_CODE_MAP.put("太原", "0351")
|
||||
CITY_CODE_MAP.put("娄底", "0738")
|
||||
CITY_CODE_MAP.put("孝感", "0712")
|
||||
CITY_CODE_MAP.put("宁波", "0574")
|
||||
CITY_CODE_MAP.put("安庆", "0556")
|
||||
CITY_CODE_MAP.put("安康", "0915")
|
||||
CITY_CODE_MAP.put("安阳", "0372")
|
||||
CITY_CODE_MAP.put("安顺", "0853")
|
||||
CITY_CODE_MAP.put("定西", "0932")
|
||||
CITY_CODE_MAP.put("宜春", "0795")
|
||||
CITY_CODE_MAP.put("宝鸡", "0917")
|
||||
CITY_CODE_MAP.put("宣城", "0563")
|
||||
CITY_CODE_MAP.put("宿州", "0557")
|
||||
CITY_CODE_MAP.put("宿迁", "0527")
|
||||
CITY_CODE_MAP.put("山南地区", "0893")
|
||||
CITY_CODE_MAP.put("崇左", "01771")
|
||||
CITY_CODE_MAP.put("巢湖", "0565")
|
||||
CITY_CODE_MAP.put("巴中", "0827")
|
||||
CITY_CODE_MAP.put("平凉", "0933")
|
||||
CITY_CODE_MAP.put("平顶山", "0375")
|
||||
CITY_CODE_MAP.put("广元", "0839")
|
||||
CITY_CODE_MAP.put("广安", "0826")
|
||||
CITY_CODE_MAP.put("庆阳", "0934")
|
||||
CITY_CODE_MAP.put("开封", "0378")
|
||||
CITY_CODE_MAP.put("张家口", "0313")
|
||||
CITY_CODE_MAP.put("张家界", "0744")
|
||||
CITY_CODE_MAP.put("张掖", "0936")
|
||||
CITY_CODE_MAP.put("德州", "0534")
|
||||
CITY_CODE_MAP.put("德阳", "0838")
|
||||
CITY_CODE_MAP.put("忻州", "0350")
|
||||
CITY_CODE_MAP.put("怀化", "0745")
|
||||
CITY_CODE_MAP.put("恩施土家族苗族自治州", "0718")
|
||||
CITY_CODE_MAP.put("惠州", "0752")
|
||||
CITY_CODE_MAP.put("抚州", "0794")
|
||||
CITY_CODE_MAP.put("拉萨", "0891")
|
||||
CITY_CODE_MAP.put("攀枝花", "0812")
|
||||
CITY_CODE_MAP.put("新余", "0790")
|
||||
CITY_CODE_MAP.put("无锡", "0510")
|
||||
CITY_CODE_MAP.put("日喀则地区", "0892")
|
||||
CITY_CODE_MAP.put("日照", "0633")
|
||||
CITY_CODE_MAP.put("昆明", "0871")
|
||||
CITY_CODE_MAP.put("昌吉回族自治州", "0994")
|
||||
CITY_CODE_MAP.put("昭通", "0870")
|
||||
CITY_CODE_MAP.put("晋中", "0354")
|
||||
CITY_CODE_MAP.put("晋城", "0356")
|
||||
CITY_CODE_MAP.put("曲靖", "0874")
|
||||
CITY_CODE_MAP.put("朔州", "0349")
|
||||
CITY_CODE_MAP.put("朝阳", "0421")
|
||||
CITY_CODE_MAP.put("来宾", "01772")
|
||||
CITY_CODE_MAP.put("松原", "0438")
|
||||
CITY_CODE_MAP.put("枣庄", "0632")
|
||||
CITY_CODE_MAP.put("柳州", "0772")
|
||||
CITY_CODE_MAP.put("梅州", "0753")
|
||||
CITY_CODE_MAP.put("梧州", "0774")
|
||||
CITY_CODE_MAP.put("楚雄彝族自治州", "0878")
|
||||
CITY_CODE_MAP.put("榆林", "0912")
|
||||
CITY_CODE_MAP.put("武威", "01935")
|
||||
CITY_CODE_MAP.put("毕节地区", "0857")
|
||||
CITY_CODE_MAP.put("永州", "0746")
|
||||
CITY_CODE_MAP.put("汉中", "0916")
|
||||
CITY_CODE_MAP.put("池州", "0566")
|
||||
CITY_CODE_MAP.put("沈阳", "024")
|
||||
CITY_CODE_MAP.put("沧州", "0317")
|
||||
CITY_CODE_MAP.put("河池", "0778")
|
||||
CITY_CODE_MAP.put("河源", "0762")
|
||||
CITY_CODE_MAP.put("泰安", "0538")
|
||||
CITY_CODE_MAP.put("泰州", "0523")
|
||||
CITY_CODE_MAP.put("济宁", "0537")
|
||||
CITY_CODE_MAP.put("济源", "01391")
|
||||
CITY_CODE_MAP.put("海东地区", "0972")
|
||||
CITY_CODE_MAP.put("海北藏族自治州", "0970")
|
||||
CITY_CODE_MAP.put("海南藏族自治州", "0974")
|
||||
CITY_CODE_MAP.put("海口", "0898")
|
||||
CITY_CODE_MAP.put("淄博", "0533")
|
||||
CITY_CODE_MAP.put("淮北", "0561")
|
||||
CITY_CODE_MAP.put("淮南", "0554")
|
||||
CITY_CODE_MAP.put("淮安", "0517")
|
||||
CITY_CODE_MAP.put("清远", "0763")
|
||||
CITY_CODE_MAP.put("渭南", "0913")
|
||||
CITY_CODE_MAP.put("湘西土家族苗族自治州", "0743")
|
||||
CITY_CODE_MAP.put("滁州", "0550")
|
||||
CITY_CODE_MAP.put("滨州", "0543")
|
||||
CITY_CODE_MAP.put("漯河", "0395")
|
||||
CITY_CODE_MAP.put("潍坊", "0536")
|
||||
CITY_CODE_MAP.put("潜江", "02728")
|
||||
CITY_CODE_MAP.put("潮州", "0768")
|
||||
CITY_CODE_MAP.put("濮阳", "0393")
|
||||
CITY_CODE_MAP.put("烟台", "0535")
|
||||
CITY_CODE_MAP.put("焦作", "0391")
|
||||
CITY_CODE_MAP.put("牡丹江", "0453")
|
||||
CITY_CODE_MAP.put("玉林", "0775")
|
||||
CITY_CODE_MAP.put("玉溪", "0877")
|
||||
CITY_CODE_MAP.put("甘孜藏族自治州", "0836")
|
||||
CITY_CODE_MAP.put("白城", "0436")
|
||||
CITY_CODE_MAP.put("白山", "0439")
|
||||
CITY_CODE_MAP.put("白银", "0943")
|
||||
CITY_CODE_MAP.put("益阳", "0737")
|
||||
CITY_CODE_MAP.put("盐城", "0515")
|
||||
CITY_CODE_MAP.put("盘锦", "0427")
|
||||
CITY_CODE_MAP.put("石嘴山", "0952")
|
||||
CITY_CODE_MAP.put("石家庄", "0311")
|
||||
CITY_CODE_MAP.put("石河子", "0993")
|
||||
CITY_CODE_MAP.put("神农架林区", "01719")
|
||||
CITY_CODE_MAP.put("红河哈尼族彝族自治州", "0873")
|
||||
CITY_CODE_MAP.put("绍兴", "0575")
|
||||
CITY_CODE_MAP.put("绥化", "0455")
|
||||
CITY_CODE_MAP.put("绵阳", "0816")
|
||||
CITY_CODE_MAP.put("聊城", "0635")
|
||||
CITY_CODE_MAP.put("肇庆", "0758")
|
||||
CITY_CODE_MAP.put("舟山", "0580")
|
||||
CITY_CODE_MAP.put("芜湖", "0553")
|
||||
CITY_CODE_MAP.put("荆州", "0716")
|
||||
CITY_CODE_MAP.put("莱芜", "0634")
|
||||
CITY_CODE_MAP.put("菏泽", "0530")
|
||||
CITY_CODE_MAP.put("萍乡", "0799")
|
||||
CITY_CODE_MAP.put("营口", "0417")
|
||||
CITY_CODE_MAP.put("衡水", "0318")
|
||||
CITY_CODE_MAP.put("衡阳", "0734")
|
||||
CITY_CODE_MAP.put("衢州", "0570")
|
||||
CITY_CODE_MAP.put("西双版纳傣族自治州", "0691")
|
||||
CITY_CODE_MAP.put("西宁", "0971")
|
||||
CITY_CODE_MAP.put("西安", "029")
|
||||
CITY_CODE_MAP.put("许昌", "0374")
|
||||
CITY_CODE_MAP.put("贵港", "01755")
|
||||
CITY_CODE_MAP.put("贺州", "01774")
|
||||
CITY_CODE_MAP.put("资阳", "0832")
|
||||
CITY_CODE_MAP.put("赣州", "0797")
|
||||
CITY_CODE_MAP.put("赤峰", "0476")
|
||||
CITY_CODE_MAP.put("辽源", "0437")
|
||||
CITY_CODE_MAP.put("达州", "0818")
|
||||
CITY_CODE_MAP.put("运城", "0359")
|
||||
CITY_CODE_MAP.put("连云港", "0518")
|
||||
CITY_CODE_MAP.put("通化", "0435")
|
||||
CITY_CODE_MAP.put("通辽", "0475")
|
||||
CITY_CODE_MAP.put("遂宁", "0825")
|
||||
CITY_CODE_MAP.put("遵义", "0852")
|
||||
CITY_CODE_MAP.put("邢台", "0319")
|
||||
CITY_CODE_MAP.put("邯郸", "0310")
|
||||
CITY_CODE_MAP.put("邵阳", "0739")
|
||||
CITY_CODE_MAP.put("郑州", "0371")
|
||||
CITY_CODE_MAP.put("郴州", "0735")
|
||||
CITY_CODE_MAP.put("鄂尔多斯", "0477")
|
||||
CITY_CODE_MAP.put("鄂州", "0711")
|
||||
CITY_CODE_MAP.put("酒泉", "0937")
|
||||
CITY_CODE_MAP.put("金华", "0579")
|
||||
CITY_CODE_MAP.put("金昌", "0935")
|
||||
CITY_CODE_MAP.put("钦州", "0777")
|
||||
CITY_CODE_MAP.put("铁岭", "0410")
|
||||
CITY_CODE_MAP.put("铜仁地区", "0856")
|
||||
CITY_CODE_MAP.put("铜川", "0919")
|
||||
CITY_CODE_MAP.put("铜陵", "0562")
|
||||
CITY_CODE_MAP.put("银川", "0951")
|
||||
CITY_CODE_MAP.put("锦州", "0416")
|
||||
CITY_CODE_MAP.put("长治", "0355")
|
||||
CITY_CODE_MAP.put("阜新", "0418")
|
||||
CITY_CODE_MAP.put("阜阳", "01558")
|
||||
CITY_CODE_MAP.put("阳泉", "0353")
|
||||
CITY_CODE_MAP.put("阿坝藏族羌族自治州", "0837")
|
||||
CITY_CODE_MAP.put("陇南", "02935")
|
||||
CITY_CODE_MAP.put("随州", "0722")
|
||||
CITY_CODE_MAP.put("雅安", "0835")
|
||||
CITY_CODE_MAP.put("韶关", "0751")
|
||||
CITY_CODE_MAP.put("马鞍山", "0555")
|
||||
CITY_CODE_MAP.put("驻马店", "0396")
|
||||
CITY_CODE_MAP.put("鸡西", "0467")
|
||||
CITY_CODE_MAP.put("鹤壁", "0392")
|
||||
CITY_CODE_MAP.put("鹤岗", "0468")
|
||||
CITY_CODE_MAP.put("鹰潭", "0701")
|
||||
CITY_CODE_MAP.put("黄冈", "0713")
|
||||
CITY_CODE_MAP.put("黑河", "0456")
|
||||
CITY_CODE_MAP.put("黔东南苗族侗族自治州", "0855")
|
||||
CITY_CODE_MAP.put("黔南布依族苗族自治州", "0854")
|
||||
CITY_CODE_MAP.put("齐齐哈尔", "0452")
|
||||
CITY_CODE_MAP.put("龙岩", "0597")
|
||||
CITY_CODE_MAP.put("甘南藏族自治州", "0941")
|
||||
CITY_CODE_MAP.put("嘉峪关", "1937")
|
||||
CITY_CODE_MAP.put("百色", "0776")
|
||||
CITY_CODE_MAP.put("六盘水", "0858")
|
||||
CITY_CODE_MAP.put("黔西南布依族苗族自治州", "0859")
|
||||
CITY_CODE_MAP.put("白沙黎族自治县", "0802")
|
||||
CITY_CODE_MAP.put("保亭黎族苗族自治县", "0801")
|
||||
CITY_CODE_MAP.put("昌江黎族自治县", "0803")
|
||||
CITY_CODE_MAP.put("澄迈县", "0804")
|
||||
CITY_CODE_MAP.put("定安县", "0806")
|
||||
CITY_CODE_MAP.put("东方", "0807")
|
||||
CITY_CODE_MAP.put("东沙群岛", "0808")
|
||||
CITY_CODE_MAP.put("乐东黎族自治县", "2802")
|
||||
CITY_CODE_MAP.put("临高县", "1896")
|
||||
CITY_CODE_MAP.put("陵水黎族自治县", "0809")
|
||||
CITY_CODE_MAP.put("南沙群岛", "1891")
|
||||
CITY_CODE_MAP.put("琼海", "1894")
|
||||
CITY_CODE_MAP.put("琼中黎族苗族自治县", "1899")
|
||||
CITY_CODE_MAP.put("屯昌县", "1892")
|
||||
CITY_CODE_MAP.put("万宁", "1898")
|
||||
CITY_CODE_MAP.put("文昌", "1893")
|
||||
CITY_CODE_MAP.put("五指山", "1897")
|
||||
CITY_CODE_MAP.put("西沙群岛", "1895")
|
||||
CITY_CODE_MAP.put("中沙群岛", "2801")
|
||||
CITY_CODE_MAP.put("儋州", "0805")
|
||||
CITY_CODE_MAP.put("阿拉善盟", "0483")
|
||||
CITY_CODE_MAP.put("巴彦淖尔市", "0478")
|
||||
CITY_CODE_MAP.put("锡林郭勒盟", "0479")
|
||||
CITY_CODE_MAP.put("固原", "0954")
|
||||
CITY_CODE_MAP.put("果洛藏族自治州", "0975")
|
||||
CITY_CODE_MAP.put("海西蒙古族藏族自治州", "0977")
|
||||
CITY_CODE_MAP.put("黄南藏族自治州", "0973")
|
||||
CITY_CODE_MAP.put("玉树藏族自治州", "0976")
|
||||
CITY_CODE_MAP.put("阿里地区", "0897")
|
||||
CITY_CODE_MAP.put("昌都地区", "0895")
|
||||
CITY_CODE_MAP.put("林芝地区", "0894")
|
||||
CITY_CODE_MAP.put("那曲地区", "0896")
|
||||
CITY_CODE_MAP.put("阿克苏地区", "0997")
|
||||
CITY_CODE_MAP.put("阿勒泰地区", "0906")
|
||||
CITY_CODE_MAP.put("巴音郭楞蒙古自治州", "0996")
|
||||
CITY_CODE_MAP.put("哈密地区", "0902")
|
||||
CITY_CODE_MAP.put("和田地区", "0903")
|
||||
CITY_CODE_MAP.put("喀什地区", "0998")
|
||||
CITY_CODE_MAP.put("克孜勒苏柯尔克孜自治州", "0908")
|
||||
CITY_CODE_MAP.put("保山", "0875")
|
||||
CITY_CODE_MAP.put("德宏傣族景颇族自治州", "0692")
|
||||
CITY_CODE_MAP.put("迪庆藏族自治州", "0887")
|
||||
CITY_CODE_MAP.put("丽江", "0888")
|
||||
CITY_CODE_MAP.put("临沧市", "0883")
|
||||
CITY_CODE_MAP.put("怒江傈僳族自治州", "0886")
|
||||
CITY_CODE_MAP.put("普洱", "0879")
|
||||
CITY_CODE_MAP.put("文山壮族苗族自治州", "0876")
|
||||
CITY_CODE_MAP.put("香港", "1852")
|
||||
CITY_CODE_MAP.put("澳门", "1853")
|
||||
CITY_CODE_MAP.put("五家渠市", "652300")
|
||||
CITY_CODE_MAP.put("阿拉尔市", "652900")
|
||||
CITY_CODE_MAP.put("图木舒克市", "653000")
|
||||
CITY_CODE_MAP.put("北京", "010")
|
||||
CITY_CODE_MAP.put("广州", "020")
|
||||
CITY_CODE_MAP.put("重庆", "023")
|
||||
CITY_CODE_MAP.put("南京", "025")
|
||||
CITY_CODE_MAP.put("武汉", "027")
|
||||
CITY_CODE_MAP.put("成都", "028")
|
||||
CITY_CODE_MAP.put("保定", "0312")
|
||||
CITY_CODE_MAP.put("承德", "0314")
|
||||
CITY_CODE_MAP.put("唐山", "0315")
|
||||
CITY_CODE_MAP.put("廊坊", "0316")
|
||||
CITY_CODE_MAP.put("秦皇岛", "0335")
|
||||
CITY_CODE_MAP.put("新乡", "0373")
|
||||
CITY_CODE_MAP.put("洛阳", "0379")
|
||||
CITY_CODE_MAP.put("大连", "0411")
|
||||
CITY_CODE_MAP.put("鞍山", "0412")
|
||||
CITY_CODE_MAP.put("抚顺", "0413")
|
||||
CITY_CODE_MAP.put("本溪", "0414")
|
||||
CITY_CODE_MAP.put("丹东", "0415")
|
||||
CITY_CODE_MAP.put("辽阳", "0419")
|
||||
CITY_CODE_MAP.put("葫芦岛", "0429")
|
||||
CITY_CODE_MAP.put("长春", "0431")
|
||||
CITY_CODE_MAP.put("哈尔滨", "0451")
|
||||
CITY_CODE_MAP.put("佳木斯", "0454")
|
||||
CITY_CODE_MAP.put("包头", "0472")
|
||||
CITY_CODE_MAP.put("镇江", "0511")
|
||||
CITY_CODE_MAP.put("苏州", "0512")
|
||||
CITY_CODE_MAP.put("昆山", "0512")
|
||||
CITY_CODE_MAP.put("南通", "0513")
|
||||
CITY_CODE_MAP.put("扬州", "0514")
|
||||
CITY_CODE_MAP.put("徐州", "0516")
|
||||
CITY_CODE_MAP.put("常州", "0519")
|
||||
CITY_CODE_MAP.put("济南", "0531")
|
||||
CITY_CODE_MAP.put("青岛", "0532")
|
||||
CITY_CODE_MAP.put("蚌埠", "0552")
|
||||
CITY_CODE_MAP.put("黄山", "0559")
|
||||
CITY_CODE_MAP.put("杭州", "0571")
|
||||
CITY_CODE_MAP.put("湖州", "0572")
|
||||
CITY_CODE_MAP.put("台州", "0576")
|
||||
CITY_CODE_MAP.put("温州", "0577")
|
||||
CITY_CODE_MAP.put("福州", "0591")
|
||||
CITY_CODE_MAP.put("厦门", "0592")
|
||||
CITY_CODE_MAP.put("宁德", "0593")
|
||||
CITY_CODE_MAP.put("莆田", "0594")
|
||||
CITY_CODE_MAP.put("泉州", "0595")
|
||||
CITY_CODE_MAP.put("漳州", "0596")
|
||||
CITY_CODE_MAP.put("威海", "0631")
|
||||
CITY_CODE_MAP.put("汕尾", "0660")
|
||||
CITY_CODE_MAP.put("汕尾岛", "0660")
|
||||
CITY_CODE_MAP.put("阳江", "0662")
|
||||
CITY_CODE_MAP.put("揭阳", "0663")
|
||||
CITY_CODE_MAP.put("茂名", "0668")
|
||||
CITY_CODE_MAP.put("襄阳", "0710")
|
||||
CITY_CODE_MAP.put("黄石", "0714")
|
||||
CITY_CODE_MAP.put("宜昌", "0717")
|
||||
CITY_CODE_MAP.put("十堰", "0719")
|
||||
CITY_CODE_MAP.put("荆门", "0724")
|
||||
CITY_CODE_MAP.put("岳阳", "0730")
|
||||
CITY_CODE_MAP.put("长沙", "0731")
|
||||
CITY_CODE_MAP.put("湘潭", "0732")
|
||||
CITY_CODE_MAP.put("株洲", "0733")
|
||||
CITY_CODE_MAP.put("常德", "0736")
|
||||
CITY_CODE_MAP.put("江门", "0750")
|
||||
CITY_CODE_MAP.put("汕头", "0754")
|
||||
CITY_CODE_MAP.put("深圳", "0755")
|
||||
CITY_CODE_MAP.put("珠海", "0756")
|
||||
CITY_CODE_MAP.put("湛江", "0759")
|
||||
CITY_CODE_MAP.put("中山", "0760")
|
||||
CITY_CODE_MAP.put("防城港", "0770")
|
||||
CITY_CODE_MAP.put("南宁", "0771")
|
||||
CITY_CODE_MAP.put("桂林", "0773")
|
||||
CITY_CODE_MAP.put("北海", "0779")
|
||||
CITY_CODE_MAP.put("南昌", "0791")
|
||||
CITY_CODE_MAP.put("九江", "0792")
|
||||
CITY_CODE_MAP.put("景德镇", "0798")
|
||||
CITY_CODE_MAP.put("自贡", "0813")
|
||||
CITY_CODE_MAP.put("泸州", "0830")
|
||||
CITY_CODE_MAP.put("宜宾", "0831")
|
||||
CITY_CODE_MAP.put("内江", "01832")
|
||||
CITY_CODE_MAP.put("眉山", "0833")
|
||||
CITY_CODE_MAP.put("贵阳", "0851")
|
||||
CITY_CODE_MAP.put("三亚", "0899")
|
||||
CITY_CODE_MAP.put("咸阳", "0910")
|
||||
CITY_CODE_MAP.put("延安", "0911")
|
||||
CITY_CODE_MAP.put("延边朝鲜族自治州", "0433")
|
||||
CITY_CODE_MAP.put("七台河", "0464")
|
||||
CITY_CODE_MAP.put("三明", "0598")
|
||||
CITY_CODE_MAP.put("三门峡", "0398")
|
||||
CITY_CODE_MAP.put("上海", "021")
|
||||
CITY_CODE_MAP.put("上饶", "0793")
|
||||
CITY_CODE_MAP.put("东莞", "0769")
|
||||
CITY_CODE_MAP.put("东营", "0546")
|
||||
CITY_CODE_MAP.put("中卫", "01953")
|
||||
CITY_CODE_MAP.put("临夏回族自治州", "0930")
|
||||
CITY_CODE_MAP.put("临汾", "0357")
|
||||
CITY_CODE_MAP.put("临沂", "0539")
|
||||
CITY_CODE_MAP.put("丽水", "0578")
|
||||
CITY_CODE_MAP.put("乌兰察布", "0474")
|
||||
CITY_CODE_MAP.put("乌海", "0473")
|
||||
CITY_CODE_MAP.put("乌鲁木齐", "0991")
|
||||
CITY_CODE_MAP.put("乐山", "0833")
|
||||
CITY_CODE_MAP.put("云浮", "0766")
|
||||
CITY_CODE_MAP.put("亳州", "0558")
|
||||
CITY_CODE_MAP.put("仙桃", "0728")
|
||||
CITY_CODE_MAP.put("伊春", "0458")
|
||||
CITY_CODE_MAP.put("伊犁哈萨克自治州", "0999")
|
||||
CITY_CODE_MAP.put("佛山", "0757")
|
||||
CITY_CODE_MAP.put("信阳", "0376")
|
||||
CITY_CODE_MAP.put("克拉玛依", "0990")
|
||||
CITY_CODE_MAP.put("六安", "0564")
|
||||
CITY_CODE_MAP.put("兰州", "0931")
|
||||
CITY_CODE_MAP.put("兴安盟", "0482")
|
||||
CITY_CODE_MAP.put("凉山彝族自治州", "0834")
|
||||
CITY_CODE_MAP.put("南充", "0817")
|
||||
CITY_CODE_MAP.put("南平", "0599")
|
||||
CITY_CODE_MAP.put("南阳", "0377")
|
||||
CITY_CODE_MAP.put("博尔塔拉蒙古自治州", "0909")
|
||||
CITY_CODE_MAP.put("双鸭山", "0469")
|
||||
CITY_CODE_MAP.put("合肥", "0551")
|
||||
CITY_CODE_MAP.put("吉安", "0796")
|
||||
CITY_CODE_MAP.put("吉林", "0432")
|
||||
CITY_CODE_MAP.put("吐鲁番地区", "0995")
|
||||
CITY_CODE_MAP.put("吕梁", "0358")
|
||||
CITY_CODE_MAP.put("吴忠", "0953")
|
||||
CITY_CODE_MAP.put("周口", "0394")
|
||||
CITY_CODE_MAP.put("呼伦贝尔", "0470")
|
||||
CITY_CODE_MAP.put("呼和浩特", "0471")
|
||||
CITY_CODE_MAP.put("咸宁", "0715")
|
||||
CITY_CODE_MAP.put("商丘", "0370")
|
||||
CITY_CODE_MAP.put("商洛", "0914")
|
||||
CITY_CODE_MAP.put("嘉兴", "0573")
|
||||
CITY_CODE_MAP.put("四平", "0434")
|
||||
CITY_CODE_MAP.put("塔城地区", "0901")
|
||||
CITY_CODE_MAP.put("大兴安岭地区", "0457")
|
||||
CITY_CODE_MAP.put("大同", "0352")
|
||||
CITY_CODE_MAP.put("大庆", "0459")
|
||||
CITY_CODE_MAP.put("大理白族自治州", "0872")
|
||||
CITY_CODE_MAP.put("天水", "0938")
|
||||
CITY_CODE_MAP.put("天津", "022")
|
||||
CITY_CODE_MAP.put("天门", "01728")
|
||||
CITY_CODE_MAP.put("太原", "0351")
|
||||
CITY_CODE_MAP.put("娄底", "0738")
|
||||
CITY_CODE_MAP.put("孝感", "0712")
|
||||
CITY_CODE_MAP.put("宁波", "0574")
|
||||
CITY_CODE_MAP.put("安庆", "0556")
|
||||
CITY_CODE_MAP.put("安康", "0915")
|
||||
CITY_CODE_MAP.put("安阳", "0372")
|
||||
CITY_CODE_MAP.put("安顺", "0853")
|
||||
CITY_CODE_MAP.put("定西", "0932")
|
||||
CITY_CODE_MAP.put("宜春", "0795")
|
||||
CITY_CODE_MAP.put("宝鸡", "0917")
|
||||
CITY_CODE_MAP.put("宣城", "0563")
|
||||
CITY_CODE_MAP.put("宿州", "0557")
|
||||
CITY_CODE_MAP.put("宿迁", "0527")
|
||||
CITY_CODE_MAP.put("山南地区", "0893")
|
||||
CITY_CODE_MAP.put("崇左", "01771")
|
||||
CITY_CODE_MAP.put("巢湖", "0565")
|
||||
CITY_CODE_MAP.put("巴中", "0827")
|
||||
CITY_CODE_MAP.put("平凉", "0933")
|
||||
CITY_CODE_MAP.put("平顶山", "0375")
|
||||
CITY_CODE_MAP.put("广元", "0839")
|
||||
CITY_CODE_MAP.put("广安", "0826")
|
||||
CITY_CODE_MAP.put("庆阳", "0934")
|
||||
CITY_CODE_MAP.put("开封", "0378")
|
||||
CITY_CODE_MAP.put("张家口", "0313")
|
||||
CITY_CODE_MAP.put("张家界", "0744")
|
||||
CITY_CODE_MAP.put("张掖", "0936")
|
||||
CITY_CODE_MAP.put("德州", "0534")
|
||||
CITY_CODE_MAP.put("德阳", "0838")
|
||||
CITY_CODE_MAP.put("忻州", "0350")
|
||||
CITY_CODE_MAP.put("怀化", "0745")
|
||||
CITY_CODE_MAP.put("恩施土家族苗族自治州", "0718")
|
||||
CITY_CODE_MAP.put("惠州", "0752")
|
||||
CITY_CODE_MAP.put("抚州", "0794")
|
||||
CITY_CODE_MAP.put("拉萨", "0891")
|
||||
CITY_CODE_MAP.put("攀枝花", "0812")
|
||||
CITY_CODE_MAP.put("新余", "0790")
|
||||
CITY_CODE_MAP.put("无锡", "0510")
|
||||
CITY_CODE_MAP.put("日喀则地区", "0892")
|
||||
CITY_CODE_MAP.put("日照", "0633")
|
||||
CITY_CODE_MAP.put("昆明", "0871")
|
||||
CITY_CODE_MAP.put("昌吉回族自治州", "0994")
|
||||
CITY_CODE_MAP.put("昭通", "0870")
|
||||
CITY_CODE_MAP.put("晋中", "0354")
|
||||
CITY_CODE_MAP.put("晋城", "0356")
|
||||
CITY_CODE_MAP.put("曲靖", "0874")
|
||||
CITY_CODE_MAP.put("朔州", "0349")
|
||||
CITY_CODE_MAP.put("朝阳", "0421")
|
||||
CITY_CODE_MAP.put("来宾", "01772")
|
||||
CITY_CODE_MAP.put("松原", "0438")
|
||||
CITY_CODE_MAP.put("枣庄", "0632")
|
||||
CITY_CODE_MAP.put("柳州", "0772")
|
||||
CITY_CODE_MAP.put("梅州", "0753")
|
||||
CITY_CODE_MAP.put("梧州", "0774")
|
||||
CITY_CODE_MAP.put("楚雄彝族自治州", "0878")
|
||||
CITY_CODE_MAP.put("榆林", "0912")
|
||||
CITY_CODE_MAP.put("武威", "01935")
|
||||
CITY_CODE_MAP.put("毕节地区", "0857")
|
||||
CITY_CODE_MAP.put("永州", "0746")
|
||||
CITY_CODE_MAP.put("汉中", "0916")
|
||||
CITY_CODE_MAP.put("池州", "0566")
|
||||
CITY_CODE_MAP.put("沈阳", "024")
|
||||
CITY_CODE_MAP.put("沧州", "0317")
|
||||
CITY_CODE_MAP.put("河池", "0778")
|
||||
CITY_CODE_MAP.put("河源", "0762")
|
||||
CITY_CODE_MAP.put("泰安", "0538")
|
||||
CITY_CODE_MAP.put("泰州", "0523")
|
||||
CITY_CODE_MAP.put("济宁", "0537")
|
||||
CITY_CODE_MAP.put("济源", "01391")
|
||||
CITY_CODE_MAP.put("海东地区", "0972")
|
||||
CITY_CODE_MAP.put("海北藏族自治州", "0970")
|
||||
CITY_CODE_MAP.put("海南藏族自治州", "0974")
|
||||
CITY_CODE_MAP.put("海口", "0898")
|
||||
CITY_CODE_MAP.put("淄博", "0533")
|
||||
CITY_CODE_MAP.put("淮北", "0561")
|
||||
CITY_CODE_MAP.put("淮南", "0554")
|
||||
CITY_CODE_MAP.put("淮安", "0517")
|
||||
CITY_CODE_MAP.put("清远", "0763")
|
||||
CITY_CODE_MAP.put("渭南", "0913")
|
||||
CITY_CODE_MAP.put("湘西土家族苗族自治州", "0743")
|
||||
CITY_CODE_MAP.put("滁州", "0550")
|
||||
CITY_CODE_MAP.put("滨州", "0543")
|
||||
CITY_CODE_MAP.put("漯河", "0395")
|
||||
CITY_CODE_MAP.put("潍坊", "0536")
|
||||
CITY_CODE_MAP.put("潜江", "02728")
|
||||
CITY_CODE_MAP.put("潮州", "0768")
|
||||
CITY_CODE_MAP.put("濮阳", "0393")
|
||||
CITY_CODE_MAP.put("烟台", "0535")
|
||||
CITY_CODE_MAP.put("焦作", "0391")
|
||||
CITY_CODE_MAP.put("牡丹江", "0453")
|
||||
CITY_CODE_MAP.put("玉林", "0775")
|
||||
CITY_CODE_MAP.put("玉溪", "0877")
|
||||
CITY_CODE_MAP.put("甘孜藏族自治州", "0836")
|
||||
CITY_CODE_MAP.put("白城", "0436")
|
||||
CITY_CODE_MAP.put("白山", "0439")
|
||||
CITY_CODE_MAP.put("白银", "0943")
|
||||
CITY_CODE_MAP.put("益阳", "0737")
|
||||
CITY_CODE_MAP.put("盐城", "0515")
|
||||
CITY_CODE_MAP.put("盘锦", "0427")
|
||||
CITY_CODE_MAP.put("石嘴山", "0952")
|
||||
CITY_CODE_MAP.put("石家庄", "0311")
|
||||
CITY_CODE_MAP.put("石河子", "0993")
|
||||
CITY_CODE_MAP.put("神农架林区", "01719")
|
||||
CITY_CODE_MAP.put("红河哈尼族彝族自治州", "0873")
|
||||
CITY_CODE_MAP.put("绍兴", "0575")
|
||||
CITY_CODE_MAP.put("绥化", "0455")
|
||||
CITY_CODE_MAP.put("绵阳", "0816")
|
||||
CITY_CODE_MAP.put("聊城", "0635")
|
||||
CITY_CODE_MAP.put("肇庆", "0758")
|
||||
CITY_CODE_MAP.put("舟山", "0580")
|
||||
CITY_CODE_MAP.put("芜湖", "0553")
|
||||
CITY_CODE_MAP.put("荆州", "0716")
|
||||
CITY_CODE_MAP.put("莱芜", "0634")
|
||||
CITY_CODE_MAP.put("菏泽", "0530")
|
||||
CITY_CODE_MAP.put("萍乡", "0799")
|
||||
CITY_CODE_MAP.put("营口", "0417")
|
||||
CITY_CODE_MAP.put("衡水", "0318")
|
||||
CITY_CODE_MAP.put("衡阳", "0734")
|
||||
CITY_CODE_MAP.put("衢州", "0570")
|
||||
CITY_CODE_MAP.put("西双版纳傣族自治州", "0691")
|
||||
CITY_CODE_MAP.put("西宁", "0971")
|
||||
CITY_CODE_MAP.put("西安", "029")
|
||||
CITY_CODE_MAP.put("许昌", "0374")
|
||||
CITY_CODE_MAP.put("贵港", "01755")
|
||||
CITY_CODE_MAP.put("贺州", "01774")
|
||||
CITY_CODE_MAP.put("资阳", "0832")
|
||||
CITY_CODE_MAP.put("赣州", "0797")
|
||||
CITY_CODE_MAP.put("赤峰", "0476")
|
||||
CITY_CODE_MAP.put("辽源", "0437")
|
||||
CITY_CODE_MAP.put("达州", "0818")
|
||||
CITY_CODE_MAP.put("运城", "0359")
|
||||
CITY_CODE_MAP.put("连云港", "0518")
|
||||
CITY_CODE_MAP.put("通化", "0435")
|
||||
CITY_CODE_MAP.put("通辽", "0475")
|
||||
CITY_CODE_MAP.put("遂宁", "0825")
|
||||
CITY_CODE_MAP.put("遵义", "0852")
|
||||
CITY_CODE_MAP.put("邢台", "0319")
|
||||
CITY_CODE_MAP.put("邯郸", "0310")
|
||||
CITY_CODE_MAP.put("邵阳", "0739")
|
||||
CITY_CODE_MAP.put("郑州", "0371")
|
||||
CITY_CODE_MAP.put("郴州", "0735")
|
||||
CITY_CODE_MAP.put("鄂尔多斯", "0477")
|
||||
CITY_CODE_MAP.put("鄂州", "0711")
|
||||
CITY_CODE_MAP.put("酒泉", "0937")
|
||||
CITY_CODE_MAP.put("金华", "0579")
|
||||
CITY_CODE_MAP.put("金昌", "0935")
|
||||
CITY_CODE_MAP.put("钦州", "0777")
|
||||
CITY_CODE_MAP.put("铁岭", "0410")
|
||||
CITY_CODE_MAP.put("铜仁地区", "0856")
|
||||
CITY_CODE_MAP.put("铜川", "0919")
|
||||
CITY_CODE_MAP.put("铜陵", "0562")
|
||||
CITY_CODE_MAP.put("银川", "0951")
|
||||
CITY_CODE_MAP.put("锦州", "0416")
|
||||
CITY_CODE_MAP.put("长治", "0355")
|
||||
CITY_CODE_MAP.put("阜新", "0418")
|
||||
CITY_CODE_MAP.put("阜阳", "01558")
|
||||
CITY_CODE_MAP.put("阳泉", "0353")
|
||||
CITY_CODE_MAP.put("阿坝藏族羌族自治州", "0837")
|
||||
CITY_CODE_MAP.put("陇南", "02935")
|
||||
CITY_CODE_MAP.put("随州", "0722")
|
||||
CITY_CODE_MAP.put("雅安", "0835")
|
||||
CITY_CODE_MAP.put("韶关", "0751")
|
||||
CITY_CODE_MAP.put("马鞍山", "0555")
|
||||
CITY_CODE_MAP.put("驻马店", "0396")
|
||||
CITY_CODE_MAP.put("鸡西", "0467")
|
||||
CITY_CODE_MAP.put("鹤壁", "0392")
|
||||
CITY_CODE_MAP.put("鹤岗", "0468")
|
||||
CITY_CODE_MAP.put("鹰潭", "0701")
|
||||
CITY_CODE_MAP.put("黄冈", "0713")
|
||||
CITY_CODE_MAP.put("黑河", "0456")
|
||||
CITY_CODE_MAP.put("黔东南苗族侗族自治州", "0855")
|
||||
CITY_CODE_MAP.put("黔南布依族苗族自治州", "0854")
|
||||
CITY_CODE_MAP.put("齐齐哈尔", "0452")
|
||||
CITY_CODE_MAP.put("龙岩", "0597")
|
||||
CITY_CODE_MAP.put("甘南藏族自治州", "0941")
|
||||
CITY_CODE_MAP.put("嘉峪关", "1937")
|
||||
CITY_CODE_MAP.put("百色", "0776")
|
||||
CITY_CODE_MAP.put("六盘水", "0858")
|
||||
CITY_CODE_MAP.put("黔西南布依族苗族自治州", "0859")
|
||||
CITY_CODE_MAP.put("白沙黎族自治县", "0802")
|
||||
CITY_CODE_MAP.put("保亭黎族苗族自治县", "0801")
|
||||
CITY_CODE_MAP.put("昌江黎族自治县", "0803")
|
||||
CITY_CODE_MAP.put("澄迈县", "0804")
|
||||
CITY_CODE_MAP.put("定安县", "0806")
|
||||
CITY_CODE_MAP.put("东方", "0807")
|
||||
CITY_CODE_MAP.put("东沙群岛", "0808")
|
||||
CITY_CODE_MAP.put("乐东黎族自治县", "2802")
|
||||
CITY_CODE_MAP.put("临高县", "1896")
|
||||
CITY_CODE_MAP.put("陵水黎族自治县", "0809")
|
||||
CITY_CODE_MAP.put("南沙群岛", "1891")
|
||||
CITY_CODE_MAP.put("琼海", "1894")
|
||||
CITY_CODE_MAP.put("琼中黎族苗族自治县", "1899")
|
||||
CITY_CODE_MAP.put("屯昌县", "1892")
|
||||
CITY_CODE_MAP.put("万宁", "1898")
|
||||
CITY_CODE_MAP.put("文昌", "1893")
|
||||
CITY_CODE_MAP.put("五指山", "1897")
|
||||
CITY_CODE_MAP.put("西沙群岛", "1895")
|
||||
CITY_CODE_MAP.put("中沙群岛", "2801")
|
||||
CITY_CODE_MAP.put("儋州", "0805")
|
||||
CITY_CODE_MAP.put("阿拉善盟", "0483")
|
||||
CITY_CODE_MAP.put("巴彦淖尔市", "0478")
|
||||
CITY_CODE_MAP.put("锡林郭勒盟", "0479")
|
||||
CITY_CODE_MAP.put("固原", "0954")
|
||||
CITY_CODE_MAP.put("果洛藏族自治州", "0975")
|
||||
CITY_CODE_MAP.put("海西蒙古族藏族自治州", "0977")
|
||||
CITY_CODE_MAP.put("黄南藏族自治州", "0973")
|
||||
CITY_CODE_MAP.put("玉树藏族自治州", "0976")
|
||||
CITY_CODE_MAP.put("阿里地区", "0897")
|
||||
CITY_CODE_MAP.put("昌都地区", "0895")
|
||||
CITY_CODE_MAP.put("林芝地区", "0894")
|
||||
CITY_CODE_MAP.put("那曲地区", "0896")
|
||||
CITY_CODE_MAP.put("阿克苏地区", "0997")
|
||||
CITY_CODE_MAP.put("阿勒泰地区", "0906")
|
||||
CITY_CODE_MAP.put("巴音郭楞蒙古自治州", "0996")
|
||||
CITY_CODE_MAP.put("哈密地区", "0902")
|
||||
CITY_CODE_MAP.put("和田地区", "0903")
|
||||
CITY_CODE_MAP.put("喀什地区", "0998")
|
||||
CITY_CODE_MAP.put("克孜勒苏柯尔克孜自治州", "0908")
|
||||
CITY_CODE_MAP.put("保山", "0875")
|
||||
CITY_CODE_MAP.put("德宏傣族景颇族自治州", "0692")
|
||||
CITY_CODE_MAP.put("迪庆藏族自治州", "0887")
|
||||
CITY_CODE_MAP.put("丽江", "0888")
|
||||
CITY_CODE_MAP.put("临沧市", "0883")
|
||||
CITY_CODE_MAP.put("怒江傈僳族自治州", "0886")
|
||||
CITY_CODE_MAP.put("普洱", "0879")
|
||||
CITY_CODE_MAP.put("文山壮族苗族自治州", "0876")
|
||||
CITY_CODE_MAP.put("香港", "1852")
|
||||
CITY_CODE_MAP.put("澳门", "1853")
|
||||
CITY_CODE_MAP.put("五家渠市", "652300")
|
||||
CITY_CODE_MAP.put("阿拉尔市", "652900")
|
||||
CITY_CODE_MAP.put("图木舒克市", "653000")
|
||||
}
|
||||
|
||||
/**
|
||||
* 从poi中获得需要的数据
|
||||
* 从poi中获得需要的数据
|
||||
* @param ret
|
||||
* @param poi
|
||||
*/
|
||||
@@ -688,7 +688,7 @@ class AddressToBaiduLngLat {
|
||||
m.mabc_lat = o[1];
|
||||
}else{
|
||||
index++;
|
||||
println "["+index+"]"+m.address +" 没有解析出经纬度 "+m.id
|
||||
println "["+index+"]"+m.address +" 没有解析出经纬度 "+m.id
|
||||
}
|
||||
|
||||
|
||||
@@ -727,7 +727,7 @@ class AddressToBaiduLngLat {
|
||||
conn.setRequestMethod("GET");
|
||||
conn.doOutput = true
|
||||
inputStream = conn.getInputStream();
|
||||
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
|
||||
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
|
||||
while((line = buf.readLine())!=null){
|
||||
sb.append(line);
|
||||
}
|
||||
|
||||
+120
-120
@@ -35,18 +35,18 @@ class AnShouDataTest {
|
||||
def task = new AnShouDataTest();
|
||||
def token = task.getToken();
|
||||
println token
|
||||
// task.run("E:\\数据测试\\数据\\", "安硕测试数据.txt");
|
||||
// task.batchTestApi("E:\\数据测试\\数据\\", "安硕测试数据", ".txt");
|
||||
// task.run("E:\\数据测试\\数据\\", "安硕测试数据.txt");
|
||||
// task.batchTestApi("E:\\数据测试\\数据\\", "安硕测试数据", ".txt");
|
||||
}
|
||||
|
||||
def batchTestApi(def path, def fileName, def fileType){
|
||||
def ret = readTxt(path, fileName+fileType);//读文件
|
||||
def ret = readTxt(path, fileName+fileType);//读文件
|
||||
def result = [];
|
||||
def index = 0;
|
||||
for(def m : ret){
|
||||
|
||||
|
||||
if(m.estateType=="住房"){
|
||||
if(m.estateType=="住房"){
|
||||
|
||||
|
||||
index ++;
|
||||
@@ -83,28 +83,28 @@ class AnShouDataTest {
|
||||
}
|
||||
|
||||
}
|
||||
//writeCSV2("E:\\数据测试\\结果\\", "批量地址测试.csv", result);
|
||||
writeCSV("E:\\数据测试\\结果\\", fileName+"(新).csv", result);
|
||||
//writeCSV2("E:\\数据测试\\结果\\", "批量地址测试.csv", result);
|
||||
writeCSV("E:\\数据测试\\结果\\", fileName+"(新).csv", result);
|
||||
}
|
||||
|
||||
|
||||
def run(def path, def fileName){
|
||||
def ret = readTxt(path, fileName);//读文件
|
||||
def citys = CommonDatautils.findAllCity();//所有城市
|
||||
def regionMap = CommonDatautils.getAllCityRegions();//所有城市
|
||||
def provinces = CommonDatautils.findAllCityProvince();//城市所对省
|
||||
def sroucesData = selectDistrict("住房", ret, citys);//选取住宅类型
|
||||
println "住宅共:"+sroucesData.size();
|
||||
def results = getResult(sroucesData, provinces, citys, regionMap);//填充结果
|
||||
writeCSV("E:\\数据测试\\结果\\", "安硕住宅数据测试结果.csv", results);
|
||||
//writeCSV3("E:\\数据测试\\结果\\", "安硕住宅数据测试结果(6-13).csv", results);
|
||||
def ret = readTxt(path, fileName);//读文件
|
||||
def citys = CommonDatautils.findAllCity();//所有城市
|
||||
def regionMap = CommonDatautils.getAllCityRegions();//所有城市
|
||||
def provinces = CommonDatautils.findAllCityProvince();//城市所对省
|
||||
def sroucesData = selectDistrict("住房", ret, citys);//选取住宅类型
|
||||
println "住宅共:"+sroucesData.size();
|
||||
def results = getResult(sroucesData, provinces, citys, regionMap);//填充结果
|
||||
writeCSV("E:\\数据测试\\结果\\", "安硕住宅数据测试结果.csv", results);
|
||||
//writeCSV3("E:\\数据测试\\结果\\", "安硕住宅数据测试结果(6-13).csv", results);
|
||||
}
|
||||
|
||||
def getResult(def sroucesData, def provinces, def citys, def regionMap){
|
||||
def token = getToken();
|
||||
def index = 0;
|
||||
def matchIndex = 0;
|
||||
// def lastMonth = CommonDatautils.findLastMonth("上海");
|
||||
// def lastMonth = CommonDatautils.findLastMonth("上海");
|
||||
for(def m : sroucesData){
|
||||
|
||||
index++;
|
||||
@@ -115,9 +115,9 @@ class AnShouDataTest {
|
||||
// continue;
|
||||
// }
|
||||
|
||||
println "【${index}】 old:"+m.oldAddress
|
||||
println "【${index}】 old:"+m.oldAddress
|
||||
m = selectCity(m, citys);
|
||||
//省
|
||||
//省
|
||||
if(m.city){
|
||||
m.address = AddressParserUtils.filterCity(m.city, m.oldAddress);
|
||||
m.province = provinces.get(m.city);
|
||||
@@ -127,9 +127,9 @@ class AnShouDataTest {
|
||||
|
||||
|
||||
def numMap = AddressParserUtils.selectHouseNumAndRoom(m.address?m.address:m.oldAddress);
|
||||
println "【旧方法】"+numMap
|
||||
println "【旧方法】"+numMap
|
||||
def numMap2 = AddressParserUtils.selectHouseNumAndRoom2(m.address?m.address:m.oldAddress);
|
||||
println "【新方法】"+numMap2
|
||||
println "【新方法】"+numMap2
|
||||
m.houseNum = numMap.houseNum;
|
||||
m.roomNum = numMap.roomNum;
|
||||
m.unitNum = numMap.unitNum;
|
||||
@@ -145,12 +145,12 @@ class AnShouDataTest {
|
||||
m.address1 = numMap2.s1?numMap2.s1:numMap2.s2;
|
||||
m.distName1 = numMap2.s2;
|
||||
|
||||
//地址匹配
|
||||
//地址匹配
|
||||
def rett = null;
|
||||
if(m.address){
|
||||
rett = getNameByAPI(m.city, m.address, token);
|
||||
}
|
||||
//地址没匹配上,如果能提出小区名,用city+name查找
|
||||
//地址没匹配上,如果能提出小区名,用city+name查找
|
||||
if(!rett && m.distName){
|
||||
rett = getNameByAPI(m.city, m.distName, token);
|
||||
if(!rett){
|
||||
@@ -176,7 +176,7 @@ class AnShouDataTest {
|
||||
if(!m.address){
|
||||
m.address = rett.address
|
||||
}else{
|
||||
if((rett.address.indexOf("弄")!=-1||rett.address.indexOf("号")!=-1||rett.address.indexOf("街")!=-1)&&m.address.indexOf("号")==-1&&m.address.indexOf("路")==-1){
|
||||
if((rett.address.indexOf("弄")!=-1||rett.address.indexOf("号")!=-1||rett.address.indexOf("街")!=-1)&&m.address.indexOf("号")==-1&&m.address.indexOf("路")==-1){
|
||||
m.address = rett.address
|
||||
}
|
||||
}
|
||||
@@ -195,24 +195,24 @@ class AnShouDataTest {
|
||||
m = getDistrictInfo(m);
|
||||
|
||||
if(m.address){
|
||||
m.address = m.address.replace("#", "#").replaceAll(",", "、");
|
||||
m.address = m.address.replace("#", "#").replaceAll(",", "、");
|
||||
}
|
||||
if(m.address){
|
||||
m.address = m.address.replace("#", "#").replaceAll(",", "、");
|
||||
m.address = m.address.replace("#", "#").replaceAll(",", "、");
|
||||
}
|
||||
|
||||
|
||||
|
||||
//if(m.houseNum ||m.roomNum){
|
||||
|
||||
println "结果1: 【${m.name}】【${m.houseNum}】【${m.unitNum}】【${m.roomNum}】"
|
||||
println "结果1: 【${m.name}】【${numMap2.houseNum}】【${numMap2.unitNum}】【${numMap2.roomNum}】"
|
||||
println "结果2:【${m.address}】【${m.distName}】【${m.houseInfo}】【${m.baiduMatch}】【${m.baiduAddress}】"
|
||||
println "结果1: 【${m.name}】【${m.houseNum}】【${m.unitNum}】【${m.roomNum}】"
|
||||
println "结果1: 【${m.name}】【${numMap2.houseNum}】【${numMap2.unitNum}】【${numMap2.roomNum}】"
|
||||
println "结果2:【${m.address}】【${m.distName}】【${m.houseInfo}】【${m.baiduMatch}】【${m.baiduAddress}】"
|
||||
println "..................................................."
|
||||
// }
|
||||
|
||||
}
|
||||
println "能匹配上的有:[ "+matchIndex+" ]";
|
||||
println "能匹配上的有:[ "+matchIndex+" ]";
|
||||
return sroucesData;
|
||||
}
|
||||
|
||||
@@ -230,7 +230,7 @@ class AnShouDataTest {
|
||||
m.avgPrice = CommonDatautils.queryDistrictPrice(m.city, m.region, m.name, weekend, m.cat);
|
||||
}
|
||||
|
||||
//房价网估价
|
||||
//房价网估价
|
||||
m.fangjiaTotalPrice = evaluation(m, token);
|
||||
if(!m.fangjiaTotalPrice && m.avgPrice){
|
||||
m.fangjiaTotalPrice = (m.avgPrice * m.area).toInteger();
|
||||
@@ -238,7 +238,7 @@ class AnShouDataTest {
|
||||
|
||||
//m.rentPrice = queryRentPriceRange(m.city, m.region, m.name, lastMonth);
|
||||
|
||||
//其它网站均价
|
||||
//其它网站均价
|
||||
def otherWebsiteMap = CommonDatautils.queryOtherWebsitePrice(m.city, m.name);
|
||||
m.sunfunPrice = otherWebsiteMap.price_soufun
|
||||
m.anjukePrice = otherWebsiteMap.price_anjuke
|
||||
@@ -267,7 +267,7 @@ class AnShouDataTest {
|
||||
m.city = m.oldCity;
|
||||
return m;
|
||||
}else{
|
||||
def city = m.oldAddress.substring(0, m.oldAddress.indexOf("市"));
|
||||
def city = m.oldAddress.substring(0, m.oldAddress.indexOf("市"));
|
||||
if(city != m.oldCity){
|
||||
m.city = city;
|
||||
}
|
||||
@@ -281,8 +281,8 @@ class AnShouDataTest {
|
||||
}
|
||||
def selectRegion(def m){
|
||||
def address = m.oldAddress;
|
||||
if(address.indexOf("市")!=-1 && address.indexOf("区")!=-1){
|
||||
m.region = address.substring(address.indexOf("市")+1, address.indexOf("区"));
|
||||
if(address.indexOf("市")!=-1 && address.indexOf("区")!=-1){
|
||||
m.region = address.substring(address.indexOf("市")+1, address.indexOf("区"));
|
||||
}
|
||||
return m;
|
||||
}
|
||||
@@ -291,25 +291,25 @@ class AnShouDataTest {
|
||||
m.roomNum = filterRoomNum(m.roomNum);
|
||||
m.unitNum = selectUnitNum(m.address!=null?m.address:m.oldAddress);
|
||||
|
||||
//所在楼层
|
||||
//所在楼层
|
||||
//m = selectHouseFloor(m);
|
||||
m = standardAddress(m);
|
||||
return m;
|
||||
}
|
||||
|
||||
def selectUnitNum(def address){
|
||||
def ret = findPattern(address, "[幢|栋|楼](\\d+|\\w+|\\S)[单元|座]",0);
|
||||
def ret = findPattern(address, "[幢|栋|楼](\\d+|\\w+|\\S)[单元|座]",0);
|
||||
if(!ret.num1){
|
||||
ret = findPattern(address, "[层](\\d+|\\w+|\\S)[单元|座]",0);
|
||||
ret = findPattern(address, "[层](\\d+|\\w+|\\S)[单元|座]",0);
|
||||
}
|
||||
|
||||
if(!ret.num1){
|
||||
ret = findPattern(address, "(\\d+|\\w+|\\S)[单元|座]",0);
|
||||
ret = findPattern(address, "(\\d+|\\w+|\\S)[单元|座]",0);
|
||||
}
|
||||
|
||||
|
||||
if(ret.num1){
|
||||
ret.num1 = ret.num1.replace("一", "1").replace("二", "2").replace("三", "3").replace("四", "4").replace("五", "5").replace("六", "6").replace("七", "7").replace("八", "8").replace("九", "9").replace("十", "10")
|
||||
ret.num1 = ret.num1.replace("一", "1").replace("二", "2").replace("三", "3").replace("四", "4").replace("五", "5").replace("六", "6").replace("七", "7").replace("八", "8").replace("九", "9").replace("十", "10")
|
||||
}
|
||||
return ret.num1;
|
||||
}
|
||||
@@ -334,8 +334,8 @@ class AnShouDataTest {
|
||||
def address = m.address!=null?m.address:m.oldAddress;
|
||||
|
||||
|
||||
if(!m.region && address.indexOf("县")!=-1){
|
||||
m.region = address.substring(0, address.indexOf("县"));
|
||||
if(!m.region && address.indexOf("县")!=-1){
|
||||
m.region = address.substring(0, address.indexOf("县"));
|
||||
if(m.region){
|
||||
m.address = address.substring(address.indexOf(m.region)+m.region.length() + 1, address.length());
|
||||
}
|
||||
@@ -343,15 +343,15 @@ class AnShouDataTest {
|
||||
|
||||
}
|
||||
|
||||
if(!m.region && address.indexOf("区")!=-1){
|
||||
m.region = address.substring(0, address.indexOf("区")+1);
|
||||
if(!m.region && address.indexOf("区")!=-1){
|
||||
m.region = address.substring(0, address.indexOf("区")+1);
|
||||
if(m.region){
|
||||
m.address = address.substring(address.indexOf(m.region)+m.region.length(), address.length());
|
||||
def index = m.address.indexOf("区");
|
||||
def index = m.address.indexOf("区");
|
||||
if(index!=-1){
|
||||
def end = m.address.indexOf("街");
|
||||
def end = m.address.indexOf("街");
|
||||
if(end==-1){
|
||||
end = m.address.indexOf("路");
|
||||
end = m.address.indexOf("路");
|
||||
}
|
||||
|
||||
if(index < end){
|
||||
@@ -362,22 +362,22 @@ class AnShouDataTest {
|
||||
m.region = findStandardRegion(m.city, m.region, regionMap);
|
||||
}
|
||||
|
||||
if(!m.region && address.indexOf("市")!=-1){
|
||||
m.region = address.substring(0, address.indexOf("市"));
|
||||
if(!m.region && address.indexOf("市")!=-1){
|
||||
m.region = address.substring(0, address.indexOf("市"));
|
||||
if(m.region){
|
||||
m.address = address.substring(address.indexOf(m.region)+m.region.length() + 1, address.length());
|
||||
}
|
||||
m.region = findStandardRegion(m.city, m.region, regionMap);
|
||||
}
|
||||
|
||||
m.oldAddress = m.oldAddress.replaceAll(",", "、");
|
||||
m.address = m.address.replace("#", "#").replaceAll(",", "、");
|
||||
m.oldAddress = m.oldAddress.replaceAll(",", "、");
|
||||
m.address = m.address.replace("#", "#").replaceAll(",", "、");
|
||||
return m;
|
||||
}
|
||||
|
||||
def standardAddress(def m){
|
||||
if(m.address.indexOf("号")!=-1){
|
||||
m.address = m.address.substring(0, m.address.indexOf("号")+1);
|
||||
if(m.address.indexOf("号")!=-1){
|
||||
m.address = m.address.substring(0, m.address.indexOf("号")+1);
|
||||
}else if(m.houseNum){
|
||||
m.address = m.address.substring(0, m.address.indexOf(m.houseNum));
|
||||
}else if(m.roomNum){
|
||||
@@ -391,53 +391,53 @@ class AnShouDataTest {
|
||||
def selectHouseNumAndRoom(def map){
|
||||
def address = map.address?map.address:map.oldAddress;
|
||||
|
||||
def ret = findPattern(address, "[弄|区](\\d+)[号|幢|栋](\\w{2,4})?[室]?",1);
|
||||
def ret = findPattern(address, "[弄|区](\\d+)[号|幢|栋](\\w{2,4})?[室]?",1);
|
||||
map.houseNum = ret.num1;
|
||||
map.roomNum = ret.num2;
|
||||
|
||||
if(!map.houseNum){
|
||||
address = map.address?map.address:map.oldAddress;
|
||||
ret = findPattern(address, "号(\\d+)[幢|栋](\\d+)?[室|房]", 1);
|
||||
ret = findPattern(address, "号(\\d+)[幢|栋](\\d+)?[室|房]", 1);
|
||||
map.houseNum = ret.num1;
|
||||
map.roomNum = ret.num2;
|
||||
}
|
||||
|
||||
if(!map.houseNum){
|
||||
address = map.address?map.address:map.oldAddress;
|
||||
ret = findPattern(address, "(\\d+|\\w+)[幢|栋]", 0);
|
||||
ret = findPattern(address, "(\\d+|\\w+)[幢|栋]", 0);
|
||||
map.houseNum = ret.num1;
|
||||
}
|
||||
|
||||
if(!map.houseNum){
|
||||
address = map.address?map.address:map.oldAddress;
|
||||
ret = findPattern(address, "(\\d+|\\w+)号楼", 0);
|
||||
ret = findPattern(address, "(\\d+|\\w+)号楼", 0);
|
||||
map.houseNum = ret.num1;
|
||||
}
|
||||
|
||||
|
||||
if(!map.houseNum){
|
||||
address = map.address?map.address:map.oldAddress;
|
||||
ret = findPattern(address, "(\\d+)#楼", 0);
|
||||
ret = findPattern(address, "(\\d+)#楼", 0);
|
||||
map.houseNum = ret.num1;
|
||||
}
|
||||
|
||||
|
||||
if(!map.roomNum){
|
||||
address = map.address?map.address:map.oldAddress;
|
||||
ret = findPattern(address, "(\\w{1,4}|\\d+)[室|房]",0);
|
||||
ret = findPattern(address, "(\\w{1,4}|\\d+)[室|房]",0);
|
||||
map.roomNum = ret.num1;
|
||||
}
|
||||
|
||||
if(!map.roomNum){
|
||||
address = map.address?map.address:map.oldAddress;
|
||||
|
||||
ret = findPattern(address, "单元(\\w{1,4})[号|室]",0);
|
||||
ret = findPattern(address, "单元(\\w{1,4})[号|室]",0);
|
||||
map.roomNum = ret.num1;
|
||||
}
|
||||
|
||||
|
||||
if(!map.houseNum&&!map.roomNum && map.oldAddress.indexOf("路")==-1 && map.oldAddress.indexOf("街")==-1){
|
||||
ret = findPattern(map.oldAddress, "(\\d+)[号|幢|栋](\\w{2,4})?[室]?",0);
|
||||
if(!map.houseNum&&!map.roomNum && map.oldAddress.indexOf("路")==-1 && map.oldAddress.indexOf("街")==-1){
|
||||
ret = findPattern(map.oldAddress, "(\\d+)[号|幢|栋](\\w{2,4})?[室]?",0);
|
||||
map.houseNum = ret.num1;
|
||||
map.roomNum = ret.num2;
|
||||
|
||||
@@ -446,7 +446,7 @@ class AnShouDataTest {
|
||||
|
||||
if(!map.roomNum){
|
||||
address = map.address?map.address:map.oldAddress;
|
||||
ret = findPattern(address, "[座|号楼|梯|层](\\d{3,5})",0);
|
||||
ret = findPattern(address, "[座|号楼|梯|层](\\d{3,5})",0);
|
||||
map.roomNum = ret.num1;
|
||||
}
|
||||
|
||||
@@ -456,7 +456,7 @@ class AnShouDataTest {
|
||||
map.houseNum = ret.num1;
|
||||
map.roomNum = ret.num2;
|
||||
}
|
||||
//println map.oldAddress+" 楼栋号结果:["+map.houseNum+"]["+map.roomNum+"]["+map.address+"]";
|
||||
//println map.oldAddress+" 楼栋号结果:["+map.houseNum+"]["+map.roomNum+"]["+map.address+"]";
|
||||
return map;
|
||||
}
|
||||
|
||||
@@ -478,7 +478,7 @@ class AnShouDataTest {
|
||||
}
|
||||
if(!map.floor){
|
||||
def address = map.address?map.address:map.oldAddress;
|
||||
def ret = findPattern(address, "(\\d+)[层|F|f]", 0);
|
||||
def ret = findPattern(address, "(\\d+)[层|F|f]", 0);
|
||||
map.floor = ret.num1;
|
||||
if(ret.address){
|
||||
map.address = ret.address;
|
||||
@@ -492,7 +492,7 @@ class AnShouDataTest {
|
||||
if(!map.floor){
|
||||
map.floor = null;
|
||||
}
|
||||
//println map.oldAddress +" 结果:"+map.floor+" ["+map.address+"]";
|
||||
//println map.oldAddress +" 结果:"+map.floor+" ["+map.address+"]";
|
||||
return map;
|
||||
}
|
||||
|
||||
@@ -575,7 +575,7 @@ class AnShouDataTest {
|
||||
|
||||
}
|
||||
|
||||
//println "周边均价:" + totalPrice +"【${url_str}】"
|
||||
//println "周边均价:" + totalPrice +"【${url_str}】"
|
||||
return totalPrice;
|
||||
}
|
||||
|
||||
@@ -592,7 +592,7 @@ class AnShouDataTest {
|
||||
List vs = vo.get("cla");
|
||||
for(String[] o : vs){
|
||||
String s = o[1];
|
||||
if(s.indexOf("小区")!=-1||s.indexOf("楼盘")!=-1){
|
||||
if(s.indexOf("小区")!=-1||s.indexOf("楼盘")!=-1){
|
||||
address = vo.get("addr");
|
||||
break;
|
||||
}
|
||||
@@ -705,7 +705,7 @@ class AnShouDataTest {
|
||||
conn.setRequestMethod("GET");
|
||||
conn.doOutput = true
|
||||
inputStream = conn.getInputStream();
|
||||
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
|
||||
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
|
||||
while((line = buf.readLine())!=null){
|
||||
sb.append(line);
|
||||
}
|
||||
@@ -743,14 +743,14 @@ class AnShouDataTest {
|
||||
for(def m : ret){
|
||||
if(m.estateType == estateType){
|
||||
result << m;
|
||||
// if(m.address.indexOf("市")!=-1){
|
||||
// def city = m.address.substring(0, m.address.lastIndexOf("市"));
|
||||
// if(city.indexOf("市")!=-1){
|
||||
// def tempCity = city.substring(city.indexOf("市")+1, city.length());
|
||||
// if(m.address.indexOf("市")!=-1){
|
||||
// def city = m.address.substring(0, m.address.lastIndexOf("市"));
|
||||
// if(city.indexOf("市")!=-1){
|
||||
// def tempCity = city.substring(city.indexOf("市")+1, city.length());
|
||||
// if(citys.contains(tempCity)){
|
||||
// city = tempCity;
|
||||
// }else{
|
||||
// city = m.address.substring(0, m.address.indexOf("市"))
|
||||
// city = m.address.substring(0, m.address.indexOf("市"))
|
||||
// }
|
||||
// }
|
||||
// if(city){
|
||||
@@ -784,11 +784,11 @@ class AnShouDataTest {
|
||||
String estateType = arrs[4]
|
||||
def m = [:];
|
||||
m.cid = cid;
|
||||
if(estateType=="住房"){
|
||||
if(estateType=="住房"){
|
||||
m.cat = "district";
|
||||
}else if(estateType=='写字楼'){
|
||||
}else if(estateType=='写字楼'){
|
||||
m.cat = "office";
|
||||
}else if(estateType=="商铺"){
|
||||
}else if(estateType=="商铺"){
|
||||
m.cat = "shop";
|
||||
}else{
|
||||
m.cat = "other";
|
||||
@@ -811,16 +811,16 @@ class AnShouDataTest {
|
||||
}
|
||||
|
||||
|
||||
file.append("城市,")
|
||||
file.append("地址,")
|
||||
file.append("楼盘名称,")
|
||||
file.append("省,")
|
||||
file.append("楼盘地址,")
|
||||
file.append("别名,")
|
||||
file.append("楼号,")
|
||||
file.append("室号,")
|
||||
file.append("单元号,")
|
||||
file.append("状态,")
|
||||
file.append("城市,")
|
||||
file.append("地址,")
|
||||
file.append("楼盘名称,")
|
||||
file.append("省,")
|
||||
file.append("楼盘地址,")
|
||||
file.append("别名,")
|
||||
file.append("楼号,")
|
||||
file.append("室号,")
|
||||
file.append("单元号,")
|
||||
file.append("状态,")
|
||||
|
||||
file.append(System.getProperty("line.separator"));
|
||||
ret.each{
|
||||
@@ -864,29 +864,29 @@ class AnShouDataTest {
|
||||
}
|
||||
|
||||
|
||||
file.append("房产编号,")
|
||||
file.append("房产地址,")
|
||||
file.append("城市,")
|
||||
file.append("建筑面积,")
|
||||
file.append("抵押物类型,")
|
||||
file.append("标准化-省,")
|
||||
file.append("标准化-区,")
|
||||
file.append("标准化-市,")
|
||||
file.append("标准化-板块,")
|
||||
file.append("标准化-楼盘,")
|
||||
file.append("标准化-楼盘地址,")
|
||||
file.append("楼栋,")
|
||||
file.append("单元(号),")
|
||||
file.append("房号,")
|
||||
file.append("楼盘单价,")
|
||||
file.append("批量估值(精准),")
|
||||
file.append("搜房网楼盘单价,")
|
||||
file.append("搜房网精确估值,")
|
||||
file.append("安居客楼盘价格,")
|
||||
//file.append("匹配状态,")
|
||||
//file.append("别名,")
|
||||
file.append("安居客精确估值,")
|
||||
// file.append("租金,")
|
||||
file.append("房产编号,")
|
||||
file.append("房产地址,")
|
||||
file.append("城市,")
|
||||
file.append("建筑面积,")
|
||||
file.append("抵押物类型,")
|
||||
file.append("标准化-省,")
|
||||
file.append("标准化-区,")
|
||||
file.append("标准化-市,")
|
||||
file.append("标准化-板块,")
|
||||
file.append("标准化-楼盘,")
|
||||
file.append("标准化-楼盘地址,")
|
||||
file.append("楼栋,")
|
||||
file.append("单元(号),")
|
||||
file.append("房号,")
|
||||
file.append("楼盘单价,")
|
||||
file.append("批量估值(精准),")
|
||||
file.append("搜房网楼盘单价,")
|
||||
file.append("搜房网精确估值,")
|
||||
file.append("安居客楼盘价格,")
|
||||
//file.append("匹配状态,")
|
||||
//file.append("别名,")
|
||||
file.append("安居客精确估值,")
|
||||
// file.append("租金,")
|
||||
file.append(System.getProperty("line.separator"));
|
||||
ret.each{
|
||||
file.append(it.cid + ",","gbk")
|
||||
@@ -993,17 +993,17 @@ class AnShouDataTest {
|
||||
}
|
||||
|
||||
|
||||
file.append("房产地址,")
|
||||
file.append("标准化-区,")
|
||||
file.append("标准化-板块,")
|
||||
file.append("标准化-楼盘,")
|
||||
file.append("标准化-楼盘地址,")
|
||||
file.append("楼栋,")
|
||||
//file.append("楼栋-新,")
|
||||
file.append("单元(号),")
|
||||
//file.append("单元(号)-新,")
|
||||
file.append("房号,")
|
||||
//file.append("房号-新,")
|
||||
file.append("房产地址,")
|
||||
file.append("标准化-区,")
|
||||
file.append("标准化-板块,")
|
||||
file.append("标准化-楼盘,")
|
||||
file.append("标准化-楼盘地址,")
|
||||
file.append("楼栋,")
|
||||
//file.append("楼栋-新,")
|
||||
file.append("单元(号),")
|
||||
//file.append("单元(号)-新,")
|
||||
file.append("房号,")
|
||||
//file.append("房号-新,")
|
||||
|
||||
file.append(System.getProperty("line.separator"));
|
||||
ret.each{
|
||||
|
||||
+3
-3
@@ -12,7 +12,7 @@ import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
/**
|
||||
*批量地址分析脚本
|
||||
*批量地址分析脚本
|
||||
*/
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
public class BulkAddressAnalyBatch extends BaseBatch{
|
||||
@@ -41,7 +41,7 @@ public class BulkAddressAnalyBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 恢复被打断的任务
|
||||
* 恢复被打断的任务
|
||||
*/
|
||||
void resumeInterrtuptedTask(){
|
||||
|
||||
@@ -57,7 +57,7 @@ public class BulkAddressAnalyBatch extends BaseBatch{
|
||||
task.status = TaskStatus.TASK_STATUS_DOING
|
||||
TASK.save(task)
|
||||
executorService.submit(new AddressAnalysisJob(task, false))
|
||||
logger.info("提交任务:${task._id.toString()}")
|
||||
logger.info("提交任务:${task._id.toString()}")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -69,7 +69,7 @@ public class BulkEvaluationForTest extends BaseBatch{
|
||||
def findNewTask(MqProvider mqProvider){
|
||||
ADDRESS_COLL.find(new BasicDBObject("finish_mark",0)).sort(new BasicDBObject("version", -1)).each {BasicDBObject dbo->
|
||||
ADDRESS_COLL.update(new BasicDBObject("_id", dbo._id), new BasicDBObject('$set', new BasicDBObject("finish_mark", 1).append("send_date", new Date())))
|
||||
println("${new Date().format("yyyy-MM-dd HH:mm:ss")}Ìá½»ÈÎÎñ${dbo.city} ${dbo.address}")
|
||||
println("${new Date().format("yyyy-MM-dd HH:mm:ss")}提交任务${dbo.city} ${dbo.address}")
|
||||
mqProvider.send(dbo.get("_id").toString())
|
||||
//executorService.submit(new BulkEvaluationTask(dbo))
|
||||
//new BulkEvaluationTask(dbo);
|
||||
|
||||
+12
-12
@@ -53,7 +53,7 @@ public class BulkEvaluationTask implements Runnable {
|
||||
Integer height = task.height
|
||||
String build_date = task.build_date
|
||||
String buildDate = null
|
||||
String estateType = "普通住宅"
|
||||
String estateType = "普通住宅"
|
||||
if(StringUtils.isNotBlank(build_date)){
|
||||
Pattern pattern = Pattern.compile("^\\d\\d\\d\\d");
|
||||
def matcher = pattern.matcher(build_date)
|
||||
@@ -62,15 +62,15 @@ public class BulkEvaluationTask implements Runnable {
|
||||
}
|
||||
}
|
||||
if (StringUtils.isBlank(city) || area == null || area <= 0 || area > 1000) {
|
||||
task.msg = "参数非法"
|
||||
task.msg = "参数非法"
|
||||
finishTask()
|
||||
return
|
||||
}
|
||||
if (city.endsWith("市")) {
|
||||
city = city.replaceAll("市\$", "")
|
||||
if (city.endsWith("市")) {
|
||||
city = city.replaceAll("市\$", "")
|
||||
}
|
||||
if((districtName && districtName.contains("别墅")) || (address && address.contains("别墅"))){
|
||||
estateType = "别墅"
|
||||
if((districtName && districtName.contains("别墅")) || (address && address.contains("别墅"))){
|
||||
estateType = "别墅"
|
||||
}
|
||||
JSONObject matchResult = null
|
||||
|
||||
@@ -117,7 +117,7 @@ public class BulkEvaluationTask implements Runnable {
|
||||
BasicDBObject district = DbUtils.seaweed.findById(matchResult.getString("id"));
|
||||
def propertyType = null;
|
||||
if(district){
|
||||
if(district.estate_type2.contains("别墅")){
|
||||
if(district.estate_type2.contains("别墅")){
|
||||
propertyType = 1;
|
||||
}else{
|
||||
propertyType = 0;
|
||||
@@ -198,7 +198,7 @@ public class BulkEvaluationTask implements Runnable {
|
||||
task.fjw_price = (task.total_price/task.area).toInteger();
|
||||
}
|
||||
|
||||
//房估估价格
|
||||
//房估估价格
|
||||
if(task.match_city && task.match_region && task.match_name){
|
||||
def priceObj = findFangguguAndYungujiaPrice(task.match_city, task.match_region, task.match_name);
|
||||
if(priceObj){
|
||||
@@ -211,7 +211,7 @@ public class BulkEvaluationTask implements Runnable {
|
||||
}
|
||||
|
||||
|
||||
//其它网站价格
|
||||
//其它网站价格
|
||||
|
||||
def startDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -6, "yyyy-MM");
|
||||
startDate = startDate.replace("-", "")
|
||||
@@ -300,7 +300,7 @@ public class BulkEvaluationTask implements Runnable {
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
println("${new Date().format("yyyy-MM-dd HH:mm:ss")}估价完成")
|
||||
println("${new Date().format("yyyy-MM-dd HH:mm:ss")}估价完成")
|
||||
}
|
||||
|
||||
def findFangguguAndYungujiaPrice(def city, def region, def name){
|
||||
@@ -514,7 +514,7 @@ public class BulkEvaluationTask implements Runnable {
|
||||
def values = [];
|
||||
def fjw_price = fjwPrice;
|
||||
|
||||
//找出最小差
|
||||
//找出最小差
|
||||
for(def i=0;i<prices.size()-1;i++){
|
||||
def a = prices[i]
|
||||
for(def j=i+1;j<prices.size();j++){
|
||||
@@ -526,7 +526,7 @@ public class BulkEvaluationTask implements Runnable {
|
||||
}
|
||||
}
|
||||
|
||||
//根据最小差找出结果
|
||||
//根据最小差找出结果
|
||||
for(def i=0;i<prices.size()-1;i++){
|
||||
def a = prices[i]
|
||||
for(def j=i+1;j<prices.size();j++){
|
||||
|
||||
+5
-5
@@ -1,7 +1,7 @@
|
||||
package com.fangjia.script.address
|
||||
/**
|
||||
* 去除相邻重复的字符串(不包括单个字符重复)
|
||||
* @author 王文艳
|
||||
* 去除相邻重复的字符串(不包括单个字符重复)
|
||||
* @author 王文艳
|
||||
*
|
||||
*/
|
||||
class DuplicateRemovalHandler extends AddressStandardizeHandler {
|
||||
@@ -11,7 +11,7 @@ class DuplicateRemovalHandler extends AddressStandardizeHandler {
|
||||
try{
|
||||
request.setAddress(duplicateRemoval(request.getAddress()));
|
||||
}catch(Exception e){
|
||||
println "DuplicateRemovalHandler: " + request.getAddress()+" 处理失败,message:"+e.getMessage();
|
||||
println "DuplicateRemovalHandler: " + request.getAddress()+" 处理失败,message:"+e.getMessage();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ class DuplicateRemovalHandler extends AddressStandardizeHandler {
|
||||
for(i=0; i< s.length();i++){
|
||||
for(int j= i+1; j < s.length() ; j++ ){
|
||||
foundStr = s.substring(i, j+1);
|
||||
if(s.indexOf(foundStr,j+1) == j+1 && !(foundStr ==~/\d+/)){//方向相邻重复且非数字的字符串片段
|
||||
if(s.indexOf(foundStr,j+1) == j+1 && !(foundStr ==~/\d+/)){//方向相邻重复且非数字的字符串片段
|
||||
s = s.replaceFirst(foundStr, "");
|
||||
continue outer;
|
||||
}
|
||||
@@ -36,7 +36,7 @@ class DuplicateRemovalHandler extends AddressStandardizeHandler {
|
||||
}
|
||||
|
||||
public static void main(String[] args){
|
||||
Request request = new Request("西南1000米)");
|
||||
Request request = new Request("西南1000米)");
|
||||
new DuplicateRemovalHandler().handleRequest(request)
|
||||
print request.getAddress();
|
||||
}
|
||||
|
||||
+100
-100
@@ -36,15 +36,15 @@ class JinDongDataTest {
|
||||
static main(args) {
|
||||
def task = new JinDongDataTest();
|
||||
|
||||
//task.batchTestApi("E:\\数据测试\\数据\\", "京东匹配成功数据", ".txt");
|
||||
task.batchTest1("E:\\数据测试\\数据\\", "商铺-京东", ".txt");
|
||||
//task.batchTest1("E:\\数据测试\\数据\\", "住宅找到小区-京东", ".txt");
|
||||
//task.batchTest2("E:\\数据测试\\数据\\", "京东测试数据", ".txt");
|
||||
//task.batchTestApi("E:\\数据测试\\数据\\", "京东匹配成功数据", ".txt");
|
||||
task.batchTest1("E:\\数据测试\\数据\\", "商铺-京东", ".txt");
|
||||
//task.batchTest1("E:\\数据测试\\数据\\", "住宅找到小区-京东", ".txt");
|
||||
//task.batchTest2("E:\\数据测试\\数据\\", "京东测试数据", ".txt");
|
||||
}
|
||||
|
||||
|
||||
def batchTestApi(def path, def fileName, def fileType){
|
||||
def ret = readTxt(path, fileName+fileType);//读文件
|
||||
def ret = readTxt(path, fileName+fileType);//读文件
|
||||
def result = [];
|
||||
def index = 0;
|
||||
for(def map : ret){
|
||||
@@ -58,40 +58,40 @@ class JinDongDataTest {
|
||||
rett = getDistrictBaseInfo(rett);
|
||||
rett = getBiduLngLat(rett.city, rett.oldAddress, rett);
|
||||
rett = getDistrictKind(rett);
|
||||
println "基本信息查询完成....."
|
||||
println "基本信息查询完成....."
|
||||
rett = getDistrictInfo(rett)
|
||||
println "小区均价查询完成....."
|
||||
println "小区均价查询完成....."
|
||||
rett = getCityAvgPriceInfo(rett);
|
||||
println "城市均价查询完成....."
|
||||
println "城市均价查询完成....."
|
||||
rett = getRentAvgPriceInfo(rett);
|
||||
println "租金信息查询完成....."
|
||||
println "租金信息查询完成....."
|
||||
rett = getOneYearDistrictPriceInfo(rett);
|
||||
println "最大最小均价完成....."
|
||||
println "最大最小均价完成....."
|
||||
rett = getPricePredict(rett);
|
||||
println "预测完成....."
|
||||
println "预测完成....."
|
||||
rett = getAvgConsumption(rett);
|
||||
println "人均消费完成....."
|
||||
println "人均消费完成....."
|
||||
rett = getOfficeCount(rett);
|
||||
println "办公人口完成....."
|
||||
println "办公人口完成....."
|
||||
rett = getPoiCount(rett);
|
||||
println "poi完成....."
|
||||
println "poi完成....."
|
||||
rett = getRegionInfo(rett);
|
||||
rett = getBlockInfo(rett);
|
||||
println "区域板块完成....."
|
||||
println "区域板块完成....."
|
||||
rett = getHouseCount(rett);
|
||||
println "房子数完成....."
|
||||
println "房子数完成....."
|
||||
result << rett;
|
||||
|
||||
println "["+index+"]"+rett
|
||||
}
|
||||
|
||||
def sysTime = System.currentTimeMillis();
|
||||
writeCSV("E:\\数据测试\\结果\\", fileName+"("+sysTime+").csv", result);
|
||||
writeCSV("E:\\数据测试\\结果\\", fileName+"("+sysTime+").csv", result);
|
||||
}
|
||||
|
||||
|
||||
def batchTest1(def path, def fileName, def fileType){
|
||||
def ret = readTxt(path, fileName+fileType);//读文件
|
||||
def ret = readTxt(path, fileName+fileType);//读文件
|
||||
def result = [];
|
||||
def index = 0;
|
||||
for(def rett : ret){
|
||||
@@ -108,54 +108,54 @@ class JinDongDataTest {
|
||||
rett.oldName = parseMap.oldName;
|
||||
rett.db_cat = parseMap.cat
|
||||
rett.area = 100;
|
||||
rett.cat = rett.type=='写字楼'?'office':(rett.type=='商铺'?'shop':(rett.type=='住宅'?"district":""));
|
||||
rett.cat = rett.type=='写字楼'?'office':(rett.type=='商铺'?'shop':(rett.type=='住宅'?"district":""));
|
||||
rett = getDistrictBaseInfo(rett);
|
||||
if(!rett.type && !rett.cat){
|
||||
rett.cat = "district";
|
||||
}
|
||||
rett = getBiduLngLat(rett.city, rett.oldAddress, rett);
|
||||
rett = getDistrictKind(rett);
|
||||
println "基本信息查询完成....."
|
||||
println "基本信息查询完成....."
|
||||
// rett = getDistrictInfo(rett)
|
||||
// println "小区均价查询完成....."
|
||||
// println "小区均价查询完成....."
|
||||
// if(rett.cat=='office'||rett.cat=="shop"){
|
||||
// rett = getShopOrOfficeCityAvgPriceInfo(rett);
|
||||
// println "城市均价查询完成....."
|
||||
// println "城市均价查询完成....."
|
||||
// rett = getShopOrOfficeRentAvgPriceInfo(rett);
|
||||
// println "租金信息查询完成....."
|
||||
// println "租金信息查询完成....."
|
||||
// }else{
|
||||
// rett = getCityAvgPriceInfo(rett);
|
||||
// println "城市均价查询完成....."
|
||||
// println "城市均价查询完成....."
|
||||
// rett = getRentAvgPriceInfo(rett);
|
||||
// println "租金信息查询完成....."
|
||||
// println "租金信息查询完成....."
|
||||
// }
|
||||
// rett = getOneYearDistrictPriceInfo(rett);
|
||||
// println "最大最小均价完成....."
|
||||
// println "最大最小均价完成....."
|
||||
// rett = getPricePredict(rett);
|
||||
// println "预测完成....."
|
||||
// println "预测完成....."
|
||||
// rett = getAvgConsumption(rett);
|
||||
// println "人均消费完成....."
|
||||
// println "人均消费完成....."
|
||||
// rett = getOfficeCount(rett);
|
||||
// println "写字楼完成....."
|
||||
// println "写字楼完成....."
|
||||
// rett = getPoiCount(rett);
|
||||
// println "poi完成....."
|
||||
// println "poi完成....."
|
||||
// rett = getRegionInfo(rett);
|
||||
// rett = getBlockInfo(rett);
|
||||
// println "区域板块完成....."
|
||||
// println "区域板块完成....."
|
||||
// rett = getHouseCount(rett);
|
||||
// println "房子数完成....."
|
||||
// println "房子数完成....."
|
||||
result << rett;
|
||||
|
||||
println "["+index+"]"+rett
|
||||
}
|
||||
|
||||
def sysTime = System.currentTimeMillis();
|
||||
writeCSV("E:\\数据测试\\结果\\", fileName+"("+sysTime+").csv", result);
|
||||
writeCSV("E:\\数据测试\\结果\\", fileName+"("+sysTime+").csv", result);
|
||||
}
|
||||
|
||||
|
||||
def batchTest2(def path, def fileName, def fileType){
|
||||
def ret = readTxt(path, fileName+fileType);//读文件
|
||||
def ret = readTxt(path, fileName+fileType);//读文件
|
||||
def index = 1;
|
||||
def result = [];
|
||||
for(def rett : ret){
|
||||
@@ -172,7 +172,7 @@ class JinDongDataTest {
|
||||
|
||||
}
|
||||
|
||||
writeCSV2("E:\\数据测试\\结果\\", fileName+"(周边人均消费).csv", result);
|
||||
writeCSV2("E:\\数据测试\\结果\\", fileName+"(周边人均消费).csv", result);
|
||||
}
|
||||
def getDistrictInfo(m){
|
||||
if(m.name){
|
||||
@@ -200,7 +200,7 @@ class JinDongDataTest {
|
||||
}
|
||||
|
||||
|
||||
//周边均价
|
||||
//周边均价
|
||||
if(!m.avg_price && m.oldAddress && m.area && m.city){
|
||||
|
||||
|
||||
@@ -219,7 +219,7 @@ class JinDongDataTest {
|
||||
if(city){
|
||||
def nearCityPrice = CommonDatautils.findCityPrice(city, null)
|
||||
|
||||
//半年前最近的城市均价
|
||||
//半年前最近的城市均价
|
||||
def weekend = DateUtil.formatByPattern(DateUtil.getFirstDayOfWeek(new Date()), "yyyy-MM-dd");
|
||||
def monthAgo = DateUtils.add(weekend, -30);
|
||||
def monthAgoPrice = CommonDatautils.findCityPrice(city, monthAgo);
|
||||
@@ -237,7 +237,7 @@ class JinDongDataTest {
|
||||
if(city){
|
||||
def nearCityPrice = CommonDatautils.findShopOrOfficeCityPrice(city, rett.cat, null)
|
||||
|
||||
//半年前最近的城市均价
|
||||
//半年前最近的城市均价
|
||||
def weekend = DateUtil.formatByPattern(DateUtil.getFirstDayOfWeek(new Date()), "yyyy-MM-dd");
|
||||
def monthAgo = DateUtils.add(weekend, -30);
|
||||
def monthAgoPrice = CommonDatautils.findShopOrOfficeCityPrice(city, rett.cat, monthAgo);
|
||||
@@ -280,7 +280,7 @@ class JinDongDataTest {
|
||||
public static def getOneYearDistrictPriceInfo(def rett){
|
||||
if(rett.city && rett.region && rett.name){
|
||||
def weekend = DateUtil.formatByPattern(DateUtil.getFirstDayOfWeek(new Date()), "yyyy-MM-dd");
|
||||
//一前内小区均价
|
||||
//一前内小区均价
|
||||
def oneYearAgo = DateUtils.add(weekend, -365);
|
||||
def oneYearPriceList = CommonDatautils.queryDistrictPrices(rett.city, rett.region, rett.name, rett.kind, oneYearAgo, weekend);
|
||||
if(oneYearPriceList.size()>0){
|
||||
@@ -345,11 +345,11 @@ class JinDongDataTest {
|
||||
|
||||
def getPoiCount(def rett){
|
||||
if(rett.lng2 && rett.lat2){
|
||||
rett.hspt_count = CommonDatautils.findPoiCount("医院", rett.lat2, rett.lng2)
|
||||
rett.bank_count = CommonDatautils.findPoiCount("银行", rett.lat2, rett.lng2)
|
||||
rett.school_count = CommonDatautils.findPoiCount("学校", rett.lat2, rett.lng2)
|
||||
rett.bus_count = CommonDatautils.findPoiCount("公交", rett.lat2, rett.lng2)
|
||||
rett.subway_count = CommonDatautils.findPoiCount("地铁", rett.lat2, rett.lng2)
|
||||
rett.hspt_count = CommonDatautils.findPoiCount("医院", rett.lat2, rett.lng2)
|
||||
rett.bank_count = CommonDatautils.findPoiCount("银行", rett.lat2, rett.lng2)
|
||||
rett.school_count = CommonDatautils.findPoiCount("学校", rett.lat2, rett.lng2)
|
||||
rett.bus_count = CommonDatautils.findPoiCount("公交", rett.lat2, rett.lng2)
|
||||
rett.subway_count = CommonDatautils.findPoiCount("地铁", rett.lat2, rett.lng2)
|
||||
}
|
||||
|
||||
return rett;
|
||||
@@ -425,21 +425,21 @@ class JinDongDataTest {
|
||||
|
||||
def getDistrictKind(def rett){
|
||||
if(rett.type){
|
||||
if(rett.type.indexOf("写字楼")!=-1){
|
||||
rett.kind = "写字楼";
|
||||
}else if(rett.type.indexOf("商铺")!=-1){
|
||||
rett.kind = "商铺";
|
||||
if(rett.type.indexOf("写字楼")!=-1){
|
||||
rett.kind = "写字楼";
|
||||
}else if(rett.type.indexOf("商铺")!=-1){
|
||||
rett.kind = "商铺";
|
||||
}else{
|
||||
if(rett.estateType){
|
||||
if(rett.estateType.indexOf("别墅")!=-1){
|
||||
rett.kind = "别墅";
|
||||
if(rett.estateType.indexOf("别墅")!=-1){
|
||||
rett.kind = "别墅";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(!rett.kind){
|
||||
rett.kind = "小区";
|
||||
rett.kind = "小区";
|
||||
}
|
||||
return rett;
|
||||
}
|
||||
@@ -487,7 +487,7 @@ class JinDongDataTest {
|
||||
|
||||
}
|
||||
|
||||
//println "周边均价:" + totalPrice +"【${url_str}】"
|
||||
//println "周边均价:" + totalPrice +"【${url_str}】"
|
||||
return totalPrice;
|
||||
}
|
||||
|
||||
@@ -504,7 +504,7 @@ class JinDongDataTest {
|
||||
conn.setRequestMethod("GET");
|
||||
conn.doOutput = true
|
||||
inputStream = conn.getInputStream();
|
||||
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
|
||||
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
|
||||
while((line = buf.readLine())!=null){
|
||||
sb.append(line);
|
||||
}
|
||||
@@ -531,11 +531,11 @@ class JinDongDataTest {
|
||||
if (file.exists()){
|
||||
file.delete();
|
||||
}
|
||||
file.append("序号,")
|
||||
file.append("地址,")
|
||||
file.append("经度,")
|
||||
file.append("纬度,")
|
||||
file.append("周边餐饮的人均消费,")
|
||||
file.append("序号,")
|
||||
file.append("地址,")
|
||||
file.append("经度,")
|
||||
file.append("纬度,")
|
||||
file.append("周边餐饮的人均消费,")
|
||||
file.append(System.getProperty("line.separator"));
|
||||
ret.each{
|
||||
file.append(it.id + ",","gbk")
|
||||
@@ -570,47 +570,47 @@ class JinDongDataTest {
|
||||
}
|
||||
|
||||
|
||||
file.append("序号,")
|
||||
file.append("地址,")
|
||||
file.append("地址属性,")
|
||||
file.append("标准化-省,")
|
||||
file.append("城市,")
|
||||
file.append("区域,")
|
||||
file.append("板块,")
|
||||
file.append("数据库类型,")
|
||||
file.append("楼盘名称,")
|
||||
file.append("匹配楼盘名称,")
|
||||
file.append("地址中楼盘名称,")
|
||||
file.append("物业地址,")
|
||||
file.append("物业类型,")
|
||||
file.append("经度,")
|
||||
file.append("纬度,")
|
||||
file.append("户数,")
|
||||
file.append("产权性质,")
|
||||
file.append("竣工时间,")
|
||||
file.append("开发商,")
|
||||
file.append("物业公司,")
|
||||
file.append("物业费,")
|
||||
file.append("建筑面积,")
|
||||
file.append("楼盘单价,")
|
||||
file.append("城市均价,")
|
||||
file.append("月城市均价涨跌幅,")
|
||||
file.append("租房均价,")
|
||||
file.append("小区月均价涨跌幅,")
|
||||
file.append("一年内小区历史最高价,")
|
||||
file.append("一年内小区历史最低价,")
|
||||
file.append("未来一年房价预测,")
|
||||
file.append("周边餐饮的人均消费,")
|
||||
file.append("板块均价排名,")
|
||||
file.append("板块楼盘个数,")
|
||||
file.append("区域均价排名,")
|
||||
file.append("区域楼盘个数,")
|
||||
file.append("写字楼数量,")
|
||||
file.append("医院数量,")
|
||||
file.append("银行数量,")
|
||||
file.append("学校数量,")
|
||||
file.append("公交车站数量,")
|
||||
file.append("地铁数量,")
|
||||
file.append("序号,")
|
||||
file.append("地址,")
|
||||
file.append("地址属性,")
|
||||
file.append("标准化-省,")
|
||||
file.append("城市,")
|
||||
file.append("区域,")
|
||||
file.append("板块,")
|
||||
file.append("数据库类型,")
|
||||
file.append("楼盘名称,")
|
||||
file.append("匹配楼盘名称,")
|
||||
file.append("地址中楼盘名称,")
|
||||
file.append("物业地址,")
|
||||
file.append("物业类型,")
|
||||
file.append("经度,")
|
||||
file.append("纬度,")
|
||||
file.append("户数,")
|
||||
file.append("产权性质,")
|
||||
file.append("竣工时间,")
|
||||
file.append("开发商,")
|
||||
file.append("物业公司,")
|
||||
file.append("物业费,")
|
||||
file.append("建筑面积,")
|
||||
file.append("楼盘单价,")
|
||||
file.append("城市均价,")
|
||||
file.append("月城市均价涨跌幅,")
|
||||
file.append("租房均价,")
|
||||
file.append("小区月均价涨跌幅,")
|
||||
file.append("一年内小区历史最高价,")
|
||||
file.append("一年内小区历史最低价,")
|
||||
file.append("未来一年房价预测,")
|
||||
file.append("周边餐饮的人均消费,")
|
||||
file.append("板块均价排名,")
|
||||
file.append("板块楼盘个数,")
|
||||
file.append("区域均价排名,")
|
||||
file.append("区域楼盘个数,")
|
||||
file.append("写字楼数量,")
|
||||
file.append("医院数量,")
|
||||
file.append("银行数量,")
|
||||
file.append("学校数量,")
|
||||
file.append("公交车站数量,")
|
||||
file.append("地铁数量,")
|
||||
file.append(System.getProperty("line.separator"));
|
||||
ret.each{
|
||||
file.append(it.id + ",","gbk")
|
||||
@@ -888,7 +888,7 @@ class JinDongDataTest {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", city);
|
||||
params.put("address", address);
|
||||
params.put("businessMatch", "1");//没有找到,进行商铺写字楼匹配
|
||||
params.put("businessMatch", "1");//没有找到,进行商铺写字楼匹配
|
||||
params.put("token", "FF231CF2F7268316D937374DB96AC91ED7478F663E9D233D7389B1F0A809038F59F52B983E627AD8E95FB644D8C43601E5FEE7F107B3BFD8F5AF2D73853591F481A34E26723B657C2DD2D05684DB12F69C37447B272F857EFE2BEC49BADB44A3A2DBDB7CDBE2AA25");
|
||||
HttpRequest http = new HttpRequest();
|
||||
Map<String, Object> map = null;
|
||||
|
||||
+4
-4
@@ -14,14 +14,14 @@ class OfficeLivePrecent {
|
||||
|
||||
static main(args) {
|
||||
def task = new OfficeLivePrecent();
|
||||
task.run("上海", "E:\\数据测试\\数据\\", "写字楼.txt");
|
||||
task.run("上海", "E:\\数据测试\\数据\\", "写字楼.txt");
|
||||
}
|
||||
|
||||
def run(def city, def path, def fileName){
|
||||
def ret = readTxt(path, fileName);
|
||||
def offices = findOffice(city);
|
||||
def results = matchOffice(ret, offices);
|
||||
writeCSV("E:\\数据测试\\结果\\", "写字楼.csv", results);
|
||||
writeCSV("E:\\数据测试\\结果\\", "写字楼.csv", results);
|
||||
}
|
||||
|
||||
def matchOffice(def ret, def offices){
|
||||
@@ -67,8 +67,8 @@ class OfficeLivePrecent {
|
||||
if (file.exists()){
|
||||
file.delete();
|
||||
}
|
||||
file.append("名称,")
|
||||
file.append("入住率,")
|
||||
file.append("名称,")
|
||||
file.append("入住率,")
|
||||
file.append(System.getProperty("line.separator"));
|
||||
list.each{m->
|
||||
|
||||
|
||||
+2
-2
@@ -21,8 +21,8 @@ public class RegionRemoveHandler extends AddressStandardizeHandler {
|
||||
}
|
||||
|
||||
public static void main(String[] args){
|
||||
Request request = new Request("팹顧팹顧쨀소領돎땜켓沽쨀베쨌宅쨀깝켓쨌슥썹뇹");
|
||||
request.setRegion("팹顧");
|
||||
Request request = new Request("普陀普陀鲁家峙岛东南涂鲁海路与鲁滨南路交界处");
|
||||
request.setRegion("普陀");
|
||||
new RegionRemoveHandler().handleRequest(request);
|
||||
println request.getAddress();
|
||||
}
|
||||
|
||||
+3184
-3184
File diff suppressed because it is too large
Load Diff
+7
-7
@@ -21,7 +21,7 @@ class ShopAddressBatch {
|
||||
|
||||
def run(){
|
||||
|
||||
def data = readExcel("E:\\数据导出\\商铺\\201412.xlsx");
|
||||
def data = readExcel("E:\\数据导出\\商铺\\201412.xlsx");
|
||||
data = getLatLng(data);
|
||||
saveData(data,"2014-12");
|
||||
}
|
||||
@@ -30,7 +30,7 @@ class ShopAddressBatch {
|
||||
for(def row : data){
|
||||
|
||||
DBObject obj = new BasicDBObject()
|
||||
obj.city = "上海";
|
||||
obj.city = "上海";
|
||||
obj.region = row[0];
|
||||
obj.block = row[1];
|
||||
obj.name = row[2];
|
||||
@@ -81,17 +81,17 @@ class ShopAddressBatch {
|
||||
def rowCount = workbook.getSheetAt(0).getLastRowNum();
|
||||
def sheet1=workbook.getSheetAt(0);
|
||||
def rett = [];
|
||||
//循环行
|
||||
//循环行
|
||||
sheet1.eachWithIndex {row, index->
|
||||
|
||||
if(index>0){
|
||||
def cells = row.physicalNumberOfCells;//取得列数
|
||||
def cells = row.physicalNumberOfCells;//取得列数
|
||||
if(cells>1){
|
||||
def ret = [];
|
||||
for(def i=0;i<10;i++){
|
||||
def value = row.getCell(i);
|
||||
def name = value.toString();
|
||||
if(name.indexOf("E")!=-1&&!name.contains("路")){
|
||||
if(name.indexOf("E")!=-1&&!name.contains("路")){
|
||||
BigDecimal db = new BigDecimal(name);
|
||||
name = db.toPlainString()+".0"
|
||||
}
|
||||
@@ -135,7 +135,7 @@ class ShopAddressBatch {
|
||||
for(def row : data){
|
||||
def address = row[3];
|
||||
|
||||
def map = getLonLngV2("上海市"+address);
|
||||
def map = getLonLngV2("上海市"+address);
|
||||
|
||||
if(map){
|
||||
row << map.lat
|
||||
@@ -175,7 +175,7 @@ class ShopAddressBatch {
|
||||
conn.setRequestMethod("GET");
|
||||
conn.doOutput = true
|
||||
inputStream = conn.getInputStream();
|
||||
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
|
||||
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
|
||||
while((line = buf.readLine())!=null){
|
||||
sb.append(line);
|
||||
}
|
||||
|
||||
+3
-3
@@ -4,14 +4,14 @@ class SymbolRemoveHandler extends AddressStandardizeHandler {
|
||||
|
||||
|
||||
|
||||
String[] SYMBOLS = ["...", "?", ",", ".", ",", "。", ";", "-", "?"];
|
||||
String[] SYMBOLS = ["...", "?", ",", ".", ",", "。", ";", "-", "?"];
|
||||
|
||||
@Override
|
||||
public void handleRequest(Request request) {
|
||||
try{
|
||||
request.setAddress(removeEllipsis(request.getAddress()));
|
||||
}catch(Exception e){
|
||||
println "SymbolRemoveHandler: " + request.getAddress()+" 处理失败,message:"+e.getMessage();
|
||||
println "SymbolRemoveHandler: " + request.getAddress()+" 处理失败,message:"+e.getMessage();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,6 +33,6 @@ class SymbolRemoveHandler extends AddressStandardizeHandler {
|
||||
}
|
||||
|
||||
public static void main(String[] arhs){
|
||||
print new SymbolRemoveHandler().removeEllipsis("...?ssss........?,,");
|
||||
print new SymbolRemoveHandler().removeEllipsis("...?ssss........?,,");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,14 +8,14 @@ import com.fangjia.core.util.DateUtil;
|
||||
|
||||
class Test {
|
||||
|
||||
private static final Pattern shihaoPattern = Pattern.compile("(\\w+)[幢|栋|号楼|号|单元|楼|座](\\w+||\\d{3,4})室?");
|
||||
private static final Pattern shihaoPattern = Pattern.compile("(\\w+)[幢|栋|号楼|号|单元|楼|座](\\w+||\\d{3,4})室?");
|
||||
|
||||
static main(args) {
|
||||
// println CosineSimilarAlgorithm.getSimilarity("沪太路1700弄","沪太路1700弄小区")
|
||||
// println CosineSimilarAlgorithm.getSimilarity("沪太路1700弄","沪太路631弄", )
|
||||
// println CosineSimilarAlgorithm.getSimilarity("沪太路1700弄","沪太路951号")
|
||||
// println CosineSimilarAlgorithm.getSimilarity("证大立方大厦","富庆国定大厦")
|
||||
// println CosineSimilarAlgorithm.getSimilarity("紫光大厦","元福大厦")
|
||||
// println CosineSimilarAlgorithm.getSimilarity("沪太路1700弄","沪太路1700弄小区")
|
||||
// println CosineSimilarAlgorithm.getSimilarity("沪太路1700弄","沪太路631弄", )
|
||||
// println CosineSimilarAlgorithm.getSimilarity("沪太路1700弄","沪太路951号")
|
||||
// println CosineSimilarAlgorithm.getSimilarity("证大立方大厦","富庆国定大厦")
|
||||
// println CosineSimilarAlgorithm.getSimilarity("紫光大厦","元福大厦")
|
||||
|
||||
|
||||
// def month = 360;
|
||||
@@ -26,12 +26,12 @@ class Test {
|
||||
// println getPayMonth1(9360000, month, rate, 0.5);
|
||||
// println getPayMonth1(9360000, month, rate, 0.6);
|
||||
|
||||
// def map = parseLouDouInfo("新泾三村A栋1501");
|
||||
// def map = parseLouDouInfo("新泾三村A栋1501");
|
||||
// println map
|
||||
|
||||
def obj = "发布时间:2016/4/9"
|
||||
obj = obj.substring(obj.indexOf("发布时间"));
|
||||
obj = obj.replace("发布时间:", "").replace("发布时间", "").replace("发布", "").replace("/","-")
|
||||
def obj = "发布时间:2016/4/9"
|
||||
obj = obj.substring(obj.indexOf("发布时间"));
|
||||
obj = obj.replace("发布时间:", "").replace("发布时间", "").replace("发布", "").replace("/","-")
|
||||
def pattern = ~/\d{4}-\d{1,2}-\d{1,2}/
|
||||
def matcher = pattern.matcher(obj)
|
||||
if (matcher.find()) {
|
||||
|
||||
+9
-9
@@ -31,20 +31,20 @@ class YongjiaDataTest {
|
||||
static main(args) {
|
||||
def task = new YongjiaDataTest()
|
||||
|
||||
task.batchTestApi("上海", "C:\\Users\\yk\\Desktop\\yongjia\\上海-0730.txt")
|
||||
task.batchTestApi("上海", "C:\\Users\\yk\\Desktop\\yongjia\\上海-0730.txt")
|
||||
}
|
||||
|
||||
def batchTestApi(def city, def fileName){
|
||||
def result = []
|
||||
def index = 0
|
||||
|
||||
File file = new File("C:\\Users\\yk\\Desktop\\yongjia\\上海-0730.csv")
|
||||
File file = new File("C:\\Users\\yk\\Desktop\\yongjia\\上海-0730.csv")
|
||||
if (file.exists()){
|
||||
file.delete();
|
||||
}
|
||||
file.append("房产地址,")
|
||||
file.append("标准化-楼盘,")
|
||||
file.append("楼盘单价,")
|
||||
file.append("房产地址,")
|
||||
file.append("标准化-楼盘,")
|
||||
file.append("楼盘单价,")
|
||||
file.append(System.getProperty("line.separator"));
|
||||
|
||||
BufferedReader reader = new File(fileName).newReader('utf-8')
|
||||
@@ -90,7 +90,7 @@ class YongjiaDataTest {
|
||||
m.avgPrice = CommonDatautils.queryDistrictPrice(m.city, m.region, m.name, weekend, m.cat);
|
||||
}
|
||||
|
||||
//房价网估价
|
||||
//房价网估价
|
||||
// m.fangjiaTotalPrice = evaluation(m, token);
|
||||
// if(!m.fangjiaTotalPrice && m.avgPrice){
|
||||
// m.fangjiaTotalPrice = (m.avgPrice * m.area).toInteger();
|
||||
@@ -98,7 +98,7 @@ class YongjiaDataTest {
|
||||
|
||||
//m.rentPrice = queryRentPriceRange(m.city, m.region, m.name, lastMonth);
|
||||
|
||||
//其它网站均价
|
||||
//其它网站均价
|
||||
// def otherWebsiteMap = CommonDatautils.queryOtherWebsitePrice(m.city, m.name);
|
||||
// m.sunfunPrice = otherWebsiteMap.price_soufun
|
||||
// m.anjukePrice = otherWebsiteMap.price_anjuke
|
||||
@@ -137,7 +137,7 @@ class YongjiaDataTest {
|
||||
|
||||
}
|
||||
|
||||
//println "周边均价:" + totalPrice +"【${url_str}】"
|
||||
//println "周边均价:" + totalPrice +"【${url_str}】"
|
||||
return totalPrice;
|
||||
}
|
||||
|
||||
@@ -153,7 +153,7 @@ class YongjiaDataTest {
|
||||
conn.setRequestMethod("GET");
|
||||
conn.doOutput = true
|
||||
inputStream = conn.getInputStream();
|
||||
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
|
||||
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
|
||||
while((line = buf.readLine())!=null){
|
||||
sb.append(line);
|
||||
}
|
||||
|
||||
+3
-3
@@ -29,7 +29,7 @@ class HouseLoudongAnaBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
/**
|
||||
* 从楼栋表分析出小区楼号、室号信息
|
||||
* 从楼栋表分析出小区楼号、室号信息
|
||||
*/
|
||||
private anaHouseLoudong() {
|
||||
def index = 0;
|
||||
@@ -44,7 +44,7 @@ class HouseLoudongAnaBatch extends BaseBatch {
|
||||
query.put('region', house.region);
|
||||
query.put('name', house.name);
|
||||
|
||||
// 处理小区内楼号
|
||||
// 处理小区内楼号
|
||||
DBObject districtHouse = DB_2.findOne(query);
|
||||
if (districtHouse) {
|
||||
BasicDBList houseNums = districtHouse.house_nums;
|
||||
@@ -73,7 +73,7 @@ class HouseLoudongAnaBatch extends BaseBatch {
|
||||
DB_2.save(obj);
|
||||
}
|
||||
|
||||
// 处理同一楼号内的室号
|
||||
// 处理同一楼号内的室号
|
||||
query.put('house_num', houseNumStr);
|
||||
DBObject houseRoom = DB_3.findOne(query);
|
||||
if (houseRoom) {
|
||||
|
||||
+219
-219
@@ -23,211 +23,211 @@ class SortDistrictPics {
|
||||
def final static DEL_HEIGHT = 82
|
||||
|
||||
static main(args) {
|
||||
// process('shop', '上海')
|
||||
process('office', '上海')
|
||||
// process('shop', '上海')
|
||||
process('office', '上海')
|
||||
|
||||
// process('district', '上海')
|
||||
// process('district', '北京')
|
||||
// process('district', '广州')
|
||||
// process('district', '深圳')
|
||||
// process('district', '贵阳')
|
||||
// process('district', '乌鲁木齐')
|
||||
// process('district', '泰州')
|
||||
// process('district', '江门')
|
||||
// process('district', '秦皇岛')
|
||||
// process('district', '温州')
|
||||
// process('district', '台州')
|
||||
// process('district', '大庆')
|
||||
// process('district', '无锡')
|
||||
// process('district', '清远')
|
||||
// process('district', '珠海')
|
||||
// process('district', '兰州')
|
||||
// process('district', '佛山')
|
||||
// process('district', '柳州')
|
||||
// process('district', '西安')
|
||||
// process('district', '湛江')
|
||||
// process('district', '镇江')
|
||||
// process('district', '武汉')
|
||||
// process('district', '湖州')
|
||||
// process('district', '南通')
|
||||
// process('district', '赣州')
|
||||
// process('district', '丹东')
|
||||
// process('district', '咸阳')
|
||||
// process('district', '丽江')
|
||||
// process('district', '张家界')
|
||||
// process('district', '银川')
|
||||
// process('district', '大理')
|
||||
// process('district', '宿迁')
|
||||
// process('district', '襄阳')
|
||||
// process('district', '唐山')
|
||||
// process('district', '大连')
|
||||
// process('district', '芜湖')
|
||||
// process('district', '马鞍山')
|
||||
// process('district', '惠州')
|
||||
// process('district', '玉林')
|
||||
// process('district', '潍坊')
|
||||
// process('district', '南昌')
|
||||
// process('district', '哈尔滨')
|
||||
// process('district', '连云港')
|
||||
// process('district', '太原')
|
||||
// process('district', '延安')
|
||||
// process('district', '宁波')
|
||||
// process('district', '九江')
|
||||
// process('district', '三亚')
|
||||
// process('district', '南京')
|
||||
// process('district', '海口')
|
||||
// process('district', '济南')
|
||||
// process('district', '宜昌')
|
||||
// process('district', '烟台')
|
||||
// process('district', '东莞')
|
||||
// process('district', '天津')
|
||||
// process('district', '常州')
|
||||
// process('district', '威海')
|
||||
// process('district', '六盘水')
|
||||
// process('district', '沈阳')
|
||||
// process('district', '吉林')
|
||||
// process('district', '徐州')
|
||||
// process('district', '鞍山')
|
||||
// process('district', '保定')
|
||||
// process('district', '宜宾')
|
||||
// process('district', '莆田')
|
||||
// process('district', '金华')
|
||||
// process('district', '西宁')
|
||||
// process('district', '郑州')
|
||||
// process('district', '盐城')
|
||||
// process('district', '自贡')
|
||||
// process('district', '石家庄')
|
||||
// process('district', '绍兴')
|
||||
// process('district', '昆明')
|
||||
// process('district', '淄博')
|
||||
// process('district', '攀枝花')
|
||||
// process('district', '南宁')
|
||||
// process('district', '衢州')
|
||||
// process('district', '扬州')
|
||||
// process('district', '泉州')
|
||||
// process('district', '长春')
|
||||
// process('district', '苏州')
|
||||
// process('district', '邯郸')
|
||||
// process('district', '北海')
|
||||
// process('district', '合肥')
|
||||
// process('district', '青岛')
|
||||
// process('district', '廊坊')
|
||||
// process('district', '洛阳')
|
||||
// process('district', '呼和浩特')
|
||||
// process('district', '嘉兴')
|
||||
// process('district', '中山')
|
||||
// process('district', '桂林')
|
||||
// process('district', '长沙')
|
||||
// process('district', '株洲')
|
||||
// process('district', '湘潭')
|
||||
// process('district', '成都')
|
||||
// process('district', '重庆')
|
||||
// process('district', '杭州')
|
||||
// process('district', '厦门')
|
||||
// process('district', '福州')
|
||||
// process('district', '绵阳')
|
||||
// process('district', '上海')
|
||||
// process('district', '北京')
|
||||
// process('district', '广州')
|
||||
// process('district', '深圳')
|
||||
// process('district', '贵阳')
|
||||
// process('district', '乌鲁木齐')
|
||||
// process('district', '泰州')
|
||||
// process('district', '江门')
|
||||
// process('district', '秦皇岛')
|
||||
// process('district', '温州')
|
||||
// process('district', '台州')
|
||||
// process('district', '大庆')
|
||||
// process('district', '无锡')
|
||||
// process('district', '清远')
|
||||
// process('district', '珠海')
|
||||
// process('district', '兰州')
|
||||
// process('district', '佛山')
|
||||
// process('district', '柳州')
|
||||
// process('district', '西安')
|
||||
// process('district', '湛江')
|
||||
// process('district', '镇江')
|
||||
// process('district', '武汉')
|
||||
// process('district', '湖州')
|
||||
// process('district', '南通')
|
||||
// process('district', '赣州')
|
||||
// process('district', '丹东')
|
||||
// process('district', '咸阳')
|
||||
// process('district', '丽江')
|
||||
// process('district', '张家界')
|
||||
// process('district', '银川')
|
||||
// process('district', '大理')
|
||||
// process('district', '宿迁')
|
||||
// process('district', '襄阳')
|
||||
// process('district', '唐山')
|
||||
// process('district', '大连')
|
||||
// process('district', '芜湖')
|
||||
// process('district', '马鞍山')
|
||||
// process('district', '惠州')
|
||||
// process('district', '玉林')
|
||||
// process('district', '潍坊')
|
||||
// process('district', '南昌')
|
||||
// process('district', '哈尔滨')
|
||||
// process('district', '连云港')
|
||||
// process('district', '太原')
|
||||
// process('district', '延安')
|
||||
// process('district', '宁波')
|
||||
// process('district', '九江')
|
||||
// process('district', '三亚')
|
||||
// process('district', '南京')
|
||||
// process('district', '海口')
|
||||
// process('district', '济南')
|
||||
// process('district', '宜昌')
|
||||
// process('district', '烟台')
|
||||
// process('district', '东莞')
|
||||
// process('district', '天津')
|
||||
// process('district', '常州')
|
||||
// process('district', '威海')
|
||||
// process('district', '六盘水')
|
||||
// process('district', '沈阳')
|
||||
// process('district', '吉林')
|
||||
// process('district', '徐州')
|
||||
// process('district', '鞍山')
|
||||
// process('district', '保定')
|
||||
// process('district', '宜宾')
|
||||
// process('district', '莆田')
|
||||
// process('district', '金华')
|
||||
// process('district', '西宁')
|
||||
// process('district', '郑州')
|
||||
// process('district', '盐城')
|
||||
// process('district', '自贡')
|
||||
// process('district', '石家庄')
|
||||
// process('district', '绍兴')
|
||||
// process('district', '昆明')
|
||||
// process('district', '淄博')
|
||||
// process('district', '攀枝花')
|
||||
// process('district', '南宁')
|
||||
// process('district', '衢州')
|
||||
// process('district', '扬州')
|
||||
// process('district', '泉州')
|
||||
// process('district', '长春')
|
||||
// process('district', '苏州')
|
||||
// process('district', '邯郸')
|
||||
// process('district', '北海')
|
||||
// process('district', '合肥')
|
||||
// process('district', '青岛')
|
||||
// process('district', '廊坊')
|
||||
// process('district', '洛阳')
|
||||
// process('district', '呼和浩特')
|
||||
// process('district', '嘉兴')
|
||||
// process('district', '中山')
|
||||
// process('district', '桂林')
|
||||
// process('district', '长沙')
|
||||
// process('district', '株洲')
|
||||
// process('district', '湘潭')
|
||||
// process('district', '成都')
|
||||
// process('district', '重庆')
|
||||
// process('district', '杭州')
|
||||
// process('district', '厦门')
|
||||
// process('district', '福州')
|
||||
// process('district', '绵阳')
|
||||
//
|
||||
//
|
||||
// process('new_district', '上海')
|
||||
// process('new_district', '北京')
|
||||
// process('new_district', '广州')
|
||||
// process('new_district', '深圳')
|
||||
// process('new_district', '贵阳')
|
||||
// process('new_district', '乌鲁木齐')
|
||||
// process('new_district', '泰州')
|
||||
// process('new_district', '江门')
|
||||
// process('new_district', '秦皇岛')
|
||||
// process('new_district', '温州')
|
||||
// process('new_district', '台州')
|
||||
// process('new_district', '大庆')
|
||||
// process('new_district', '无锡')
|
||||
// process('new_district', '清远')
|
||||
// process('new_district', '珠海')
|
||||
// process('new_district', '兰州')
|
||||
// process('new_district', '佛山')
|
||||
// process('new_district', '柳州')
|
||||
// process('new_district', '西安')
|
||||
// process('new_district', '湛江')
|
||||
// process('new_district', '镇江')
|
||||
// process('new_district', '武汉')
|
||||
// process('new_district', '湖州')
|
||||
// process('new_district', '南通')
|
||||
// process('new_district', '赣州')
|
||||
// process('new_district', '丹东')
|
||||
// process('new_district', '咸阳')
|
||||
// process('new_district', '丽江')
|
||||
// process('new_district', '张家界')
|
||||
// process('new_district', '银川')
|
||||
// process('new_district', '大理')
|
||||
// process('new_district', '宿迁')
|
||||
// process('new_district', '襄阳')
|
||||
// process('new_district', '唐山')
|
||||
// process('new_district', '大连')
|
||||
// process('new_district', '芜湖')
|
||||
// process('new_district', '马鞍山')
|
||||
// process('new_district', '惠州')
|
||||
// process('new_district', '玉林')
|
||||
// process('new_district', '潍坊')
|
||||
// process('new_district', '南昌')
|
||||
// process('new_district', '哈尔滨')
|
||||
// process('new_district', '连云港')
|
||||
// process('new_district', '太原')
|
||||
// process('new_district', '延安')
|
||||
// process('new_district', '宁波')
|
||||
// process('new_district', '九江')
|
||||
// process('new_district', '三亚')
|
||||
// process('new_district', '南京')
|
||||
// process('new_district', '海口')
|
||||
// process('new_district', '济南')
|
||||
// process('new_district', '宜昌')
|
||||
// process('new_district', '烟台')
|
||||
// process('new_district', '东莞')
|
||||
// process('new_district', '天津')
|
||||
// process('new_district', '常州')
|
||||
// process('new_district', '威海')
|
||||
// process('new_district', '六盘水')
|
||||
// process('new_district', '沈阳')
|
||||
// process('new_district', '吉林')
|
||||
// process('new_district', '徐州')
|
||||
// process('new_district', '鞍山')
|
||||
// process('new_district', '保定')
|
||||
// process('new_district', '宜宾')
|
||||
// process('new_district', '莆田')
|
||||
// process('new_district', '金华')
|
||||
// process('new_district', '西宁')
|
||||
// process('new_district', '郑州')
|
||||
// process('new_district', '盐城')
|
||||
// process('new_district', '自贡')
|
||||
// process('new_district', '石家庄')
|
||||
// process('new_district', '绍兴')
|
||||
// process('new_district', '昆明')
|
||||
// process('new_district', '淄博')
|
||||
// process('new_district', '攀枝花')
|
||||
// process('new_district', '南宁')
|
||||
// process('new_district', '衢州')
|
||||
// process('new_district', '扬州')
|
||||
// process('new_district', '泉州')
|
||||
// process('new_district', '长春')
|
||||
// process('new_district', '苏州')
|
||||
// process('new_district', '邯郸')
|
||||
// process('new_district', '北海')
|
||||
// process('new_district', '合肥')
|
||||
// process('new_district', '青岛')
|
||||
// process('new_district', '廊坊')
|
||||
// process('new_district', '洛阳')
|
||||
// process('new_district', '呼和浩特')
|
||||
// process('new_district', '嘉兴')
|
||||
// process('new_district', '中山')
|
||||
// process('new_district', '桂林')
|
||||
// process('new_district', '长沙')
|
||||
// process('new_district', '株洲')
|
||||
// process('new_district', '湘潭')
|
||||
// process('new_district', '成都')
|
||||
// process('new_district', '重庆')
|
||||
// process('new_district', '杭州')
|
||||
// process('new_district', '厦门')
|
||||
// process('new_district', '福州')
|
||||
// process('new_district', '绵阳')
|
||||
// process('new_district', '上海')
|
||||
// process('new_district', '北京')
|
||||
// process('new_district', '广州')
|
||||
// process('new_district', '深圳')
|
||||
// process('new_district', '贵阳')
|
||||
// process('new_district', '乌鲁木齐')
|
||||
// process('new_district', '泰州')
|
||||
// process('new_district', '江门')
|
||||
// process('new_district', '秦皇岛')
|
||||
// process('new_district', '温州')
|
||||
// process('new_district', '台州')
|
||||
// process('new_district', '大庆')
|
||||
// process('new_district', '无锡')
|
||||
// process('new_district', '清远')
|
||||
// process('new_district', '珠海')
|
||||
// process('new_district', '兰州')
|
||||
// process('new_district', '佛山')
|
||||
// process('new_district', '柳州')
|
||||
// process('new_district', '西安')
|
||||
// process('new_district', '湛江')
|
||||
// process('new_district', '镇江')
|
||||
// process('new_district', '武汉')
|
||||
// process('new_district', '湖州')
|
||||
// process('new_district', '南通')
|
||||
// process('new_district', '赣州')
|
||||
// process('new_district', '丹东')
|
||||
// process('new_district', '咸阳')
|
||||
// process('new_district', '丽江')
|
||||
// process('new_district', '张家界')
|
||||
// process('new_district', '银川')
|
||||
// process('new_district', '大理')
|
||||
// process('new_district', '宿迁')
|
||||
// process('new_district', '襄阳')
|
||||
// process('new_district', '唐山')
|
||||
// process('new_district', '大连')
|
||||
// process('new_district', '芜湖')
|
||||
// process('new_district', '马鞍山')
|
||||
// process('new_district', '惠州')
|
||||
// process('new_district', '玉林')
|
||||
// process('new_district', '潍坊')
|
||||
// process('new_district', '南昌')
|
||||
// process('new_district', '哈尔滨')
|
||||
// process('new_district', '连云港')
|
||||
// process('new_district', '太原')
|
||||
// process('new_district', '延安')
|
||||
// process('new_district', '宁波')
|
||||
// process('new_district', '九江')
|
||||
// process('new_district', '三亚')
|
||||
// process('new_district', '南京')
|
||||
// process('new_district', '海口')
|
||||
// process('new_district', '济南')
|
||||
// process('new_district', '宜昌')
|
||||
// process('new_district', '烟台')
|
||||
// process('new_district', '东莞')
|
||||
// process('new_district', '天津')
|
||||
// process('new_district', '常州')
|
||||
// process('new_district', '威海')
|
||||
// process('new_district', '六盘水')
|
||||
// process('new_district', '沈阳')
|
||||
// process('new_district', '吉林')
|
||||
// process('new_district', '徐州')
|
||||
// process('new_district', '鞍山')
|
||||
// process('new_district', '保定')
|
||||
// process('new_district', '宜宾')
|
||||
// process('new_district', '莆田')
|
||||
// process('new_district', '金华')
|
||||
// process('new_district', '西宁')
|
||||
// process('new_district', '郑州')
|
||||
// process('new_district', '盐城')
|
||||
// process('new_district', '自贡')
|
||||
// process('new_district', '石家庄')
|
||||
// process('new_district', '绍兴')
|
||||
// process('new_district', '昆明')
|
||||
// process('new_district', '淄博')
|
||||
// process('new_district', '攀枝花')
|
||||
// process('new_district', '南宁')
|
||||
// process('new_district', '衢州')
|
||||
// process('new_district', '扬州')
|
||||
// process('new_district', '泉州')
|
||||
// process('new_district', '长春')
|
||||
// process('new_district', '苏州')
|
||||
// process('new_district', '邯郸')
|
||||
// process('new_district', '北海')
|
||||
// process('new_district', '合肥')
|
||||
// process('new_district', '青岛')
|
||||
// process('new_district', '廊坊')
|
||||
// process('new_district', '洛阳')
|
||||
// process('new_district', '呼和浩特')
|
||||
// process('new_district', '嘉兴')
|
||||
// process('new_district', '中山')
|
||||
// process('new_district', '桂林')
|
||||
// process('new_district', '长沙')
|
||||
// process('new_district', '株洲')
|
||||
// process('new_district', '湘潭')
|
||||
// process('new_district', '成都')
|
||||
// process('new_district', '重庆')
|
||||
// process('new_district', '杭州')
|
||||
// process('new_district', '厦门')
|
||||
// process('new_district', '福州')
|
||||
// process('new_district', '绵阳')
|
||||
|
||||
|
||||
// def floors = ['http://img1.soufunimg.com/agents1/2011_05/23/17/53/94/sh/houseinfo/404243101600.jpg',
|
||||
@@ -423,7 +423,7 @@ class SortDistrictPics {
|
||||
DBObject query = new BasicDBObject('_id', district._id);
|
||||
DBObject data = new BasicDBObject();
|
||||
|
||||
// 图片总数超过上限,则取top
|
||||
// 图片总数超过上限,则取top
|
||||
if (retFloors.size() + retOuts.size() > maxSize) {
|
||||
if (retFloors.size() > (maxSize / 2)) {
|
||||
data.put('floorplan_pics', getTop(retFloors, maxSize / 2));
|
||||
@@ -433,7 +433,7 @@ class SortDistrictPics {
|
||||
|
||||
data.put('out_pics', getTop(retOuts, maxSize - data.floorplan_pics.size()));
|
||||
} else {
|
||||
// 没超过上限,全部保留
|
||||
// 没超过上限,全部保留
|
||||
data.put('floorplan_pics', retFloors);
|
||||
data.put('out_pics', retOuts);
|
||||
}
|
||||
@@ -587,7 +587,7 @@ class SortDistrictPics {
|
||||
}
|
||||
|
||||
/**
|
||||
* 取得fs中不存在的图片URL
|
||||
* 取得fs中不存在的图片URL
|
||||
*/
|
||||
static remove404Pics(picInfos) {
|
||||
def ret = [];
|
||||
@@ -603,7 +603,7 @@ class SortDistrictPics {
|
||||
static doSort(picInfos) {
|
||||
// Collections.sort(picInfos, new Comparator<PicInfo>() {
|
||||
// public int compare(PicInfo info1, PicInfo info2) {
|
||||
// // 图片不存在或者没正确读取信息,往后排
|
||||
// // 图片不存在或者没正确读取信息,往后排
|
||||
// if (info1.is404 == true || info1.isEmpty == true || info1.isSmall == true) {
|
||||
// return 1;
|
||||
// }
|
||||
@@ -611,7 +611,7 @@ class SortDistrictPics {
|
||||
// return -1;
|
||||
// }
|
||||
//
|
||||
// // 图片长宽比较大,往后排
|
||||
// // 图片长宽比较大,往后排
|
||||
// if (info1.hdw < 0.5 || info1.hdw > 2) {
|
||||
// return 1;
|
||||
// }
|
||||
@@ -626,7 +626,7 @@ class SortDistrictPics {
|
||||
// });
|
||||
|
||||
picInfos.sort { info1, info2 ->
|
||||
// 图片不存在或者没正确读取信息,往后排
|
||||
// 图片不存在或者没正确读取信息,往后排
|
||||
if (info1.isF0f() || info1.isSmall()) {
|
||||
return 1;
|
||||
}
|
||||
@@ -634,7 +634,7 @@ class SortDistrictPics {
|
||||
return -1;
|
||||
}
|
||||
|
||||
// 图片长宽比较大,往后排
|
||||
// 图片长宽比较大,往后排
|
||||
if (info1.getHdw() < 0.5 || info1.getHdw() > 2) {
|
||||
return 1;
|
||||
}
|
||||
@@ -663,7 +663,7 @@ class SortDistrictPics {
|
||||
}
|
||||
|
||||
/**
|
||||
* 房型图列表转换为string列表
|
||||
* 房型图列表转换为string列表
|
||||
*
|
||||
* @param srcs
|
||||
* @return
|
||||
@@ -709,19 +709,19 @@ class SortDistrictPics {
|
||||
// if(directories.hasNext()){
|
||||
// Directory directory = (Directory) directories.next();
|
||||
//
|
||||
// // 如果 imageInfo中无法读取,从exif中读取dpi
|
||||
// // 如果 imageInfo中无法读取,从exif中读取dpi
|
||||
// if(imageInfo.getPhysicalWidthDpi() <= 0 || imageInfo.getPhysicalHeightDpi() <= 0){
|
||||
// dipx = new BigDecimal(directory.getString(ExifDirectory.TAG_X_RESOLUTION));
|
||||
// dipy = new BigDecimal(directory.getString(ExifDirectory.TAG_Y_RESOLUTION));
|
||||
// if(dipx.equals(new BigDecimal(72)) || dipy.equals(new BigDecimal(72))) {
|
||||
// // 如果dpi为72的话,抛出错误,因为暂时无法确定从exif中读出的数据是对的
|
||||
// // 如果dpi为72的话,抛出错误,因为暂时无法确定从exif中读出的数据是对的
|
||||
// } else{
|
||||
// dipx = new BigDecimal(imageInfo.getPhysicalWidthDpi());
|
||||
// dipy = new BigDecimal(imageInfo.getPhysicalHeightDpi());
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// } catch (Exception e) { // 处理出错
|
||||
// } catch (Exception e) { // 处理出错
|
||||
//// e.printStackTrace();
|
||||
// }
|
||||
|
||||
@@ -732,12 +732,12 @@ class SortDistrictPics {
|
||||
|
||||
public static class PicInfo {
|
||||
private String imgUrl;
|
||||
private int hxw; // 图片高度*宽度(像素)
|
||||
private double hdw; // 图片高度÷宽度(像素)
|
||||
private double dpi; // 图片DPI
|
||||
private boolean f0f; // fs库中不存在
|
||||
private boolean small; // 是否小于120*82
|
||||
private boolean vertical; // 是否竖向图片
|
||||
private int hxw; // 图片高度*宽度(像素)
|
||||
private double hdw; // 图片高度÷宽度(像素)
|
||||
private double dpi; // 图片DPI
|
||||
private boolean f0f; // fs库中不存在
|
||||
private boolean small; // 是否小于120*82
|
||||
private boolean vertical; // 是否竖向图片
|
||||
public double score;
|
||||
|
||||
public String getImgUrl() {
|
||||
|
||||
+43
-43
@@ -30,8 +30,8 @@ import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 对解析好的网页进行分析
|
||||
* 获取小区解析的网站数量
|
||||
* 对解析好的网页进行分析
|
||||
* 获取小区解析的网站数量
|
||||
* @author gaojuan
|
||||
*
|
||||
*/
|
||||
@@ -46,23 +46,23 @@ class AnalyDistrictLog extends BaseBatch {
|
||||
public static final DBCollection VERIFY_RULE_COLL = MongoUtils.getCollectionByName("coredata", "verify_rule", "coredata.mongo.db.fangjia.com")
|
||||
public static final DSCollection SEABIRD = new DataStore("datastore.db.fangjia.com").getCollection("fangjia.seabird")
|
||||
public static final DSCollection seawater = new DataStore("192.168.0.65", 27777).getCollection("fangjia.seawater")
|
||||
/**人工删除*/
|
||||
/**人工删除*/
|
||||
private static final int MANUAL_DELETE = 1;
|
||||
/**区域错误删除*/
|
||||
/**区域错误删除*/
|
||||
private static final int REGION_ERROR_DELETE = 2;
|
||||
/**别名关联删除*/
|
||||
/**别名关联删除*/
|
||||
private static final int ALIAS_RELATE_DELETE = 3;
|
||||
/**名称非法删除*/
|
||||
/**名称非法删除*/
|
||||
private static final int NAME_ILLEGAL_DELETE = 4;
|
||||
/**通过百度API判定类型不合法删除*/
|
||||
/**通过百度API判定类型不合法删除*/
|
||||
private static final int BAIDU_JUDGE_DELETE = 5;
|
||||
/**CPDB已经删除*/
|
||||
/**CPDB已经删除*/
|
||||
private static final int CPDB_DELETE = 6;
|
||||
/**正常数据*/
|
||||
/**正常数据*/
|
||||
private static final int NORMAL_DATA = 0;
|
||||
|
||||
private static final int MACHINE_DETELE = 8
|
||||
/**数据抓取失败标示*/
|
||||
/**数据抓取失败标示*/
|
||||
private static final int ANALY_ROAD_STATUS_ERROR = 2;
|
||||
private Date now = new Date()
|
||||
private int THREE_MONTHS_AGO = DateUtil.add(now, Calendar.MONTH, -6).format("yyyyMMdd").toInteger()
|
||||
@@ -73,14 +73,14 @@ class AnalyDistrictLog extends BaseBatch {
|
||||
private static List districtUnContainsWords = []
|
||||
|
||||
|
||||
static List<Pattern> illegalDistirctNamePatterns = [Pattern.compile(".*地图定位.*"), Pattern.compile(".*地块.*"), Pattern.compile(".*售罄.*"),
|
||||
Pattern.compile(".*-\$"), Pattern.compile(".*….*"), Pattern.compile(".*\\d#.*")
|
||||
, /.*(东|南|西|北|一|二|三|四|五|正|前|后)+门.*/, /.*出售.*/, /.*(\d|一|二|三|四|五)号楼$/]
|
||||
static List<Pattern> illegalDistirctNamePatterns = [Pattern.compile(".*地图定位.*"), Pattern.compile(".*地块.*"), Pattern.compile(".*售罄.*"),
|
||||
Pattern.compile(".*-\$"), Pattern.compile(".*….*"), Pattern.compile(".*\\d#.*")
|
||||
, /.*(东|南|西|北|一|二|三|四|五|正|前|后)+门.*/, /.*出售.*/, /.*(\d|一|二|三|四|五)号楼$/]
|
||||
|
||||
static main(args) {
|
||||
def s = System.currentTimeMillis()
|
||||
runBatch(new AnalyDistrictLog(), args)
|
||||
println("运行时间:${(int) ((System.currentTimeMillis() - s)) / (1000 * 60)}分钟")
|
||||
println("运行时间:${(int) ((System.currentTimeMillis() - s)) / (1000 * 60)}分钟")
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -90,8 +90,8 @@ class AnalyDistrictLog extends BaseBatch {
|
||||
districtUnContainsWords << dbo.getString("value")
|
||||
}
|
||||
def allCity = CommonUtils.findAllCity()
|
||||
if (SystemUtils.IS_OS_WINDOWS) {//本地手动跑特定城市数据
|
||||
allCity = ["南京"]
|
||||
if (SystemUtils.IS_OS_WINDOWS) {//本地手动跑特定城市数据
|
||||
allCity = ["南京"]
|
||||
}else{
|
||||
updateTempData()
|
||||
}
|
||||
@@ -102,19 +102,19 @@ class AnalyDistrictLog extends BaseBatch {
|
||||
def index = 1;
|
||||
for (def city : allCity) {
|
||||
def s = System.currentTimeMillis()
|
||||
println "[${index}]开始分析 ${city} ${new Date().format('yyyy/MM/dd HH:mm:ss')}"
|
||||
println "[${index}]开始分析 ${city} ${new Date().format('yyyy/MM/dd HH:mm:ss')}"
|
||||
for (def dataType : dataTypes) {
|
||||
process(city, dataType);
|
||||
// if(allOfflineCity.contains(city) && dataType == "district"){//统计线下城市房源
|
||||
// if(allOfflineCity.contains(city) && dataType == "district"){//统计线下城市房源
|
||||
// calcHouseCount(city)
|
||||
// }
|
||||
}
|
||||
index++;
|
||||
println "[${index}]分析结束 ${city} ${new Date().format('yyyy/MM/dd HH:mm:ss')} 耗时:${(int) ((System.currentTimeMillis() - s) / (1000 * 60))}分钟"
|
||||
println "[${index}]分析结束 ${city} ${new Date().format('yyyy/MM/dd HH:mm:ss')} 耗时:${(int) ((System.currentTimeMillis() - s) / (1000 * 60))}分钟"
|
||||
}
|
||||
}
|
||||
|
||||
/*统计线下城市房源的数量*/
|
||||
/*统计线下城市房源的数量*/
|
||||
def calcHouseCount(def city){
|
||||
|
||||
def startDate = DateUtil.add(new Date(), Calendar.MONTH, -6).format("yyyyMMdd").toInteger()
|
||||
@@ -132,10 +132,10 @@ class AnalyDistrictLog extends BaseBatch {
|
||||
long count = (long) seawater.count(houseQuery)
|
||||
ANALY_SITE_COLL.update(new BasicDBObject("_id",dbo._id), new BasicDBObject('$set',new BasicDBObject('house_num',count)))
|
||||
if(idx % 100 == 0){
|
||||
println("${city}统计进度:${DataUtil.roundDouble((idx *100)/total,2)}%")
|
||||
println("${city}统计进度:${DataUtil.roundDouble((idx *100)/total,2)}%")
|
||||
}
|
||||
}
|
||||
println("${city}分析结束")
|
||||
println("${city}分析结束")
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -152,7 +152,7 @@ class AnalyDistrictLog extends BaseBatch {
|
||||
query.append("version", new BasicDBObject('$lt', version))
|
||||
DBCursor cursor = ANALY_SITE_COLL.find(query, new BasicDBObject("city", 1).append("region", 1).append("district_name", 1).append("cat", 1)).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(300)
|
||||
while (cursor.hasNext()) {
|
||||
println("删除:${cursor.next()}")
|
||||
println("删除:${cursor.next()}")
|
||||
}
|
||||
ANALY_SITE_COLL.remove(query)
|
||||
query.removeField("version")
|
||||
@@ -182,13 +182,13 @@ class AnalyDistrictLog extends BaseBatch {
|
||||
updater.put("region", formateRegion)
|
||||
}
|
||||
}
|
||||
String key = city + JOIN_SEPARATOR + formateRegion + JOIN_SEPARATOR + name.toLowerCase().replaceAll(' ', " ").replaceAll(" ", " ").replaceAll('(^\\s+)|(\\s+$)', '');
|
||||
String key = city + JOIN_SEPARATOR + formateRegion + JOIN_SEPARATOR + name.toLowerCase().replaceAll(' ', " ").replaceAll(" ", " ").replaceAll('(^\\s+)|(\\s+$)', '');
|
||||
|
||||
int audit_status = analySite.getInt("audit_status", 0)//审阅状态 0:未审核 1:已审核 4:已审阅
|
||||
int audit_status = analySite.getInt("audit_status", 0)//审阅状态 0:未审核 1:已审核 4:已审阅
|
||||
int data_status = analySite.getInt("data_status", NORMAL_DATA)
|
||||
int exists
|
||||
def district = allRightData.get(key);
|
||||
//判断小区是否在CPDB库中
|
||||
//判断小区是否在CPDB库中
|
||||
if (district != null) {
|
||||
exists = 1
|
||||
} else {
|
||||
@@ -215,7 +215,7 @@ class AnalyDistrictLog extends BaseBatch {
|
||||
} else if (deletedData.containsKey(key)) {
|
||||
data_status = CPDB_DELETE
|
||||
audit_status = 1
|
||||
} else if (name.endsWith("宿舍") && analySite.containsField("map_compare_result") && analySite.getString("map_compare_result") == "not_exist") {
|
||||
} else if (name.endsWith("宿舍") && analySite.containsField("map_compare_result") && analySite.getString("map_compare_result") == "not_exist") {
|
||||
data_status = NAME_ILLEGAL_DELETE
|
||||
audit_status = 1
|
||||
} else {
|
||||
@@ -240,7 +240,7 @@ class AnalyDistrictLog extends BaseBatch {
|
||||
ANALY_SITE_COLL.update(new BasicDBObject("_id", analySite._id), new BasicDBObject('$set', updater))
|
||||
}
|
||||
if (progress % 1000 == 0) {
|
||||
println("状态进度:${DataUtil.roundDouble((progress / count) * 100, 1)}%")
|
||||
println("状态进度:${DataUtil.roundDouble((progress / count) * 100, 1)}%")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -277,10 +277,10 @@ class AnalyDistrictLog extends BaseBatch {
|
||||
*/
|
||||
public def process(def city, def cat) {
|
||||
|
||||
def analyRoads = findAnalyRoads(city, cat);//查找抓取数据
|
||||
def analyRoads = findAnalyRoads(city, cat);//查找抓取数据
|
||||
if (analyRoads.size() > 0) {
|
||||
def districts = getAllDistricts(city, cat);
|
||||
//按网页合并
|
||||
//按网页合并
|
||||
def List<BasicDBObject> mergedRoad = analyzeAnalyRoads(analyRoads);
|
||||
saveAnalySite(mergedRoad, cat, city);
|
||||
updateOldVersionAnalySiteDataStatus(city, cat, districts)
|
||||
@@ -288,7 +288,7 @@ class AnalyDistrictLog extends BaseBatch {
|
||||
}
|
||||
|
||||
/**
|
||||
* 通过城市区域小区名对Road数据进行分组
|
||||
* 通过城市区域小区名对Road数据进行分组
|
||||
* @param analyRoads
|
||||
* @return
|
||||
*/
|
||||
@@ -314,7 +314,7 @@ class AnalyDistrictLog extends BaseBatch {
|
||||
item.get("site_names") << analyRoad.site_name
|
||||
}
|
||||
if (progress % 10000 == 0) {
|
||||
println("分组进度:${DataUtil.roundDouble((progress / analyRoads.size()) * 100, 0)}%")
|
||||
println("分组进度:${DataUtil.roundDouble((progress / analyRoads.size()) * 100, 0)}%")
|
||||
}
|
||||
|
||||
}
|
||||
@@ -326,7 +326,7 @@ class AnalyDistrictLog extends BaseBatch {
|
||||
dbo.put("site_count", siteNames.size())
|
||||
result << dbo
|
||||
}
|
||||
println "[analyzeAnalyRoads]耗时:${System.currentTimeMillis() - start}"
|
||||
println "[analyzeAnalyRoads]耗时:${System.currentTimeMillis() - start}"
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -344,7 +344,7 @@ class AnalyDistrictLog extends BaseBatch {
|
||||
def item = new BasicDBObject()
|
||||
if(!obj.primary_map){
|
||||
ANALY_ROAD_TEMP_COLL.remove(new BasicDBObject("_id", obj._id))
|
||||
println("移除TEMP:" + obj.toString())
|
||||
println("移除TEMP:" + obj.toString())
|
||||
continue
|
||||
}
|
||||
BasicDBObject primaryMap = obj.get("primary_map")
|
||||
@@ -397,7 +397,7 @@ class AnalyDistrictLog extends BaseBatch {
|
||||
def name = obj.getString("name");
|
||||
List alias = obj.get("alias");
|
||||
def status = obj.getInt("status", 0);
|
||||
String key = city + JOIN_SEPARATOR + region + JOIN_SEPARATOR + name.toLowerCase().replaceAll(' ', " ").replaceAll(" ", " ").replaceAll('(^\\s+)|(\\s+$)', '');
|
||||
String key = city + JOIN_SEPARATOR + region + JOIN_SEPARATOR + name.toLowerCase().replaceAll(' ', " ").replaceAll(" ", " ").replaceAll('(^\\s+)|(\\s+$)', '');
|
||||
if (0 != status) {
|
||||
deletedData.put(key, obj)
|
||||
} else {
|
||||
@@ -406,7 +406,7 @@ class AnalyDistrictLog extends BaseBatch {
|
||||
if (alias) {
|
||||
for (String aliaName : alias) {
|
||||
if (StringUtils.isNotBlank(aliaName) && aliaName != name) {
|
||||
key = city + JOIN_SEPARATOR + region + JOIN_SEPARATOR + aliaName.toLowerCase().replaceAll(' ', " ").replaceAll(" ", " ").replaceAll('(^\\s+)|(\\s+$)', '');
|
||||
key = city + JOIN_SEPARATOR + region + JOIN_SEPARATOR + aliaName.toLowerCase().replaceAll(' ', " ").replaceAll(" ", " ").replaceAll('(^\\s+)|(\\s+$)', '');
|
||||
if (0 != status) {
|
||||
deletedData.put(key, obj);
|
||||
} else {
|
||||
@@ -417,7 +417,7 @@ class AnalyDistrictLog extends BaseBatch {
|
||||
}
|
||||
}
|
||||
}
|
||||
//移除拥有和正常数据一样KEY的被删除数据
|
||||
//移除拥有和正常数据一样KEY的被删除数据
|
||||
for (String k : allRightData.keySet()) {
|
||||
deletedData.remove(k)
|
||||
}
|
||||
@@ -495,7 +495,7 @@ class AnalyDistrictLog extends BaseBatch {
|
||||
}
|
||||
progress.getAndIncrement()
|
||||
if (progress.get() % 1000 == 0) {
|
||||
println("入库进度:${DataUtil.roundDouble((progress.get() / mergedRoads.size()) * 100, 1)}%")
|
||||
println("入库进度:${DataUtil.roundDouble((progress.get() / mergedRoads.size()) * 100, 1)}%")
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -520,7 +520,7 @@ class AnalyDistrictLog extends BaseBatch {
|
||||
if (StringUtils.isBlank(region)) {
|
||||
return false;
|
||||
}
|
||||
if (region.contains("周边") || region.contains("其他")) {
|
||||
if (region.contains("周边") || region.contains("其他")) {
|
||||
return false;
|
||||
}
|
||||
return RegionData.isRegion(city, region)
|
||||
@@ -528,7 +528,7 @@ class AnalyDistrictLog extends BaseBatch {
|
||||
|
||||
def updateTempData() {
|
||||
|
||||
println("开始更新临时数据")
|
||||
println("开始更新临时数据")
|
||||
DBObject query = new BasicDBObject();
|
||||
query.append("status", new BasicDBObject('$ne', ANALY_ROAD_STATUS_ERROR))
|
||||
query.append("c_date", new BasicDBObject('$gt', DateUtil.add(now, Calendar.DATE, -2)))
|
||||
@@ -572,10 +572,10 @@ class AnalyDistrictLog extends BaseBatch {
|
||||
}
|
||||
}
|
||||
if(sb.length() > 0 && sb.length() < 10000){
|
||||
sendEmail("zhoutongyun@fangjia.com", "疑似解析错误小区", sb.toString())
|
||||
sendEmail("wangwenyan@fangjia.com", "疑似解析错误小区", sb.toString())
|
||||
sendEmail("zhoutongyun@fangjia.com", "疑似解析错误小区", sb.toString())
|
||||
sendEmail("wangwenyan@fangjia.com", "疑似解析错误小区", sb.toString())
|
||||
}
|
||||
println("临时数据更新结束")
|
||||
println("临时数据更新结束")
|
||||
}
|
||||
|
||||
def assembleObj(BasicDBObject srcDbo) {
|
||||
|
||||
+35
-35
@@ -27,8 +27,8 @@ import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 对解析好的网页进行分析
|
||||
* 获取小区解析的网站数量
|
||||
* 对解析好的网页进行分析
|
||||
* 获取小区解析的网站数量
|
||||
* @author gaojuan
|
||||
*
|
||||
*/
|
||||
@@ -42,21 +42,21 @@ class AnalyDistrictLogTest extends BaseBatch{
|
||||
public static final DBCollection REGION_BLOCK_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com")
|
||||
public static final DBCollection VERIFY_RULE_COLL = MongoUtils.getCollectionByName("coredata", "verify_rule", "coredata.mongo.db.fangjia.com")
|
||||
public static DSCollection seaweed = new DataStore("192.168.0.251", 27007).getCollection("fangjia.seaweed")
|
||||
/**人工删除*/
|
||||
/**人工删除*/
|
||||
private static final int MANUAL_DELETE = 1;
|
||||
/**区域错误删除*/
|
||||
/**区域错误删除*/
|
||||
private static final int REGION_ERROR_DELETE = 2;
|
||||
/**别名关联删除*/
|
||||
/**别名关联删除*/
|
||||
private static final int ALIAS_RELATE_DELETE = 3;
|
||||
/**名称非法删除*/
|
||||
/**名称非法删除*/
|
||||
private static final int NAME_ILLEGAL_DELETE = 4;
|
||||
/**通过百度API判定类型不合法删除*/
|
||||
/**通过百度API判定类型不合法删除*/
|
||||
private static final int BAIDU_JUDGE_DELETE = 5;
|
||||
/**CPDB已经删除*/
|
||||
/**CPDB已经删除*/
|
||||
private static final int CPDB_DELETE = 6;
|
||||
/**正常数据*/
|
||||
/**正常数据*/
|
||||
private static final int NORMAL_DATA = 0;
|
||||
/**数据抓取失败标示*/
|
||||
/**数据抓取失败标示*/
|
||||
private static final int ANALY_ROAD_STATUS_ERROR = 2;
|
||||
|
||||
private Date now = new Date()
|
||||
@@ -66,14 +66,14 @@ class AnalyDistrictLogTest extends BaseBatch{
|
||||
|
||||
private static List districtUnContainsWords = []
|
||||
|
||||
static List<Pattern> illegalDistirctNamePatterns = [Pattern.compile(".*地图定位.*"),Pattern.compile(".*地块.*"),Pattern.compile(".*售罄.*"),
|
||||
Pattern.compile(".*-\$"),Pattern.compile(".*….*"),Pattern.compile(".*\\d#.*")
|
||||
,/.*(东|南|西|北|一|二|三|四|五|正|前|后)+门.*/,/.*出售.*/,/.*(\d|一|二|三|四|五)号楼$/]
|
||||
static List<Pattern> illegalDistirctNamePatterns = [Pattern.compile(".*地图定位.*"),Pattern.compile(".*地块.*"),Pattern.compile(".*售罄.*"),
|
||||
Pattern.compile(".*-\$"),Pattern.compile(".*….*"),Pattern.compile(".*\\d#.*")
|
||||
,/.*(东|南|西|北|一|二|三|四|五|正|前|后)+门.*/,/.*出售.*/,/.*(\d|一|二|三|四|五)号楼$/]
|
||||
|
||||
static main(args) {
|
||||
def s = System.currentTimeMillis()
|
||||
runBatch(new AnalyDistrictLogTest(), args)
|
||||
println("运行时间:${(int)((System.currentTimeMillis() - s))/(1000*60)}分钟")
|
||||
println("运行时间:${(int)((System.currentTimeMillis() - s))/(1000*60)}分钟")
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -84,19 +84,19 @@ class AnalyDistrictLogTest extends BaseBatch{
|
||||
}
|
||||
def allCity = CommonUtils.findAllCity()
|
||||
updateTempData()
|
||||
if(SystemUtils.IS_OS_WINDOWS){//本地手动跑特定城市数据
|
||||
allCity = ["石家庄", "沈阳", "上海", "杭州", "济南", "青岛", "广州", "重庆", "成都", "西安"]
|
||||
if(SystemUtils.IS_OS_WINDOWS){//本地手动跑特定城市数据
|
||||
allCity = ["石家庄", "沈阳", "上海", "杭州", "济南", "青岛", "广州", "重庆", "成都", "西安"]
|
||||
}
|
||||
def dataTypes = ["district","office"];
|
||||
def index = 1;
|
||||
for(def city : allCity){
|
||||
def s = System.currentTimeMillis()
|
||||
println "[${index}]开始分析 ${city} ${new Date().format('yyyy/MM/dd HH:mm:ss')}"
|
||||
println "[${index}]开始分析 ${city} ${new Date().format('yyyy/MM/dd HH:mm:ss')}"
|
||||
for(def dataType : dataTypes){
|
||||
process(city, dataType);
|
||||
}
|
||||
index ++;
|
||||
println "[${index}]分析结束 ${city} ${new Date().format('yyyy/MM/dd HH:mm:ss')} 耗时:${(int)((System.currentTimeMillis() - s)/(1000 * 60))}分钟"
|
||||
println "[${index}]分析结束 ${city} ${new Date().format('yyyy/MM/dd HH:mm:ss')} 耗时:${(int)((System.currentTimeMillis() - s)/(1000 * 60))}分钟"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,7 +115,7 @@ class AnalyDistrictLogTest extends BaseBatch{
|
||||
query.append("version",new BasicDBObject('$lt',version))
|
||||
DBCursor cursor = ANALY_SITE_COLL.find(query,new BasicDBObject("city", 1).append("region",1).append("district_name",1).append("cat",1)).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(300)
|
||||
while(cursor.hasNext()){
|
||||
println("删除:${cursor.next()}")
|
||||
println("删除:${cursor.next()}")
|
||||
}
|
||||
ANALY_SITE_COLL.remove(query)
|
||||
query.removeField("version")
|
||||
@@ -145,13 +145,13 @@ class AnalyDistrictLogTest extends BaseBatch{
|
||||
updater.put("region",formateRegion)
|
||||
}
|
||||
}
|
||||
String key = city + JOIN_SEPARATOR + formateRegion + JOIN_SEPARATOR + name.toLowerCase().replaceAll(' '," ").replaceAll(" "," ").replaceAll('(^\\s+)|(\\s+$)','');
|
||||
String key = city + JOIN_SEPARATOR + formateRegion + JOIN_SEPARATOR + name.toLowerCase().replaceAll(' '," ").replaceAll(" "," ").replaceAll('(^\\s+)|(\\s+$)','');
|
||||
|
||||
int audit_status = analySite.getInt("audit_status",0)//审阅状态 0:未审核 1:已审核 4:已审阅
|
||||
int audit_status = analySite.getInt("audit_status",0)//审阅状态 0:未审核 1:已审核 4:已审阅
|
||||
int data_status = analySite.getInt("data_status",NORMAL_DATA)
|
||||
int exists
|
||||
def district = allRightData.get(key);
|
||||
//判断小区是否在CPDB库中
|
||||
//判断小区是否在CPDB库中
|
||||
if(district != null){
|
||||
exists = 1
|
||||
}else{
|
||||
@@ -200,7 +200,7 @@ class AnalyDistrictLogTest extends BaseBatch{
|
||||
ANALY_SITE_COLL.update(new BasicDBObject("_id",analySite._id), new BasicDBObject('$set',updater))
|
||||
}
|
||||
if(progress % 1000 == 0){
|
||||
println("状态进度:${DataUtil.roundDouble((progress/count)*100,1)}%")
|
||||
println("状态进度:${DataUtil.roundDouble((progress/count)*100,1)}%")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -212,10 +212,10 @@ class AnalyDistrictLogTest extends BaseBatch{
|
||||
*/
|
||||
public def process(def city, def cat){
|
||||
|
||||
def analyRoads = findAnalyRoads(city, cat);//查找抓取数据
|
||||
def analyRoads = findAnalyRoads(city, cat);//查找抓取数据
|
||||
if(analyRoads.size() > 0){
|
||||
def districts = getAllDistricts(city, cat);
|
||||
//按网页合并
|
||||
//按网页合并
|
||||
def List<BasicDBObject> mergedRoad = analyzeAnalyRoads(analyRoads);
|
||||
saveAnalySite(mergedRoad,cat);
|
||||
updateOldVersionAnalySiteDataStatus(city,cat,districts)
|
||||
@@ -223,7 +223,7 @@ class AnalyDistrictLogTest extends BaseBatch{
|
||||
}
|
||||
|
||||
/**
|
||||
* 通过城市区域小区名对Road数据进行分组
|
||||
* 通过城市区域小区名对Road数据进行分组
|
||||
* @param analyRoads
|
||||
* @return
|
||||
*/
|
||||
@@ -249,7 +249,7 @@ class AnalyDistrictLogTest extends BaseBatch{
|
||||
item.get("site_names") << analyRoad.site_name
|
||||
}
|
||||
if(progress % 10000 == 0){
|
||||
println("分组进度:${DataUtil.roundDouble((progress/analyRoads.size())*100,0)}%")
|
||||
println("分组进度:${DataUtil.roundDouble((progress/analyRoads.size())*100,0)}%")
|
||||
}
|
||||
|
||||
}
|
||||
@@ -261,7 +261,7 @@ class AnalyDistrictLogTest extends BaseBatch{
|
||||
dbo.put("site_count",siteNames.size())
|
||||
result << dbo
|
||||
}
|
||||
println "[analyzeAnalyRoads]耗时:${System.currentTimeMillis() - start}"
|
||||
println "[analyzeAnalyRoads]耗时:${System.currentTimeMillis() - start}"
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -327,7 +327,7 @@ class AnalyDistrictLogTest extends BaseBatch{
|
||||
def name = obj.getString("name");
|
||||
List alias = obj.get("alias");
|
||||
def status = obj.getInt("status", 0);
|
||||
String key = city + JOIN_SEPARATOR + region + JOIN_SEPARATOR + name.toLowerCase().replaceAll(' '," ").replaceAll(" "," ").replaceAll('(^\\s+)|(\\s+$)','');
|
||||
String key = city + JOIN_SEPARATOR + region + JOIN_SEPARATOR + name.toLowerCase().replaceAll(' '," ").replaceAll(" "," ").replaceAll('(^\\s+)|(\\s+$)','');
|
||||
if(0 != status){
|
||||
deletedData.put(key,obj)
|
||||
}else{
|
||||
@@ -336,7 +336,7 @@ class AnalyDistrictLogTest extends BaseBatch{
|
||||
if(alias){
|
||||
for(String aliaName : alias){
|
||||
if(StringUtils.isNotBlank(aliaName) && aliaName != name){
|
||||
key = city + JOIN_SEPARATOR +region+ JOIN_SEPARATOR + aliaName.toLowerCase().replaceAll(' '," ").replaceAll(" "," ").replaceAll('(^\\s+)|(\\s+$)','');
|
||||
key = city + JOIN_SEPARATOR +region+ JOIN_SEPARATOR + aliaName.toLowerCase().replaceAll(' '," ").replaceAll(" "," ").replaceAll('(^\\s+)|(\\s+$)','');
|
||||
if(0 != status){
|
||||
deletedData.put(key,obj);
|
||||
}else{
|
||||
@@ -347,7 +347,7 @@ class AnalyDistrictLogTest extends BaseBatch{
|
||||
}
|
||||
}
|
||||
}
|
||||
//移除拥有和正常数据一样KEY的被删除数据
|
||||
//移除拥有和正常数据一样KEY的被删除数据
|
||||
for(String k : allRightData.keySet()){
|
||||
deletedData.remove(k)
|
||||
}
|
||||
@@ -415,7 +415,7 @@ class AnalyDistrictLogTest extends BaseBatch{
|
||||
}
|
||||
progress.getAndIncrement()
|
||||
if(progress.get() % 1000 == 0){
|
||||
println("入库进度:${DataUtil.roundDouble((progress.get()/mergedRoads.size())*100,1)}%")
|
||||
println("入库进度:${DataUtil.roundDouble((progress.get()/mergedRoads.size())*100,1)}%")
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -438,7 +438,7 @@ class AnalyDistrictLogTest extends BaseBatch{
|
||||
if(StringUtils.isBlank(region)){
|
||||
return false;
|
||||
}
|
||||
if(region.contains("周边")||region.contains("其他")){
|
||||
if(region.contains("周边")||region.contains("其他")){
|
||||
return false;
|
||||
}
|
||||
if(regions.containsKey(region)){
|
||||
@@ -455,7 +455,7 @@ class AnalyDistrictLogTest extends BaseBatch{
|
||||
|
||||
def updateTempData(){
|
||||
|
||||
println("开始更新临时数据")
|
||||
println("开始更新临时数据")
|
||||
DBObject query = new BasicDBObject();
|
||||
query.append("status",new BasicDBObject('$ne',ANALY_ROAD_STATUS_ERROR))
|
||||
query.append("c_date", new BasicDBObject('$gt',DateUtil.add(now, Calendar.DATE, -2)))
|
||||
@@ -488,7 +488,7 @@ class AnalyDistrictLogTest extends BaseBatch{
|
||||
println(count)
|
||||
}
|
||||
}
|
||||
println("临时数据更新结束")
|
||||
println("临时数据更新结束")
|
||||
}
|
||||
|
||||
def assembleObj(BasicDBObject srcDbo){
|
||||
|
||||
+6
-6
@@ -39,12 +39,12 @@ class AutoAddDistrict {
|
||||
}
|
||||
|
||||
def run(){
|
||||
def ret = findNeedAddDistrict("南京","玄武");
|
||||
println "玄武:" + ret.size();
|
||||
def ret = findNeedAddDistrict("南京","玄武");
|
||||
println "玄武:" + ret.size();
|
||||
def distRet = findAnalyRoads(ret);
|
||||
|
||||
def ret1 = findNeedAddDistrict("南京","建邺");
|
||||
println "建邺:" + ret1.size();
|
||||
def ret1 = findNeedAddDistrict("南京","建邺");
|
||||
println "建邺:" + ret1.size();
|
||||
def distRet1 = findAnalyRoads(ret1);
|
||||
//updateDistrict(distRet);
|
||||
}
|
||||
@@ -96,8 +96,8 @@ class AutoAddDistrict {
|
||||
for(def m : rett){
|
||||
def siteName = m.key;
|
||||
def values = m.values;
|
||||
if(siteName=='网上房地产'){
|
||||
println "网上房地产"
|
||||
if(siteName=='网上房地产'){
|
||||
println "网上房地产"
|
||||
}else{
|
||||
|
||||
for(def field : fields){
|
||||
|
||||
+41
-41
@@ -17,9 +17,9 @@ public class AutoAddSouFangDistrict{
|
||||
|
||||
DBCollection analySiteColl = MongoUtils.getCollectionByName("coredata","analy_site", "coredata.mongo.db.fangjia.com",27017)
|
||||
|
||||
def static citys = ["温州","金华","衢州","湖州","绍兴","嘉兴","舟山","泰州","徐州","镇江","淮安","连云港","宿迁","扬州","盐城","南通","无锡","丽水","重庆","佛山","江门","珠海","郑州","洛阳","许昌","新乡","开封","安阳","大连","鞍山","营口","丹东","西安","榆林","渭南","宝鸡","太原","忻州","运城","朔州","长治","晋中","哈尔滨","齐齐哈尔","大庆","南宁","柳州","桂林","呼和浩特","鄂尔多斯","呼伦贝尔","包头"]
|
||||
def static citys = ["温州","金华","衢州","湖州","绍兴","嘉兴","舟山","泰州","徐州","镇江","淮安","连云港","宿迁","扬州","盐城","南通","无锡","丽水","重庆","佛山","江门","珠海","郑州","洛阳","许昌","新乡","开封","安阳","大连","鞍山","营口","丹东","西安","榆林","渭南","宝鸡","太原","忻州","运城","朔州","长治","晋中","哈尔滨","齐齐哈尔","大庆","南宁","柳州","桂林","呼和浩特","鄂尔多斯","呼伦贝尔","包头"]
|
||||
def static final DBCollection ANALY_ROAD_COLL = MongoUtils.getCollectionByName("coredata", "analy_road", "coredata.mongo.db.fangjia.com")
|
||||
def districtNameNoise = [".","·","。","#","@"," ","?"]
|
||||
def districtNameNoise = [".","·","。","#","@"," ","?"]
|
||||
def static main(args){
|
||||
def obj = new AutoAddSouFangDistrict()
|
||||
citys.each {
|
||||
@@ -34,7 +34,7 @@ public class AutoAddSouFangDistrict{
|
||||
},new BasicDBObject("city",1).append("region",1).append("name",1))
|
||||
|
||||
def keys = new BasicDBObject("city",1).append("region",1).append("district_name",1).append("site_names",1).append("site_count",1)
|
||||
DBCursor cursor = analySiteColl.find(new BasicDBObject("city",city).append("cat","district").append("exists",0).append("audit_status",0).append("data_status",0),keys)//append("site_names",Pattern.compile(".*搜房.*")
|
||||
DBCursor cursor = analySiteColl.find(new BasicDBObject("city",city).append("cat","district").append("exists",0).append("audit_status",0).append("data_status",0),keys)//append("site_names",Pattern.compile(".*搜房.*")
|
||||
cursor.batchSize(10000)
|
||||
cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
cursor.each {BasicDBObject dbo ->
|
||||
@@ -43,10 +43,10 @@ public class AutoAddSouFangDistrict{
|
||||
def name = formatName(orgName)
|
||||
def validSite = isSelectedSites(dbo);
|
||||
if(!validSite){
|
||||
println("网站不合法:" + dbo)
|
||||
println("网站不合法:" + dbo)
|
||||
}
|
||||
if(validSite && !isEndsWith(name) && !isContainsChars(name)){
|
||||
name = name.replaceAll("\\(|(","#").replaceAll("\\)|)","@")
|
||||
name = name.replaceAll("\\(|(","#").replaceAll("\\)|)","@")
|
||||
if(name.indexOf("#") > 0 && name.indexOf("@") > 0){
|
||||
name = name.substring(0,name.indexOf("#"))
|
||||
}
|
||||
@@ -68,8 +68,8 @@ public class AutoAddSouFangDistrict{
|
||||
seaweed.append("status",0)
|
||||
def info = findDistrictInfo(city,region,name)
|
||||
def baiDuEstateType = getBaiDuEstateType(city,name)
|
||||
if(info[0] == "写字楼"|| "写字楼" == baiDuEstateType || "商铺" == baiDuEstateType){
|
||||
println("写字楼/商铺:" + seaweed)
|
||||
if(info[0] == "写字楼"|| "写字楼" == baiDuEstateType || "商铺" == baiDuEstateType){
|
||||
println("写字楼/商铺:" + seaweed)
|
||||
}else{
|
||||
seaweed.append("estate_type2",info[0])
|
||||
if(StringUtils.isNotBlank(info[1])){
|
||||
@@ -92,10 +92,10 @@ public class AutoAddSouFangDistrict{
|
||||
if(dbo.getInt("site_count") > 2){
|
||||
return true;
|
||||
}
|
||||
if(dbo.getInt("site_count") == 1 &&(dbo.getString("site_names").contains("图吧") || dbo.getString("site_names").contains("千万间"))){
|
||||
if(dbo.getInt("site_count") == 1 &&(dbo.getString("site_names").contains("图吧") || dbo.getString("site_names").contains("千万间"))){
|
||||
return false
|
||||
}
|
||||
if(dbo.getInt("site_count") == 2 && dbo.getString("site_names").contains("图吧") && dbo.getString("site_names").contains("千万间")){
|
||||
if(dbo.getInt("site_count") == 2 && dbo.getString("site_names").contains("图吧") && dbo.getString("site_names").contains("千万间")){
|
||||
return false
|
||||
}
|
||||
return true;
|
||||
@@ -120,9 +120,9 @@ public class AutoAddSouFangDistrict{
|
||||
if(StringUtils.isNotBlank(estate_type)){
|
||||
estateTypes << estate_type
|
||||
}
|
||||
if(dbo.getString("site_name") == "搜房" && address == null){
|
||||
if(dbo.getString("site_name") == "搜房" && address == null){
|
||||
address = result.get("address")
|
||||
}else if(otherAddress == null ||(result.get("address") && result.getString("address") ==~ /.*\d+号.*/)){
|
||||
}else if(otherAddress == null ||(result.get("address") && result.getString("address") ==~ /.*\d+号.*/)){
|
||||
otherAddress = result.get("address")
|
||||
}
|
||||
|
||||
@@ -133,16 +133,16 @@ public class AutoAddSouFangDistrict{
|
||||
int liangYongCount = 0
|
||||
int zhuZhaiCount = 0
|
||||
estateTypes.each {
|
||||
if(it.contains("商铺") || it.contains("写字楼") || it.contains("商业") || it.contains("办公")){
|
||||
if(it.contains("商铺") || it.contains("写字楼") || it.contains("商业") || it.contains("办公")){
|
||||
officeCount ++
|
||||
}
|
||||
if(it.contains("别墅")){
|
||||
if(it.contains("别墅")){
|
||||
bieshuCount ++
|
||||
}
|
||||
if(it.contains("商住两用")){
|
||||
if(it.contains("商住两用")){
|
||||
liangYongCount ++
|
||||
}
|
||||
if(it.contains("住宅")){
|
||||
if(it.contains("住宅")){
|
||||
zhuZhaiCount ++
|
||||
}
|
||||
}
|
||||
@@ -151,29 +151,29 @@ public class AutoAddSouFangDistrict{
|
||||
}
|
||||
if(zhuZhaiCount > 0){
|
||||
if(bieshuCount > 0){
|
||||
return ["别墅",address]
|
||||
return ["别墅",address]
|
||||
}else if(liangYongCount > 0){
|
||||
return ["商住两用",address]
|
||||
return ["商住两用",address]
|
||||
}
|
||||
return ["普通住宅",address]
|
||||
return ["普通住宅",address]
|
||||
}else if(officeCount > zhuZhaiCount){
|
||||
return ["写字楼",address]
|
||||
return ["写字楼",address]
|
||||
}else if(bieshuCount > 0){
|
||||
return ["别墅",address]
|
||||
return ["别墅",address]
|
||||
}else if(liangYongCount > 0){
|
||||
return ["商住两用",address]
|
||||
return ["商住两用",address]
|
||||
}else{
|
||||
return ["普通住宅",address]
|
||||
return ["普通住宅",address]
|
||||
}
|
||||
}
|
||||
|
||||
def formateAddress(String city,String address){
|
||||
if(address.endsWith("交汇")){
|
||||
if(address.endsWith("交汇")){
|
||||
return address
|
||||
}
|
||||
String orgAddress = address
|
||||
if(address ==~ /.*\d+$/){
|
||||
address += "号"
|
||||
address += "号"
|
||||
}
|
||||
JSONObject result = OpenApiClient.addressStandardize(city, address)
|
||||
String formateAddr;
|
||||
@@ -183,7 +183,7 @@ public class AutoAddSouFangDistrict{
|
||||
formateAddr += result.getString("streetNumber")
|
||||
}
|
||||
}
|
||||
if(formateAddr && (formateAddr.contains("市") || formateAddr.contains("县"))){
|
||||
if(formateAddr && (formateAddr.contains("市") || formateAddr.contains("县"))){
|
||||
result = OpenApiClient.addressStandardize(city, formateAddr)
|
||||
if(result.containsKey("street")){
|
||||
formateAddr = result.getString("street")
|
||||
@@ -198,7 +198,7 @@ public class AutoAddSouFangDistrict{
|
||||
formateAddr = addrs[0] + addrs[1]
|
||||
return formateAddr
|
||||
}else if(addrs.size() == 3){
|
||||
formateAddr = addrs[0] + "与" + addrs[1].replaceFirst("^(与|和)","") + "交汇"
|
||||
formateAddr = addrs[0] + "与" + addrs[1].replaceFirst("^(与|和)","") + "交汇"
|
||||
return formateAddr
|
||||
}
|
||||
}
|
||||
@@ -214,12 +214,12 @@ public class AutoAddSouFangDistrict{
|
||||
if(item.containsKey("detail_info") && item.getString("name") == name){
|
||||
String tag = item.getJSONObject("detail_info")?.get("tag")?.toString();
|
||||
if (StringUtils.isNotBlank(tag)) {
|
||||
if(tag.contains("写字楼")){
|
||||
return "写字楼"
|
||||
}else if(tag.contains("商场") || tag.contains("购物") || tag.contains("商铺")){
|
||||
return "商铺"
|
||||
}else if(tag.contains("住宅区")){
|
||||
return "普通住宅"
|
||||
if(tag.contains("写字楼")){
|
||||
return "写字楼"
|
||||
}else if(tag.contains("商场") || tag.contains("购物") || tag.contains("商铺")){
|
||||
return "商铺"
|
||||
}else if(tag.contains("住宅区")){
|
||||
return "普通住宅"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -233,22 +233,22 @@ public class AutoAddSouFangDistrict{
|
||||
districtNameNoise.each {
|
||||
name = name.replace(it,"")
|
||||
}
|
||||
if(name ==~ /.*(村|小区)\d+号$/){
|
||||
name = name.replaceAll("\\d+号\$","")
|
||||
if(name ==~ /.*(村|小区)\d+号$/){
|
||||
name = name.replaceAll("\\d+号\$","")
|
||||
}
|
||||
if(name ==~ /.*路\d+号$/){
|
||||
name = name + "小区"
|
||||
if(name ==~ /.*路\d+号$/){
|
||||
name = name + "小区"
|
||||
}
|
||||
name = name.replaceAll("\\d+幢\$","")
|
||||
name = name.replaceAll("\\d+幢\$","")
|
||||
return name;
|
||||
}
|
||||
|
||||
def isEndsWith(def s){
|
||||
def endChars = ["路","街","店","贸区","巷","村","设计院","社区","地块","交易中心","综合大楼","附近","对面","安置房"];
|
||||
def endChars = ["路","街","店","贸区","巷","村","设计院","社区","地块","交易中心","综合大楼","附近","对面","安置房"];
|
||||
def flag = false;
|
||||
for(def key : endChars){
|
||||
if(s.endsWith(key)){
|
||||
if(key=="村"||key=="巷"){
|
||||
if(key=="村"||key=="巷"){
|
||||
if(s.length()<=3){
|
||||
flag = true;
|
||||
}
|
||||
@@ -263,7 +263,7 @@ public class AutoAddSouFangDistrict{
|
||||
}
|
||||
|
||||
def isContainsChars(s) {
|
||||
def chars = ["世贸中心", "商务中心", "家具中心", "商业中心", "建材市场", "生活广场", "购物广场", "商城", "电器城"]
|
||||
def chars = ["世贸中心", "商务中心", "家具中心", "商业中心", "建材市场", "生活广场", "购物广场", "商城", "电器城"]
|
||||
def flag = false;
|
||||
for (def key : chars) {
|
||||
if (s.indexOf(key) != -1) {
|
||||
@@ -276,7 +276,7 @@ public class AutoAddSouFangDistrict{
|
||||
|
||||
def List<String> splitAddress(String addr){
|
||||
List<String> addres = []
|
||||
def keys = ["街道","大道","国道","道路","路","街","线"]
|
||||
def keys = ["街道","大道","国道","道路","路","街","线"]
|
||||
String remain = addr;
|
||||
while(true){
|
||||
int p = Integer.MAX_VALUE
|
||||
|
||||
@@ -25,7 +25,7 @@ class AutoDistrict {
|
||||
|
||||
public static void main(String[] args){
|
||||
def task = new AutoDistrict();
|
||||
task.run(["西安", "成都"]);//"北京",
|
||||
task.run(["西安", "成都"]);//"北京",
|
||||
|
||||
}
|
||||
|
||||
@@ -154,7 +154,7 @@ class AutoDistrict {
|
||||
s = s + " "+ estateType;
|
||||
}
|
||||
}
|
||||
if(s.indexOf("写字楼")!=-1||s.indexOf("办字楼")!=-1||s.indexOf("商铺")!=-1||s.indexOf("商住楼")!=-1){
|
||||
if(s.indexOf("写字楼")!=-1||s.indexOf("办字楼")!=-1||s.indexOf("商铺")!=-1||s.indexOf("商住楼")!=-1){
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
@@ -233,7 +233,7 @@ class AutoDistrict {
|
||||
}
|
||||
|
||||
def writeCSV(city, results, fields){
|
||||
File file = new File("e:\\自动入库("+city+").csv");
|
||||
File file = new File("e:\\自动入库("+city+").csv");
|
||||
if (file.exists()){
|
||||
file.delete();
|
||||
}
|
||||
|
||||
+5
-5
@@ -13,7 +13,7 @@ class CityAnaDataBatch extends BaseBatch {
|
||||
|
||||
DBCollection CITY_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "city_sell_stat", "mongo.db.fangjia.com")
|
||||
DBCollection CITY_ANA_DATA = MongoUtils.getCollectionByName("fangjia_stat", "city_ana_data", "mongo.db.fangjia.com")
|
||||
List CENTER_CITY = ["石家庄","太原","呼和浩特","沈阳","长春","哈尔滨","南京","杭州","合肥","福州","南昌","济南","郑州","武汉","长沙","广州","南宁","海口","成都","贵阳","昆明","拉萨","西安","兰州","西宁","银川","乌鲁木齐","台北"]
|
||||
List CENTER_CITY = ["石家庄","太原","呼和浩特","沈阳","长春","哈尔滨","南京","杭州","合肥","福州","南昌","济南","郑州","武汉","长沙","广州","南宁","海口","成都","贵阳","昆明","拉萨","西安","兰州","西宁","银川","乌鲁木齐","台北"]
|
||||
|
||||
static void main(String[] args) {
|
||||
runBatch(new CityAnaDataBatch(), args)
|
||||
@@ -21,7 +21,7 @@ class CityAnaDataBatch extends BaseBatch {
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
Integer yearMonth = CITY_SELL_STAT.find(new BasicDBObject("city", "上海")).sort(new BasicDBObject("date", -1)).limit(1).toList()[0].date
|
||||
Integer yearMonth = CITY_SELL_STAT.find(new BasicDBObject("city", "上海")).sort(new BasicDBObject("date", -1)).limit(1).toList()[0].date
|
||||
List orders = []
|
||||
CITY_SELL_STAT.find(new BasicDBObject("date", yearMonth)).each {BasicDBObject dbo->
|
||||
orders << new CityOrderItem(dbo.city, dbo.avg_price)
|
||||
@@ -31,11 +31,11 @@ class CityAnaDataBatch extends BaseBatch {
|
||||
CityOrderItem item = orders.get(i - 1)
|
||||
def data = new BasicDBObject('city_rank', i).append("city_total_rank", orders.size()).append("avg_price", item.avgPrice)
|
||||
if(ProvinceData.isGovernmentCity(item.city)){
|
||||
data.put("level", "直辖市")
|
||||
data.put("level", "直辖市")
|
||||
}else if(CENTER_CITY.contains(item.city)){
|
||||
data.put("level", "省会城市")
|
||||
data.put("level", "省会城市")
|
||||
}else{
|
||||
data.put("level", "地址市")
|
||||
data.put("level", "地址市")
|
||||
}
|
||||
CITY_ANA_DATA.update(new BasicDBObject("city", item.city), new BasicDBObject('$set', data), true,true)
|
||||
}
|
||||
|
||||
+3
-3
@@ -11,7 +11,7 @@ import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
/**
|
||||
* 城市区域人均消费统计脚本
|
||||
* 城市区域人均消费统计脚本
|
||||
* @author
|
||||
*/
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
@@ -38,7 +38,7 @@ public class CityAndRegionAverageConsume extends BaseBatch{
|
||||
|
||||
def analyRegionAverageConsume(def city, def region){
|
||||
List<Integer> numbers = new ArrayList<Integer>();
|
||||
BUSINESS_SHOP_COLL.find(new BasicDBObject("business_type", "美食").append("city", city).append("region", region), new BasicDBObject("consumption", 1)).each {BasicDBObject dbo->
|
||||
BUSINESS_SHOP_COLL.find(new BasicDBObject("business_type", "美食").append("city", city).append("region", region), new BasicDBObject("consumption", 1)).each {BasicDBObject dbo->
|
||||
if(dbo.consumption && dbo.consumption > 0){
|
||||
numbers << dbo.consumption
|
||||
}
|
||||
@@ -59,7 +59,7 @@ public class CityAndRegionAverageConsume extends BaseBatch{
|
||||
def analyCityAverageConsume(def city){
|
||||
|
||||
List<Integer> numbers = new ArrayList<Integer>();
|
||||
BUSINESS_SHOP_COLL.find(new BasicDBObject("business_type", "美食").append("city", city), new BasicDBObject("consumption", 1)).each {BasicDBObject dbo->
|
||||
BUSINESS_SHOP_COLL.find(new BasicDBObject("business_type", "美食").append("city", city), new BasicDBObject("consumption", 1)).each {BasicDBObject dbo->
|
||||
if(dbo.consumption && dbo.consumption > 0){
|
||||
numbers << dbo.consumption
|
||||
}
|
||||
|
||||
+5
-5
@@ -15,8 +15,8 @@ import java.sql.Connection
|
||||
import java.sql.DriverManager
|
||||
|
||||
/**
|
||||
* 小区均价涨跌幅纠正脚本
|
||||
* 功能说明:
|
||||
* 小区均价涨跌幅纠正脚本
|
||||
* 功能说明:
|
||||
*/
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
class ClickHouseDistrictStatMarkupCalcBatch extends BaseBatch {
|
||||
@@ -39,10 +39,10 @@ class ClickHouseDistrictStatMarkupCalcBatch extends BaseBatch {
|
||||
query.city = district.city
|
||||
query.region = district.region
|
||||
query.district_name = district.name
|
||||
if(district.estate_type2 && "别墅" == district.estate_type2){
|
||||
query.estate_type = "别墅"
|
||||
if(district.estate_type2 && "别墅" == district.estate_type2){
|
||||
query.estate_type = "别墅"
|
||||
}else{
|
||||
query.estate_type = "小区"
|
||||
query.estate_type = "小区"
|
||||
}
|
||||
List<BasicDBObject> items = DISTRICT_STAT.find(query, new BasicDBObject("avg_price", 1).append("weekend_markup", 1).append("weekend", 1)).batchSize(2000).addOption(Bytes.QUERYOPTION_SLAVEOK).sort(new BasicDBObject("weekend", 1)).toList()
|
||||
if(items.size() < 2){
|
||||
|
||||
+3
-3
@@ -11,7 +11,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* Created by wenyan on 2016/1/22.
|
||||
*/
|
||||
def cityList = ["南昌","包头","安庆","泸州","遵义","常德","南充","牡丹江","锦州","岳阳","韶关","济宁","平顶山"]
|
||||
def cityList = ["南昌","包头","安庆","泸州","遵义","常德","南充","牡丹江","锦州","岳阳","韶关","济宁","平顶山"]
|
||||
DBCollection ANALY_SITE_COLL = MongoUtils.getCollectionByName("coredata", "analy_site", "coredata.mongo.db.fangjia.com")
|
||||
DSCollection seawater = new DataStore("192.168.0.65", 27777).getCollection("fangjia.seawater")
|
||||
def startDate = DateUtil.add(new Date(), Calendar.MONTH, -6).format("yyyyMMdd").toInteger()
|
||||
@@ -30,9 +30,9 @@ cityList.each {def city->
|
||||
long count = (long) seawater.count(houseQuery)
|
||||
ANALY_SITE_COLL.update(new BasicDBObject("_id",dbo._id), new BasicDBObject('$set',new BasicDBObject('house_num',count)))
|
||||
if(idx % 100 == 0){
|
||||
println("${city}统计进度:${DataUtil.roundDouble((idx *100)/total,2)}%")
|
||||
println("${city}统计进度:${DataUtil.roundDouble((idx *100)/total,2)}%")
|
||||
}
|
||||
}
|
||||
println("${city}分析结束")
|
||||
println("${city}分析结束")
|
||||
|
||||
}
|
||||
+19
-19
@@ -18,8 +18,8 @@ class CoverPrecent {
|
||||
public static final DBCollection OFFICE_STAT_COLL = MongoUtils.getCollectionByName("fangjia", "office_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
//def citys = ["天津","重庆","成都","大连","沈阳","哈尔滨","牡丹江","大庆","齐齐哈尔","佳木斯","伊春","鸡西","鹤岗","双鸭山","七台河","绥化","黑河"];
|
||||
def citys = ["上海"]
|
||||
//def citys = ["天津","重庆","成都","大连","沈阳","哈尔滨","牡丹江","大庆","齐齐哈尔","佳木斯","伊春","鸡西","鹤岗","双鸭山","七台河","绥化","黑河"];
|
||||
def citys = ["上海"]
|
||||
def cat = "office";
|
||||
def weekend = "2014-11-16";
|
||||
|
||||
@@ -41,7 +41,7 @@ class CoverPrecent {
|
||||
|
||||
result << ret;
|
||||
}
|
||||
task.writeCSV3("E:\\数据测试\\结果\\商铺写字楼覆盖率\\","覆盖率-"+cat+".csv", result)
|
||||
task.writeCSV3("E:\\数据测试\\结果\\商铺写字楼覆盖率\\","覆盖率-"+cat+".csv", result)
|
||||
}
|
||||
|
||||
|
||||
@@ -52,18 +52,18 @@ class CoverPrecent {
|
||||
}
|
||||
|
||||
|
||||
file.append("城市,")
|
||||
file.append("楼盘数量,")
|
||||
file.append("售价数量,")
|
||||
file.append("租金数量,")
|
||||
file.append("售价占比,")
|
||||
file.append("租金占比,")
|
||||
file.append("租房房源数,")
|
||||
file.append("租房房源小区量,")
|
||||
file.append("出售房源数,")
|
||||
file.append("出售房源小区量,")
|
||||
file.append("出租房源小区占比,")
|
||||
file.append("出售房源小区占比,")
|
||||
file.append("城市,")
|
||||
file.append("楼盘数量,")
|
||||
file.append("售价数量,")
|
||||
file.append("租金数量,")
|
||||
file.append("售价占比,")
|
||||
file.append("租金占比,")
|
||||
file.append("租房房源数,")
|
||||
file.append("租房房源小区量,")
|
||||
file.append("出售房源数,")
|
||||
file.append("出售房源小区量,")
|
||||
file.append("出租房源小区占比,")
|
||||
file.append("出售房源小区占比,")
|
||||
file.append(System.getProperty("line.separator"));
|
||||
ret.each{ map->
|
||||
file.append(map.city+ ",","gbk")
|
||||
@@ -87,9 +87,9 @@ class CoverPrecent {
|
||||
def totalCount = findTotalCount(city, cat);
|
||||
def kind = "";
|
||||
if(cat == "shop"){
|
||||
kind = "商铺";
|
||||
kind = "商铺";
|
||||
}else if(cat == "office"){
|
||||
kind = "写字楼";
|
||||
kind = "写字楼";
|
||||
}
|
||||
def sellCount = findSellPriceCountByCat(city, kind, weekend);
|
||||
def rentCount = findRentPriceCountByCat(city, kind, weekend);
|
||||
@@ -107,7 +107,7 @@ class CoverPrecent {
|
||||
ret.rentCount = rentCount;
|
||||
ret.sellCoverPecent = DataUtil.getRound(2, sellCoverPecent) ;
|
||||
ret.rentCoverPecent = DataUtil.getRound(2, rentCoverPecent) ;
|
||||
println "【${city}】:"+totalCount+" sellCount:"+sellCount+" rentCount:"+rentCount+" sellCoverPecent:"+sellCoverPecent+" rentCoverPecent:"+rentCoverPecent
|
||||
println "【${city}】:"+totalCount+" sellCount:"+sellCount+" rentCount:"+rentCount+" sellCoverPecent:"+sellCoverPecent+" rentCoverPecent:"+rentCoverPecent
|
||||
return ret;
|
||||
}
|
||||
|
||||
@@ -134,7 +134,7 @@ class CoverPrecent {
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("city", city);
|
||||
q.put('weekend', DateUtils.parseFromTightYYYYMMDD(weekend));
|
||||
if(kind == '商铺'){
|
||||
if(kind == '商铺'){
|
||||
return SHOP_STAT_COLL.count(q);
|
||||
}else{
|
||||
return OFFICE_STAT_COLL.count(q);
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@ import com.mongodb.Mongo
|
||||
import com.mongodb.MongoURI
|
||||
|
||||
/**
|
||||
* 功能说明:监控数据库数据量
|
||||
* 功能说明:监控数据库数据量
|
||||
*/
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
class DatabaseStatWatcherBatch extends BaseBatch {
|
||||
|
||||
+6
-6
@@ -16,7 +16,7 @@ import com.mongodb.DBObject
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
|
||||
/**
|
||||
* 小区周边数据统计脚本
|
||||
* 小区周边数据统计脚本
|
||||
*/
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
public class DistrictAroundDataAnaly extends BaseBatch {
|
||||
@@ -58,7 +58,7 @@ public class DistrictAroundDataAnaly extends BaseBatch {
|
||||
DbUtils.multiThreadingExecuteQueryOnSeaweed(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), { BasicDBObject dbo ->
|
||||
def data = new BasicDBObject()
|
||||
if (dbo.lng && polygonMap.size() > 0) {
|
||||
def economics = ["中心城区", "普通城区", "郊区"]
|
||||
def economics = ["中心城区", "普通城区", "郊区"]
|
||||
for (def economic : economics) {
|
||||
def latlngs = polygonMap.get(economic)
|
||||
if (latlngs != null) {
|
||||
@@ -75,10 +75,10 @@ public class DistrictAroundDataAnaly extends BaseBatch {
|
||||
}
|
||||
}
|
||||
if (!data.containsField("economic")) {
|
||||
data.put("economic", "郊外")
|
||||
data.put("economic", "郊外")
|
||||
}
|
||||
}
|
||||
def types = ["商场": "060101", "麦当劳": "050302", "肯德基": "050301", "星巴克": "050501", "地铁站": "150500", "小学": "141202", "中学": "141203", "市中心": "190500"]
|
||||
def types = ["商场": "060101", "麦当劳": "050302", "肯德基": "050301", "星巴克": "050501", "地铁站": "150500", "小学": "141202", "中学": "141203", "市中心": "190500"]
|
||||
if (dbo.lng) {
|
||||
def nearest_classic_pois = []
|
||||
types.each { def type, def code ->
|
||||
@@ -98,7 +98,7 @@ public class DistrictAroundDataAnaly extends BaseBatch {
|
||||
AMAP_POI_COLL.find(query, new BasicDBObject("name", 1).append("lat", 1).append("lng", 1)).limit(1).each { BasicDBObject poi ->
|
||||
def classicPoi = new BasicDBObject()
|
||||
classicPoi.code = code
|
||||
classicPoi.name = poi.name.toString().replaceAll("((|\\().*()|\\))", "")
|
||||
classicPoi.name = poi.name.toString().replaceAll("((|\\().*()|\\))", "")
|
||||
classicPoi.lng = poi.lng
|
||||
classicPoi.lat = poi.lat
|
||||
classicPoi.distance = GeoUtils.calculateDistance(dbo.lng / 10000000000d, dbo.lat / 10000000000d, classicPoi.lng, classicPoi.lat)
|
||||
@@ -186,7 +186,7 @@ public class DistrictAroundDataAnaly extends BaseBatch {
|
||||
def query = new BasicDBObject()
|
||||
query.append('lat', new BasicDBObject('$gte', arr[0]).append('$lte', arr[1]))
|
||||
query.append('lng', new BasicDBObject('$gte', arr[2]).append('$lte', arr[3]))
|
||||
query.append("business_type", "美食")
|
||||
query.append("business_type", "美食")
|
||||
def keys = new BasicDBObject("consumption", 1)
|
||||
List<Integer> numbers = new ArrayList<Integer>();
|
||||
BUSINESS_SHOP_COLL.find(query, keys).each { BasicDBObject dbo ->
|
||||
|
||||
+7
-7
@@ -31,14 +31,14 @@ class DistrictBackupReset {
|
||||
def startDate = "2014-12-07";
|
||||
def index = 0;
|
||||
for(def city : citys){
|
||||
if(city!="上海"){
|
||||
if(city!="上海"){
|
||||
index ++;
|
||||
def backUpMap = findBackUp(city);
|
||||
println "【${city}】备份数据:"+backUpMap.size();
|
||||
println "【${city}】备份数据:"+backUpMap.size();
|
||||
def districts = findUpdateDistrictData(city, startDate)
|
||||
println "【${city}】可能修改的小区数据有:"+districts.size();
|
||||
println "【${city}】可能修改的小区数据有:"+districts.size();
|
||||
updateDistrict(districts, backUpMap);
|
||||
println "${index} 更新完成..........................."
|
||||
println "${index} 更新完成..........................."
|
||||
}
|
||||
|
||||
}
|
||||
@@ -58,8 +58,8 @@ class DistrictBackupReset {
|
||||
|
||||
}else{
|
||||
index ++;
|
||||
println "[${index}]当前:"+obj;
|
||||
println "[${index}]备份:"+map;
|
||||
println "[${index}]当前:"+obj;
|
||||
println "[${index}]备份:"+map;
|
||||
|
||||
DBObject udata = new BasicDBObject();
|
||||
udata.put("m_date", new Date());
|
||||
@@ -134,7 +134,7 @@ class DistrictBackupReset {
|
||||
if(cursor){
|
||||
for(def dbo : cursor){
|
||||
def city = dbo.get("display");
|
||||
if(city!='全国'){
|
||||
if(city!='全国'){
|
||||
citys << city;
|
||||
}
|
||||
}
|
||||
|
||||
+14
-14
@@ -11,7 +11,7 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.core.util.DateUtil;
|
||||
import com.fangjia.core.util.DataUtil
|
||||
/**
|
||||
* 分期竣工时间覆盖率
|
||||
* 分期竣工时间覆盖率
|
||||
* @author gaojuan
|
||||
*
|
||||
*/
|
||||
@@ -25,7 +25,7 @@ class DistrictFinishDateCover {
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = ["上海"];//findAllCity()
|
||||
def citys = ["上海"];//findAllCity()
|
||||
for(def city : citys){
|
||||
def districts = getAllDistricts(city, "district");
|
||||
findFenQiData(districts);
|
||||
@@ -43,24 +43,24 @@ class DistrictFinishDateCover {
|
||||
|
||||
def findFenQiData(districts){
|
||||
def fenqiDatas = [];
|
||||
//查找分期数据
|
||||
//查找分期数据
|
||||
if(districts && districts.size() > 0){
|
||||
for(def map : districts){
|
||||
def keys = map.key
|
||||
def dbo = map.value;
|
||||
def name = dbo.name;
|
||||
if(name.indexOf("期")!=-1){
|
||||
if(name.indexOf("期")!=-1){
|
||||
fenqiDatas << dbo;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//查找总小区
|
||||
//查找总小区
|
||||
def dists = [];
|
||||
for(def dbo : fenqiDatas){
|
||||
def name = dbo.name
|
||||
def region = dbo.region
|
||||
name = name.replaceAll("[一|二|三|四|五|六|七|八|九|十]期", "")
|
||||
name = name.replaceAll("[一|二|三|四|五|六|七|八|九|十]期", "")
|
||||
def obj = districts.get(region + "@"+name);
|
||||
if(obj){
|
||||
if(!dists.contains(obj)){
|
||||
@@ -103,8 +103,8 @@ class DistrictFinishDateCover {
|
||||
|
||||
def rett1 = calDateCover(dists);
|
||||
//def rett2 = calDateCover(rets);
|
||||
writeCSV("E:\\数据测试\\结果\\","上海分期(总).csv", rett1)
|
||||
//writeCSV("E:\\数据测试\\结果\\","上海分期.csv", rett2)
|
||||
writeCSV("E:\\数据测试\\结果\\","上海分期(总).csv", rett1)
|
||||
//writeCSV("E:\\数据测试\\结果\\","上海分期.csv", rett2)
|
||||
}
|
||||
|
||||
def calDateCover(dists){
|
||||
@@ -160,8 +160,8 @@ class DistrictFinishDateCover {
|
||||
def count = haveTimeDists.size();
|
||||
def precent = count * 100/total;
|
||||
precent = DataUtil.getRound(2,precent);
|
||||
println "覆盖率:" + precent
|
||||
println "共:" + total
|
||||
println "覆盖率:" + precent
|
||||
println "共:" + total
|
||||
return haveTimeDists;
|
||||
}
|
||||
|
||||
@@ -172,9 +172,9 @@ class DistrictFinishDateCover {
|
||||
}
|
||||
|
||||
|
||||
file.append("区域,")
|
||||
file.append("名称,")
|
||||
file.append("时间,")
|
||||
file.append("区域,")
|
||||
file.append("名称,")
|
||||
file.append("时间,")
|
||||
|
||||
file.append(System.getProperty("line.separator"));
|
||||
ret.each{ map->
|
||||
@@ -217,7 +217,7 @@ class DistrictFinishDateCover {
|
||||
if(cursor){
|
||||
for(def dbo : cursor){
|
||||
def city = dbo.get("display");
|
||||
if(city!='全国'){
|
||||
if(city!='全国'){
|
||||
citys << city;
|
||||
}
|
||||
}
|
||||
|
||||
+4
-4
@@ -58,7 +58,7 @@ class DistrictPriceRatingBatch extends BaseBatch {
|
||||
if(item.maxArea && item.maxArea < 40){
|
||||
continue
|
||||
}
|
||||
item.tag = "봉빽遼酪"
|
||||
item.tag = "豪华住宅"
|
||||
haoHuaList << item
|
||||
}
|
||||
list.removeAll(haoHuaList)
|
||||
@@ -78,19 +78,19 @@ class DistrictPriceRatingBatch extends BaseBatch {
|
||||
if(item.maxArea && item.maxArea < 60){
|
||||
continue
|
||||
}
|
||||
item.tag = "멕도遼酪"
|
||||
item.tag = "高档住宅"
|
||||
gaoDangList << item
|
||||
}
|
||||
for(int i = regionList.size() * 0.75 ; i <= regionList.size(); i++){
|
||||
DistrictPriceOrderItem item = regionList.get(i - 1)
|
||||
item.tag = "됴도遼酪"
|
||||
item.tag = "低档住宅"
|
||||
diDangList << item
|
||||
}
|
||||
}
|
||||
list.removeAll(gaoDangList)
|
||||
list.removeAll(diDangList)
|
||||
list.each {DistrictPriceOrderItem item->
|
||||
item.tag = "팹繫遼酪"
|
||||
item.tag = "普通住宅"
|
||||
}
|
||||
haoHuaList.each {
|
||||
DISTRICT_ANALY_DATA_COLL.update(new BasicDBObject("city", city).append("region", it.region).append("district_name", it.name), new BasicDBObject('$set', new BasicDBObject('tag', it.tag)), true, true)
|
||||
|
||||
+7
-7
@@ -12,8 +12,8 @@ import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
/**
|
||||
* 小区均价涨跌幅纠正脚本
|
||||
* 功能说明:
|
||||
* 小区均价涨跌幅纠正脚本
|
||||
* 功能说明:
|
||||
*/
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
class DistrictStatMarkupCalcBatch extends BaseBatch {
|
||||
@@ -29,19 +29,19 @@ class DistrictStatMarkupCalcBatch extends BaseBatch {
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def cityList = CommonUtils.findAllCity()
|
||||
cityList =["北海"]
|
||||
cityList =["北海"]
|
||||
int total = cityList.size()
|
||||
int count = 1
|
||||
cityList.each {def city ->
|
||||
DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city,"海城","东旭华庭"),{BasicDBObject district->
|
||||
DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city,"海城","东旭华庭"),{BasicDBObject district->
|
||||
def query = new BasicDBObject()
|
||||
query.city = district.city
|
||||
query.region = district.region
|
||||
query.district_name = district.name
|
||||
if(district.estate_type2 && "别墅" == district.estate_type2){
|
||||
query.estate_type = "别墅"
|
||||
if(district.estate_type2 && "别墅" == district.estate_type2){
|
||||
query.estate_type = "别墅"
|
||||
}else{
|
||||
query.estate_type = "小区"
|
||||
query.estate_type = "小区"
|
||||
}
|
||||
List<BasicDBObject> items = DISTRICT_STAT.find(query, new BasicDBObject("avg_price", 1).append("weekend_markup", 1).append("weekend", 1)).batchSize(2000).addOption(Bytes.QUERYOPTION_SLAVEOK).sort(new BasicDBObject("weekend", 1)).toList()
|
||||
if(items.size() < 2){
|
||||
|
||||
+4
-4
@@ -37,7 +37,7 @@ class DistrictStatRankBatch extends BaseBatch {
|
||||
public void process(Object args) {
|
||||
|
||||
|
||||
def cities = ["深圳"]//ServerAreaMap.findAllCities()
|
||||
def cities = ["深圳"]//ServerAreaMap.findAllCities()
|
||||
for(String city : cities) {
|
||||
// DISTRICT_STAT_MAP.clear()
|
||||
// Map<String, List<String>> regions = RegionData.getAllRegions(city)
|
||||
@@ -187,7 +187,7 @@ class DistrictStatRankBatch extends BaseBatch {
|
||||
String city = district.city
|
||||
String region = district.region
|
||||
String name = district.name
|
||||
String estateType = (district.estate_type2 != null && district.estate_type2.indexOf('别墅') > -1 ? '别墅' : '小区')
|
||||
String estateType = (district.estate_type2 != null && district.estate_type2.indexOf('别墅') > -1 ? '别墅' : '小区')
|
||||
|
||||
Date momWeekend = DateUtil.getThisSunday(DateUtil.getBeforeDay(new Date(), 30))
|
||||
Date sosWeekend = DateUtil.getThisSunday(DateUtil.getBeforeDay(new Date(), 90))
|
||||
@@ -339,9 +339,9 @@ class DistrictStatRankBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
public static int calculatePercentile(def values, def targetValue) {
|
||||
// 找出数字在List中的排名
|
||||
// 找出数字在List中的排名
|
||||
def rank = values.indexOf(targetValue) + 1
|
||||
// 计算百分位数
|
||||
// 计算百分位数
|
||||
def percentile = (int)((rank / values.size()) * 100)
|
||||
return percentile;
|
||||
}
|
||||
|
||||
+15
-15
@@ -25,7 +25,7 @@ class EstateTypeChange {
|
||||
|
||||
def offices = getAllDistricts(city, "office");
|
||||
updateEstateType("office", offices);
|
||||
println "【${city}】 完成......"
|
||||
println "【${city}】 完成......"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,21 +38,21 @@ class EstateTypeChange {
|
||||
def new_estate_type2 = "";
|
||||
|
||||
if(cat == "shop"){
|
||||
if(estate_type2 == "市场类商铺"||estate_type2 == "专项卖场"){
|
||||
new_estate_type2 = "专业市场";
|
||||
}else if(estate_type2 == "百货购物中心商铺" || estate_type2 == "商业综合体" || estate_type2 == "商务区商铺"){
|
||||
new_estate_type2 = "商场商铺";
|
||||
}else if(estate_type2 == "社区内铺"){
|
||||
new_estate_type2 = "社区商铺";
|
||||
if(estate_type2 == "市场类商铺"||estate_type2 == "专项卖场"){
|
||||
new_estate_type2 = "专业市场";
|
||||
}else if(estate_type2 == "百货购物中心商铺" || estate_type2 == "商业综合体" || estate_type2 == "商务区商铺"){
|
||||
new_estate_type2 = "商场商铺";
|
||||
}else if(estate_type2 == "社区内铺"){
|
||||
new_estate_type2 = "社区商铺";
|
||||
}
|
||||
}else if(cat == "office"){
|
||||
|
||||
if(estate_type2 == "园区写字楼"){
|
||||
new_estate_type2 = "创意园区";
|
||||
}else if(estate_type2 == "酒店商务楼" || estate_type2 == "商业综合楼"){
|
||||
new_estate_type2 = "标准写字楼";
|
||||
}else if(estate_type2 == "商住两用楼"){
|
||||
new_estate_type2 = "商住两用";
|
||||
if(estate_type2 == "园区写字楼"){
|
||||
new_estate_type2 = "创意园区";
|
||||
}else if(estate_type2 == "酒店商务楼" || estate_type2 == "商业综合楼"){
|
||||
new_estate_type2 = "标准写字楼";
|
||||
}else if(estate_type2 == "商住两用楼"){
|
||||
new_estate_type2 = "商住两用";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,10 +69,10 @@ class EstateTypeChange {
|
||||
|
||||
}
|
||||
// else{
|
||||
// if(estate_type2!="社区商铺" && estate_type2!="商务区商铺" && estate_type2!="商业街商铺"){
|
||||
// if(estate_type2!="社区商铺" && estate_type2!="商务区商铺" && estate_type2!="商业街商铺"){
|
||||
// println "[ ${index} ]"+ estate_type2+" "
|
||||
// }
|
||||
// if(estate_type2!="标准写字楼"){
|
||||
// if(estate_type2!="标准写字楼"){
|
||||
// println "[ ${index} ]"+ estate_type2+" "
|
||||
// }
|
||||
//
|
||||
|
||||
+5
-5
@@ -10,12 +10,12 @@ import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.regex.Pattern
|
||||
// 分期分区与主小区映射关系表
|
||||
// 分期分区与主小区映射关系表
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
class FindStageDistrictBatch extends BaseBatch {
|
||||
|
||||
Pattern STAGE_PATTERN = Pattern.compile("(([一二三四五六七八九十百千\\d]+)期?[,、至到]?)?([一二三四五六七八九十百千\\d]+)期\$");
|
||||
Pattern PARTITION_PATTERN = Pattern.compile("(\\w|东|南|北|中|西|甲|乙|丙|丁|一|二|三|四|五|六|七|八|九|十)区\$");
|
||||
Pattern STAGE_PATTERN = Pattern.compile("(([一二三四五六七八九十百千\\d]+)期?[,、至到]?)?([一二三四五六七八九十百千\\d]+)期\$");
|
||||
Pattern PARTITION_PATTERN = Pattern.compile("(\\w|东|南|北|中|西|甲|乙|丙|丁|一|二|三|四|五|六|七|八|九|十)区\$");
|
||||
|
||||
|
||||
static void main(String[] args) {
|
||||
@@ -27,14 +27,14 @@ class FindStageDistrictBatch extends BaseBatch {
|
||||
CommonUtils.findAllCity().each { def city ->
|
||||
CommonDatautils.SEAWEED_COLL.find(DbUtils.buildSeaweedQuery2(city), new BasicDBObject("region", 1).append("name", 1).append("master_district_name", 1).append("city", 1)).each { BasicDBObject dbo ->
|
||||
if (dbo.name =~ STAGE_PATTERN && StringUtils.isBlank(dbo.master_district_name)) {
|
||||
def mainName = dbo.name.toString().replaceAll("([一二三四五六七八九十百千\\d]+)期\$", "")
|
||||
def mainName = dbo.name.toString().replaceAll("([一二三四五六七八九十百千\\d]+)期\$", "")
|
||||
def query = DbUtils.buildSeaweedQuery(city, dbo.region, mainName)
|
||||
if (DbUtils.seaweed.count(query) != 0) {
|
||||
DbUtils.executeUpdateOnSeaweed(new BasicDBObject("_id", dbo._id), new BasicDBObject("master_district_name", mainName))
|
||||
println(dbo)
|
||||
}
|
||||
} else if (dbo.name =~ PARTITION_PATTERN && StringUtils.isBlank(dbo.master_district_name)) {
|
||||
def mainName = dbo.name.toString().replaceAll("(\\w|东|南|北|中|西|甲|乙|丙|丁|一|二|三|四|五|六|七|八|九|十)区\$", "")
|
||||
def mainName = dbo.name.toString().replaceAll("(\\w|东|南|北|中|西|甲|乙|丙|丁|一|二|三|四|五|六|七|八|九|十)区\$", "")
|
||||
def query = DbUtils.buildSeaweedQuery(city, dbo.region, mainName)
|
||||
if (DbUtils.seaweed.count(query) != 0) {
|
||||
DbUtils.executeUpdateOnSeaweed(new BasicDBObject("_id", dbo._id), new BasicDBObject("master_district_name", mainName))
|
||||
|
||||
+11
-11
@@ -59,13 +59,13 @@ public class ManualOperationDataStat extends BaseBatch{
|
||||
aliasAddQuery.append("cat", "district")
|
||||
aliasAddQuery.append("c_date", new BasicDBObject('$gte', startDate).append('$lt', endDate))
|
||||
aliasAddQuery.append("city", city)
|
||||
aliasAddQuery.append("opt", "Ìí¼Ó")
|
||||
aliasAddQuery.append("opt", "添加")
|
||||
|
||||
def aliasRemoveQuery = new BasicDBObject()
|
||||
aliasRemoveQuery.append("cat", "district")
|
||||
aliasRemoveQuery.append("c_date", new BasicDBObject('$gte', startDate).append('$lt', endDate))
|
||||
aliasRemoveQuery.append("city", city)
|
||||
aliasRemoveQuery.append("opt", "ɾ³ý")
|
||||
aliasRemoveQuery.append("opt", "删除")
|
||||
data << [city, DbUtils.seaweed.count(createQuery), DbUtils.seaweed.count(removeQuery), ALIAS_UPDATE_OPERATION.count(aliasAddQuery), ALIAS_UPDATE_OPERATION.count(aliasRemoveQuery)]
|
||||
println([city, DbUtils.seaweed.count(createQuery), DbUtils.seaweed.count(removeQuery), ALIAS_UPDATE_OPERATION.count(aliasAddQuery), ALIAS_UPDATE_OPERATION.count(aliasRemoveQuery)])
|
||||
|
||||
@@ -122,15 +122,15 @@ public class ManualOperationDataStat extends BaseBatch{
|
||||
|
||||
def content = sd.getContent(map)
|
||||
println(content)
|
||||
sendEmail(Watcher.WangWenYan.email, "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
sendEmail("wangchencen@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
// sendEmail("caoyuyan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
// sendEmail("shenyuan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
// sendEmail("xuminxia@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
// sendEmail("huangcunliang@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
// sendEmail("yaojingmeng@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
// sendEmail("muliangshu@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
// sendEmail("sunjian@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
sendEmail(Watcher.WangWenYan.email, "后台统计量", content)
|
||||
sendEmail("wangchencen@fangjia.com", "后台统计量", content)
|
||||
// sendEmail("caoyuyan@fangjia.com", "后台统计量", content)
|
||||
// sendEmail("shenyuan@fangjia.com", "后台统计量", content)
|
||||
// sendEmail("xuminxia@fangjia.com", "后台统计量", content)
|
||||
// sendEmail("huangcunliang@fangjia.com", "后台统计量", content)
|
||||
// sendEmail("yaojingmeng@fangjia.com", "后台统计量", content)
|
||||
// sendEmail("muliangshu@fangjia.com", "后台统计量", content)
|
||||
// sendEmail("sunjian@fangjia.com", "后台统计量", content)
|
||||
}
|
||||
|
||||
}
|
||||
+5
-5
@@ -13,7 +13,7 @@ import com.mongodb.DBCollection
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
*MongoDB慢日志分析脚本
|
||||
*MongoDB慢日志分析脚本
|
||||
*/
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
class MongoAccessLogAnalyzer extends BaseBatch {
|
||||
@@ -49,7 +49,7 @@ class MongoAccessLogAnalyzer extends BaseBatch {
|
||||
MONGO_SLOW_LOG.save(dbo)
|
||||
}
|
||||
}catch (Exception e){
|
||||
println(line + "\t\t发生异常")
|
||||
println(line + "\t\t发生异常")
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
@@ -129,9 +129,9 @@ class MongoAccessLogAnalyzer extends BaseBatch {
|
||||
map.put('totalCount', totalCount)
|
||||
map.put('avgTime', totalCountTime/totalCount)
|
||||
def content = sd.getContent(map)
|
||||
sendEmail(Watcher.WangWenYan.email, DateUtil.add(new Date(), Calendar.DATE, -1).format("yyyy年MM月dd日")+ "超时日志报表", content)
|
||||
sendEmail(Watcher.DuXiaoYu.email, DateUtil.add(new Date(), Calendar.DATE, -1).format("yyyy年MM月dd日")+ "超时日志报表", content)
|
||||
sendEmail("joe@fangjia.com", DateUtil.add(new Date(), Calendar.DATE, -1).format("yyyy年MM月dd日")+ "超时日志报表", content)
|
||||
sendEmail(Watcher.WangWenYan.email, DateUtil.add(new Date(), Calendar.DATE, -1).format("yyyy年MM月dd日")+ "超时日志报表", content)
|
||||
sendEmail(Watcher.DuXiaoYu.email, DateUtil.add(new Date(), Calendar.DATE, -1).format("yyyy年MM月dd日")+ "超时日志报表", content)
|
||||
sendEmail("joe@fangjia.com", DateUtil.add(new Date(), Calendar.DATE, -1).format("yyyy年MM月dd日")+ "超时日志报表", content)
|
||||
}
|
||||
|
||||
static final COUNTERS_PATTERNS = ['ntoreturn': Pattern.compile('ntoreturn:\\d+'), 'ntoskip': Pattern.compile('ntoskip:\\d+'),
|
||||
|
||||
@@ -26,14 +26,14 @@ class PriceStat {
|
||||
|
||||
static main(args) {
|
||||
def task = new PriceStat();
|
||||
//task.run("南京", "shop");
|
||||
//task.houseCountStat("北京","写字楼");
|
||||
def citys = ["天津","重庆","成都","大连","沈阳","哈尔滨","牡丹江","大庆","齐齐哈尔","佳木斯","伊春","鸡西","鹤岗","双鸭山","七台河","绥化","黑河"];
|
||||
//task.run("南京", "shop");
|
||||
//task.houseCountStat("北京","写字楼");
|
||||
def citys = ["天津","重庆","成都","大连","沈阳","哈尔滨","牡丹江","大庆","齐齐哈尔","佳木斯","伊春","鸡西","鹤岗","双鸭山","七台河","绥化","黑河"];
|
||||
|
||||
|
||||
// task.run("上海", "district");
|
||||
// task.run("上海", "district");
|
||||
|
||||
// def citys = ["重庆","大连","哈尔滨","沈阳","大庆","牡丹江","齐齐哈尔"];
|
||||
// def citys = ["重庆","大连","哈尔滨","沈阳","大庆","牡丹江","齐齐哈尔"];
|
||||
// for(def city : citys){
|
||||
// task.backupStat(city, "district");
|
||||
// }
|
||||
@@ -55,7 +55,7 @@ class PriceStat {
|
||||
def backupStat(def city, def cat, def prefix){
|
||||
def resultMap = getAllDistricts(city, cat);
|
||||
def ret = resultMap.ret;
|
||||
println "[${city}] "+cat + " 共"+ret.size();
|
||||
println "[${city}] "+cat + " 共"+ret.size();
|
||||
//def detail = houseBackupStat(city);
|
||||
def detail = houseStat(city, cat +"_"+ prefix);
|
||||
def res = [];
|
||||
@@ -69,7 +69,7 @@ class PriceStat {
|
||||
def result = [:];
|
||||
result.put(prefix+"HouseCount", ret.size());
|
||||
result.put(prefix+"HouseDistCount", res.size());
|
||||
println "[${city}]"+" 共:"+ret.size()+" 有房源的[${cat}]:"+res.size();
|
||||
println "[${city}]"+" 共:"+ret.size()+" 有房源的[${cat}]:"+res.size();
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -169,15 +169,15 @@ class PriceStat {
|
||||
}
|
||||
|
||||
|
||||
// println "共"+index;
|
||||
// println "有售价共:"+sellIndex
|
||||
// println "有租价共:"+rentIndex
|
||||
// println "竣工时间共:"+finishDateIndex
|
||||
// println "建筑时间共:"+buildDateIndex
|
||||
// println "居住时间共:"+liveDateIndex
|
||||
// println "有时间共:"+dateIndex
|
||||
// println "经纬度共:"+latlngIndex
|
||||
// println "图片共:"+picIndex
|
||||
// println "共"+index;
|
||||
// println "有售价共:"+sellIndex
|
||||
// println "有租价共:"+rentIndex
|
||||
// println "竣工时间共:"+finishDateIndex
|
||||
// println "建筑时间共:"+buildDateIndex
|
||||
// println "居住时间共:"+liveDateIndex
|
||||
// println "有时间共:"+dateIndex
|
||||
// println "经纬度共:"+latlngIndex
|
||||
// println "图片共:"+picIndex
|
||||
resultMap.sellPriceOffices = sellPriceOffices;
|
||||
resultMap.ret = ret;
|
||||
|
||||
@@ -199,7 +199,7 @@ class PriceStat {
|
||||
detail << key;
|
||||
|
||||
}
|
||||
println "参与计算的共:"+sum+"房源!"
|
||||
println "参与计算的共:"+sum+"房源!"
|
||||
return detail;
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -22,7 +22,7 @@ class RegionAnaDataBatch extends BaseBatch {
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
Integer yearMonth = REGION_SELL_STAT.find(new BasicDBObject("city", "上海").append("region", "长宁")).sort(new BasicDBObject("date", -1)).limit(1).toList()[0].date
|
||||
Integer yearMonth = REGION_SELL_STAT.find(new BasicDBObject("city", "上海").append("region", "长宁")).sort(new BasicDBObject("date", -1)).limit(1).toList()[0].date
|
||||
CommonUtils.findAllCity().each {def city->
|
||||
List orders = []
|
||||
for(def region : RegionData.getAllRegions(city).keySet()){
|
||||
|
||||
@@ -37,7 +37,7 @@ public boolean isLegalRegion(String city,String region){
|
||||
if(StringUtils.isBlank(region)){
|
||||
return false;
|
||||
}
|
||||
if(region.contains("Öܱß")||region.contains("ÆäËû")){
|
||||
if(region.contains("周边")||region.contains("其他")){
|
||||
return false;
|
||||
}
|
||||
if(regions.containsKey(region)){
|
||||
|
||||
+41
-41
@@ -14,9 +14,9 @@ import org.bson.types.ObjectId
|
||||
class SeaweedFieldStat {
|
||||
DSCollection dataStoreConnection = null;
|
||||
public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
|
||||
def distTypes = ["多层","小高层","超高层","高层","花园洋房","老洋房","四合院"];
|
||||
def villaTypes = ["独栋","双拼","联排","叠加"];
|
||||
def shopTypes = ["多层","小高层","超高层","高层","独栋","双拼","联排","叠加"];
|
||||
def distTypes = ["多层","小高层","超高层","高层","花园洋房","老洋房","四合院"];
|
||||
def villaTypes = ["独栋","双拼","联排","叠加"];
|
||||
def shopTypes = ["多层","小高层","超高层","高层","独栋","双拼","联排","叠加"];
|
||||
|
||||
|
||||
static main(args) {
|
||||
@@ -25,7 +25,7 @@ class SeaweedFieldStat {
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = ["上海"]//findAllCity();
|
||||
def citys = ["上海"]//findAllCity();
|
||||
for(def city : citys){
|
||||
def districts = getAllDistricts(city, "district");
|
||||
//processEstateType(districts);
|
||||
@@ -46,7 +46,7 @@ class SeaweedFieldStat {
|
||||
|
||||
public def statBlock(districts){
|
||||
|
||||
println "小区共:"+districts.size();
|
||||
println "小区共:"+districts.size();
|
||||
|
||||
def index=0;
|
||||
for(def district : districts){
|
||||
@@ -55,7 +55,7 @@ class SeaweedFieldStat {
|
||||
index++;
|
||||
}
|
||||
}
|
||||
println "无block:"+index;
|
||||
println "无block:"+index;
|
||||
}
|
||||
public def getAllDistricts(def city, def cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
@@ -91,35 +91,35 @@ class SeaweedFieldStat {
|
||||
for(def map : districts){
|
||||
def estate_type2 = map.estate_type;
|
||||
if(estate_type2){
|
||||
if(!estate_type2.equals("普通住宅")&&!estate_type2.equals("别墅")&&!estate_type2.equals("商住")){
|
||||
if(!estate_type2.equals("普通住宅")&&!estate_type2.equals("别墅")&&!estate_type2.equals("商住")){
|
||||
DBObject udata = new BasicDBObject();
|
||||
udata.put("m_date", new Date());
|
||||
udata.put("estate_type2", "普通住宅");
|
||||
udata.put("estate_type2", "普通住宅");
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("_id", map.id);
|
||||
println "update ${map.id} "+estate_type2
|
||||
getDataStoreConnection().update(q, new BasicDBObject('$set', udata))
|
||||
}
|
||||
}else{//物业类弄为空
|
||||
}else{//物业类弄为空
|
||||
def name = map.name;
|
||||
if(name.indexOf("别墅")!=-1){
|
||||
if(name.indexOf("别墅")!=-1){
|
||||
DBObject udata = new BasicDBObject();
|
||||
udata.put("m_date", new Date());
|
||||
udata.put("estate_type2", "别墅");
|
||||
udata.put("estate_type2", "别墅");
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("_id", map.id);
|
||||
println "update ${map.id} 改为:别墅"
|
||||
println "update ${map.id} 改为:别墅"
|
||||
getDataStoreConnection().update(q, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
DBObject udata = new BasicDBObject();
|
||||
udata.put("m_date", new Date());
|
||||
udata.put("estate_type2", "普通住宅");
|
||||
udata.put("estate_type2", "普通住宅");
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("_id", map.id);
|
||||
println "update ${map.id} 无物业类型"
|
||||
println "update ${map.id} 无物业类型"
|
||||
getDataStoreConnection().update(q, new BasicDBObject('$set', udata))
|
||||
|
||||
}
|
||||
@@ -132,15 +132,15 @@ class SeaweedFieldStat {
|
||||
for(def map : districts){
|
||||
def build_type = map.build_type;
|
||||
def estate_type2 = map.estate_type;
|
||||
if(map.name == "佘山东郡"){
|
||||
if(map.name == "佘山东郡"){
|
||||
println "ssssss"
|
||||
}
|
||||
def result ;
|
||||
if(estate_type2 == "普通住宅"){
|
||||
if(estate_type2 == "普通住宅"){
|
||||
result = changeValue(estate_type2, build_type, distTypes);
|
||||
}else if(estate_type2 == "别墅"){
|
||||
}else if(estate_type2 == "别墅"){
|
||||
result = changeValue(estate_type2, build_type, villaTypes);
|
||||
}else if(estate_type2 == "商住"){
|
||||
}else if(estate_type2 == "商住"){
|
||||
result = changeValue(estate_type2, build_type, shopTypes);
|
||||
}
|
||||
if(build_type != result){
|
||||
@@ -159,50 +159,50 @@ class SeaweedFieldStat {
|
||||
|
||||
if(buildType && buildType!='null'){
|
||||
def tempBuildTypes = [];
|
||||
println "(${estate_type2})处理前:"+buildType;
|
||||
println "(${estate_type2})处理前:"+buildType;
|
||||
|
||||
if(buildType.indexOf("别墅")!=-1){
|
||||
buildType = buildType.replace("别墅","");
|
||||
if(buildType.indexOf("别墅")!=-1){
|
||||
buildType = buildType.replace("别墅","");
|
||||
}
|
||||
|
||||
if(buildType.indexOf("叠拼")!=-1){
|
||||
buildType = buildType.replace("叠拼","");
|
||||
if(buildType.indexOf("叠拼")!=-1){
|
||||
buildType = buildType.replace("叠拼","");
|
||||
}
|
||||
|
||||
if(buildType.indexOf("(")!=-1){
|
||||
def startIndex = buildType.indexOf("(");
|
||||
def endIndex = buildType.indexOf(")")+1;
|
||||
if(buildType.indexOf("(")!=-1){
|
||||
def startIndex = buildType.indexOf("(");
|
||||
def endIndex = buildType.indexOf(")")+1;
|
||||
def s = buildType.substring(startIndex, endIndex)
|
||||
buildType = buildType.replace(s, "");
|
||||
}
|
||||
if(buildType.indexOf(",")!=-1||buildType.indexOf(",")!=-1||buildType.indexOf(";")!=-1||buildType.indexOf(";")!=-1||buildType.indexOf(" ")!=-1||buildType.indexOf("、")!=-1||buildType.indexOf("|")!=-1){
|
||||
tempBuildTypes = buildType.split(",|;| |、|,|;||")
|
||||
if(buildType.indexOf(",")!=-1||buildType.indexOf(",")!=-1||buildType.indexOf(";")!=-1||buildType.indexOf(";")!=-1||buildType.indexOf(" ")!=-1||buildType.indexOf("、")!=-1||buildType.indexOf("|")!=-1){
|
||||
tempBuildTypes = buildType.split(",|;| |、|,|;||")
|
||||
}else{
|
||||
tempBuildTypes << buildType
|
||||
}
|
||||
|
||||
if(tempBuildTypes.size()==1 && buildType.length() > 5){
|
||||
println "可能含有其它分隔符......"
|
||||
println "可能含有其它分隔符......"
|
||||
}
|
||||
|
||||
if(tempBuildTypes.size() > 0){
|
||||
buildType = standardType(tempBuildTypes, types);
|
||||
println "处理后:"+buildType;
|
||||
println "处理后:"+buildType;
|
||||
if(!buildType){
|
||||
println "不合要求"
|
||||
println "不合要求"
|
||||
}
|
||||
}
|
||||
|
||||
}else{
|
||||
if(estate_type2 == "普通住宅"){
|
||||
buildType = "小高层";
|
||||
println "默认 "
|
||||
}else if(estate_type2 == "别墅"){
|
||||
buildType = "联排";
|
||||
println "默认 "
|
||||
}else if(estate_type2 == "商住"){
|
||||
buildType = "小高层";
|
||||
println "默认 "
|
||||
if(estate_type2 == "普通住宅"){
|
||||
buildType = "小高层";
|
||||
println "默认 "
|
||||
}else if(estate_type2 == "别墅"){
|
||||
buildType = "联排";
|
||||
println "默认 "
|
||||
}else if(estate_type2 == "商住"){
|
||||
buildType = "小高层";
|
||||
println "默认 "
|
||||
}
|
||||
}
|
||||
return buildType;
|
||||
@@ -228,7 +228,7 @@ class SeaweedFieldStat {
|
||||
if(cursor){
|
||||
for(def dbo : cursor){
|
||||
def city = dbo.get("display");
|
||||
if(city!='全国'){
|
||||
if(city!='全国'){
|
||||
citys << city;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ class SeedsScore {
|
||||
|
||||
|
||||
private def run(){
|
||||
// def ids = findWebsitByScope("全国类");
|
||||
// def ids = findWebsitByScope("全国类");
|
||||
// updateSeedsScoreByParentId(ids, 0);
|
||||
// def ret = readText("anjuke.txt");
|
||||
// def ret = readText("focus.txt");
|
||||
@@ -74,12 +74,12 @@ class SeedsScore {
|
||||
private def readText(def fileName){
|
||||
File file = new File('h:\\'+fileName)
|
||||
def ret = [];
|
||||
//对文件内容的每一行进行分割处理,比较常用在处理csv文件
|
||||
//对文件内容的每一行进行分割处理,比较常用在处理csv文件
|
||||
file.splitEachLine(" ") {
|
||||
def temp = it
|
||||
ret << temp;
|
||||
}
|
||||
//转为Writable对象,可重定向输出
|
||||
//转为Writable对象,可重定向输出
|
||||
file.asWritable()
|
||||
return ret;
|
||||
}
|
||||
|
||||
+7
-7
@@ -12,8 +12,8 @@ import com.mongodb.DBCollection
|
||||
|
||||
|
||||
/**
|
||||
* @author 文艳
|
||||
* 短息使用量监控脚本
|
||||
* @author 文艳
|
||||
* 短息使用量监控脚本
|
||||
*/
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
public class SmsUsageMonitorBatch extends BaseBatch{
|
||||
@@ -34,21 +34,21 @@ public class SmsUsageMonitorBatch extends BaseBatch{
|
||||
for(int i = 0; i < 7; i++){
|
||||
def date = calendar.getTime();
|
||||
def item = [:]
|
||||
item.day = date.format("yyyy年MM月dd日")
|
||||
item.day = date.format("yyyy年MM月dd日")
|
||||
item.count = countUsageByDay(date)
|
||||
weekUsage << item
|
||||
calendar.add(Calendar.DATE, 1);
|
||||
}
|
||||
/**获取剩余短信数量*/
|
||||
/**获取剩余短信数量*/
|
||||
int remain = (int)(SingletonClient.getClient().balance * 10)
|
||||
EmailTemplate sd = new EmailTemplate("smsUsage.ftl")
|
||||
Map map = new HashMap()
|
||||
map.put('weekUsage', weekUsage)
|
||||
map.put('remain', remain)
|
||||
def content = sd.getContent(map)
|
||||
sendEmail(Watcher.WangWenYan.email, "短信近期使用情况", content)
|
||||
sendEmail(Watcher.DuXiaoYu.email, "短信近期使用情况", content)
|
||||
sendEmail("yangqixing@fangjia.com", "短信近期使用情况", content)
|
||||
sendEmail(Watcher.WangWenYan.email, "短信近期使用情况", content)
|
||||
sendEmail(Watcher.DuXiaoYu.email, "短信近期使用情况", content)
|
||||
sendEmail("yangqixing@fangjia.com", "短信近期使用情况", content)
|
||||
}
|
||||
|
||||
def countUsageByDay(Date date){
|
||||
|
||||
+20
-20
@@ -33,7 +33,7 @@ class SplitWebsiteTable {
|
||||
def task = new SplitWebsiteTable();
|
||||
// task.run();
|
||||
|
||||
String s="周围商场:超市、便利商店等一应俱全,国美电器、易初莲花休闲生活:中小学:新城小学侨光中学,川沙中学华夏西校 商场:超市、便利商店等一应俱全,国美电器、易初莲花 医院:安平医院、川沙镇中心医院 邮局:中国邮政唐镇邮政所(浦东新区川沙分局) 银行:交通银行、工商银行、建设银行、农业银行 其他:六团邮政所、唐镇邮政所、避风塘茶楼、川沙体育场羽毛球馆餐饮设施:暂无资料周围学校:暂无资料医院:安平医院、川沙镇中心医院"
|
||||
String s="周围商场:超市、便利商店等一应俱全,国美电器、易初莲花休闲生活:中小学:新城小学侨光中学,川沙中学华夏西校 商场:超市、便利商店等一应俱全,国美电器、易初莲花 医院:安平医院、川沙镇中心医院 邮局:中国邮政唐镇邮政所(浦东新区川沙分局) 银行:交通银行、工商银行、建设银行、农业银行 其他:六团邮政所、唐镇邮政所、避风塘茶楼、川沙体育场羽毛球馆餐饮设施:暂无资料周围学校:暂无资料医院:安平医院、川沙镇中心医院"
|
||||
def ss = task.test2(s);
|
||||
println ss ;
|
||||
}
|
||||
@@ -41,27 +41,27 @@ class SplitWebsiteTable {
|
||||
def test2(obj){
|
||||
String s = obj.toString();
|
||||
String ret = "";
|
||||
if(s=='周围商场:休闲生活:餐饮设施:周围学校:医院:'){
|
||||
if(s=='周围商场:休闲生活:餐饮设施:周围学校:医院:'){
|
||||
return "";
|
||||
}else{
|
||||
String s1 = s.substring(0, s.indexOf("休闲生活:"));
|
||||
String s2 = s.substring(s.indexOf("休闲生活:"), s.indexOf("餐饮设施:"));
|
||||
String s3 = s.substring(s.indexOf("餐饮设施:"), s.indexOf("周围学校:"));
|
||||
String s4 = s.substring(s.indexOf("周围学校:"), s.indexOf("医院:"));
|
||||
String s5 = s.substring(s.indexOf("医院:"), s.length());
|
||||
if(s1!="周围商场:"){
|
||||
String s1 = s.substring(0, s.indexOf("休闲生活:"));
|
||||
String s2 = s.substring(s.indexOf("休闲生活:"), s.indexOf("餐饮设施:"));
|
||||
String s3 = s.substring(s.indexOf("餐饮设施:"), s.indexOf("周围学校:"));
|
||||
String s4 = s.substring(s.indexOf("周围学校:"), s.indexOf("医院:"));
|
||||
String s5 = s.substring(s.indexOf("医院:"), s.length());
|
||||
if(s1!="周围商场:"){
|
||||
ret = s1+"<br/>";
|
||||
}
|
||||
if(s2!="休闲生活:"){
|
||||
if(s2!="休闲生活:"){
|
||||
ret = ret + s2 +"<br/>";
|
||||
}
|
||||
if(s3!="餐饮设施:"){
|
||||
if(s3!="餐饮设施:"){
|
||||
ret = ret + s3 +"<br/>";
|
||||
}
|
||||
if(s4!="周围学校:"){
|
||||
if(s4!="周围学校:"){
|
||||
ret = ret + s4 +"<br/>";
|
||||
}
|
||||
if(s5!="医院:"){
|
||||
if(s5!="医院:"){
|
||||
ret = ret + s5 +"<br/>";
|
||||
}
|
||||
}
|
||||
@@ -73,19 +73,19 @@ class SplitWebsiteTable {
|
||||
def test(obj){
|
||||
String s = obj.toString();
|
||||
String ret = "";
|
||||
String[] traffices = s.split("周围商场:");
|
||||
String[] traffices = s.split("周围商场:");
|
||||
String traffice = traffices[0];
|
||||
if(traffice=='公交线路:地铁线路:'){
|
||||
if(traffice=='公交线路:地铁线路:'){
|
||||
return "";
|
||||
}else{
|
||||
String[] arr = traffice.split("地铁线路:");
|
||||
if(arr[0]!='公交线路:'){
|
||||
String[] arr = traffice.split("地铁线路:");
|
||||
if(arr[0]!='公交线路:'){
|
||||
ret = arr[0]+"<br/>";
|
||||
}
|
||||
if(arr.length==2){
|
||||
arr[1] = arr[1].trim();
|
||||
if(arr[1].length()>2){
|
||||
ret = ret +"地铁线路:"+arr[1];
|
||||
ret = ret +"地铁线路:"+arr[1];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,9 +126,9 @@ class SplitWebsiteTable {
|
||||
|
||||
private List<Map<String, Object>> relatedJobs;
|
||||
|
||||
def siteScores = ["搜房网":9,"新浪乐居":9,"安居客":8.5,"搜狐焦点":8.5,"网上房地产":10,"网易":8,
|
||||
"链家在线":8,"带看":8,"腾讯":7.5,"推推99房产网":7.5,"淘房屋":7,"我来房产":6.5,"星空地产网":6,"百度乐居":6,"房之窗":5,"房产之门":5,"上海热线":5,"吉屋网":4,"楼盘网":4,"聊宅网":3.5,"爱易网":3.5,"淘房":3,
|
||||
"房王":3,"赶集":2,"58同城":2,"猎房网":1];
|
||||
def siteScores = ["搜房网":9,"新浪乐居":9,"安居客":8.5,"搜狐焦点":8.5,"网上房地产":10,"网易":8,
|
||||
"链家在线":8,"带看":8,"腾讯":7.5,"推推99房产网":7.5,"淘房屋":7,"我来房产":6.5,"星空地产网":6,"百度乐居":6,"房之窗":5,"房产之门":5,"上海热线":5,"吉屋网":4,"楼盘网":4,"聊宅网":3.5,"爱易网":3.5,"淘房":3,
|
||||
"房王":3,"赶集":2,"58同城":2,"猎房网":1];
|
||||
public def saveNewData(website){
|
||||
for(def obj : website){
|
||||
DBObject udata = new BasicDBObject();
|
||||
|
||||
@@ -39,8 +39,8 @@ public class Test {
|
||||
public static void main(String[] ars){
|
||||
def test = new Test();
|
||||
// test.test();
|
||||
// //test.run("district", "上海");
|
||||
// test.analyDistrictPk("district", "天津");//小区统计与小区的对比
|
||||
// //test.run("district", "上海");
|
||||
// test.analyDistrictPk("district", "天津");//小区统计与小区的对比
|
||||
// println new Date(new ObjectId('5469bb4cda29ee73010fa577').getTime())
|
||||
// test.addTestData();
|
||||
test.findRepeatGuidingPrice()
|
||||
@@ -62,16 +62,16 @@ public class Test {
|
||||
}
|
||||
|
||||
def datas = []
|
||||
datas << ["id", "城市","区域","名称","添加人","指导价","日期"]
|
||||
datas << ["id", "城市","区域","名称","添加人","指导价","日期"]
|
||||
map.each{key,list->
|
||||
if(list.size()>1){
|
||||
def userNames = []
|
||||
for(def ret : list){
|
||||
userNames << ret.guiding_person
|
||||
}
|
||||
if(userNames.contains("付贵生")){
|
||||
if(userNames.contains("付贵生")){
|
||||
for(def r : list){
|
||||
if(r.guiding_person!="付贵生"){
|
||||
if(r.guiding_person!="付贵生"){
|
||||
GUIDING_PRICE_COLL.remove(new BasicDBObject("_id", r._id))
|
||||
}
|
||||
}
|
||||
@@ -80,15 +80,15 @@ public class Test {
|
||||
}
|
||||
}
|
||||
|
||||
// ExcelUtils.write("d:\\数据\\福州\\指导价.xlsx",datas, "datas")
|
||||
// ExcelUtils.write("d:\\数据\\福州\\指导价.xlsx",datas, "datas")
|
||||
}
|
||||
|
||||
|
||||
public void addTestData(){
|
||||
DBObject obj = new BasicDBObject();
|
||||
obj.put("city", "北京");
|
||||
obj.put("region", "朝阳");
|
||||
obj.put("name", "朝阳广场");
|
||||
obj.put("city", "北京");
|
||||
obj.put("region", "朝阳");
|
||||
obj.put("name", "朝阳广场");
|
||||
obj.put("guiding_date", new Date());
|
||||
obj.put("guiding_value", 60000);
|
||||
obj.put("category", "office");
|
||||
@@ -99,7 +99,7 @@ public class Test {
|
||||
}
|
||||
|
||||
public void test(){
|
||||
// 所有城市
|
||||
// 所有城市
|
||||
def allCitys = [:];
|
||||
SERVER_AREA_CONN.find(new BasicDBObject('status', new BasicDBObject('$in', [0, 2]))).each { allCitys.put(it.display, it); }
|
||||
|
||||
@@ -108,7 +108,7 @@ public class Test {
|
||||
citys << k.toString();
|
||||
}
|
||||
|
||||
// 当期周末
|
||||
// 当期周末
|
||||
Date weekend = DateUtil.getFirstDayOfWeek(new Date());
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("weekend", weekend);
|
||||
@@ -132,8 +132,8 @@ public class Test {
|
||||
public analyDistrictPk(def cat, def city){
|
||||
def seaweedDists = findDistrictIds(cat, city);
|
||||
def districtInfos = findDistrictInfo(city, "2014-10-26");
|
||||
println "小区数:"+seaweedDists.size();
|
||||
println "小区分析表数:"+districtInfos.size();
|
||||
println "小区数:"+seaweedDists.size();
|
||||
println "小区分析表数:"+districtInfos.size();
|
||||
def index = 0;
|
||||
for(def dist : districtInfos){
|
||||
if(!seaweedDists.contains(dist)){
|
||||
@@ -191,13 +191,13 @@ public class Test {
|
||||
}
|
||||
|
||||
public String run(def cat, def city) {
|
||||
def analyDists = findAnalySiteDistrict(cat, city);//分析表
|
||||
def analyDists = findAnalySiteDistrict(cat, city);//分析表
|
||||
|
||||
def surplusDists = findSurplusDistrict(cat, city);//不存在小区表
|
||||
def surplusDists = findSurplusDistrict(cat, city);//不存在小区表
|
||||
|
||||
analyDists = analyDists + surplusDists;
|
||||
|
||||
def seaweedDists = CommonDatautils.getSimpleDistricts(cat, city);//总小区表
|
||||
def seaweedDists = CommonDatautils.getSimpleDistricts(cat, city);//总小区表
|
||||
println analyDists.size()
|
||||
println seaweedDists.size();
|
||||
def index = 0;
|
||||
@@ -205,14 +205,14 @@ public class Test {
|
||||
|
||||
if(!seaweedDists.contains(site)){
|
||||
index++;
|
||||
println "[${index}] 不在小区表:"+site;
|
||||
println "[${index}] 不在小区表:"+site;
|
||||
}
|
||||
}
|
||||
|
||||
for(def dist : seaweedDists){
|
||||
if(!analyDists.contains(dist)){
|
||||
index++;
|
||||
println "[${index}] 小区不在分析表:"+dist;
|
||||
println "[${index}] 小区不在分析表:"+dist;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@ class UpdateCityLevel {
|
||||
def arr = line.split("\t");
|
||||
def map = [:];
|
||||
map.city = arr[0];
|
||||
map.level = arr[1].replace("级", "").replace("一", "1").replace("二", "2").replace("三", "3").replace("四", "4").replace("五", "5").replace("六", "6")
|
||||
map.level = arr[1].replace("级", "").replace("一", "1").replace("二", "2").replace("三", "3").replace("四", "4").replace("五", "5").replace("六", "6")
|
||||
ret << map;
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -53,7 +53,7 @@ public class UserAgentAnalysisBatch extends BaseBatch{
|
||||
|
||||
int times = 3
|
||||
while(!logFile.exists() && times > 0){
|
||||
println("${new Date().format("yyyy-MM-dd HH:mm:ss")}\t${logFile.getAbsolutePath()} 未找到!!!!!")
|
||||
println("${new Date().format("yyyy-MM-dd HH:mm:ss")}\t${logFile.getAbsolutePath()} 未找到!!!!!")
|
||||
Thread.sleep(1000 * 60 * 60)
|
||||
times --
|
||||
}
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
/**
|
||||
* 微众银行房源统计脚本
|
||||
* 微众银行房源统计脚本
|
||||
*/
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
class WeBankHouseAnalyBatch extends BaseBatch {
|
||||
|
||||
+18
-18
@@ -20,8 +20,8 @@ class AnalyCommonUtils {
|
||||
public static final DBCollection ANALY_TASK_COLL = MongoUtils.getCollectionByName("coredata", "analy_task", "coredata.mongo.db.fangjia.com")
|
||||
public static final DBCollection ANALY_ROAD_COLL = MongoUtils.getCollectionByName("coredata", "analy_road", "coredata.mongo.db.fangjia.com")
|
||||
|
||||
static def villas = ["独栋","联排","双拼","叠加"]
|
||||
static def districts = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院"];
|
||||
static def villas = ["独栋","联排","双拼","叠加"]
|
||||
static def districts = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院"];
|
||||
|
||||
static def standardRegion(def regionMap,def region){
|
||||
def lastRegion = "";
|
||||
@@ -154,7 +154,7 @@ class AnalyCommonUtils {
|
||||
if(taskId!="null"&&taskId){
|
||||
saveAnalyRoad(city, ret, taskId, site_name, page_name, cat);
|
||||
}else{
|
||||
println "没有相对应该的task_id";
|
||||
println "没有相对应该的task_id";
|
||||
}
|
||||
return taskId;
|
||||
}
|
||||
@@ -247,7 +247,7 @@ class AnalyCommonUtils {
|
||||
def newEstateType = "";
|
||||
def newBuildType = "";
|
||||
if(build_type){
|
||||
def buildTypes = build_type.split(",|、|;")
|
||||
def buildTypes = build_type.split(",|、|;")
|
||||
def distTypes = []
|
||||
def villaTypes = []
|
||||
for(def type : buildTypes){
|
||||
@@ -260,35 +260,35 @@ class AnalyCommonUtils {
|
||||
}
|
||||
}
|
||||
|
||||
if(estate_type2.indexOf("别墅")!=-1 && estate_type2.indexOf("普通住宅")!=-1){
|
||||
if(estate_type2.indexOf("别墅")!=-1 && estate_type2.indexOf("普通住宅")!=-1){
|
||||
if(villaTypes.size() > 0){
|
||||
newEstateType = "别墅";
|
||||
newEstateType = "别墅";
|
||||
newBuildType = villaTypes.unique().join(",");
|
||||
}else{
|
||||
newEstateType = "普通住宅";
|
||||
newEstateType = "普通住宅";
|
||||
if(distTypes.size() > 0){
|
||||
newBuildType = distTypes.unique().join(",");
|
||||
}
|
||||
}
|
||||
}else if(estate_type2.indexOf("别墅")==-1 && estate_type2.indexOf("普通住宅")!=-1){
|
||||
newEstateType = "普通住宅";
|
||||
}else if(estate_type2.indexOf("别墅")==-1 && estate_type2.indexOf("普通住宅")!=-1){
|
||||
newEstateType = "普通住宅";
|
||||
if(distTypes.size() > 0){
|
||||
newBuildType = distTypes.unique().join(",");
|
||||
}
|
||||
}else if(estate_type2.indexOf("别墅")!=-1 && estate_type2.indexOf("普通住宅")==-1){
|
||||
newEstateType = "别墅";
|
||||
}else if(estate_type2.indexOf("别墅")!=-1 && estate_type2.indexOf("普通住宅")==-1){
|
||||
newEstateType = "别墅";
|
||||
if(villaTypes.size()>0){
|
||||
newBuildType = villaTypes.unique().join(",");
|
||||
}
|
||||
}else{
|
||||
newEstateType = "普通住宅";
|
||||
newEstateType = "普通住宅";
|
||||
}
|
||||
|
||||
}else{
|
||||
if(estate_type2.indexOf("别墅")!=-1){
|
||||
newEstateType = "别墅";
|
||||
if(estate_type2.indexOf("别墅")!=-1){
|
||||
newEstateType = "别墅";
|
||||
}else{
|
||||
newEstateType = "普通住宅";
|
||||
newEstateType = "普通住宅";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -301,7 +301,7 @@ class AnalyCommonUtils {
|
||||
def index = 1;
|
||||
for(def map : results){
|
||||
updateDistrict(map, city);
|
||||
println "[ "+city+" ]"+index +" 修改完成....."
|
||||
println "[ "+city+" ]"+index +" 修改完成....."
|
||||
index++;
|
||||
}
|
||||
}
|
||||
@@ -327,7 +327,7 @@ class AnalyCommonUtils {
|
||||
if (!dbValue ) {
|
||||
udata.put(k, v);
|
||||
}else{
|
||||
if(dbValue.toString().indexOf("暂无")!=-1){
|
||||
if(dbValue.toString().indexOf("暂无")!=-1){
|
||||
udata.put(k, v);
|
||||
}
|
||||
}
|
||||
@@ -343,7 +343,7 @@ class AnalyCommonUtils {
|
||||
udata.put("estate_type2", typeMap.estate_type);
|
||||
udata.put("build_type", typeMap.build_type);
|
||||
}
|
||||
println id.toString() + " " + obj.get("region") + " " + obj.get("name") + " 修改的值 :" + udata
|
||||
println id.toString() + " " + obj.get("region") + " " + obj.get("name") + " 修改的值 :" + udata
|
||||
udata.put("m_date", new Date());
|
||||
getDataStoreConnection().update(q, new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
+2
-2
@@ -36,7 +36,7 @@ class BkoffActionSelectAnaly{
|
||||
char[] chars = new char[1];
|
||||
chars[0] = str.charAt(0);
|
||||
String temp = new String(chars);
|
||||
if (chars[0] >= 'A' && chars[0] <= 'Z') {//当为字母时,则转换为小写
|
||||
if (chars[0] >= 'A' && chars[0] <= 'Z') {//当为字母时,则转换为小写
|
||||
return str.replaceFirst(temp, temp.toLowerCase());
|
||||
}
|
||||
return str;
|
||||
@@ -91,7 +91,7 @@ class BkoffActionSelectAnaly{
|
||||
files << tempList[i];
|
||||
}
|
||||
// if (tempList[i].isDirectory()) {
|
||||
// System.out.println("文件夹:"+tempList[i]);
|
||||
// System.out.println("文件夹:"+tempList[i]);
|
||||
// }
|
||||
}
|
||||
return files;
|
||||
|
||||
+4
-4
@@ -26,7 +26,7 @@ import static com.mongodb.client.model.Projections.include
|
||||
import static com.mongodb.client.model.Updates.combine
|
||||
import static com.mongodb.client.model.Updates.set
|
||||
|
||||
@BatchOwner(value = Watcher.WangWenYan, describe = "分析小区特征")
|
||||
@BatchOwner(value = Watcher.WangWenYan, describe = "分析小区特征")
|
||||
class DistrictFeatureAnalyBatch extends BaseBatch {
|
||||
|
||||
|
||||
@@ -51,7 +51,7 @@ class DistrictFeatureAnalyBatch extends BaseBatch {
|
||||
keys.finish_building_date = 1
|
||||
UpdateOptions options = new UpdateOptions()
|
||||
options.upsert(true)
|
||||
["佛山", "济南", "北京", "广州", "深圳"].each { def city ->
|
||||
["佛山", "济南", "北京", "广州", "深圳"].each { def city ->
|
||||
println(city)
|
||||
def statMap = new ConcurrentHashMap<>()
|
||||
def query = and(eq("city", city), eq("weekend", DateUtil.getThisSunday(new Date())))
|
||||
@@ -108,8 +108,8 @@ class DistrictFeatureAnalyBatch extends BaseBatch {
|
||||
sqlBuffer.append("SELECT count(*) as count from esf_deal_price_lansi WHERE 1=1")
|
||||
sqlBuffer.append(" and fjCity = '${city}'")
|
||||
sqlBuffer.append(" and fjRegion ='${region}'")
|
||||
sqlBuffer.append(" and (districtType != '写字楼' and districtType != '商铺' and districtType != '办公' and districtType !='工厂' and districtType != '其它' " +
|
||||
" and houseType != '写字楼' and houseType != '商铺' and houseType != '办公' and houseType !='工厂' and houseType != '其它')")
|
||||
sqlBuffer.append(" and (districtType != '写字楼' and districtType != '商铺' and districtType != '办公' and districtType !='工厂' and districtType != '其它' " +
|
||||
" and houseType != '写字楼' and houseType != '商铺' and houseType != '办公' and houseType !='工厂' and houseType != '其它')")
|
||||
List<String> namePars = new ArrayList<String>()
|
||||
for (String alia : names) {
|
||||
if (StringUtils.isNotBlank(alia) && alia.trim().length() > 1) {
|
||||
|
||||
+2
-2
@@ -5,7 +5,7 @@ import com.mongodb.DBCollection;
|
||||
import com.mongodb.DBObject
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
//估价数据分析
|
||||
//估价数据分析
|
||||
class EvaluationAnaly{
|
||||
|
||||
public static final DBCollection EVALUATION_COLL = MongoUtils.getCollectionByName("fangjia", "evaluation", "mongo.db.fangjia.com")
|
||||
@@ -32,7 +32,7 @@ class EvaluationAnaly{
|
||||
map.put(ip, detailMap)
|
||||
}
|
||||
|
||||
//ip估价次数统计
|
||||
//ip估价次数统计
|
||||
def totalCount = detailMap.get("total_count");
|
||||
if(!totalCount){
|
||||
detailMap.put("total_count", 1)
|
||||
|
||||
+2
-2
@@ -8,8 +8,8 @@ import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
/**
|
||||
* 房价网首页
|
||||
* 估价记录和API总量统计
|
||||
* 房价网首页
|
||||
* 估价记录和API总量统计
|
||||
*
|
||||
*/
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
|
||||
+30
-30
@@ -19,7 +19,7 @@ import org.jsoup.select.Elements;
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
/**
|
||||
* 解析克而瑞小区概况页(二手小区)
|
||||
* 解析克而瑞小区概况页(二手小区)
|
||||
* @author gaojuan
|
||||
*
|
||||
*/
|
||||
@@ -39,25 +39,25 @@ class NewKerEsfCrawlTask extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
run(["沈阳","大连","天津","成都","重庆"]);
|
||||
run(["沈阳","大连","天津","成都","重庆"]);
|
||||
}
|
||||
|
||||
|
||||
def run(def citys){
|
||||
def site_name = "克而瑞房价";
|
||||
def page_name = "首页";
|
||||
def site_name = "克而瑞房价";
|
||||
def page_name = "首页";
|
||||
def cat = "district"
|
||||
|
||||
for(def city : citys){
|
||||
|
||||
def webpages = findWebpages(site_name, page_name, city, cat);
|
||||
println "【${city}】共找到搜房 "+webpages.size()+"个网页";
|
||||
println "【${city}】共找到搜房 "+webpages.size()+"个网页";
|
||||
|
||||
def regions = AnalyCommonUtils.findRegion(city);
|
||||
def blocks = AnalyCommonUtils.findRegionBlock(city);
|
||||
|
||||
def ret = analyWebpages(city, webpages, regions, blocks);
|
||||
AnalyCommonUtils.addAnalyRoad(site_name, "概况", city, ret, cat);
|
||||
AnalyCommonUtils.addAnalyRoad(site_name, "概况", city, ret, cat);
|
||||
}
|
||||
|
||||
|
||||
@@ -78,22 +78,22 @@ class NewKerEsfCrawlTask extends BaseBatch{
|
||||
def doc = Jsoup.parse(s);
|
||||
resultMap.city = city;
|
||||
|
||||
def alias = doc.select(".esgrid_left p:contains(别名:) em").text().trim()
|
||||
def alias = doc.select(".esgrid_left p:contains(别名:) em").text().trim()
|
||||
def ele = doc.select(".descr_cont_info")
|
||||
resultMap.name = ele.select("li:contains(小区名称) a").text().trim()
|
||||
def addressInfo = ele.select("li:contains(小区地址) cite").text().trim();
|
||||
resultMap.build_area = AnalyCommonUtils.getNumeric(ele.select("li:contains(建筑面积) cite").text().replace("-", "").replace(",", "").trim(), "(\\d+)");
|
||||
resultMap.developer = ele.select("li:contains(开发商) cite").text().replace("-", "").trim();
|
||||
resultMap.estate_company = ele.select("li:contains(物业公司) cite").text().replace("-", "").trim();
|
||||
resultMap.virescence_percent = AnalyCommonUtils.getNumeric(ele.select("li:contains(绿化率) cite").text().replace("-", "").replace("%", "").trim(), "(\\d+[\\.]?[\\d+]?(\\S+)(.*)?)");
|
||||
resultMap.cubage_percent = AnalyCommonUtils.getNumeric(ele.select("li:contains(容积率) cite").text().replace("-", "").trim(), "(\\d+[\\.]?[\\d+]?(\\S+)(.*)?)");
|
||||
resultMap.name = ele.select("li:contains(小区名称) a").text().trim()
|
||||
def addressInfo = ele.select("li:contains(小区地址) cite").text().trim();
|
||||
resultMap.build_area = AnalyCommonUtils.getNumeric(ele.select("li:contains(建筑面积) cite").text().replace("-", "").replace(",", "").trim(), "(\\d+)");
|
||||
resultMap.developer = ele.select("li:contains(开发商) cite").text().replace("-", "").trim();
|
||||
resultMap.estate_company = ele.select("li:contains(物业公司) cite").text().replace("-", "").trim();
|
||||
resultMap.virescence_percent = AnalyCommonUtils.getNumeric(ele.select("li:contains(绿化率) cite").text().replace("-", "").replace("%", "").trim(), "(\\d+[\\.]?[\\d+]?(\\S+)(.*)?)");
|
||||
resultMap.cubage_percent = AnalyCommonUtils.getNumeric(ele.select("li:contains(容积率) cite").text().replace("-", "").trim(), "(\\d+[\\.]?[\\d+]?(\\S+)(.*)?)");
|
||||
|
||||
if(alias){
|
||||
def aliasList = [];
|
||||
aliasList << alias;
|
||||
resultMap.alias = aliasList;
|
||||
}
|
||||
//区域及地址解析
|
||||
//区域及地址解析
|
||||
if(addressInfo){
|
||||
def region = addressInfo.substring(addressInfo.indexOf("[")+1, addressInfo.indexOf("]"));
|
||||
resultMap.region = AnalyCommonUtils.standardRegion(regions, region);
|
||||
@@ -108,8 +108,8 @@ class NewKerEsfCrawlTask extends BaseBatch{
|
||||
resultMap.address = addressInfo.substring(addressInfo.lastIndexOf("]")+1, addressInfo.size());
|
||||
}
|
||||
|
||||
//物业费解析
|
||||
def estate_charge = ele.select("li:contains(物业费) cite").text().replace("元/㎡/月", "").trim();
|
||||
//物业费解析
|
||||
def estate_charge = ele.select("li:contains(物业费) cite").text().replace("元/㎡/月", "").trim();
|
||||
if(estate_charge){
|
||||
if(estate_charge.indexOf("-")==-1){
|
||||
resultMap.estate_charge = estate_charge;
|
||||
@@ -125,8 +125,8 @@ class NewKerEsfCrawlTask extends BaseBatch{
|
||||
|
||||
}
|
||||
|
||||
//建造时间解析
|
||||
def houseAge = ele.select("li:contains(小区房龄) cite").text().replace("年", "").trim();
|
||||
//建造时间解析
|
||||
def houseAge = ele.select("li:contains(小区房龄) cite").text().replace("年", "").trim();
|
||||
if(houseAge && AnalyCommonUtils.isNumeric(houseAge)){
|
||||
def age = houseAge.toInteger();
|
||||
if(age > 1000){
|
||||
@@ -139,18 +139,18 @@ class NewKerEsfCrawlTask extends BaseBatch{
|
||||
|
||||
}
|
||||
|
||||
//户数解析
|
||||
def houseInfo = ele.select("li:contains(小区房源) cite").text().replace("-", "").trim();
|
||||
//户数解析
|
||||
def houseInfo = ele.select("li:contains(小区房源) cite").text().replace("-", "").trim();
|
||||
if(houseInfo){
|
||||
def household_count = AnalyCommonUtils.getNumeric(houseInfo, "共(\\d+)套")
|
||||
def household_count = AnalyCommonUtils.getNumeric(houseInfo, "共(\\d+)套")
|
||||
resultMap.household_count = household_count;
|
||||
}
|
||||
|
||||
//页面异步加载,拿不到相关信息
|
||||
//交通
|
||||
//页面异步加载,拿不到相关信息
|
||||
//交通
|
||||
// resultMap.traffic = doc.select("#data_list_station")
|
||||
//
|
||||
// //周边配套
|
||||
// //周边配套
|
||||
// def facilities = "";
|
||||
// def facilities_edu = doc.select("#data_list_edu cite")
|
||||
// def facilities_life = doc.select("#data_list_life cite")
|
||||
@@ -175,24 +175,24 @@ class NewKerEsfCrawlTask extends BaseBatch{
|
||||
|
||||
resultMap.url = url+"_des";
|
||||
resultMap = AnalyCommonUtils.filerValueIsNull(resultMap);
|
||||
println "【 ${index} 】"+resultMap;
|
||||
println "【 ${index} 】"+resultMap;
|
||||
if(resultMap.name){
|
||||
sucCount++;
|
||||
results << resultMap;
|
||||
}else{
|
||||
failCount++;
|
||||
println "小区名称没解析出来"
|
||||
println "小区名称没解析出来"
|
||||
}
|
||||
|
||||
|
||||
}else{
|
||||
failCount++;
|
||||
println "没有网页内容"
|
||||
println "没有网页内容"
|
||||
}
|
||||
}
|
||||
|
||||
println "失败:"+failCount
|
||||
println "成功:"+sucCount
|
||||
println "失败:"+failCount
|
||||
println "成功:"+sucCount
|
||||
return results;
|
||||
}
|
||||
|
||||
|
||||
+37
-37
@@ -17,7 +17,7 @@ import org.jsoup.nodes.Document
|
||||
import org.jsoup.select.Elements;
|
||||
|
||||
/**
|
||||
* 克而瑞新房抓取,解析,空值替换
|
||||
* 克而瑞新房抓取,解析,空值替换
|
||||
* @author gaojuan
|
||||
*
|
||||
*/
|
||||
@@ -34,11 +34,11 @@ class NewKerXfCrawlTask {
|
||||
static main(args) {
|
||||
def task = new NewKerXfCrawlTask();
|
||||
def ret = [
|
||||
"沈阳":"http://www.fangjiadp.com/shenyang/advanced/search",
|
||||
"大连":"http://www.fangjiadp.com/dalian/advanced/search",
|
||||
"天津":"http://www.fangjiadp.com/tianjin/advanced/search",
|
||||
"成都":"http://www.fangjiadp.com/chengdu/advanced/search",
|
||||
"重庆":"http://www.fangjiadp.com/chongqing/advanced/search"
|
||||
"沈阳":"http://www.fangjiadp.com/shenyang/advanced/search",
|
||||
"大连":"http://www.fangjiadp.com/dalian/advanced/search",
|
||||
"天津":"http://www.fangjiadp.com/tianjin/advanced/search",
|
||||
"成都":"http://www.fangjiadp.com/chengdu/advanced/search",
|
||||
"重庆":"http://www.fangjiadp.com/chongqing/advanced/search"
|
||||
];
|
||||
task.run(ret, "district");
|
||||
}
|
||||
@@ -61,14 +61,14 @@ class NewKerXfCrawlTask {
|
||||
def blocks = AnalyCommonUtils.findRegionBlock(city);
|
||||
|
||||
def ret = analyWebpages(city, detailUrls, regions, blocks);
|
||||
AnalyCommonUtils.addAnalyRoad("克而瑞房价", "详情", city, ret, cat);
|
||||
AnalyCommonUtils.addAnalyRoad("克而瑞房价", "详情", city, ret, cat);
|
||||
}
|
||||
|
||||
def getNextPageLink(def url, def urls){
|
||||
String s = http.get(url, [:], "utf-8");
|
||||
if(s){
|
||||
def doc = Jsoup.parse(s);
|
||||
def nextPageElement = doc.select("a:contains(下一页)");
|
||||
def nextPageElement = doc.select("a:contains(下一页)");
|
||||
if(nextPageElement){
|
||||
def nextPageUrl = nextPageElement.attr("href");
|
||||
def newUrl = "http://www.fangjiadp.com/"+nextPageUrl;
|
||||
@@ -108,7 +108,7 @@ class NewKerXfCrawlTask {
|
||||
String s = http.get(url, [:], "utf-8");
|
||||
if(s){
|
||||
def doc = Jsoup.parse(s);
|
||||
def detailUrl = doc.select("a:contains(楼盘详情)");
|
||||
def detailUrl = doc.select("a:contains(楼盘详情)");
|
||||
if(detailUrl){
|
||||
detailUrl = detailUrl.attr("href");
|
||||
allUrls << "http://www.fangjiadp.com" + detailUrl;
|
||||
@@ -140,30 +140,30 @@ class NewKerXfCrawlTask {
|
||||
def doc = Jsoup.parse(s);
|
||||
resultMap.city = city;
|
||||
|
||||
resultMap.name = doc.select("li:contains(楼盘名称:) cite,.a_t").text().trim();
|
||||
resultMap.address = doc.select(".property_m>dl:contains(地址:)>dd").text().replace(";", "").trim();
|
||||
resultMap.estate_type2 = doc.select(".property_f>ul>li:contains(物业类型:)>span").text().replace(";", "").trim();
|
||||
resultMap.name = doc.select("li:contains(楼盘名称:) cite,.a_t").text().trim();
|
||||
resultMap.address = doc.select(".property_m>dl:contains(地址:)>dd").text().replace(";", "").trim();
|
||||
resultMap.estate_type2 = doc.select(".property_f>ul>li:contains(物业类型:)>span").text().replace(";", "").trim();
|
||||
|
||||
resultMap.live_date = doc.select(".property_f>ul>li:contains(交房日期:)>span").text().trim();
|
||||
resultMap.developer = doc.select(".property_f>ul>li:contains(开发商:)>span").text().replace("-", "").trim();
|
||||
resultMap.build_area = AnalyCommonUtils.getNumeric(doc.select(".property_f>ul>li:contains(建筑面积:)>span").text().replace("-", "").replace(",", "").trim(), "(\\d+)");
|
||||
resultMap.build_type = doc.select(".property_f>ul>li:contains(建筑形态:)>span").text().trim();
|
||||
resultMap.household_count = AnalyCommonUtils.getNumeric(doc.select(".property_f>ul>li:contains(规划户数:)>span").text().trim(), "(\\d+)");
|
||||
resultMap.property_life = doc.select("li:contains(产权信息:) cite").text().replace(";", "").trim();
|
||||
resultMap.fitment = doc.select(".property_f>ul>li:contains(装修情况:)>span").text().replace(";", "").trim();
|
||||
resultMap.fitment_level = doc.select(".property_f>ul>li:contains(装修标准:)>span").text().replace("-元/㎡", "").trim();
|
||||
resultMap.virescence_percent = AnalyCommonUtils.getNumeric(doc.select(".property_f>ul>li:contains(绿化率:)>span").text().replace("-", "").replace("%", "").trim(), "(\\d+[\\.]?[\\d+]?(\\S+)(.*)?)");
|
||||
resultMap.cubage_percent = AnalyCommonUtils.getNumeric(doc.select(".property_f>ul>li:contains(容积率:)>span").text().replace("-", "").trim(), "(\\d+[\\.]?[\\d+]?(\\S+)(.*)?)");
|
||||
resultMap.estate_company = doc.select(".property_f>ul>li:contains(物业公司:)>span").text().trim();
|
||||
resultMap.live_date = doc.select(".property_f>ul>li:contains(交房日期:)>span").text().trim();
|
||||
resultMap.developer = doc.select(".property_f>ul>li:contains(开发商:)>span").text().replace("-", "").trim();
|
||||
resultMap.build_area = AnalyCommonUtils.getNumeric(doc.select(".property_f>ul>li:contains(建筑面积:)>span").text().replace("-", "").replace(",", "").trim(), "(\\d+)");
|
||||
resultMap.build_type = doc.select(".property_f>ul>li:contains(建筑形态:)>span").text().trim();
|
||||
resultMap.household_count = AnalyCommonUtils.getNumeric(doc.select(".property_f>ul>li:contains(规划户数:)>span").text().trim(), "(\\d+)");
|
||||
resultMap.property_life = doc.select("li:contains(产权信息:) cite").text().replace(";", "").trim();
|
||||
resultMap.fitment = doc.select(".property_f>ul>li:contains(装修情况:)>span").text().replace(";", "").trim();
|
||||
resultMap.fitment_level = doc.select(".property_f>ul>li:contains(装修标准:)>span").text().replace("-元/㎡", "").trim();
|
||||
resultMap.virescence_percent = AnalyCommonUtils.getNumeric(doc.select(".property_f>ul>li:contains(绿化率:)>span").text().replace("-", "").replace("%", "").trim(), "(\\d+[\\.]?[\\d+]?(\\S+)(.*)?)");
|
||||
resultMap.cubage_percent = AnalyCommonUtils.getNumeric(doc.select(".property_f>ul>li:contains(容积率:)>span").text().replace("-", "").trim(), "(\\d+[\\.]?[\\d+]?(\\S+)(.*)?)");
|
||||
resultMap.estate_company = doc.select(".property_f>ul>li:contains(物业公司:)>span").text().trim();
|
||||
|
||||
resultMap.car_site = doc.select(".property_f>ul>li:contains(车位数:)>span").text().trim();
|
||||
resultMap.sell_address = doc.select(".property_f>ul>li:contains(售楼处:)>span").text().trim();
|
||||
resultMap.mobile = doc.select(".property_f>ul>li:contains(售楼电话:)>span").text().trim();
|
||||
resultMap.car_site = doc.select(".property_f>ul>li:contains(车位数:)>span").text().trim();
|
||||
resultMap.sell_address = doc.select(".property_f>ul>li:contains(售楼处:)>span").text().trim();
|
||||
resultMap.mobile = doc.select(".property_f>ul>li:contains(售楼电话:)>span").text().trim();
|
||||
|
||||
def regionHtml = doc.select(".property_m>dl:contains(区域:)>dd").html();
|
||||
def regionHtml = doc.select(".property_m>dl:contains(区域:)>dd").html();
|
||||
|
||||
|
||||
//区域板块及环线
|
||||
//区域板块及环线
|
||||
if(regionHtml){
|
||||
def array = regionHtml.split("</em><em>")
|
||||
if(array.size()>=1){
|
||||
@@ -185,8 +185,8 @@ class NewKerXfCrawlTask {
|
||||
}
|
||||
}
|
||||
|
||||
//物业费解析
|
||||
def estate_charge = doc.select(".property_f>ul>li:contains(物业费用:)>span").text().replace("元/㎡/月", "").trim();
|
||||
//物业费解析
|
||||
def estate_charge = doc.select(".property_f>ul>li:contains(物业费用:)>span").text().replace("元/㎡/月", "").trim();
|
||||
if(estate_charge){
|
||||
if(estate_charge.indexOf("-")==-1){
|
||||
resultMap.estate_charge = estate_charge;
|
||||
@@ -202,30 +202,30 @@ class NewKerXfCrawlTask {
|
||||
|
||||
}
|
||||
|
||||
//开盘时间
|
||||
def open_date_str = doc.select(".property_f>ul>li:contains(开盘日期:)>span").text().trim();
|
||||
//开盘时间
|
||||
def open_date_str = doc.select(".property_f>ul>li:contains(开盘日期:)>span").text().trim();
|
||||
if(open_date_str){
|
||||
resultMap.open_date = DateUtil.parseStringToDate(open_date_str)
|
||||
}
|
||||
|
||||
resultMap.url = url;
|
||||
resultMap = AnalyCommonUtils.filerValueIsNull(resultMap);
|
||||
println "【 ${index} 】"+resultMap;
|
||||
println "【 ${index} 】"+resultMap;
|
||||
if(resultMap.name){
|
||||
sucCount++;
|
||||
results << resultMap;
|
||||
}else{
|
||||
failCount++;
|
||||
println "小区名称没解析出来"
|
||||
println "小区名称没解析出来"
|
||||
}
|
||||
}else{
|
||||
failCount++;
|
||||
println "没有网页内容"
|
||||
println "没有网页内容"
|
||||
}
|
||||
}
|
||||
|
||||
println "失败:"+failCount
|
||||
println "成功:"+sucCount
|
||||
println "失败:"+failCount
|
||||
println "成功:"+sucCount
|
||||
return results;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ class Test {
|
||||
def task = new Test();
|
||||
|
||||
|
||||
def obj = "23平米"
|
||||
def obj = "23平米"
|
||||
Pattern pa = Pattern.compile("(\\d+(.\\d+)?)")
|
||||
def ma=pa.matcher(obj)
|
||||
def value = null;
|
||||
@@ -27,10 +27,10 @@ class Test {
|
||||
println value
|
||||
|
||||
|
||||
// def names = task.findDistricts("shop", "北京", "房山");
|
||||
// println "库中:"+ names.size();
|
||||
// def names = task.findDistricts("shop", "北京", "房山");
|
||||
// println "库中:"+ names.size();
|
||||
// def ret = task.readTxt("H:\\", "shop.txt");
|
||||
// println "Excel:"+ ret.size();
|
||||
// println "Excel:"+ ret.size();
|
||||
//
|
||||
// def index = 0;
|
||||
// for(def exclName : ret){
|
||||
|
||||
@@ -87,13 +87,13 @@ return;
|
||||
//// }
|
||||
//
|
||||
// String h = new String(s.getBytes("iso-8859-1"),"UTF-8");
|
||||
// if(StringUtils.contains(h, "opt:更新")){
|
||||
// if(StringUtils.contains(h, "opt:更新")){
|
||||
// count_update++;
|
||||
// } else if(StringUtils.contains(h, "opt:放弃")){
|
||||
// } else if(StringUtils.contains(h, "opt:放弃")){
|
||||
// count_abort++;
|
||||
// } else if(StringUtils.contains(h, "opt:保存")){
|
||||
// } else if(StringUtils.contains(h, "opt:保存")){
|
||||
// count_save++;
|
||||
// } else if(StringUtils.contains(h, "opt:过滤")){
|
||||
// } else if(StringUtils.contains(h, "opt:过滤")){
|
||||
// count_filter++;
|
||||
// }
|
||||
//
|
||||
|
||||
+34
-34
@@ -42,14 +42,14 @@ class WebAnalyLogBatch extends BaseBatch {
|
||||
|
||||
if (file.exists()) {
|
||||
def retMap = analyFile(filePath);
|
||||
println fileName + " 文件分析完成......"
|
||||
println fileName + " 文件分析完成......"
|
||||
|
||||
saveToDB(retMap, yesterDay);
|
||||
//移除10天前的文件
|
||||
//移除10天前的文件
|
||||
removeFile(path);
|
||||
|
||||
} else {
|
||||
println fileName + " 没有找到文件!";
|
||||
println fileName + " 没有找到文件!";
|
||||
}
|
||||
|
||||
}
|
||||
@@ -63,9 +63,9 @@ class WebAnalyLogBatch extends BaseBatch {
|
||||
|
||||
if (file.exists()) {
|
||||
file.delete();
|
||||
println "删除文件 ......" + fileName;
|
||||
println "删除文件 ......" + fileName;
|
||||
} else {
|
||||
println "要删的文件不存在 ......" + fileName;
|
||||
println "要删的文件不存在 ......" + fileName;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -92,29 +92,29 @@ class WebAnalyLogBatch extends BaseBatch {
|
||||
obj.put("city", keys[1]);
|
||||
obj.put("site_name", keys[2]);
|
||||
obj.put("stat_date", DateUtils.parseFromTightYYYYMMDD(yesterDay));
|
||||
if (v.getAt("总量")) {
|
||||
obj.put("total_count", v.getAt("总量"));
|
||||
if (v.getAt("总量")) {
|
||||
obj.put("total_count", v.getAt("总量"));
|
||||
}
|
||||
|
||||
if (v.getAt("保存")) {
|
||||
obj.put("add_count", v.getAt("保存"));
|
||||
if (v.getAt("保存")) {
|
||||
obj.put("add_count", v.getAt("保存"));
|
||||
}
|
||||
|
||||
|
||||
if (v.getAt("更新")) {
|
||||
obj.put("update_count", v.getAt("更新"));
|
||||
if (v.getAt("更新")) {
|
||||
obj.put("update_count", v.getAt("更新"));
|
||||
}
|
||||
|
||||
if (v.getAt("过滤")) {
|
||||
obj.put("filter_count", v.getAt("过滤"));
|
||||
if (v.getAt("过滤")) {
|
||||
obj.put("filter_count", v.getAt("过滤"));
|
||||
}
|
||||
|
||||
if (v.getAt("异常")) {
|
||||
obj.put("error_count", v.getAt("异常"));
|
||||
if (v.getAt("异常")) {
|
||||
obj.put("error_count", v.getAt("异常"));
|
||||
}
|
||||
|
||||
if (v.getAt("放弃")) {
|
||||
obj.put("lost_count", v.getAt("放弃"));
|
||||
if (v.getAt("放弃")) {
|
||||
obj.put("lost_count", v.getAt("放弃"));
|
||||
}
|
||||
|
||||
obj.put("c_date", new Date());
|
||||
@@ -132,7 +132,7 @@ class WebAnalyLogBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
def writeTxt(def s, def day) {
|
||||
FileUtils.writeStringToFile(new File("E:\\数据\\logs\\结果\\" + day + ".txt"), s, "GBK")
|
||||
FileUtils.writeStringToFile(new File("E:\\数据\\logs\\结果\\" + day + ".txt"), s, "GBK")
|
||||
}
|
||||
|
||||
def printlnBaseInfo(retMap) {
|
||||
@@ -143,7 +143,7 @@ class WebAnalyLogBatch extends BaseBatch {
|
||||
|
||||
|
||||
def readTxt1(def file) {
|
||||
//方式一
|
||||
//方式一
|
||||
def retMap = [:];
|
||||
FileInputStream fis = new FileInputStream(file);
|
||||
RandomAccessFile raf = new RandomAccessFile(new File(file), "r");
|
||||
@@ -173,11 +173,11 @@ class WebAnalyLogBatch extends BaseBatch {
|
||||
def s = line.substring(line.indexOf("-") + 2, line.length());
|
||||
def infoMap = splitToMap(s)
|
||||
// def array = s.split("\\s+");
|
||||
//房源类型
|
||||
//房源类型
|
||||
def cat = infoMap['cat'];
|
||||
//城市详情
|
||||
//城市详情
|
||||
def city = infoMap['city'];
|
||||
//解析操作
|
||||
//解析操作
|
||||
def optType = infoMap['opt'];
|
||||
/*if(array[5].toString().endsWith('"city":"')){
|
||||
array[5] = array[5] + array[6];
|
||||
@@ -191,14 +191,14 @@ class WebAnalyLogBatch extends BaseBatch {
|
||||
detailMap = [:]
|
||||
retMap.putAt(key, detailMap);
|
||||
}
|
||||
//房源类型总量
|
||||
def totalCount = detailMap["总量"];
|
||||
//房源类型总量
|
||||
def totalCount = detailMap["总量"];
|
||||
if (!totalCount) {
|
||||
totalCount = 0;
|
||||
}
|
||||
detailMap.put("总量", totalCount + 1);
|
||||
detailMap.put("总量", totalCount + 1);
|
||||
|
||||
//各种操作类型统计
|
||||
//各种操作类型统计
|
||||
def count = detailMap.get(optType);
|
||||
if (!count) {
|
||||
count = 0;
|
||||
@@ -244,13 +244,13 @@ class WebAnalyLogBatch extends BaseBatch {
|
||||
webDetailMap.put(key, infoMap);
|
||||
}
|
||||
|
||||
def totalCount = infoMap.get("总量");
|
||||
def totalCount = infoMap.get("总量");
|
||||
if (!totalCount) {
|
||||
totalCount = 0;
|
||||
}
|
||||
infoMap.put("总量", totalCount + 1);
|
||||
infoMap.put("总量", totalCount + 1);
|
||||
|
||||
//解析操作
|
||||
//解析操作
|
||||
def optType = getOptionType(array[0]);
|
||||
def count = infoMap.get(optType);
|
||||
if (!count) {
|
||||
@@ -288,7 +288,7 @@ class WebAnalyLogBatch extends BaseBatch {
|
||||
|
||||
def readTxt3(def file) {
|
||||
Charset charset = Charset.forName("UTF-8");
|
||||
//Java.nio.charset.Charset处理了字符转换问题。它通过构造CharsetEncoder和CharsetDecoder将字符序列转换成字节和逆转换。
|
||||
//Java.nio.charset.Charset处理了字符转换问题。它通过构造CharsetEncoder和CharsetDecoder将字符序列转换成字节和逆转换。
|
||||
CharsetDecoder decoder = charset.newDecoder();
|
||||
FileInputStream fis = null;
|
||||
def retMap = [:];
|
||||
@@ -319,7 +319,7 @@ class WebAnalyLogBatch extends BaseBatch {
|
||||
|
||||
println byteBuffer.toString();
|
||||
|
||||
if (charBuffer.toString().contains("[1009 10:39:22 711 INFO ] [exe1] - opt:异")) {
|
||||
if (charBuffer.toString().contains("[1009 10:39:22 711 INFO ] [exe1] - opt:异")) {
|
||||
println "sss"
|
||||
}
|
||||
while (true) {
|
||||
@@ -429,7 +429,7 @@ class WebAnalyLogBatch extends BaseBatch {
|
||||
def readTxt4(def file) {
|
||||
// try {
|
||||
// BufferedInputStream bis = new BufferedInputStream(new FileInputStream(new File(file)));
|
||||
// BufferedReader ino = new BufferedReader(new InputStreamReader(bis, "utf-8"), 10 * 1024 * 1024);//10M缓存
|
||||
// BufferedReader ino = new BufferedReader(new InputStreamReader(bis, "utf-8"), 10 * 1024 * 1024);//10M缓存
|
||||
// int index = 0;
|
||||
// while(ino.ready()) {
|
||||
// String line = ino.readLine();
|
||||
@@ -446,7 +446,7 @@ class WebAnalyLogBatch extends BaseBatch {
|
||||
// ex.printStackTrace();
|
||||
// }
|
||||
//
|
||||
//非一行一行读取
|
||||
//非一行一行读取
|
||||
// File inFile = new File(file);
|
||||
// FileChannel inFileChannel = new FileInputStream(inFile).getChannel();
|
||||
// byte[] bs = new byte[1024];
|
||||
@@ -460,7 +460,7 @@ class WebAnalyLogBatch extends BaseBatch {
|
||||
// buffer.flip();
|
||||
// def s = charset.decode(buffer);
|
||||
// sb.append(s);
|
||||
// if(s.contains("[1009 10:39:22 711 INFO ] [exe1] - opt:异")){
|
||||
// if(s.contains("[1009 10:39:22 711 INFO ] [exe1] - opt:异")){
|
||||
// println "sss"
|
||||
// }
|
||||
// println "[${index}]"+s;
|
||||
|
||||
+30
-30
@@ -36,10 +36,10 @@ class XingyeDataBatch {
|
||||
|
||||
public static void main(String[] args) {
|
||||
List<String> cities = ServerAreaMap.findAllCities()
|
||||
List<List<Object>> rows = ExcelUtils.read("D:/房产测试合并数据21.xls", 0, 0)
|
||||
//新序号 序号 地区 楼宇名称 房产地址 建筑面积 单价 总价值 城市 名称 地址 经度 纬度
|
||||
rows.get(0).add('计算类型')
|
||||
rows.get(0).add('可信度')
|
||||
List<List<Object>> rows = ExcelUtils.read("D:/房产测试合并数据21.xls", 0, 0)
|
||||
//新序号 序号 地区 楼宇名称 房产地址 建筑面积 单价 总价值 城市 名称 地址 经度 纬度
|
||||
rows.get(0).add('计算类型')
|
||||
rows.get(0).add('可信度')
|
||||
for (int i = 1; i < rows.size(); i++) {
|
||||
println rows.get(i).get(0)
|
||||
List<Object> cloumns = rows.get(i)
|
||||
@@ -57,15 +57,15 @@ class XingyeDataBatch {
|
||||
String realCity = cloumns.get(11).toString().trim()
|
||||
realCity = formatCity(realCity, cities)
|
||||
|
||||
//todo 过滤掉不是小区的数据
|
||||
if('其他' == cat || StringUtils.isBlank(realCity) || !cities.contains(realCity)) {
|
||||
//todo 过滤掉不是小区的数据
|
||||
if('其他' == cat || StringUtils.isBlank(realCity) || !cities.contains(realCity)) {
|
||||
rows.get(i).add('')
|
||||
rows.get(i).add('')
|
||||
}else {
|
||||
Res res = null;
|
||||
if ('商铺' == cat) {
|
||||
if ('商铺' == cat) {
|
||||
res = getShopPriceResult(realCity, realName, address, lng, lat)
|
||||
} else if ('写字楼' == cat) {
|
||||
} else if ('写字楼' == cat) {
|
||||
res = getOfficePriceResult(realCity, realName, address, lng, lat)
|
||||
} else {
|
||||
res = getDistrictPriceResult(realCity, realName, address, lng, lat)
|
||||
@@ -85,7 +85,7 @@ class XingyeDataBatch {
|
||||
}
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:/房产测试合并数据21-output.xls", rows, "数据")
|
||||
ExcelUtils.write("D:/房产测试合并数据21-output.xls", rows, "数据")
|
||||
}
|
||||
|
||||
private static String formatCity(String city, List<String> cities) {
|
||||
@@ -93,8 +93,8 @@ class XingyeDataBatch {
|
||||
return city
|
||||
}
|
||||
|
||||
if(city.endsWith('市')) {
|
||||
city = StringUtils.substringBeforeLast(city, "市")
|
||||
if(city.endsWith('市')) {
|
||||
city = StringUtils.substringBeforeLast(city, "市")
|
||||
if(cities.contains(city)) {
|
||||
return city
|
||||
}
|
||||
@@ -121,13 +121,13 @@ class XingyeDataBatch {
|
||||
}
|
||||
}
|
||||
|
||||
/**推测真正的正式名*/
|
||||
/**推测真正的正式名*/
|
||||
static String speculateCityNameByRegion(String province, String orgName){
|
||||
List<String> foundCityList = new ArrayList<String>();
|
||||
/**根据区域推测所在的城市*/
|
||||
/**根据区域推测所在的城市*/
|
||||
String temp;
|
||||
for(Map.Entry<String,Set<String>> entry : CITY_REGION_MAP.entrySet()){
|
||||
temp = orgName.replaceFirst("^" + entry.getKey(),"");//排除无"市"标示的城市信息干扰 如:北京海定区
|
||||
temp = orgName.replaceFirst("^" + entry.getKey(),"");//排除无"市"标示的城市信息干扰 如:北京海定区
|
||||
if(entry.getValue().contains(temp)){
|
||||
if(province == null){
|
||||
foundCityList.add(entry.getKey());
|
||||
@@ -183,10 +183,10 @@ class XingyeDataBatch {
|
||||
private static Res getDistrictPriceResult(String realCity, String realName, String realAddr, Double lng, Double lat) {
|
||||
Res res = null;
|
||||
DBObject district = chooseDistrict(realCity, realName, realAddr, 'district')
|
||||
if(district != null) {//利用计算的均价
|
||||
if(district != null) {//利用计算的均价
|
||||
String proRegion = district.get('region')
|
||||
String proName = district.get('name')
|
||||
String proEstateType = (district.get('estate_type2') != null && district.get('estate_type2').indexOf('别墅') > -1 ? '别墅' : '小区');
|
||||
String proEstateType = (district.get('estate_type2') != null && district.get('estate_type2').indexOf('别墅') > -1 ? '别墅' : '小区');
|
||||
DBObject statq = new BasicDBObject()
|
||||
statq.put('city', realCity)
|
||||
statq.put('region', proRegion)
|
||||
@@ -200,7 +200,7 @@ class XingyeDataBatch {
|
||||
}
|
||||
}
|
||||
|
||||
//利用样本房源进行计算
|
||||
//利用样本房源进行计算
|
||||
if(res == null) {
|
||||
List<DBObject> houses = fetchHousesInSeawater(realCity, realName, 1000)
|
||||
if(houses != null && houses.size() > 10) {
|
||||
@@ -212,7 +212,7 @@ class XingyeDataBatch {
|
||||
}
|
||||
}
|
||||
|
||||
//利用周边计算
|
||||
//利用周边计算
|
||||
if(res == null && lat != null && lng != null) {
|
||||
String js = HTTP.invoke('/evaluation/lngLatEvaluate', [city : realCity, lng:lng, lat:lat, lngLatType:1], "UTF-8")
|
||||
if(js != null) {
|
||||
@@ -241,7 +241,7 @@ class XingyeDataBatch {
|
||||
}
|
||||
|
||||
/**
|
||||
* 在seaweed中选择一个最优的小区
|
||||
* 在seaweed中选择一个最优的小区
|
||||
* @param city
|
||||
* @param name
|
||||
* @param address
|
||||
@@ -341,7 +341,7 @@ class XingyeDataBatch {
|
||||
|
||||
|
||||
/**
|
||||
* 数量得分(最高得分0.1)
|
||||
* 数量得分(最高得分0.1)
|
||||
*
|
||||
* @param points
|
||||
* @return
|
||||
@@ -350,14 +350,14 @@ class XingyeDataBatch {
|
||||
if (points.size() == 0) {
|
||||
return 0;
|
||||
}
|
||||
// 每10个样本=0.01
|
||||
// 每10个样本=0.01
|
||||
double c1 = (double) points.size() / 10.0 / 100.0;
|
||||
c1 = (c1 > 0.1 ? 0.1 : c1);
|
||||
return c1;
|
||||
}
|
||||
|
||||
/**
|
||||
* 质量得分(最高得分0.1)
|
||||
* 质量得分(最高得分0.1)
|
||||
*
|
||||
* @param points
|
||||
* @return
|
||||
@@ -373,21 +373,21 @@ class XingyeDataBatch {
|
||||
for (DBObject p : points) {
|
||||
Date d = p.get('publish_date')
|
||||
long diff = calcDate.getTime() - d.getTime();
|
||||
if (diff <= 30 * 24 * 3600 * 1000) {// 一个月内
|
||||
if (diff <= 30 * 24 * 3600 * 1000) {// 一个月内
|
||||
c2 = c2 + perc * 1.00d;
|
||||
} else if (diff <= 60 * 24 * 3600 * 1000) {// 两个月内
|
||||
} else if (diff <= 60 * 24 * 3600 * 1000) {// 两个月内
|
||||
c2 = c2 + perc * 0.90d;
|
||||
} else if (diff <= 90 * 24 * 3600 * 1000) {// 三个月内
|
||||
} else if (diff <= 90 * 24 * 3600 * 1000) {// 三个月内
|
||||
c2 = c2 + perc * 0.85d;
|
||||
} else if (diff <= 180 * 24 * 3600 * 1000) {// 半年内
|
||||
} else if (diff <= 180 * 24 * 3600 * 1000) {// 半年内
|
||||
c2 = c2 + perc * 0.80d;
|
||||
} else if (diff <= 365 * 24 * 3600 * 1000) {// 一年内
|
||||
} else if (diff <= 365 * 24 * 3600 * 1000) {// 一年内
|
||||
c2 = c2 + perc * 0.70d;
|
||||
} else if (diff <= 540 * 24 * 3600 * 1000) {// 一年半内
|
||||
} else if (diff <= 540 * 24 * 3600 * 1000) {// 一年半内
|
||||
c2 = c2 + perc * 0.60d;
|
||||
} else if (diff <= 720 * 24 * 3600 * 1000) {// 两年内
|
||||
} else if (diff <= 720 * 24 * 3600 * 1000) {// 两年内
|
||||
c2 = c2 + perc * 0.40d;
|
||||
} else {// 超过两年
|
||||
} else {// 超过两年
|
||||
c2 = c2 + perc * 0.10d;
|
||||
}
|
||||
}
|
||||
|
||||
+12
-12
@@ -24,7 +24,7 @@ class XingyeDataPrice {
|
||||
public static HttpHelper HTTP = new HttpHelper("192.168.10.66:7180");
|
||||
|
||||
public static void main(String[] args) {
|
||||
String fileName = "介入数据跑价格补城市价格";
|
||||
String fileName = "介入数据跑价格补城市价格";
|
||||
List<String> cities = ServerAreaMap.findAllCities()
|
||||
List<List<Object>> rows = ExcelUtils.read("D:/${fileName}.xls", 0, 0)
|
||||
for (int i = 1; i < rows.size(); i++) {
|
||||
@@ -78,14 +78,14 @@ class XingyeDataPrice {
|
||||
cloumns.set(16, "0.01")
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:/${fileName}-final.xls", rows, "数据")
|
||||
ExcelUtils.write("D:/${fileName}-final.xls", rows, "数据")
|
||||
}
|
||||
|
||||
|
||||
public static void maindd3(String[] args) {
|
||||
List<String> idList = FileUtils.readLines(new File("D:/介入ID.txt"))
|
||||
List<String> idList = FileUtils.readLines(new File("D:/介入ID.txt"))
|
||||
|
||||
String fileName = "房价调查完成合并数据0324终-final16";
|
||||
String fileName = "房价调查完成合并数据0324终-final16";
|
||||
List<String> cities = ServerAreaMap.findAllCities()
|
||||
List<List<Object>> rows = ExcelUtils.read("D:/${fileName}.xls", 0, 0)
|
||||
List<List<Object>> newRows = [rows.get(0)];
|
||||
@@ -161,10 +161,10 @@ class XingyeDataPrice {
|
||||
}
|
||||
|
||||
|
||||
ExcelUtils.write("D:/${fileName}-final.xls", newRows, "数据")
|
||||
ExcelUtils.write("D:/${fileName}-final.xls", newRows, "数据")
|
||||
}
|
||||
public static void maind(String[] args) {
|
||||
String fileName = "其他类";
|
||||
String fileName = "其他类";
|
||||
List<String> cities = ServerAreaMap.findAllCities()
|
||||
List<List<Object>> rows = ExcelUtils.read("D:/${fileName}.xlsx", 0, 0)
|
||||
def eschars = [] as String[]
|
||||
@@ -243,7 +243,7 @@ class XingyeDataPrice {
|
||||
cloumns.set(16, "0.01")
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:/${fileName}-final.xlsx", rows, "数据")
|
||||
ExcelUtils.write("D:/${fileName}-final.xlsx", rows, "数据")
|
||||
}
|
||||
|
||||
|
||||
@@ -252,8 +252,8 @@ class XingyeDataPrice {
|
||||
return city
|
||||
}
|
||||
|
||||
if(city.endsWith('市')) {
|
||||
city = StringUtils.substringBeforeLast(city, "市")
|
||||
if(city.endsWith('市')) {
|
||||
city = StringUtils.substringBeforeLast(city, "市")
|
||||
if(cities.contains(city)) {
|
||||
return city
|
||||
}
|
||||
@@ -279,13 +279,13 @@ class XingyeDataPrice {
|
||||
}
|
||||
}
|
||||
|
||||
/**推测真正的正式名*/
|
||||
/**推测真正的正式名*/
|
||||
static String speculateCityNameByRegion(String province, String orgName){
|
||||
List<String> foundCityList = new ArrayList<String>();
|
||||
/**根据区域推测所在的城市*/
|
||||
/**根据区域推测所在的城市*/
|
||||
String temp;
|
||||
for(Map.Entry<String,Set<String>> entry : CITY_REGION_MAP.entrySet()){
|
||||
temp = orgName.replaceFirst("^" + entry.getKey(),"");//排除无"市"标示的城市信息干扰 如:北京海定区
|
||||
temp = orgName.replaceFirst("^" + entry.getKey(),"");//排除无"市"标示的城市信息干扰 如:北京海定区
|
||||
if(entry.getValue().contains(temp)){
|
||||
if(province == null){
|
||||
foundCityList.add(entry.getKey());
|
||||
|
||||
+118
-118
@@ -28,95 +28,95 @@ import com.mongodb.QueryOperators
|
||||
|
||||
class DistrictExportBatch {
|
||||
public static final FILELD_MAP = [
|
||||
"_id":"楼盘编号",
|
||||
"city":"所在城市",
|
||||
"region":"所在地区",
|
||||
"block":"所在板块",
|
||||
"name":"楼盘名称",
|
||||
"estate_type2":"物业类型",
|
||||
"build_type":"建筑类型",
|
||||
"live_date":"竣工时间(建成年份)",
|
||||
"lng2":"经度",
|
||||
"lat2":"纬度",
|
||||
"developer":"开发商名称",
|
||||
"estate_company":"物业公司",
|
||||
"estate_charge":"物业费",
|
||||
"licence":"商品房销售许可证号",
|
||||
"address":"楼盘地址",
|
||||
"area":"面积",
|
||||
"avg_price":"楼盘单价",
|
||||
"totalPrice":"评估总价",
|
||||
"city_price":"城市均价",
|
||||
"city_price_markup":"近半年城市均价涨跌幅",
|
||||
"rent_price":"租房均价",
|
||||
"year_markup":"均价涨跌幅",
|
||||
"month_markup":"环比上月",
|
||||
"max_price":"历史最高单价",
|
||||
"min_price":"历史最低单价",
|
||||
"futrue_price":"未来一年房价预测",
|
||||
"futrue_markup":"未来一年房价涨跌幅预测",
|
||||
"last_month_house_count":"上个月挂牌量",
|
||||
"half_year_house_count":"半年挂牌量",
|
||||
"attention_level":"近三个月买家关注度",
|
||||
"block_rank":"板块排名",
|
||||
"block_count":"板块个数",
|
||||
"region_rank":"区域排名",
|
||||
"region_count":"区域个数",
|
||||
"consumption":"周边餐饮的人均消费",
|
||||
"consumption_level":"综合消费水平高于所属城市的楼盘数",
|
||||
"_id":"楼盘编号",
|
||||
"city":"所在城市",
|
||||
"region":"所在地区",
|
||||
"block":"所在板块",
|
||||
"name":"楼盘名称",
|
||||
"estate_type2":"物业类型",
|
||||
"build_type":"建筑类型",
|
||||
"live_date":"竣工时间(建成年份)",
|
||||
"lng2":"经度",
|
||||
"lat2":"纬度",
|
||||
"developer":"开发商名称",
|
||||
"estate_company":"物业公司",
|
||||
"estate_charge":"物业费",
|
||||
"licence":"商品房销售许可证号",
|
||||
"address":"楼盘地址",
|
||||
"area":"面积",
|
||||
"avg_price":"楼盘单价",
|
||||
"totalPrice":"评估总价",
|
||||
"city_price":"城市均价",
|
||||
"city_price_markup":"近半年城市均价涨跌幅",
|
||||
"rent_price":"租房均价",
|
||||
"year_markup":"均价涨跌幅",
|
||||
"month_markup":"环比上月",
|
||||
"max_price":"历史最高单价",
|
||||
"min_price":"历史最低单价",
|
||||
"futrue_price":"未来一年房价预测",
|
||||
"futrue_markup":"未来一年房价涨跌幅预测",
|
||||
"last_month_house_count":"上个月挂牌量",
|
||||
"half_year_house_count":"半年挂牌量",
|
||||
"attention_level":"近三个月买家关注度",
|
||||
"block_rank":"板块排名",
|
||||
"block_count":"板块个数",
|
||||
"region_rank":"区域排名",
|
||||
"region_count":"区域个数",
|
||||
"consumption":"周边餐饮的人均消费",
|
||||
"consumption_level":"综合消费水平高于所属城市的楼盘数",
|
||||
|
||||
"KTV_count":"KTV数量",
|
||||
"cinema_count":"电影院数量",
|
||||
"jianshen_count":"健身中心数量",
|
||||
"zhonghesc_count":"综合商场数量",
|
||||
"spa_count":"美容院数量",
|
||||
"hotel_count":"星级酒店数量",
|
||||
"bigmarket_count":"大型超市数量",
|
||||
"gef_count":"高尔夫球场数量",
|
||||
"slschool_count":"私立学校数量",
|
||||
"zrzx_count":"整容整形数量",
|
||||
"coffee_count":"咖啡厅数量",
|
||||
"bath_count":"洗浴数量",
|
||||
"office_count":"写字楼数量",
|
||||
"hspt_count":"医院数量",
|
||||
"bank_count":"银行数量",
|
||||
"atm_count":"ATM数量",
|
||||
"park_count":"公园绿地数量",
|
||||
"comm_count":"商圈数量",
|
||||
"bus_count":"公交车站数量",
|
||||
"subway_count":"地铁数量",
|
||||
"school_count":"学校数量",
|
||||
"air_count":"机场数量",
|
||||
"market_count":"超市数量",
|
||||
"wb_count":"网吧数量",
|
||||
"ywss_count":"厌恶设施数量",
|
||||
"all_area":"楼盘占地总面积",
|
||||
"build_area":"楼盘建筑总面积",
|
||||
"household_count":"总套数",
|
||||
// "":"可售套数",
|
||||
// "":"所在省份",
|
||||
// "alias1":"楼盘别名1",
|
||||
// "alias2":"楼盘别名2",
|
||||
// "alias3":"楼盘别名3",
|
||||
"kind":"小区类型",
|
||||
"virescence_percent":"绿化率",
|
||||
"cubage_percent":"容积率",
|
||||
// "facilities":"配套信息",
|
||||
"traffic":"交通情况",
|
||||
"household_count":"户数",
|
||||
"car_site":"车位情况",
|
||||
// "":"具体楼盘号",
|
||||
"property_life":"产权性质",
|
||||
"build_structure":"房屋结构",
|
||||
"open_date":"开盘时间",
|
||||
"sell_address":"售楼地址",
|
||||
"discount":"折扣信息",
|
||||
// "fitment_materials":"建材装修",
|
||||
"floor_info":"楼层状况",
|
||||
"description":"描述",
|
||||
// "c_date":"登记日期",
|
||||
// "m_date":"更新日期",
|
||||
// "out_pics":"外景图",
|
||||
"KTV_count":"KTV数量",
|
||||
"cinema_count":"电影院数量",
|
||||
"jianshen_count":"健身中心数量",
|
||||
"zhonghesc_count":"综合商场数量",
|
||||
"spa_count":"美容院数量",
|
||||
"hotel_count":"星级酒店数量",
|
||||
"bigmarket_count":"大型超市数量",
|
||||
"gef_count":"高尔夫球场数量",
|
||||
"slschool_count":"私立学校数量",
|
||||
"zrzx_count":"整容整形数量",
|
||||
"coffee_count":"咖啡厅数量",
|
||||
"bath_count":"洗浴数量",
|
||||
"office_count":"写字楼数量",
|
||||
"hspt_count":"医院数量",
|
||||
"bank_count":"银行数量",
|
||||
"atm_count":"ATM数量",
|
||||
"park_count":"公园绿地数量",
|
||||
"comm_count":"商圈数量",
|
||||
"bus_count":"公交车站数量",
|
||||
"subway_count":"地铁数量",
|
||||
"school_count":"学校数量",
|
||||
"air_count":"机场数量",
|
||||
"market_count":"超市数量",
|
||||
"wb_count":"网吧数量",
|
||||
"ywss_count":"厌恶设施数量",
|
||||
"all_area":"楼盘占地总面积",
|
||||
"build_area":"楼盘建筑总面积",
|
||||
"household_count":"总套数",
|
||||
// "":"可售套数",
|
||||
// "":"所在省份",
|
||||
// "alias1":"楼盘别名1",
|
||||
// "alias2":"楼盘别名2",
|
||||
// "alias3":"楼盘别名3",
|
||||
"kind":"小区类型",
|
||||
"virescence_percent":"绿化率",
|
||||
"cubage_percent":"容积率",
|
||||
// "facilities":"配套信息",
|
||||
"traffic":"交通情况",
|
||||
"household_count":"户数",
|
||||
"car_site":"车位情况",
|
||||
// "":"具体楼盘号",
|
||||
"property_life":"产权性质",
|
||||
"build_structure":"房屋结构",
|
||||
"open_date":"开盘时间",
|
||||
"sell_address":"售楼地址",
|
||||
"discount":"折扣信息",
|
||||
// "fitment_materials":"建材装修",
|
||||
"floor_info":"楼层状况",
|
||||
"description":"描述",
|
||||
// "c_date":"登记日期",
|
||||
// "m_date":"更新日期",
|
||||
// "out_pics":"外景图",
|
||||
]
|
||||
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
public static final DBCollection DIST_STAT_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "district_stat_rent", "mongo.db.fangjia.com")
|
||||
@@ -134,7 +134,7 @@ class DistrictExportBatch {
|
||||
public static def radius = 1000;
|
||||
|
||||
public static void main(String[] args) {
|
||||
for(String city : ["上海"]) {//"北京",,
|
||||
for(String city : ["上海"]) {//"北京",,
|
||||
areas = [];
|
||||
StringBuilder builder = new StringBuilder()
|
||||
FILELD_MAP.each() {column, display->
|
||||
@@ -160,7 +160,7 @@ class DistrictExportBatch {
|
||||
c ++
|
||||
|
||||
def kind = dbo.get("kind").toString();
|
||||
if(kind.indexOf("别墅")!=-1){
|
||||
if(kind.indexOf("别墅")!=-1){
|
||||
continue;
|
||||
}
|
||||
if(!dbo.get("lng2") || !dbo.get("lat2")){
|
||||
@@ -170,7 +170,7 @@ class DistrictExportBatch {
|
||||
//if(!regions.contains(dbo.get("region"))){
|
||||
// regions << dbo.get("region").toString()
|
||||
|
||||
//房子估价
|
||||
//房子估价
|
||||
def params = findLoudongInfo(city, dbo.get("region"), dbo.get("name"));
|
||||
params.city = city;
|
||||
params.region = dbo.get("region")
|
||||
@@ -180,33 +180,33 @@ class DistrictExportBatch {
|
||||
continue;
|
||||
}
|
||||
|
||||
//最近城市均价
|
||||
//最近城市均价
|
||||
def nearCityPrice = findCityPrice(city, null);
|
||||
//半年前最近的城市均价
|
||||
//半年前最近的城市均价
|
||||
def weekend = DateUtil.formatByPattern(DateUtil.getFirstDayOfWeek(new Date()), "yyyy-MM-dd");
|
||||
def halfYearAgo = DateUtils.add(weekend, -180);
|
||||
def halfYearAgoPrice = findCityPrice(city, halfYearAgo);
|
||||
|
||||
//租金价格
|
||||
//租金价格
|
||||
def endMonthDate = DateUtils.add(DateUtils.formatToYYYYMMDD(DateUtils.getFirstDayOfMonth(new Date())),-1);
|
||||
def houseTypePriceMap = findDistrictRentStat(city, dbo.get("region"), dbo.get("name"),endMonthDate);
|
||||
//最近期小区均价
|
||||
//最近期小区均价
|
||||
def nearDistAvgPrice = queryDistrictPrice(city, dbo.get("region"), dbo.get("name"), null, 1);
|
||||
//一前内小区均价
|
||||
//一前内小区均价
|
||||
def oneYearAgo = DateUtils.add(weekend, -365);
|
||||
def oneYearPriceList = queryDistrictPrices(city, dbo.get("region"), dbo.get("name"), oneYearAgo, weekend);
|
||||
|
||||
//小区在所在区域或板块中的点击量排名
|
||||
//小区在所在区域或板块中的点击量排名
|
||||
def clickRank = findRegionBlockRank(city, dbo.get("region"), dbo.get("name"));
|
||||
|
||||
//某板块中的小区量
|
||||
//某板块中的小区量
|
||||
def blockCount = findRegionBlockDistrictCount(city, dbo.get("region"), dbo.get("block"));
|
||||
//某区域中的小区量
|
||||
//某区域中的小区量
|
||||
def regionCount = findRegionBlockDistrictCount(city, dbo.get("region"), null);
|
||||
|
||||
//一年后的预测价格
|
||||
//一年后的预测价格
|
||||
def futurePrice = findPricePredict(city, dbo.get("region"), dbo.get("name"));
|
||||
//人均消费水平
|
||||
//人均消费水平
|
||||
def avgConsumption = 0;
|
||||
|
||||
FILELD_MAP.each() {column, display->
|
||||
@@ -216,15 +216,15 @@ class DistrictExportBatch {
|
||||
val = ''
|
||||
}else if(column == 'address'){
|
||||
def address = dbo.get("address");
|
||||
if(address.indexOf("号")!=-1){
|
||||
val = address + params.houseNumber +"栋"+params.roomNumber+"室";
|
||||
if(address.indexOf("号")!=-1){
|
||||
val = address + params.houseNumber +"栋"+params.roomNumber+"室";
|
||||
}else{
|
||||
val = address + params.houseNumber +"号"+params.roomNumber+"室";
|
||||
val = address + params.houseNumber +"号"+params.roomNumber+"室";
|
||||
}
|
||||
}else if(column == 'area'){
|
||||
val = params.area
|
||||
}else if(column == 'totalPrice'){
|
||||
val = DataUtil.getRound(2, totalPrice*1.0/10000) +"万元";
|
||||
val = DataUtil.getRound(2, totalPrice*1.0/10000) +"万元";
|
||||
}else if(column.startsWith('alias')) {
|
||||
if(dbo.get('alias')) {
|
||||
List aliList = (List)dbo.get('alias');
|
||||
@@ -247,7 +247,7 @@ class DistrictExportBatch {
|
||||
}else if(column == 'estate_charge'){
|
||||
def s = dbo.get(column);
|
||||
if(s){
|
||||
val = s.toString().replaceAll("元/㎡/月","").replaceAll("元/月/平米","")
|
||||
val = s.toString().replaceAll("元/㎡/月","").replaceAll("元/月/平米","")
|
||||
}else{
|
||||
val = "";
|
||||
}
|
||||
@@ -309,20 +309,20 @@ class DistrictExportBatch {
|
||||
def three_room = houseTypePriceMap.get("3");
|
||||
val = "";
|
||||
if(one_room){
|
||||
val = "1室:"+one_room+"元";
|
||||
val = "1室:"+one_room+"元";
|
||||
}
|
||||
if(two_room){
|
||||
if(val){
|
||||
val = val + " ";
|
||||
}
|
||||
val = val + "2室:"+two_room+"元";
|
||||
val = val + "2室:"+two_room+"元";
|
||||
}
|
||||
|
||||
if(three_room){
|
||||
if(val){
|
||||
val = val + " ";
|
||||
}
|
||||
val = val + "3室:"+three_room+"元";
|
||||
val = val + "3室:"+three_room+"元";
|
||||
}
|
||||
|
||||
|
||||
@@ -330,11 +330,11 @@ class DistrictExportBatch {
|
||||
if(clickRank){
|
||||
def attent = clickRank.get("region_click_rank")/regionCount * 1.0
|
||||
if(attent<=0.3){
|
||||
val = "高";
|
||||
val = "高";
|
||||
}else if(attent>0.3 && attent<=0.7){
|
||||
val = "一般"
|
||||
val = "一般"
|
||||
}else{
|
||||
val = "较低";
|
||||
val = "较低";
|
||||
}
|
||||
}else{
|
||||
val = "";
|
||||
@@ -353,11 +353,11 @@ class DistrictExportBatch {
|
||||
val = "";
|
||||
}
|
||||
|
||||
}else if(column.indexOf("_count")!=-1 && display.indexOf("数量")!=-1){
|
||||
}else if(column.indexOf("_count")!=-1 && display.indexOf("数量")!=-1){
|
||||
if(dbo.get("lng2") && dbo.get("lat2")){
|
||||
def lat = (double)dbo.get("lat2") / 10000000000.0
|
||||
def lng = (double)dbo.get("lng2") / 10000000000.0
|
||||
val = findPoiCount(display.toString(), lat.toString(), lng.toString()) +"个";
|
||||
val = findPoiCount(display.toString(), lat.toString(), lng.toString()) +"个";
|
||||
}else{
|
||||
val = "";
|
||||
}
|
||||
@@ -380,7 +380,7 @@ class DistrictExportBatch {
|
||||
}
|
||||
|
||||
}else if(column == "month_markup"){
|
||||
//一个月以前最近的小区均价
|
||||
//一个月以前最近的小区均价
|
||||
def monthAgo = DateUtils.add(weekend, -30);
|
||||
def monthAgoAvgPrice = queryDistrictPrice(city, dbo.get("region"), dbo.get("name"), monthAgo, 1);
|
||||
if(oneYearPriceList.size()>0 && monthAgoAvgPrice>0){
|
||||
@@ -551,7 +551,7 @@ class DistrictExportBatch {
|
||||
|
||||
|
||||
public static def findPoiCount(display, lat, lng){
|
||||
display = display.toString().replaceAll("数量","");
|
||||
display = display.toString().replaceAll("数量","");
|
||||
|
||||
def ret = 0;
|
||||
def page_num = 0;
|
||||
@@ -591,7 +591,7 @@ class DistrictExportBatch {
|
||||
conn.setRequestMethod("GET");
|
||||
conn.doOutput = true
|
||||
inputStream = conn.getInputStream();
|
||||
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
|
||||
BufferedReader buf = new BufferedReader(new InputStreamReader(inputStream,"UTF-8"));// 创建UTF-8的BufferedReader
|
||||
while((line = buf.readLine())!=null){
|
||||
sb.append(line);
|
||||
}
|
||||
@@ -703,7 +703,7 @@ class DistrictExportBatch {
|
||||
public static def findBusinessShop(city, maxLat, minLat, maxLng, minLng){
|
||||
DBObject q = new BasicDBObject()
|
||||
q.put("city", city);
|
||||
q.put("business_type", "餐厅");
|
||||
q.put("business_type", "餐厅");
|
||||
q.put("consumption", new BasicDBObject(QueryOperators.GT, 0));
|
||||
q.put('lat', new BasicDBObject(QueryOperators.LTE, maxLat).append(QueryOperators.GTE, minLat));
|
||||
q.put('lng', new BasicDBObject(QueryOperators.LTE, maxLng).append(QueryOperators.GTE, minLng));
|
||||
|
||||
+13
-13
@@ -57,12 +57,12 @@ class HouseLoudongExportBatch {
|
||||
static final String XML_FILE_NAME = "fangzi"
|
||||
|
||||
static def FOCUS_FIELDS = [
|
||||
"city":"城市", "region":"区域", "name":"小区名", "house_area":"面积", "house_num":"房号", "room_num":"室号", "house_num_unit":"单元",
|
||||
"house_height":"楼高", "house_floor":"所在楼层", "room":"几室", "hall":"几厅", "toilet":"几卫", "direction":"朝向", "build_type":"建筑类型",
|
||||
"garden_area":"花园面积", "outer_area":"露台面积", "attic_area":"阁楼面积", "worry_facility":"厌恶设施",
|
||||
"noise":"噪音情况", "sight":"景观情况", "inside_pos":"小区位置", "house_property":"产权性质",
|
||||
"fitment_cost":"装修花费", "fitment_age":"装修年限", "fitment_rank":"装修级别", "lighting_ventilation":"采光通风"
|
||||
//"basement_area":"地下室面积", "sideset":"边套设施","car_port":"",
|
||||
"city":"城市", "region":"区域", "name":"小区名", "house_area":"面积", "house_num":"房号", "room_num":"室号", "house_num_unit":"单元",
|
||||
"house_height":"楼高", "house_floor":"所在楼层", "room":"几室", "hall":"几厅", "toilet":"几卫", "direction":"朝向", "build_type":"建筑类型",
|
||||
"garden_area":"花园面积", "outer_area":"露台面积", "attic_area":"阁楼面积", "worry_facility":"厌恶设施",
|
||||
"noise":"噪音情况", "sight":"景观情况", "inside_pos":"小区位置", "house_property":"产权性质",
|
||||
"fitment_cost":"装修花费", "fitment_age":"装修年限", "fitment_rank":"装修级别", "lighting_ventilation":"采光通风"
|
||||
//"basement_area":"地下室面积", "sideset":"边套设施","car_port":"",
|
||||
]
|
||||
|
||||
private static final String API_HOST = "open.fangjia.com"//"sha-66.qa.fangjia.com/fangjia-open";
|
||||
@@ -76,7 +76,7 @@ class HouseLoudongExportBatch {
|
||||
static HttpClient client = null
|
||||
|
||||
public static void main(String[] args) {
|
||||
def cities = ['北京']
|
||||
def cities = ['北京']
|
||||
for(String city : cities) {
|
||||
exportLoudong(city)
|
||||
}
|
||||
@@ -142,7 +142,7 @@ class HouseLoudongExportBatch {
|
||||
|
||||
List<Element> elementList = new ArrayList<Element>()
|
||||
int inx = 1
|
||||
String fileName = "D:/安硕数据/${XML_FILE_NAME}_${city}(${inx}).xml"
|
||||
String fileName = "D:/安硕数据/${XML_FILE_NAME}_${city}(${inx}).xml"
|
||||
|
||||
while(cur.hasNext()) {
|
||||
cc++
|
||||
@@ -186,7 +186,7 @@ class HouseLoudongExportBatch {
|
||||
|
||||
String kind = getDistrictKind(city, region, districtName)
|
||||
if(!kind) {
|
||||
kind = "小区"
|
||||
kind = "小区"
|
||||
}
|
||||
|
||||
Date currentWeekend = DateUtil.getThisSunday(new Date())
|
||||
@@ -251,7 +251,7 @@ class HouseLoudongExportBatch {
|
||||
|
||||
if(c % 300000 == 0) {
|
||||
inx ++
|
||||
fileName = "D:/安硕数据/${XML_FILE_NAME}_${city}(${inx}).xml"
|
||||
fileName = "D:/安硕数据/${XML_FILE_NAME}_${city}(${inx}).xml"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -267,9 +267,9 @@ class HouseLoudongExportBatch {
|
||||
}
|
||||
|
||||
private static def getFromEvaluation(String city, String region, String districtName, double area, String houseNum, String roomNum, String kind) {
|
||||
String buildType = "公寓"
|
||||
if(kind == '小区') {
|
||||
buildType = "公寓"
|
||||
String buildType = "公寓"
|
||||
if(kind == '小区') {
|
||||
buildType = "公寓"
|
||||
}else {
|
||||
buildType = kind
|
||||
}
|
||||
|
||||
+4
-4
@@ -9,7 +9,7 @@ import groovy.sql.Sql
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
/**
|
||||
* 功能说明:
|
||||
* 功能说明:
|
||||
*/
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
class ApiUserDeadTimeMonitorBatch extends BaseBatch {
|
||||
@@ -24,11 +24,11 @@ class ApiUserDeadTimeMonitorBatch extends BaseBatch {
|
||||
void process(Object args) {
|
||||
def sql = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia", "sa", "G00jia", "com.mysql.jdbc.Driver")
|
||||
def tenDayAgo = DateUtil.add(new Date(), Calendar.DATE, 10).format("yyyy-MM-dd")
|
||||
String body = "API用户到期通知:\n\n"
|
||||
String body = "API用户到期通知:\n\n"
|
||||
int count = 0
|
||||
sql.eachRow("select ID,COMPANY,DEAD_TIME from fangjia.api_user where status = 0 and DEAD_TIME = '${tenDayAgo}'",{row->
|
||||
println(row)
|
||||
String msg = "通知:用户ID为${row['ID']},用户名为:${row['COMPANY']}的API账户将于${((Date)row['DEAD_TIME']).format("yyyy年MM月dd日")}到期!"
|
||||
String msg = "通知:用户ID为${row['ID']},用户名为:${row['COMPANY']}的API账户将于${((Date)row['DEAD_TIME']).format("yyyy年MM月dd日")}到期!"
|
||||
body = body + msg + "\n\n"
|
||||
println(msg)
|
||||
count++
|
||||
@@ -37,7 +37,7 @@ class ApiUserDeadTimeMonitorBatch extends BaseBatch {
|
||||
JSONObject requestBody = new JSONObject()
|
||||
requestBody.put("msgtype", "markdown")
|
||||
JSONObject markdown = new JSONObject()
|
||||
markdown.put("title", "API监控")
|
||||
markdown.put("title", "API监控")
|
||||
markdown.put("text", body)
|
||||
requestBody.put("markdown", markdown)
|
||||
println(requestBody.toString())
|
||||
|
||||
+5
-5
@@ -12,7 +12,7 @@ import groovy.sql.Sql
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
/**
|
||||
* 功能说明:API使用状态监控脚本
|
||||
* 功能说明:API使用状态监控脚本
|
||||
*/
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
class OpenApiUseStatMonitorBatch extends BaseBatch {
|
||||
@@ -55,8 +55,8 @@ class OpenApiUseStatMonitorBatch extends BaseBatch {
|
||||
}
|
||||
}
|
||||
|
||||
def title = "API性能监控"
|
||||
String body = """失败过多警报[${startDate.format("HH:mm")}-${endDate.format("HH:mm")}]\n"""
|
||||
def title = "API性能监控"
|
||||
String body = """失败过多警报[${startDate.format("HH:mm")}-${endDate.format("HH:mm")}]\n"""
|
||||
def hasWarn = false
|
||||
def sql = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia", "sa", "G00jia", "com.mysql.jdbc.Driver")
|
||||
statMap.each {String userId,List<BasicDBObject> logs->
|
||||
@@ -84,9 +84,9 @@ class OpenApiUseStatMonitorBatch extends BaseBatch {
|
||||
if(!name){
|
||||
name = userId
|
||||
}
|
||||
item = "> - 用户ID:${userId},用户名:${name}, 访问异常,访问次数:${sucCount + errCount},失败次数:${errCount},失败率:${errRate}\n\n"
|
||||
item = "> - 用户ID:${userId},用户名:${name}, 访问异常,访问次数:${sucCount + errCount},失败次数:${errCount},失败率:${errRate}\n\n"
|
||||
}else{
|
||||
item = "> - 用户ID:${userId},访问异常,访问次数:${sucCount + errCount},失败次数:${errCount},失败率:${errRate}\n\n"
|
||||
item = "> - 用户ID:${userId},访问异常,访问次数:${sucCount + errCount},失败次数:${errCount},失败率:${errRate}\n\n"
|
||||
}
|
||||
body = body + item
|
||||
hasWarn = true
|
||||
|
||||
+5
-5
@@ -11,7 +11,7 @@ import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.util.DbUtils
|
||||
|
||||
/**
|
||||
* 填充小区的楼栋总数
|
||||
* 填充小区的楼栋总数
|
||||
* @author gaojuan
|
||||
*
|
||||
*/
|
||||
@@ -28,10 +28,10 @@ class AddDistrictLouDongCount {
|
||||
def allCity = CommonUtils.findAllCity();
|
||||
def cat = "district";
|
||||
def field = ["city","region","name", "loudong_count", "alias"]
|
||||
def siteNames = ["链家在线","搜狐焦点"]
|
||||
def siteNames = ["链家在线","搜狐焦点"]
|
||||
for(def city : allCity){
|
||||
def ret = CommonDatautils.findAllDistricts(city, "district", field);
|
||||
println "[ ${city} ] : "+ret.count();
|
||||
println "[ ${city} ] : "+ret.count();
|
||||
def distIndex = 0;
|
||||
for(def m : ret){
|
||||
distIndex++;
|
||||
@@ -187,9 +187,9 @@ class AddDistrictLouDongCount {
|
||||
}
|
||||
|
||||
def getNewValue(result){
|
||||
def values = result.get("链家在线");
|
||||
def values = result.get("链家在线");
|
||||
if(!values){
|
||||
values = result.get("搜狐焦点");
|
||||
values = result.get("搜狐焦点");
|
||||
}
|
||||
return values[0];
|
||||
}
|
||||
|
||||
+5
-5
@@ -10,7 +10,7 @@ import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
import org.bson.types.ObjectId
|
||||
/**
|
||||
* api调用量每半个月自动添加一次
|
||||
* api调用量每半个月自动添加一次
|
||||
* @author gaojuan
|
||||
*
|
||||
*/
|
||||
@@ -28,11 +28,11 @@ class AutoAddApiCountBatch extends BaseBatch{
|
||||
|
||||
println "running date:"+new Date();
|
||||
|
||||
//查询相关接口自增配置
|
||||
//查询相关接口自增配置
|
||||
def addMap = getApiAddPrecent();
|
||||
//查询所有接口调用量
|
||||
//查询所有接口调用量
|
||||
def allRows = findApiCount();
|
||||
//获取新调用量
|
||||
//获取新调用量
|
||||
allRows = autoAddCalCount(allRows, addMap);
|
||||
|
||||
saveResult(allRows);
|
||||
@@ -69,7 +69,7 @@ class AutoAddApiCountBatch extends BaseBatch{
|
||||
println retMap.service_id +" " +addCount+" "
|
||||
|
||||
}else{
|
||||
println "没找到相关自增配置:"+retMap.service_id
|
||||
println "没找到相关自增配置:"+retMap.service_id
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user