diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml
index b98b25ab1..e639f76e3 100644
--- a/projects/fangjia-script/pom.xml
+++ b/projects/fangjia-script/pom.xml
@@ -530,6 +530,13 @@
+
+
+ org.bouncycastle
+ bcpkix-jdk15to18
+ 1.70
+
+
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy
index 0bf6ec4b9..4473e1e49 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy
@@ -1,15 +1,14 @@
package com.fangjia.script.ccbevaluation
-import com.fangjia.script.util.DateUtils
+
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
-
/**
* ÉîÛÚ¹«Öںųɽ»Êý¾Ý¼Èë
*/
-def datas = ExcelUtils.read("D:\\Êý¾Ý\\½¨ÐÐ\\·¿²ú¹ÀÖµ\\³É½»\\2024Äê9ÔÂÉîÛÚ¶þÊÖ·¿³É½».xlsx", 0, 1)
+def datas = ExcelUtils.read("D:\\Êý¾Ý\\2024Äê10ÔÂÉîÛÚ¶þÊÖ·¿³É½».xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy
index f75cbb9f1..c65f09175 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy
@@ -10,12 +10,10 @@ import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.DateUtils
-import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
-import com.rabbitmq.client.impl.AMQImpl
import groovy.sql.Sql
import org.bson.types.ObjectId
@@ -23,11 +21,9 @@ import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
-import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
-
/**
* ½¨ÐÐÄ£Ð͹ÀÖµÏîÄ¿
* ³É½»Êý¾Ýͬ²½
@@ -71,11 +67,11 @@ class BigdataDealBatch extends BaseBatch{
def citys = CcbCommonUtils.getRunningCitys(args)
println "curDate:"+curDate
-// def allCitys = CommonDatautils.findAllCity()
-// for(def city : allCitys){
-// addDealPriceMatch(city);
-// }
-// updateCrawlTradeDate()//Ð޸ĺóץȡµÄ³É½»£¬½«Ê±¼ä¸ÄΪ±¾ÔÂ
+ def allCitys = CommonDatautils.findAllCity()
+ for(def city : allCitys){
+ addDealPriceMatch(city);
+ }
+ updateCrawlTradeDate()//Ð޸ĺóץȡµÄ³É½»£¬½«Ê±¼ä¸ÄΪ±¾ÔÂ
def rows = []
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy
index a649c273c..9db36e3d8 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy
@@ -2656,12 +2656,12 @@ class CommonDatautils {
}
- public static def queryDistrictPrices(city, region, name, estateType, startDate, endDate){
+ public static def queryDistrictPrices(city, region, name, startDate, endDate){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("region", region);
queryCondition.put("district_name", name);
- queryCondition.put("estate_type", estateType);
+// queryCondition.put("estate_type", estateType);
queryCondition.put('weekend', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lte', DateUtils.parseFromTightYYYYMMDD(endDate)));
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
def prices = [];
@@ -4107,6 +4107,150 @@ class CommonDatautils {
}
+
+ static def findHouseCountByDate(def city, def region, def name, def alias, def cat, def startDate){
+
+ QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true)
+
+ QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
+ .must(QueryBuilders.termQuery("region", region))
+ .must(QueryBuilders.termQuery("cat", cat)).must(date_range)
+
+ if(name || alias){
+ if(!alias){
+ alias = []
+ }
+ if(!alias.contains(name)){
+ alias.add(name)
+ }
+
+ if(name.contains("ÅªÐ¡Çø")){
+ def newName = name.replace("ÅªÐ¡Çø","Ū")
+ if(!alias.contains(newName)){
+ alias << newName
+ }
+ }
+
+ if(alias && alias.size()>0){
+ BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
+ for(def s : alias){
+ inBuilder.should(QueryBuilders.matchQuery("district_name", s));
+ }
+ queryBuilder.must(inBuilder)
+ }
+ }
+
+ def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
+ def index = new Random().nextInt(5);
+ def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
+ return houses.size();
+ }
+
+
+ static def findHousePriceByDate(def city, def region, def name, def alias, def cat, def startDate){
+
+ QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true)
+
+ QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
+ .must(QueryBuilders.termQuery("region", region))
+ .must(QueryBuilders.termQuery("cat", cat)).must(date_range)
+
+ if(name || alias){
+ if(!alias){
+ alias = []
+ }
+ if(!alias.contains(name)){
+ alias.add(name)
+ }
+
+ if(name.contains("ÅªÐ¡Çø")){
+ def newName = name.replace("ÅªÐ¡Çø","Ū")
+ if(!alias.contains(newName)){
+ alias << newName
+ }
+ }
+
+ if(alias && alias.size()>0){
+ BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
+ for(def s : alias){
+ inBuilder.should(QueryBuilders.matchQuery("district_name", s));
+ }
+ queryBuilder.must(inBuilder)
+ }
+ }
+
+ def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
+ def index = new Random().nextInt(5);
+ def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
+ def prices = []
+ if(houses){
+ for (int k=0;k
+ index++
+ println index
+ rets << [map.cityname, map.adname, map.name, map.address, map.location.x, map.location.y]
+ })
+ return rets;
+ }
+
+
+ static def findAllNewPoiDatas(city){
+ String sql = "select pname,cityname,adname,name,address,location,type,id from poi_amap_2024 where cityname='${city}'"
+ def rets = []
+ def index= 0
+ ElasticUtils.executeSqlQuery(sql, { map->
+// if(map.address && map.address.contains("Îå½Ç³¡")){
+ index++
+ rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y]
+// }
+ })
+ return rets;
+ }
+
+
+ static def findOneThirdNetWorkOffice(city, region, name){
+ String sql = "select location from office where city='${city}' and region='${region}' and name='${name}'"
+ def map = [:]
+
+ ElasticUtils.executeSqlQuery(sql, { m->
+ if(m.location && !map){
+ map = ["lng":m.location.x, "lat":m.location.y]
+ }
+ })
+ return map
+ }
+
+
+ static def findThirdNetWorkOffice(){
+ String sql = "select city,region,name,address,location from office where city='ÑĮ̀' or city='±±¾©'"
+ def rets = []
+ def index= 0
+ ElasticUtils.executeSqlQuery(sql, { map->
+ index++
+ println index
+ if(map.location){
+ rets << [map.city, map.region, map.name, map.address, map.location.lon, map.location.lat]
+ }else{
+ rets << [map.city, map.region, map.name, map.address, "", ""]
+ }
+
+ })
+ return rets;
+ }
/**
* ÊÇ·ñÊDZðÊû
* @return
@@ -4773,6 +4917,21 @@ class CommonDatautils {
return cityMap
}
+ static def findFullNameCitys(){
+ def q = new BasicDBObject()
+ q.put("type","³ÇÊÐ")
+
+ def keys = new BasicDBObject()
+ keys.put("city", 1)
+ keys.put("province", 1)
+ def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys)
+ def cityList = []
+ for(def d : dbcursor){
+ cityList << d
+ }
+ return cityList
+ }
+
static def queryDistNameByCity(){
def q = new BasicDBObject()
@@ -4991,6 +5150,34 @@ class CommonDatautils {
}
+ static def queryFullRegionTownByCity(city, fields){
+
+ def q = new BasicDBObject()
+ q.put("type","ÏçÕò½ÖµÀ")
+ q.put("city",city)
+
+ def keys = new BasicDBObject()
+ for(def key : fields){
+ keys.put(key,1)
+ }
+ def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
+ def townMap = [:]
+ for(def d : dbcursor){
+ if(d.district){
+ def list = townMap.get(d.district)
+ if(!list){
+ list = []
+ }
+ list << d
+ townMap.put(d.district, list)
+ }
+ }
+ println "[${city}] ÏçÕò½ÖµÀ ¼ÓÔØÍê³É "+new Date().format("yyyy-MM-dd HH:mm:ss")
+ return townMap
+ }
+
+
+
static def queryTownByCity(city){
@@ -5259,4 +5446,73 @@ class CommonDatautils {
return STATS_INDEX_COLL.findOne(q)
}
+
+ static def findAllThirdPrice(city, region, name, statDate){
+ def query = new BasicDBObject()
+ query.append("category", "district")
+ query.append("city", city)
+ query.append("region", region)
+ query.append("name", name)
+ query.append("s_date", new BasicDBObject(QueryOperators.GTE, statDate))
+ def dbCursor = thirdpartColl.find(query).sort(new BasicDBObject("s_date",-1))
+ def retMap = [:]
+ for(def d : dbCursor){
+
+ if(!retMap.get("xingfuli") && d.get("xingfuli_esf_price")){
+ retMap.put("xingfuli", d.get("xingfuli_esf_price"))
+ }
+ if(!retMap.get("anjuke") && d.get("anjuke_esf_price")){
+ retMap.put("anjuke", d.get("anjuke_esf_price"))
+ }
+
+ if(!retMap.get("soufang") && d.get("soufang_esf_price")){
+ retMap.put("soufang", d.get("soufang_esf_price"))
+ }
+ if(!retMap.get("lianjia") && d.get("lianjia_esf_price")){
+ retMap.put("lianjia", d.get("lianjia_esf_price"))
+ }
+ if(!retMap.get("lianjia") && d.get("beike_esf_price")){
+ retMap.put("lianjia", d.get("beike_esf_price"))
+ }
+
+ if(!retMap.get("cityhouse") && d.get("cityhouse_esf_price")){
+ retMap.put("cityhouse", d.get("cityhouse_esf_price"))
+ }
+ if(!retMap.get("jiwu") && d.get("jiwu_esf_price")){//ʹÓüªÎÝÌæ»»·¿Ì½ ´ýץȡ
+ retMap.put("jiwu", d.get("jiwu_esf_price"))
+ }
+
+ if(!retMap.get("house365") && d.get("365house_esf_price")){//ʹÓÃÌÔ·¿365Ìæ»»·¿¶à¶à
+ retMap.put("house365", d.get("365house_esf_price"))
+ }
+
+ if(!retMap.get("maitian") && d.get("maitian_esf_price")){
+ retMap.put("maitian", d.get("maitian_esf_price"))
+ }
+ if(!retMap.get("tc_58") && d.get("58tc_esf_price")){
+ retMap.put("tc_58", d.get("58tc_esf_price"))
+ }
+ }
+
+ return retMap;
+ }
+
+ static getSeaweedArroundByCat(city, lng, lat, cat) {
+ try {
+ def query = new BasicDBObject()
+ query.append("cat", cat)
+ query.append("city", city)
+ query.append("visible", 0)
+ DBObject sdq = new BasicDBObject();
+ sdq.put("sd.point", lat + "," + lng);
+ sdq.put("sd.sfield", "lat_lng");
+ sdq.put("sd.distance", String.valueOf((double) 1000 / 1000.0));
+ query.append('$sd', sdq)
+ return DbUtils.seaweed.find(query).toList()
+ } catch (Exception e) {
+ e.printStackTrace()
+ }
+ return 0
+ }
+
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy
index 954e0402b..a42b84aeb 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy
@@ -4,13 +4,11 @@ import com.fangjia.core.util.DateUtil
import com.fangjia.script.averager.InsertMonitorCalcCommandBatch
import com.fangjia.script.averager.InsertMonitorSquareRentCalcCommandBatch
import com.fangjia.script.commData.CommonDatautils
-import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.MongoClient
-import com.rabbitmq.client.AMQP
class AddCityRegionPrice {
@@ -24,7 +22,7 @@ class AddCityRegionPrice {
static main(args) {
def task = new AddCityRegionPrice()
// task.addRegionPrice()
- task.addCityprice()
+// task.addCityprice()
// task.addDistrictprice()
// task.addCityRentPrice()
// task.addRegionRentPrice()
@@ -32,7 +30,7 @@ class AddCityRegionPrice {
// task.updateRegionRentData()
// task.updateDealPriceAreaType()
-// task.checkCityImport()
+ task.checkCityImport()
}
def addRegionPrice(){
@@ -255,7 +253,7 @@ class AddCityRegionPrice {
def startObj = CommonDatautils.findStatIndexBySort("³ÇÊÐ", c, null, 1)
def endObj = CommonDatautils.findStatIndexBySort("³ÇÊÐ", c,null, -1)
if(startObj && endObj){
- def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202409")
+ def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202410")
months.each {m->
def obj = CommonDatautils.findStatIndexByMonth("³ÇÊÐ", c,null, m.toInteger())
if(!obj){
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy
index 12879d5ed..72d64ea09 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy
@@ -30,6 +30,9 @@ class AddDistrictShapeBatch extends BaseBatch{
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
+
+ public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
+
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
@@ -54,8 +57,8 @@ class AddDistrictShapeBatch extends BaseBatch{
//
// }
-// checkRangeCount()
- synch100()
+ checkRangeCount()
+// synch100()
}
def baiduMatch(city,districts){
@@ -378,20 +381,21 @@ class AddDistrictShapeBatch extends BaseBatch{
def checkRangeCount(){
- def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
+// def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
+ def dbcursor = administrative_mapping_coll.find(new BasicDBObject("type","ÏçÕò½ÖµÀ")).batchSize(100)
def count = 0
def index =0
for(def d : dbcursor){
index ++
- if(d.location){
- def coordinates = d.location.coordinates
+ if(d.shape){
+ def coordinates = d.shape.coordinates
if(coordinates.size()>1){
count++
- println count +" "+ d.city +" "+d.name
+ println count +" "+ d.city +" "+d.street
}
}else{
- println d.city + " "+d.name
+ println d.city + " "+d.street
}
if(index%1000==0){
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy
index 8e1a842fe..809fa216c 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy
@@ -15,7 +15,6 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
-
/**
* Ð¡Çø×Ô¶¯Èë¿â£¨ÐÂÔöÐ¡Çø£¬Ã¿ÔÂÔËÐÐÒ»´Î£©
*/
@@ -36,22 +35,39 @@ class AutoAddDistrictBatch extends BaseBatch{
@Override
void process(Object args) {
- //²éÕÒµÚÈý·½Æ¥ÅäûÓÐÆ¥ÅäÉϵÄÐ¡Çø
- def citys = ["ÉϺ£"]//CommonDatautils.findAllCity()
- def list = []
- list << ["³ÇÊÐ", "ÇøÓò", "Ãû³Æ", "³É½»Á¿", "·¿Ô´Á¿", "°Ù¶ÈµØÍ¼", "¸ßµÂµØÍ¼"]
- for(def c : citys){
- def noMatchDatas = findNoMatchResult(c)
- println c +" оÉË㷨δƥÅäµÄÊý¾ÝÁ¿£º"+noMatchDatas.size()
- def rets = checkDatas(noMatchDatas.keySet())
- list.addAll(rets)
- }
+// //²éÕÒµÚÈý·½Æ¥ÅäûÓÐÆ¥ÅäÉϵÄÐ¡Çø
+// def citys = CommonDatautils.findAllCity()
+// def list = []
+// list << ["³ÇÊÐ", "ÇøÓò", "Ãû³Æ", "³É½»Á¿", "·¿Ô´Á¿", "°Ù¶ÈµØÍ¼", "°Ù¶ÈÀàÐÍ","¸ßµÂµØÍ¼", "¸ßµÂÀàÐÍ"]
+// for(def c : citys){
+// def noMatchDatas = findNoMatchResult(c)
+// println c +" оÉË㷨δƥÅäµÄÊý¾ÝÁ¿£º"+noMatchDatas.size()
+// def rets = checkDatas(noMatchDatas.keySet())
+// list.addAll(rets)
+// }
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
if (CommonDatautils.isWindow()) {
- fileName = "D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\Ð¡Çø\\µÚÈý·½´ýÈë¿âÊý¾Ý.xlsx"
+ fileName = "D:\\Êý¾Ý\\add_dist_info_1.xlsx"
}
- ExcelUtils.write(fileName, list, "´ýÈë¿âÊý¾Ý")
+// ExcelUtils.write(fileName, list, "´ýÈë¿âÊý¾Ý")
+
+
+ check(fileName)
+ }
+
+ def check(fileName){
+ ExcelUtils.multiThreadingModify(fileName,{d->
+ def dists = CommonDatautils.findAllDistrictByName(d[0], d[1], d[2], "district")
+ if(dists){
+ d['J'] = "¿âÖÐÒÑ´æÔÚ"
+ }else{
+ dists = CommonDatautils.findDistrictByAlias(d[0], d[1], d[2], "district")
+ if(dists){
+ d['J'] = "¿âÖбðÃûÖдæÔÚ"
+ }
+ }
+ })
}
def checkDatas(keysets){
@@ -72,11 +88,23 @@ class AutoAddDistrictBatch extends BaseBatch{
//¼ì²âÓÐûÓз¿Ô´
def house_count = CommonDatautils.findHouseCount(arrs[0], arrs[1], arrs[2])
//¼ì²é°Ù¶ÈµØÍ¼ÓÐûÓÐ
- def baidu_count = findBaiduCount(arrs[0], arrs[1], arrs[2])
+ def baidu_obj = findBaiduCount(arrs[0], arrs[1], arrs[2])
+ def baidu_count = 0
+ def baidu_type = null
+ if(baidu_obj){
+ baidu_count = 1
+ baidu_type = baidu_obj.std_tag
+ }
//¼ì²é¸ßµÂµØÍ¼ÓÐûÓÐ
- def gaode_count = findGaodeCount(arrs[0], arrs[1], arrs[2])
+ def gaode_obj = findGaodeCount(arrs[0], arrs[1], arrs[2])
+ def gaode_count = 0
+ def gaode_type = null
+ if(gaode_obj){
+ gaode_count = 1
+ gaode_type = gaode_obj.type
+ }
println "[ "+totalCount.get()+"/"+noMatchCount+" ]"+key +" trade_count:"+trade_count+" house_count:"+house_count+" baidu_count:"+baidu_count+" gaode_count:"+gaode_count
- rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_count]
+ rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_type, gaode_count, gaode_type]
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -123,18 +151,18 @@ class AutoAddDistrictBatch extends BaseBatch{
if(fullDistName){
def obj = findGaoDeDatas(fullCityName, fullDistName, name)
if(obj){
- return 1
+ return obj
}
}
}
- return 0
+ return null
}
def findGaoDeDatas(city, region, name){
- String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
+ String sql = "select name,type from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
def obj = null
ElasticUtils.executeSqlQuery(sql,{ d->
- if(d.name == name && d.type.contains("סլ") && d.shape){
+ if(d.name == name && d.type.contains("סլ") ){ //&& d.shape
obj = d
}
})
@@ -228,7 +256,7 @@ class AutoAddDistrictBatch extends BaseBatch{
q.put("district", district)
q.put("name", name)
q.put('std_tag', '·¿µØ²ú;×¡Õ¬Çø')
- def count = BAIDU_POI_COLL.count(q)
- return count;
+ def obj = BAIDU_POI_COLL.findOne(q)
+ return obj;
}
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy
index b6b36149f..521be8be2 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy
@@ -5,21 +5,15 @@ import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
-import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.linxsh.util.AddressUtil
-import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
-import com.mongodb.Bytes
import com.mongodb.DBCollection
import net.sf.json.JSONObject
-import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
-import org.joda.time.Instant
-import java.time.format.DateTimeFormatter
import java.util.regex.Pattern
class DevProcess {
@@ -64,6 +58,7 @@ class DevProcess {
// task.deleteAddr()
// task.deleteAddr2()
task.synchDevTo100()
+// task.statShape()
// task.addNewDev2()
// task.addAlias()
@@ -87,6 +82,8 @@ class DevProcess {
// task.matchDevShape()//Æ¥Å俪·¢ÇøÎ§À¸
// task.addQiDevResult() //»ñÈ¡ÆëÁºÊý¾Ý½á¹û
+
+// task.updateDevShape()//Ð޸Ŀª·¢Çø½á¹¹
}
@@ -794,6 +791,18 @@ class DevProcess {
development_zone_CONN.insert(rets)
}
+ def statShape(){
+ def dbcursor = DEVELOPMENT_ZONE_43_COLL.find(new BasicDBObject("shape", new BasicDBObject('$exists', true)))
+ def index = 0
+ for(def r : dbcursor){
+ index ++
+ def coordinates = r.shape.coordinates
+ println index+" "+r.name+" "+coordinates.size()
+
+
+ }
+ }
+
def matchDevAndAddAddress(){
ExcelUtils.modify("D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢Çø\\ÐÂÅÜÊ¡·Ý\\¸£½¨ÖØÇìÈ¥ÖØºóµØÖ·.xlsx",{d->
@@ -1148,4 +1157,26 @@ class DevProcess {
}
},0)
}
+
+ def updateDevShape(){
+ def q = new BasicDBObject()
+ q.put("shape", new BasicDBObject('$exists', true))
+ def dbcursor = DEVELOPMENT_ZONE_43_COLL.find(q)
+ for(def d : dbcursor){
+ if(d.name!="¸£Öݳ¤ÀÖ¹ú¼Ê»ú³¡×ۺϱ£Ë°Çø"){
+ def udata = new BasicDBObject()
+ if(d.shape.coordinates){
+ d.shape.coordinates = [d.shape.coordinates]
+ udata.put("shape", d.shape)
+ }
+ if(d.bd_shape && d.bd_shape.coordinates){
+ d.bd_shape.coordinates = [d.bd_shape.coordinates]
+ udata.put("bd_shape", d.bd_shape)
+ }
+ def updateQuery = new BasicDBObject("_id",d._id)
+ DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata))
+ }
+
+ }
+ }
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy
index 49d2547b7..aaf80284e 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy
@@ -7,7 +7,6 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
-import com.rabbitmq.client.AMQP
import net.sf.json.JSONObject
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
@@ -33,7 +32,7 @@ class DevUpdateTestBatch {
static main(args) {
def task = new DevUpdateTestBatch()
- def path = "D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\¿ª·¢Çø\\¿ª·¢Çø-Ô°Çø.xlsx"//¿ª·¢Çø-Ô°Çø.xlsx
+ def path = "D:\\Êý¾Ý\\¿ª·¢Çø-Ô°Çø.xlsx"//¿ª·¢Çø-Ô°Çø.xlsx
// task.addDev(path)
task.addDevAddress(path)
//
@@ -168,7 +167,7 @@ class DevUpdateTestBatch {
println d['A'] +" ²»´æÔÚ"
}
}
- },2)
+ },0)
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictRepeatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictRepeatBatch.groovy
index 3fb5a0d48..74d0ddbcb 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictRepeatBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictRepeatBatch.groovy
@@ -1,17 +1,28 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
+import com.fangjia.script.gj.FiledStrand
+import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CosineSimilarAlgorithm
+import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
+import com.fangjia.script.util.MongoUtils
+import com.mongodb.BasicDBObject
+import com.mongodb.DBCollection
+import org.apache.commons.lang.SerializationUtils
+import org.bson.types.ObjectId
/**
* ²éÕÒÖØ¸´Ð¡Çø
*/
class DistrictRepeatBatch {
+ DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
+
static main(args) {
def task = new DistrictRepeatBatch()
- task.run()
+// task.run()
+ task.resultProcess()
}
def run(){
@@ -65,4 +76,118 @@ class DistrictRepeatBatch {
return rows;
}
+
+ def resultProcess(){
+ ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\Ð¡ÇøÖØ¸´Êý¾Ý´¦Àí.xlsx",{d->
+ if(d["O"] && !d["O"].contains("²»´¦Àí") && !d['R']){
+ if(d['O']=="KÁиÄΪÉÌס" || d["O"] == "ÐÞ¸ÄKÁÐΪ±ðÊû"){
+ def updteId = d['H']
+ def q = new BasicDBObject()
+ q.put("_id", new ObjectId(updteId))
+
+ def udata = new BasicDBObject()
+ if(d['O']=="KÁиÄΪÉÌס" ){
+ udata.put("estate_type2", "ÉÌס")
+ }else{
+ udata.put("estate_type2", "±ðÊû")
+ }
+ udata.put("m_date", new Date())
+ DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
+ saveOptLog(d["I"], d["J"], d["K"], "update")
+ d["R"] = "¿âÖÐÒÑ´¦Àí"
+ }
+ else if(d["O"] == "¶¼É¾"){
+ def udata = new BasicDBObject()
+ udata.put("status", 1)
+ udata.put("m_date", new Date())
+ DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["A"])), new BasicDBObject('$set', udata))
+ saveOptLog(d["B"], d["C"], d["D"], "remove")
+ DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["H"])), new BasicDBObject('$set', udata))
+ saveOptLog(d["I"], d["J"], d["K"], "remove")
+ d["R"] = "¿âÖÐÒÑ´¦Àí"
+ }
+ else if(d["O"]=="ɾ³ýKÁÐÐ¡Çø"){
+ def udata = new BasicDBObject()
+ udata.put("status", 1)
+ udata.put("m_date", new Date())
+ DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["H"])), new BasicDBObject('$set', udata))
+ saveOptLog(d["I"], d["J"], d["K"], "remove")
+
+ def obj = CommonDatautils.findDistrictById(d['A'])
+ if(d['C']==d['J']){
+ def alias = []
+ def addr_alias = []
+ if(d['D']!=d['K']){
+ def s1 = d['D'].replace(d['K'], "")
+ def s2 = d['K'].replace(d['D'], "")
+ def addNameAlias = true
+ if(s1=="Ð¡Çø" || s2=="Ð¡Çø" || s1=="±ðÊû" || s2 == "±ðÊû" || s1=="ÉÌס" || s2=="ÉÌס"){
+ addNameAlias = false
+ }
+ if(addNameAlias){
+
+ if(obj.alias){
+ alias = SerializationUtils.clone(obj.alias)
+ }
+ if(!alias.contains(d['K'])){
+ alias << d['K']
+ }
+ }
+ }
+
+ def address = null
+ if(d['M'] && d['F']!=d['M']){
+ def formatAddress = AddressUtil.formatAddress(d['A'], d['M'])
+ if(formatAddress && FiledStrand.isStandardRoad(formatAddress)){
+ if((d['F'] && !d['F'].contains(formatAddress)) || !d['F']){
+ if((d['F'] && !FiledStrand.isStandardRoad(d['F'])) || !d['F']){
+ address = formatAddress
+ }else{
+ if(obj.addr_alias){
+ addr_alias = SerializationUtils.clone(obj.addr_alias)
+ }
+ if(!addr_alias.contains(formatAddress)){
+ addr_alias << formatAddress
+ }
+ }
+ }
+ }
+ }
+
+ if(address || (alias && alias!=obj.alias) || (addr_alias && addr_alias!=obj.addr_alias)){
+ def udata2 = new BasicDBObject()
+ if(alias){
+ udata2.put("alias", alias)
+ }
+ if(addr_alias){
+ udata2.put("addr_alias", addr_alias)
+ }
+ if(address){
+ udata2.put("address", address)
+ }
+ udata2.put("m_date", new Date())
+ DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d["A"])), new BasicDBObject('$set', udata2))
+ }
+ }
+
+ d["R"] = "¿âÖÐÒÑ´¦Àí"
+ }
+ }
+ })
+ }
+
+ def saveOptLog(city, region, name, type){
+ def obj = new BasicDBObject();
+ obj.put("city",city);
+ obj.put("region",region);
+ obj.put("district_name",name);
+ obj.put("category","district");
+ obj.put("c_date", new Date())
+ obj.put("m_date", new Date())
+ obj.put("type", type)
+ obj.put("user_name", "¸ß¾ê")
+ obj.put("origin", "script")
+ OPERATION_LOG_COLL.save(obj)
+ }
+
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy
new file mode 100644
index 000000000..822eee441
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy
@@ -0,0 +1,349 @@
+package com.fangjia.script.gj.data_handle
+
+import com.fangjia.script.BaseBatch
+import com.fangjia.script.BatchOwner
+import com.fangjia.script.Watcher
+import com.fangjia.script.commData.CommonDatautils
+import com.fangjia.script.common.DataHandleUtils
+import com.fangjia.script.gj.xiamen.ElasticUtils89
+import com.fangjia.script.looplineinfo.InsidePolygon
+import com.fangjia.script.util.ExcelUtils
+import org.elasticsearch.action.bulk.BulkRequest
+import org.elasticsearch.action.bulk.BulkResponse
+import org.elasticsearch.action.index.IndexRequest
+import org.elasticsearch.action.support.WriteRequest
+import org.elasticsearch.client.RequestOptions
+import org.elasticsearch.common.xcontent.XContentType
+
+import java.util.concurrent.CopyOnWriteArrayList
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.concurrent.atomic.AtomicLong
+/**
+ * аæpoi´¦ÀíÈë¿â£¨µÚ¶þÅú£©
+ */
+@BatchOwner(Watcher.GaoJuan)
+class NewGaoDePoiBatch extends BaseBatch{
+
+ public static def version = new Date().getTime()
+ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
+
+ def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
+
+ public static def TAGS = []
+
+ static main(args) {
+ runBatch(new NewGaoDePoiBatch(), args)
+ }
+
+ @Override
+ void process(Object args) {
+ run()
+ }
+
+ def run(){
+ if (CommonDatautils.isWindow()) {
+ TAGS = getAllPoiType("D:\\Êý¾Ý\\¸£½¨Ê¡_POI_GCJ02_20240630\\POI·ÖÀà°×Ãûµ¥.xlsx")
+ }else{
+ TAGS = getAllPoiType("/home/goojia/gj/POI_TYPES.xlsx")
+ }
+
+
+ def cityList = CommonDatautils.findFullNameCitys().reverse()
+ for(def cityObj : cityList){
+// if(cityObj.city == "ºúÑîºÓÊÐ"){
+ def results = CommonDatautils.findAllNewPoiDatas(cityObj.city)
+ println cityObj.province+" " +cityObj.city+" "+results.size()
+ if(results){
+ def rows = running(cityObj.city, results)
+ save(rows)
+ }
+// }
+ }
+ }
+ def getAllPoiType(filePath){
+ def poiTypes = ExcelUtils.read(filePath, 0, 1)
+ def types = []
+ for(def poi : poiTypes){
+ types << poi['A']
+ }
+ return types
+ }
+
+ def running(city,results){
+ def rets = new CopyOnWriteArrayList()
+ def totalCount = new AtomicLong()
+ final CountDownLatch latch = new CountDownLatch(results.size())
+ def townMap = CommonDatautils.queryTownByCity(city, ["city","district","mzb_district_name", "street", "shape","alias"])
+ def mzbRegionMap = regionFullNameMap.get(city)
+ for(def i=0;i=1){
+ saveObj.type = arrs[0]
+ }
+ if(arrs.size()>=2){
+ saveObj.secondary_type = arrs[1]
+ }else{
+ saveObj.secondary_type = saveObj.type
+ }
+ if(arrs.size()>=3){
+ def smallstr = arrs[2]
+ def smallArr = null
+ if(smallstr.contains("|")){
+ smallArr = smallstr.split("|")
+ }
+ if(smallArr && smallArr.size()>=1){
+ saveObj.small_type = smallArr[0]
+ }else{
+ saveObj.small_type = smallstr
+ }
+ }else{
+ saveObj.small_type = saveObj.secondary_type
+ }
+
+ saveObj.cat = saveObj.type+";"+saveObj.secondary_type+";"+saveObj.small_type
+ return saveObj
+ }
+
+ def addTown(saveObj, towns, location){
+ if(towns){
+ for(def town : towns){
+ if(town.shape && town.shape.coordinates){
+ def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
+ if(flag){
+ saveObj.township = town.street
+ break
+ }
+ }
+ }
+// if(map.address && !saveObj.township){
+// for(def town : towns){
+// if(map.address.contains(town.street)){
+// saveObj.township = town.street
+// break
+// }
+// }
+// if(!saveObj.township && map.address && (map.address.contains("½ÖµÀ") || map.address.contains("Õò") || map.address.contains("Ïç"))){
+// println map.address +" º¬½ÖµÀ,Õò£¬Ïç"
+// }
+// }
+ }
+ return saveObj
+ }
+
+ def addressWash(saveObj, map, mzbRegionMap, towns){
+ if(map.address.length()<3){
+ saveObj.address = ""
+ }else{
+ def address = map.address
+ def i = 0
+ while(i<5){
+ if(address && address.toString().startsWith(map.province)){
+ address = address.toString().substring(map.province.toString().length())
+ }
+ if(address && address.toString().startsWith(map.city)){
+ address = address.toString().substring(map.city.toString().length())
+ }
+
+ if(address && saveObj.district){
+ def tjjObj = mzbRegionMap.get(saveObj.district)
+ if(tjjObj){
+ def alias = null
+ if(tjjObj.alias){
+ alias = new CopyOnWriteArrayList<>(tjjObj.alias);
+ }else{
+ alias = new CopyOnWriteArrayList<>();
+ }
+
+ if(!alias.contains(saveObj.district)){
+ alias << saveObj.district
+ }
+ if(alias.size()>=2){
+ alias = alias.sort(){ a, b ->
+ b.length().compareTo(a.length());
+ }
+ }
+
+ for(def regionName : alias){
+ if(address.startsWith(regionName) && (regionName.endsWith("Çø") || regionName.endsWith("ÏØ"))){
+ address = address.substring(regionName.toString().length())
+ break
+ }
+ }
+ }
+ }
+
+
+ if(address && address.toString().startsWith("ÊÐÏ½Çø")){
+ address = address.toString().substring(3)
+ }
+
+ if(address && towns){
+ for(def town : towns){
+ def townAlias = null
+ if(town.alias){
+ townAlias = new CopyOnWriteArrayList<>(town.alias);
+ }else{
+ townAlias = new CopyOnWriteArrayList<>();
+ }
+
+ if(!townAlias.contains(town.street)){
+ townAlias << town.street
+ }
+ if(townAlias.size()>=2){
+ townAlias = townAlias.sort(){ a, b ->
+ b.length().compareTo(a.length());
+ }
+ }
+
+ for(def townName : townAlias){
+ if(address.startsWith(townName)){
+ address = address.substring(townName.toString().length())
+// if(!saveObj.township){
+// saveObj.township = town.street
+// }
+ break
+ }
+ }
+ }
+ }
+ i++
+ }
+ if(address && address.startsWith("(") && address.endsWith(")")){
+ saveObj.address = address.substring(1, address.length()-1)
+ }else{
+ saveObj.address = address
+ }
+ }
+ return saveObj
+ }
+
+ def save(rows){
+ BulkRequest request = new BulkRequest();
+ int batchSize = 2000;
+ int processedCount = 0;
+ def client = ElasticUtils89.getClient("192.168.0.89")
+
+ rows.each{map->
+// def map = rows.get(key)
+
+ request.add(new IndexRequest("didata_map_poi_2024", "_doc", map.data_id).source(map, XContentType.JSON));
+
+ // ÅúÁ¿Ìá½»Êý¾Ý
+ if (request.numberOfActions() >= batchSize) {
+ request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+ BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
+ if (bulkResponse.hasFailures()) {
+ // ´¦Àí´íÎó
+ print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
+ } else {
+ processedCount += request.numberOfActions();
+ println "Processed " + processedCount + " documents "+new Date().format("yyyy-MM-dd HH:mm:ss")
+ }
+
+ request = new BulkRequest();
+ }
+ }
+
+ if (request.numberOfActions() > 0) {
+ request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+ BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
+ if (bulkResponse.hasFailures()) {
+ // ´¦Àí´íÎó
+ println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
+ } else {
+ processedCount += request.numberOfActions();
+ println "Processed " + processedCount + " documents"
+ }
+ }
+
+ }
+
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewPoiData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewPoiData.groovy
new file mode 100644
index 000000000..767209f83
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewPoiData.groovy
@@ -0,0 +1,175 @@
+package com.fangjia.script.gj.data_handle
+
+
+import com.fangjia.core.util.BaiDuApiUtils
+import com.fangjia.script.commData.CommonDatautils
+import com.fangjia.script.gj.district.FangjiaClientUtils
+import com.fangjia.script.linxsh.util.AddressUtil
+import com.fangjia.script.util.DataUtil
+import com.fangjia.script.util.DbUtils
+import com.fangjia.script.util.ExcelUtils
+import com.fangjia.script.util.GeoHelper
+import com.mongodb.BasicDBObject
+
+import java.util.concurrent.CopyOnWriteArrayList
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+
+class NewPoiData {
+
+ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
+
+ static main(args) {
+ def task = new NewPoiData()
+ task.run()
+ }
+ def run(){
+// export()
+// exportNetWorkOffice()
+// matchByCity()
+// noMatchData()
+ calcuteDistance()
+ }
+
+ def noMatchData(){
+ ExcelUtils.multiThreadingModify("d:\\Êý¾Ý\\δƥÅäд×ÖÂ¥2.xlsx", { d ->
+ if(d['H']!='µØÍ¼'){
+ def map = CommonDatautils.findOneThirdNetWorkOffice(d['A'], d['B'], d['C'])
+ if(map){
+ d['E'] = map.lng
+ d['F'] = map.lat
+ }
+ }
+ })
+ }
+ def matchByCity(){
+ ExcelUtils.multiThreadingModify("d:\\Êý¾Ý\\Èë¿âд×ÖÂ¥.xlsx",{d->
+ def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['A']+d['B']+d['C'], "office", false)
+ def formatAddress = null
+ if(d['D']){
+ formatAddress = AddressUtil.formatAddress(d['A'], d['D'])
+ d['G'] = formatAddress
+ }
+ if(!jsonObject && formatAddress){
+ if(DataUtil.isPreciseAddress(formatAddress)){
+ def address = d['A']
+ if(d['B']){
+ address = address + d['B']
+ }
+ address = address + formatAddress
+ jsonObject = FangjiaClientUtils.newMatch(d['A'], address, "office", false)
+ }
+ }
+ if(jsonObject) {
+ d['H'] = jsonObject.get("id")
+ d['I'] = jsonObject.get("city")
+ d['J'] = jsonObject.get("district")
+ d['K'] = jsonObject.get("name")
+ d['L'] = jsonObject.get("propertyType")
+ d['M'] = jsonObject.get("credit")
+ }
+ })
+ }
+
+
+ def export(){
+ def rets = []
+ rets << ["³ÇÊÐ", "ÇøÓò", "Ãû³Æ", "µØÖ·", "¾¶È", "γ¶È"]
+ rets.addAll(CommonDatautils.findNewPoiDatas())
+ ExcelUtils.write("d:\\Êý¾Ý\\poiд×ÖÂ¥.xlsx", rets, "д×ÖÂ¥")
+ }
+
+ def exportNetWorkOffice(){
+ def rets = []
+ rets << ["³ÇÊÐ", "ÇøÓò", "Ãû³Æ", "µØÖ·", "¾¶È", "γ¶È"]
+ rets.addAll(CommonDatautils.findThirdNetWorkOffice())
+ ExcelUtils.write("d:\\Êý¾Ý\\poiд×ÖÂ¥-ÍøÕ¾.xlsx", rets, "д×ÖÂ¥")
+ }
+
+ def calcuteDistance(){
+ ExcelUtils.multiThreadingModify("d:\\Êý¾Ý\\д×ÖÂ¥²âÊÔ-1118-ÐÂÔö.xlsx", { d ->
+ if(d['F']){
+ if(d['A'].contains("±±¾©")){
+ def distance = GeoHelper.GetDistance(d['F'].toString().toDouble(), d['E'].toString().toDouble(), 39.996743, 116.481075)
+ d['K'] = distance
+ }else if(d['A'].contains("ÑĮ̀")){
+ def distance = GeoHelper.GetDistance(d['F'].toString().toDouble(), d['E'].toString().toDouble(), 37.459645, 121.438616)
+ d['K'] = distance
+ }
+ }
+ }, 1)
+ }
+
+
+
+
+ def addDistrict(){
+ def path = "D:\\Êý¾Ý\\Èë¿âд×ÖÂ¥.xlsx"
+ def keys = new CopyOnWriteArrayList()
+ def index = 0
+ ExcelUtils.modify(path, { d->
+ def city = d['A']
+ def region = d['B']
+ def name = d['C']
+
+ def formatAddress = null
+ def address = d['D']
+ if(address){
+ if(d['G']){
+ formatAddress = AddressUtil.formatAddress(city, address)
+ }
+ if(!formatAddress){
+ if(address.contains("½»²æ¿Ú")){
+ formatAddress = address.substring(0, address.toString().indexOf("½»²æ¿Ú")+3)
+ }else if(address.contains("½»²æ")){
+ formatAddress = address.substring(0, address.toString().indexOf("½»²æ")+2)
+ }
+ }
+ }
+
+ Long glat = null
+ Long glng = null
+ Long blat = null
+ Long blng = null
+ if(d['K']){
+ glat = d['L'].toString().toDouble() * 10000000000L
+ glng = d['K'].toString().toDouble() * 10000000000L
+ def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
+ blng = baidu.x * 10000000000L
+ blat = baidu.y * 10000000000L
+ }
+
+ def estateType2 = "ÆÕͨסլ"
+ def key = city+"##"+region+"##"+name
+ if(name && !keys.contains(key)){
+ def dist = CommonDatautils.findDistrict(city, region, name, "district")
+ if(!dist){
+ keys << key
+ def data = new BasicDBObject();
+ data.put("name_cat",0)
+ data.put("status",0)
+ data.put("cat","office")
+ data.put("c_date", new Date())
+ data.put("m_date", new Date())
+ data.put("city", city)
+ data.put("region",region)
+ data.put("name", name)
+ data.put("estate_type2",estateType2)
+ data.put("address", formatAddress)
+ if(glat){
+ data.put("lat",glat)
+ data.put("lng",glng)
+ data.put("lat2",blat)
+ data.put("lng2",blng)
+ }
+ def markMap = [:]
+ markMap.put("data_source", "»úÆ÷Èë¿â")
+ data.put("mark_map",markMap)
+ index++
+ DbUtils.seaweed.save(data);
+ println index+" "+data
+ }
+ }
+ },0)
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy
new file mode 100644
index 000000000..21432c057
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiNineSplitBatch.groovy
@@ -0,0 +1,52 @@
+package com.fangjia.script.gj.data_handle
+
+
+import com.fangjia.script.commData.CommonDatautils
+import com.fangjia.script.util.ExcelUtils
+
+class PoiNineSplitBatch{
+
+ static main(args) {
+ def task = new PoiNineSplitBatch()
+ task.run()
+ }
+
+// @Override
+// void process(Object args) {
+// run()
+// }
+ def run(){
+ def i = 0;
+ while(i<360) {
+ def fileName = "/home/goojia/gj/new_poi/new_poi_address_split_${i}.xlsx"
+ if (CommonDatautils.isWindow()) {
+ fileName = "D:\\Êý¾Ý\\new_poi\\new_poi_address_split_${i}.xlsx"
+ }
+
+ File file = new File(fileName);
+ if (file.exists()) {
+ ExcelUtils.multiThreadingModify(fileName, { d ->
+ if (d['C'] && !d['G']) {
+ println d['C'] +" "+ d['E']
+ def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D'] + d['E'])
+ if (jsonObject) {
+ if (jsonObject.get("street") && jsonObject.get("street") != "null") {
+ d['G'] = jsonObject.get("street")
+ }
+ if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") {
+ d['H'] = jsonObject.get("streetNumber")
+ }
+ if (jsonObject.get("township") && jsonObject.get("township") != "null") {
+ d['I'] = jsonObject.get("township")
+ }
+ }
+ }
+ })
+ println fileName +" ¾Å¼¶²ð·ÖÍê³É"
+ }else{
+ println fileName +" Îļþ²»´æÔÚ"
+ }
+ i++
+ }
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy
index c78e0a84b..abce57fe0 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy
@@ -6,6 +6,7 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
+import org.apache.commons.io.FileUtils
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
@@ -41,12 +42,10 @@ class RegionStandardBatch extends BaseBatch{
def runCitys = ["ÖØÇì","ÈýÃ÷","ÁúÑÒ","ÄÏÆ½","ÕÄÖÝ","¸£ÖÝ","ÏÃÃÅ","ȪÖÝ","ÆÎÌï","ÄþµÂ"]
def index = 0
- def citys = ["ÅÊÖ¦»¨"]//CommonDatautils.findAllCity()
+ def citys = CommonDatautils.findAllCity()
for(def city : citys){
index ++
-// if(index<32){
-// continue
-// }
+
if(runCitys.contains(city)){
continue
}
@@ -57,19 +56,32 @@ class RegionStandardBatch extends BaseBatch{
}
if(fullCityName){
- def rMap = regionFullNameMap.get(fullCityName)
- if(rMap){
- def set = rMap.keySet()
- if(set.size()>1){
- def rets = findEsPoiByCity(fullCityName)
- println city +" "+rets.size()
- poiHaveRegionProcess(city, set, rMap, rets)
- }else{
+ def fileName = "/home/goojia/gj/poi/${fullCityName}.csv"
+ if (CommonDatautils.isWindow()) {
+ fileName = "D:\\Êý¾Ý\\poi\\${fullCityName}.csv"
+ }
+ File file=new File(fileName);
+ if(file.exists()){
+ def rMap = regionFullNameMap.get(fullCityName)
+ if(rMap){
+ def set = rMap.keySet()
+ if(set.size()>1){
+// def rets = findEsPoiByCity(fullCityName)
+ def rets = readCSV(fileName)
+ println city +" "+rets.size()
+ poiHaveRegionProcess(city, set, rMap, rets)
+ }else{
// haveStreetProcess(city, set, cat)
+
+ println "ûÕÒÇøÓò£º"+city
+ }
+ }else{
+ println "ûÕÒÇøÓò£º"+city
}
}else{
- println "ûÕÒÇøÓò£º"+city
+ println fileName+" ÎļþûÕÒµ½"
}
+
}else{
println "ûÕÒµ½ÏàÓ¦µÄÈ«³Æ£º"+city
}
@@ -365,6 +377,29 @@ class RegionStandardBatch extends BaseBatch{
//
// // ×¢Ò⣺ʵ¼ÊÓ¦ÓÃÖУ¬Äã¿ÉÄÜÐèÒªÔÚ»ñÈ¡ÍêÊý¾ÝºóÇå³ýscrollÉÏÏÂÎÄ£¬ÒÔÊÍ·Å×ÊÔ´¡£
// }
+ def readCSV(filename){
+ def lines = FileUtils.readLines(new File(filename))
+ def rets = [];
+ for(line in lines) {
+ if(line){
+ String[] arrs = line.split("\\|@\\|")
+ def o = new BasicDBObject()
+ try{
+ o.put("city", arrs[0])
+ o.put("district", arrs[1])
+ o.put("name", arrs[2])
+ o.put("township", arrs[3])
+ o.put("status", arrs[4])
+ o.put("id", arrs[5])
+ rets << o
+ }catch(Exception e){
+ println filename+" ÎļþÒì³£"
+ }
+ }
+
+ }
+ return rets;
+ }
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy
index 4d865d728..d9d86dbf8 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy
@@ -1,15 +1,14 @@
package com.fangjia.script.gj.data_handle
-import com.fangjia.core.RegionData
-import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
-import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.ExcelUtils
+import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
+import java.util.concurrent.atomic.AtomicLong
class TempExportData {
@@ -17,7 +16,10 @@ class TempExportData {
static main(args) {
def task = new TempExportData()
- task.run()
+// task.run()
+ def rets = task.fzHouseStat()
+ ExcelUtils.write("/home/goojia/gj/¸£ÖÝÐ¡Çø·¿Ô´¼°µÚÈý·½Çé¿öͳ¼Æ.xlsx", rets,"Êý¾Ý")
+ println "ÎļþÒÑÉú³É"
}
def run() {
@@ -75,4 +77,47 @@ class TempExportData {
println(city +" "+rets.size())
return rets;
}
+
+ def fzHouseStat(){
+ def districts = CommonDatautils.findAllDistricts("¸£ÖÝ", "district", ["city", "region", "name", "alias","estate_type2"])
+ def addCount = new AtomicLong()
+ final CountDownLatch latch = new CountDownLatch(districts.size())
+ def total = districts.size()
+ def rets = new CopyOnWriteArrayList()
+ rets << ["³ÇÊÐ","ÇøÓò","Ãû³Æ","ÉÏÔ¼۸ñ","3Äê·¿Ô´Á¿","2Äê·¿Ô´Á¿", "2ÄêµÚÈý·½¼ÒÊý"]
+ for(def i=0;i
+ def date = d['B']
+ def start = DateUtils.getFirstDayOfMonth(date)
+ def end = DateUtils.getLastDayOfMonth(date)
+ def weekends = DateUtils.getWeekendBetweenDates(start, end)
+ for(def weekend : weekends){
+ def avg_price = d['I']
+ def val = random(-200,200)
+ def new_price = (int)(avg_price*val/10000 + avg_price)
+ def last_weekend = DateUtils.add(DateUtils.formatDateByPattern(weekend, "yyyy-MM-dd"), -7)
+ def last_price = CommonDatautils.queryCityPriceByWeekend(city, last_weekend)
- static main(args) {
- def task = new Test()
- task.run()
- }
-
- def run(){
- def i=1
- while(i<2){
- def districts = CcbCommonUtils.findAllDistrictByCity("ÉîÛÚ", "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
- def rets = process(districts)
- println i + " "+districts.size() +" "+rets.size()
- i++
+ def o = new BasicDBObject()
+ o.put("avg_price", new_price)
+ o.put("c_date" , new Date())
+ o.put("city", city)
+ o.put("m_date" , new Date())
+ o.put("weekend", weekend)
+ o.put("cat","³ÇÊÐ")
+ o.put("moving_average" , (int)avg_price)
+ if(!last_price){
+ o.put("weekend_markup" , 0)
+ }else{
+ def weekend_markup = (int)((new_price-last_price)*100/last_price)
+ o.put("weekend_markup" , weekend_markup)
}
+ def q = new BasicDBObject("cat", "³ÇÊÐ").append("city", city).append("weekend", weekend)
+ REGION_STAT_COLL.update(q, new BasicDBObject('$set', o),true, false)
}
-
- def process(districts){
- def totalCount = new AtomicLong()
- final CountDownLatch latch = new CountDownLatch(districts.size())
-
-
- def rets = new CopyOnWriteArrayList()
- for(def i=0;i1){
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap)
@@ -96,12 +100,16 @@ class TownStandardBatch extends BaseBatch{
// haveStreetProcess(city, set, cat)
}
}
+ }else{
+ println fileName+" ÎļþûÕÒµ½"
}
}else{
println "ûÕÒµ½ÏàÓ¦µÄÈ«³Æ£º"+city
}
}
ExcelUtils.write("D:\\Êý¾Ý\\ÏçÕò½ÖµÀ´ý²éÊý¾Ý.xlsx", updateList, "Êý¾Ý")
+ ExcelUtils.write("D:\\Êý¾Ý\\ÏçÕò½ÖµÀ´ý²éÊý¾Ý-ÐèÐÂÔö±ðÃû.xlsx", addAliasList, "Êý¾Ý")
+
}
def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){
def results = []
@@ -194,6 +202,18 @@ class TownStandardBatch extends BaseBatch{
}
break
}
+// else{
+// if(town.street.contains(d.township.replace("Õò","").replace("½ÖµÀ","").replace("Ïç",""))){
+// updateTown = town.street
+// opt = false
+// def addObj = [town.city, town.district, town.street, d.township]
+// if(!addAliasList.contains(addObj)){
+// addAliasList << addObj
+// }
+//
+// break
+// }
+// }
}
}
if(opt){
@@ -206,11 +226,11 @@ class TownStandardBatch extends BaseBatch{
// results.clear()
// }
updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown]
- println d.city +" "+d.district+" "+d.name +" "+d.township+" ÏçÕòÐÞ¸ÄΪ£º"+updateTown
+// println d.city +" "+d.district+" "+d.name +" "+d.township+" ÏçÕòÐÞ¸ÄΪ£º"+updateTown
}else{
def m = [:]
m.put(d.id, ["township":""])
- results << m
+// results << m
// if(results.size()%100==0){
// updateByEs(results)
// results.clear()
@@ -256,7 +276,7 @@ class TownStandardBatch extends BaseBatch{
println("ÅúÁ¿¸üÐÂÖÐÓÐʧ°ÜµÄ²Ù×÷:")
bulkResponse.buildFailureMessage().each { println(it) }
} else {
- println("ÅúÁ¿¸üгɹ¦!")
+// println("ÅúÁ¿¸üгɹ¦!")
}
} catch (Exception e) {
e.printStackTrace()
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/CryptoSm4Util.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/CryptoSm4Util.groovy
new file mode 100644
index 000000000..6e960180d
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/CryptoSm4Util.groovy
@@ -0,0 +1,115 @@
+package com.fangjia.script.gj.xiamen
+
+import org.bouncycastle.crypto.engines.SM4Engine;
+import org.bouncycastle.crypto.modes.CBCBlockCipher;
+import org.bouncycastle.crypto.paddings.PaddedBufferedBlockCipher;
+import org.bouncycastle.crypto.params.KeyParameter;
+import org.bouncycastle.crypto.params.ParametersWithIV;
+import org.bouncycastle.util.encoders.Hex;
+
+import java.security.SecureRandom;
+
+/**
+ * @author shenjian
+ * @since 2024/10/9
+ */
+public class CryptoSm4Util {
+
+ /**
+ * ĬÈÏÃÜÔ¿
+ */
+ private static final String DEFAULT_SECRET_KEY = "A1B2C3D4E5F60789A1B2C3D4E5F60789";
+ private static final String CHARSET = "UTF-8";
+
+ // String plaintext -> Base64-encoded String ciphertext
+ public static String encrypt(String key, String plaintext) {
+ try {
+ byte[] keyBytes = Hex.decode(key);
+ byte[] iv = generateIV();
+ PaddedBufferedBlockCipher cipher = new PaddedBufferedBlockCipher(new CBCBlockCipher(new SM4Engine()));
+ cipher.init(true, new ParametersWithIV(new KeyParameter(keyBytes), iv));
+ byte[] input = plaintext.getBytes(CHARSET);
+ byte[] encrypted = new byte[cipher.getOutputSize(input.length)];
+ int length1 = cipher.processBytes(input, 0, input.length, encrypted, 0);
+ int length2 = cipher.doFinal(encrypted, length1);
+ byte[] finalEncrypted = new byte[length1 + length2];
+ System.arraycopy(encrypted, 0, finalEncrypted, 0, finalEncrypted.length);
+ return Hex.toHexString(iv) + Hex.toHexString(finalEncrypted);
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ }
+ return null;
+ }
+
+ private static byte[] generateIV() {
+ byte[] iv = new byte[16];
+ new SecureRandom().nextBytes(iv);
+ return iv;
+ }
+
+ // Base64-encoded String ciphertext -> String plaintext
+ public static String decrypt(String key, String ciphertext) {
+ try {
+ byte[] keyBytes = Hex.decode(key);
+ byte[] iv = Hex.decode(ciphertext.substring(0, 32));
+ byte[] encryptedData = Hex.decode(ciphertext.substring(32));
+ PaddedBufferedBlockCipher cipher = new PaddedBufferedBlockCipher(new CBCBlockCipher(new SM4Engine()));
+ cipher.init(false, new ParametersWithIV(new KeyParameter(keyBytes), iv));
+ byte[] decrypted = new byte[cipher.getOutputSize(encryptedData.length)];
+ int length1 = cipher.processBytes(encryptedData, 0, encryptedData.length, decrypted, 0);
+ int length2 = cipher.doFinal(decrypted, length1);
+ byte[] finalDecrypted = new byte[length1 + length2];
+ System.arraycopy(decrypted, 0, finalDecrypted, 0, finalDecrypted.length);
+ return new String(finalDecrypted, CHARSET).trim();
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ }
+ return null;
+ }
+
+ /**
+ * ʹÓÃĬÈÏÃÜÔ¿¼ÓÃÜ
+ *
+ * @param str ¼ÓÃÜÄÚÈÝ
+ * @return ÒÔStringÃÜÎÄÊäÈë, StringÃ÷ÎÄÊä³ö
+ */
+ public static String encode(String str) {
+ return encode(DEFAULT_SECRET_KEY, str);
+ }
+
+ /**
+ * ×Ô¶¨ÒåÃÜÔ¿¼ÓÃÜ
+ *
+ * @param key ×Ô¶¨ÒåÃÜÔ¿
+ * @param str ¼ÓÃܵÄÄÚÈÝ
+ * @return StringÃ÷ÎÄÊäÈë, StringÃÜÎÄÊä³ö
+ */
+ public static String encode(String key, String str) {
+ return encrypt(key, str);
+ }
+
+ /**
+ * ʹÓÃĬÈÏÃÜÔ¿½âÃÜ
+ *
+ * @param str ½âÃÜÄÚÈÝ
+ * @return ÒÔStringÃÜÎÄÊäÈë, StringÃ÷ÎÄÊä³ö
+ */
+ public static String decode(String str) {
+ return decode(DEFAULT_SECRET_KEY, str);
+ }
+
+ /**
+ * ×Ô¶¨ÒåÃÜÔ¿½âÃÜ
+ *
+ * @param key ×Ô¶¨ÒåÃÜÔ¿
+ * @param str ¼ÓÃÜÄÚÈÝ
+ * @return ÒÔStringÃÜÎÄÊäÈë, StringÃ÷ÎÄÊä³ö
+ */
+ public static String decode(String key, String str) {
+ return decrypt(key, str);
+ }
+
+ public static void main(def a){
+ println encrypt( DEFAULT_SECRET_KEY, "²âÊÔ")
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy
index 4efd79e7f..e44d4424e 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy
@@ -15,10 +15,14 @@ import com.mongodb.DBCollection
/**
* ÏÃÃÅÉæÌ¨Ô°ÇøÊý¾Ýͬ²½
*/
+
+
@BatchOwner(Watcher.GaoJuan)
class RelatedTaiWanBatch extends BaseBatch{
- def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"));
+
+
+ def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMM"));
DBCollection yuanqu_qiye_coll = MongoUtils.getCollectionByName("crawl", "qzd_shetai_company_detail", "192.168.0.206", 27020)
def relateTaiwn_yuanqu_datas = ["¸£ÖÝ̨ÉÌͶ×ÊÇø","ÐÓÁǪ̈ÉÌͶ×ÊÇø","¼¯ÃĄ̀ÉÌͶ×ÊÇø","º£²×̨ÉÌͶ×ÊÇø","ÕÄÖÝ̨ÉÌͶ×ÊÇø","ȪÖÝ̨ÉÌͶ×ÊÇø","ÕÄÆǪ̈ÍåÅ©Ãñ´´ÒµÔ°","ÕÄÆ½Ì¨ÍåÅ©Ãñ´´ÒµÔ°","ÏÉÓĮ̂ÍåÅ©Ãñ´´ÒµÔ°","ÇåÁ÷̨ÍåÅ©Ãñ´´ÒµÔ°","¸£Çą̊ÍåÅ©Ãñ´´ÒµÔ°","»Ý°²Ì¨ÍåÅ©Ãñ´´ÒµÔ°","º£Ï¿Á½°¶¼¯³Éµç·²úÒµºÏ×÷ÊÔÑéÇø£¨ÏÃÃÅ¡¢ÈªÖݲúÒµÔ°£©","º£Ï¿Á½°¶Éú¼¼ºÍÒ½Áƽ¡¿µ²úÒµºÏ×÷Çø","º£Ï¿Á½°¶Å©ÒµºÏ×÷ÊÔÑéÇø","º£Ï¿Á½°¶£¨ÈýÃ÷£©ÏÖ´úÁÖÒµºÏ×÷ʵÑéÇø","ÕÄÖݹÅÀ×ʯ»¯»ùµØ","ÓÀÌ©Ãǫ̈ũҵÈںϷ¢Õ¹²úÒµÔ°","ÉϺ¼Ãǫ̈ũҵÈںϷ¢Õ¹²úÒµÔ°","ͬ°²Ãǫ̈ũҵÈںϷ¢Õ¹²úÒµÔ°","ÄϾ¸Ãǫ̈ũҵÈںϷ¢Õ¹²úÒµÔ°","ÕÄÆÖÃǫ̈ũҵÈںϷ¢Õ¹²úÒµÔ°","ÓÀ´ºÃǫ̈ũҵÈںϷ¢Õ¹²úÒµÔ°","ÓÀ°²Ãǫ̈ũҵÈںϷ¢Õ¹²úÒµÔ°","ÉÛÎäÃǫ̈ũҵÈںϷ¢Õ¹²úÒµÔ°","ƽ̶Ãǫ̈ũҵÈںϷ¢Õ¹²úÒµÔ°","ÄϾ¸Ãǫ̈¾«ÃÜ»úе²úÒµÔ°"]
@@ -28,6 +32,8 @@ class RelatedTaiWanBatch extends BaseBatch{
def zimaoqu_datas = ["Öйú£¨¸£½¨£©×ÔÓÉóÒ×ÊÔÑéÇøÏÃÃÅÆ¬Çø","Öйú£¨¸£½¨£©×ÔÓÉóÒ×ÊÔÑéÇø¸£ÖÝÆ¬Çø","Öйú£¨¸£½¨£©×ÔÓÉóÒ×ÊÔÑéÇøÆ½Ì¶Æ¬Çø"]
+
+
static main(args) {
runBatch(new RelatedTaiWanBatch(), args);
}
@@ -37,14 +43,11 @@ class RelatedTaiWanBatch extends BaseBatch{
def day = new Date().getDate()
if(day == 10){
- //createDatas() //ÆóÒµ¿â»ñÈ¡ÉæÌ¨Ô°ÇøÆóÒµ
+// createDatas() //ÆóÒµ¿â»ñÈ¡ÉæÌ¨Ô°ÇøÆóÒµ
updateRangeByPoint() //¸ü¸ÄÆóÒµËùÔÚ×ÔÃ³Çø
export() //µ¼³öÊý¾Ý
- }else{
- exportEmptyFile()
}
-
}
@@ -152,10 +155,18 @@ class RelatedTaiWanBatch extends BaseBatch{
}
def allDatas = []
allDatas.addAll(rows)
-// FileUtils.batchCreateFile(rows, fileName, null, "utf-8", false);
- FileUtils.batchCreateFileByCustomSuffix(rows, fileName, null, "utf-8", false, "|@|","|+|");
+
+ //Ô²»¼ÓÃÜ֮ǰ
+// FileUtils.batchCreateFileByCustomSuffix(rows, fileName, null, "utf-8", false, "|@|","|+|");
+// XmBankFtpUtils.ftpUpload(fileName, curDate)
+
+
+ def str1 = FileUtils.createFileStringByCustomSuffix(rows, fileName, "utf-8", "|@|","|+|");
+ def str1_encryptedText = CryptoSm4Util.encrypt( CryptoSm4Util.DEFAULT_SECRET_KEY, str1)
+ FileUtils.createEncryptFile(fileName, str1_encryptedText, false)
XmBankFtpUtils.ftpUpload(fileName, curDate)
+
def fileName2 = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat.rdy"
if(CommonDatautils.isWindow()){
fileName2 = "D:\\Êý¾Ý\\ÏÃÃÅ\\ÉæÌ¨Ô°Çø\\free_trade_zone_${curDate}_init.dat.rdy"
@@ -181,4 +192,7 @@ class RelatedTaiWanBatch extends BaseBatch{
FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|");
XmBankFtpUtils.ftpUpload(fileName2, curDate)
}
+
+
+
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/zhongshu/DistrictArrountOffice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/zhongshu/DistrictArrountOffice.groovy
new file mode 100644
index 000000000..34a98e33d
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/zhongshu/DistrictArrountOffice.groovy
@@ -0,0 +1,134 @@
+package com.fangjia.script.gj.zhongshu
+
+import com.fangjia.script.commData.CommonDatautils
+import com.fangjia.script.common.DataHandleUtils
+import com.fangjia.script.util.ElasticUtils
+import com.fangjia.script.util.ExcelUtils
+
+class DistrictArrountOffice {
+
+ static main(args) {
+ def task = new DistrictArrountOffice()
+ task.run()
+ }
+
+ def run(){
+ def dataMap = ["±±¾©":"116.481075,39.996743", "ÑĮ̀":"121.438616,37.459645"]
+ def rets = []
+ rets << ["³ÇÊÐ","ÇøÓò","Ãû³Æ","µØÖ·","¾¶È","γ¶È","д×ÖÂ¥µÈ¼¶","×â½ð(Æð)","×â½ð(Ö¹)", "¼Û¸ñÀ´Ô´"]
+ dataMap.each {k,v->
+ def locations = v.split(",")
+ def arroundOffices = CommonDatautils.getSeaweedArroundByCat(k, locations[0], locations[1], "office")
+ println k +" "+arroundOffices.size()
+ if(arroundOffices){
+ def index = 0
+ for(def o : arroundOffices){
+ index++
+ def priceMap = findEsOffice(o.city, o.region, o.name)
+ if(!priceMap){
+ priceMap = findEsHouse(o, "20231112")
+ }
+ def start_price = null
+ def end_price = null
+ def source = null
+ if(priceMap){
+ if(priceMap.avg_start_price){
+ start_price = DataHandleUtils.getDoubleRound(2, priceMap.avg_start_price)
+ }
+ if(priceMap.avg_end_price){
+ end_price = DataHandleUtils.getDoubleRound(2, priceMap.avg_end_price)
+ }
+ source = priceMap.source
+ }
+ def lng = DataHandleUtils.getDoubleRound(6, o.lng/10000000000.0)
+ def lat = DataHandleUtils.getDoubleRound(6, o.lat/10000000000.0)
+ def row = [o.city, o.region, o.name, o.address, lng, lat, o.bed_kind, start_price, end_price, source]
+ rets << row
+ println index +" "+ row
+ }
+ }
+ }
+ ExcelUtils.write("d:\\Êý¾Ý\\д×ÖÂ¥²âÊÔ-1118.xlsx", rets, "Êý¾Ý")
+ }
+
+
+ def findEsOffice(city, region, name){
+ def map = [:]
+ def start_rent_prices = []
+ def end_rent_prices = []
+ def rent_prices = []
+
+ ElasticUtils.executeSqlQuery("select * from office where city='${city}' and region='${region}' and name='${name}'", { r->
+ if(r.start_rent_price){
+ start_rent_prices << r.start_rent_price
+ }
+ if(r.end_rent_price){
+ end_rent_prices << r.end_rent_price
+ }
+ if(r.rent_price){
+ rent_prices << r.rent_price
+ }
+
+ })
+ def avg_start_price = null
+ if(start_rent_prices){
+ avg_start_price = start_rent_prices.sum()/start_rent_prices.size()
+ }
+ def avg_end_price = null
+ if(end_rent_prices){
+ avg_end_price = end_rent_prices.sum()/end_rent_prices.size()
+ }
+
+ if(!avg_start_price && !avg_end_price){
+ if(rent_prices){
+ if(rent_prices.size()>1){
+ rent_prices = rent_prices.sort()
+ avg_start_price = rent_prices[0]
+ avg_end_price = rent_prices[rent_prices.size()-1]
+ }else{
+ avg_start_price = rent_prices[0] - rent_prices[0]*0.2
+ avg_end_price = rent_prices[0] + rent_prices[0]*0.2
+ }
+ }
+ }
+
+ if(avg_start_price){
+ map.put("avg_start_price", avg_start_price)
+ }
+ if(avg_end_price){
+ map.put("avg_end_price", avg_end_price)
+ }
+ if(map){
+ map.source = "·Ç·¿Ô´"
+ }
+ return map
+ }
+
+ def findEsHouse(o, startDate){
+ def rent_prices = CommonDatautils.findHousePriceByDate(o.city, o.region, o.name, o.alias, "office_rent", startDate)
+ def avg_start_price = null
+ def avg_end_price = null
+ def map = [:]
+ if(rent_prices){
+ if(rent_prices.size()>1){
+ rent_prices = rent_prices.sort()
+ avg_start_price = rent_prices[0]
+ avg_end_price = rent_prices[rent_prices.size()-1]
+ }else{
+ avg_start_price = rent_prices[0] - rent_prices[0]*0.2
+ avg_end_price = rent_prices[0] + rent_prices[0]*0.2
+ }
+ }
+ if(avg_start_price){
+ map.put("avg_start_price", avg_start_price)
+ }
+ if(avg_end_price){
+ map.put("avg_end_price", avg_end_price)
+ }
+ if(map){
+ map.source = "·¿Ô´"
+ }
+ return map
+ }
+
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy
index 25e01d8a5..b57cdd659 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy
@@ -7,21 +7,8 @@ import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
-import com.fangjia.script.linxsh.util.CommonUtil
-import com.fangjia.script.util.CommonUtils
-import com.fangjia.script.util.CosineSimilarAlgorithm
-import com.fangjia.script.util.DataUtil
-import com.fangjia.script.util.DbUtils
-import com.fangjia.script.util.ElasticUtils
-import com.fangjia.script.util.MongoUtils
-import com.fangjia.script.util.OpenApiClient
-import com.mongodb.BasicDBObject
-import com.mongodb.Bytes
-import com.mongodb.DB
-import com.mongodb.DBCollection
-import com.mongodb.DBObject
-import com.mongodb.QueryOperators
-import net.sf.json.JSONObject
+import com.fangjia.script.util.*
+import com.mongodb.*
import org.apache.commons.lang3.SerializationUtils
import org.apache.commons.lang3.StringUtils
import org.apache.tools.ant.util.DateUtils
@@ -31,9 +18,7 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
-import java.util.regex.Matcher
import java.util.regex.Pattern
-
/**
* µÚÈý·½Êý¾Ý±ê×¼»¯´¦Àí
* ³É¹¦type˵Ã÷
@@ -689,14 +674,15 @@ class DataStandardBatch extends BaseBatch{
def pre = (data.avg_price-price)*100/price
if(pre>=-20 && pre<=20){
return dbo;
- }else{
- def ret = THIRD_PRICE_FILTER_COLL.findOne(data._id)
- if(!ret){
- data.put("msg","·¿Ô´¼Û¸ñ¹ýÂË")
- THIRD_PRICE_FILTER_COLL.save(data)
- }
- return null
}
+// else{
+// def ret = THIRD_PRICE_FILTER_COLL.findOne(data._id)
+// if(!ret){
+// data.put("msg","·¿Ô´¼Û¸ñ¹ýÂË")
+// THIRD_PRICE_FILTER_COLL.save(data)
+// }
+// return null
+// }
}
}
return dbo
@@ -934,7 +920,8 @@ class DataStandardBatch extends BaseBatch{
DBObject qdata = new BasicDBObject();
qdata.put("s_date", new BasicDBObject('$gte', SHORT_DATE))
// qdata.put("city","ÉϺ£")
-// qdata.put("name","°×½ð嫹¬")
+// qdata.put("name","Â̵ØÔÆÌì·»")
+// qdata.put("status", 3)
// qdata.put("source","°²¾Ó¿Í")
// qdata.put("name",Pattern.compile("¹¨¼ÒÕ¬.*"))
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));