Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into script_gj

This commit is contained in:
2022-08-19 15:19:30 +08:00
162 changed files with 14122 additions and 7838 deletions
@@ -16,6 +16,7 @@ import com.fangjia.script.util.ServerAreaMap
import com.google.common.collect.MapDifference
import com.google.common.collect.Maps
import com.mongodb.*
import org.apache.commons.lang.StringUtils
import org.bson.types.ObjectId
/**
@@ -47,9 +48,19 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
}
void process(Object args) {
def cities = ServerAreaMap.findAllCities()
String cityParam = System.getProperty("city");
String calcDate = System.getProperty("calcDate");
def cities;
if(StringUtils.isNotBlank(cityParam)){
cities = new ArrayList()
for(String c in cityParam.split(",")){
cities.add(c);
}
}else{
cities = ServerAreaMap.findAllCities()
}
println cities
Date weekend = getLastDayOfMonth()
Date weekend = getLastDayOfMonth(calcDate)
println weekend
int cityCount = 0
int regionCount = 0
@@ -58,31 +69,22 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
//开始时间
final Date startDate = new Date()
for (String city : cities) {
List<String> idList = getAllDistrictIds(city)
logInfo(city + ":" + idList.size())
List<BasicDBObject> districts = getAllDistricts(city)
logInfo(city + ":" + districts.size())
def buffer = []
//生成计算小区队列
for (String id : idList) {
DBObject fields = new BasicDBObject()
fields.put("city", 1)
fields.put("region", 1)
fields.put("name", 1)
fields.put("estate_type2", 1)
DBObject district = SEAWEED_COLL.findOne(new BasicDBObject("_id", new ObjectId(id)), fields)
if (district != null) {
String region = district.get('region')
String name = district.get('name')
String estateType = district.get('estate_type2')
districtCount++
String ip = WORKER_MACHINE_IPS[districtCount % WORKER_MACHINE_IPS.size()]
BasicDBObject cmd = saveDistrictCalcCommand(city, region, name, estateType, weekend, ip)
logInfo("district:" + city + "-" + region + "-" + name + "-" + estateType + ",weekend:" + DateUtil.formatToYYYYMMDD(weekend) + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date()))
buffer << cmd
if(buffer.size() == 1000){
COMMAND_COLL.insert(buffer)
buffer.clear()
}
for (BasicDBObject district : districts) {
String region = district.get('region')
String name = district.get('name')
String estateType = district.get('estate_type2')
districtCount++
String ip = WORKER_MACHINE_IPS[districtCount % WORKER_MACHINE_IPS.size()]
BasicDBObject cmd = saveDistrictCalcCommand(city, region, name, estateType, weekend, ip)
logInfo("district:" + city + "-" + region + "-" + name + "-" + estateType + ",weekend:" + DateUtil.formatToYYYYMMDD(weekend) + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date()))
buffer << cmd
if(buffer.size() == 1000){
COMMAND_COLL.insert(buffer)
buffer.clear()
}
}
if(!buffer.isEmpty()){
@@ -148,7 +150,7 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
query.append("weekend", weekend)
query.append("cat", "block")
blockCount = SQMETRE_RENT_COLL.count(query)
def msg = "本月小区平米租金计算量:" + districtCount + ";本月城市平米租金计算量:" + cityCount + ";本月区域平米租金计算量:" + regionCount + ";本月版块平米租金计算量:" + blockCount + ";运行时间:" + secondToTime(time) + "."
def msg = "本月(${weekend.format("yyyyMM")})小区平米租金计算量:".toString() + districtCount + ";本月城市平米租金计算量:" + cityCount + ";本月区域平米租金计算量:" + regionCount + ";本月版块平米租金计算量:" + blockCount + ";运行时间:" + secondToTime(time) + "."
sendDingDingMsg(TOKEN, msg)
}
/**
@@ -209,31 +211,34 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
}
private List<String> getAllDistrictIds(String city) {
DBObject q = new BasicDBObject()
q.put("city", city)
q.put("cat", "district")
q.put("visible", 0)
private List<BasicDBObject> getAllDistricts(String city) {
DBObject query = new BasicDBObject()
query.put("city", city)
query.put("cat", "district")
query.put("visible", 0)
int trytimes = 5
Set<String> uniqueIdList = new LinkedHashSet<String>()
DBObject fields = new BasicDBObject()
fields.put("city", 1)
fields.put("region", 1)
fields.put("name", 1)
fields.put("estate_type2", 1)
fields.put("_id", 1)
for(int i=0; i<trytimes; i++) {
DSCursor cur = SEAWEED_COLL.find(q, new BasicDBObject("_id", 1)).batchSize(2000)
int tryTimes = 5
List<BasicDBObject> districts = new ArrayList<BasicDBObject>()
for(int i=0; i < tryTimes; i++) {
DSCursor cur = SEAWEED_COLL.find(query, fields).batchSize(5000)
int totalCount = cur.count()
Iterator<DBObject> iter = cur.iterator()
while(iter.hasNext()) {
DBObject dbo = iter.next()
String id = dbo.get('_id')
uniqueIdList.add(id)
BasicDBObject dbo = (BasicDBObject)iter.next()
districts.add(dbo)
}
if(uniqueIdList.size() == totalCount) {
if(districts.size() == totalCount) {
break
}
}
return new ArrayList<String>(uniqueIdList)
return districts
}
private List<DBObject> getRegions(String city) {
@@ -316,7 +321,13 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
}
private Date getLastDayOfMonth() {
private Date getLastDayOfMonth(String calcDateStr) {
if(StringUtils.isNotBlank(calcDateStr)){
Date calcDate = DateUtil.parseDate(calcDateStr, "yyyyMM")
if(calcDate != null){
return DateUtil.truncateDay(DateUtil.getLastDayOfMonth(calcDate, 0))
}
}
return DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
}
@@ -15,7 +15,8 @@ import org.apache.commons.lang.StringUtils
public class DistrictRegionTransfer {
static final DBCollection TASK_COLL = MongoUtils.getCollectionByName("fangjia", "district_update_task", "mongo.db.fangjia.com")
static final DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection SQMETRE_RENT = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com")
static final DBCollection REGION_MIGRATE_COLL = MongoUtils.getCollectionByName("fangjia_log", "region_migrate", "mongo.db.fangjia.com")
static final List<String> IGNORE_FIELDS = ["_id", "region", "city", "flag", "estate_type2", "c_date", "m_date", "other_info", "mark_map", "addr_alias", "alias", "name", "address", "status"]
static List ALL_CITY = CommonUtils.findAllCity()
@@ -31,15 +32,24 @@ public class DistrictRegionTransfer {
copyDistrictInfo(orgDistrict, destDistrict)
DbUtils.seaweed.save(destDistrict)
DbUtils.deleteOnSeaweed(orgDistrict._id.toString())
sendRelatedInfoMigrateCommand(orgDistrict, destDistrict)
saveMigrateLog(orgDistrict, destDistrict)
} else {
destDistrict = (BasicDBObject) orgDistrict.clone()
destDistrict.city = newCity
destDistrict.region = newRegion
DbUtils.seaweed.save(destDistrict)
sendRelatedInfoMigrateCommand(orgDistrict, destDistrict)
saveMigrateLog(orgDistrict, destDistrict)
DISTRICT_STAT.remove(new BasicDBObject("city", newCity).append("region", newRegion).append("district_name", orgDistrict.name))
DISTRICT_STAT.find(new BasicDBObject("city", newCity).append("region", orgDistrict.region).append("district_name", orgDistrict.name)).each {BasicDBObject stat->
stat.region = newRegion
DISTRICT_STAT.save(stat)
}
SQMETRE_RENT.remove(new BasicDBObject("city", newCity).append("region", newRegion).append("district_name", orgDistrict.name))
SQMETRE_RENT.find(new BasicDBObject("city", newCity).append("region", orgDistrict.region).append("district_name", orgDistrict.name)).each {BasicDBObject stat->
stat.region = newRegion
SQMETRE_RENT.save(stat)
}
}
}
@@ -60,72 +70,72 @@ public class DistrictRegionTransfer {
}
}
def sendRelatedInfoMigrateCommand(BasicDBObject orgDistrict, BasicDBObject destDistrict) {
BasicDBObject task = new BasicDBObject()
task.append("city", orgDistrict.city)
task.append("new_city", destDistrict.city)
task.append("region", orgDistrict.region)
task.append("new_region", destDistrict.region)
task.append("new_name", orgDistrict.name)
task.append("name", destDistrict.name)
task.append("type", 2)
task.append("c_date", new Date())
task.append("status", 1)
String estateType;
if ("district" == orgDistrict.cat) {
if (StringUtils.isNotBlank(destDistrict.estate_type2)) {
if (destDistrict.getString("estate_type2").contains("别墅")) {
estateType = "villa"
} else {
estateType = "district"
}
} else {
if (StringUtils.isNotBlank(destDistrict.estate_type2)) {
if (destDistrict.getString("estate_type").contains("别墅")) {
estateType = "villa"
} else {
estateType = "district"
}
} else {
estateType = "district"
}
}
} else {
estateType = destDistrict.getString("cat")
}
task.append("estate_type", estateType)
def query = new BasicDBObject()
query.append("city", task.city)
query.append("region", task.region)
query.append("name", task.name)
query.append("type", task.type)
TASK_COLL.update(query, new BasicDBObject('$set', task), true, false)
def params = [:]
params.city = orgDistrict.city
params.region = orgDistrict.region
params.name = orgDistrict.name
params.estateType = orgDistrict.estate_type2
params.newCity = destDistrict.city
params.newRegion = destDistrict.region
params.newName = destDistrict.name
params.newEstateType = destDistrict.estate_type2
int count = 3
while (count-- > 0) {
try {
JSONObject json = new HttpRequest().getJsonContent("http://192.168.0.24:9080/command/transfer", params, "UTF-8")
if (json && json.getInt("code") == 200) {
println("均价重算命令发从成功:" + params)
break
}
} catch (Exception e) {
}
}
}
// def sendRelatedInfoMigrateCommand(BasicDBObject orgDistrict, BasicDBObject destDistrict) {
//
// BasicDBObject task = new BasicDBObject()
// task.append("city", orgDistrict.city)
// task.append("new_city", destDistrict.city)
// task.append("region", orgDistrict.region)
// task.append("new_region", destDistrict.region)
// task.append("new_name", orgDistrict.name)
// task.append("name", destDistrict.name)
// task.append("type", 2)
// task.append("c_date", new Date())
// task.append("status", 1)
// String estateType;
// if ("district" == orgDistrict.cat) {
// if (StringUtils.isNotBlank(destDistrict.estate_type2)) {
// if (destDistrict.getString("estate_type2").contains("别墅")) {
// estateType = "villa"
// } else {
// estateType = "district"
// }
// } else {
// if (StringUtils.isNotBlank(destDistrict.estate_type2)) {
// if (destDistrict.getString("estate_type").contains("别墅")) {
// estateType = "villa"
// } else {
// estateType = "district"
// }
// } else {
// estateType = "district"
// }
// }
// } else {
// estateType = destDistrict.getString("cat")
// }
// task.append("estate_type", estateType)
// def query = new BasicDBObject()
// query.append("city", task.city)
// query.append("region", task.region)
// query.append("name", task.name)
// query.append("type", task.type)
// TASK_COLL.update(query, new BasicDBObject('$set', task), true, false)
//
// def params = [:]
// params.city = orgDistrict.city
// params.region = orgDistrict.region
// params.name = orgDistrict.name
// params.estateType = orgDistrict.estate_type2
//
// params.newCity = destDistrict.city
// params.newRegion = destDistrict.region
// params.newName = destDistrict.name
// params.newEstateType = destDistrict.estate_type2
// int count = 3
// while (count-- > 0) {
// try {
// JSONObject json = new HttpRequest().getJsonContent("http://192.168.0.24:9080/command/transfer", params, "UTF-8")
// if (json && json.getInt("code") == 200) {
// println("均价重算命令发从成功:" + params)
// break
// }
// } catch (Exception e) {
//
// }
//
// }
// }
def saveMigrateLog(BasicDBObject orgDistrict, BasicDBObject destDistrict) {
@@ -8,10 +8,10 @@ import static com.mongodb.client.model.Filters.*
import com.fangjia.core.util.Mongo3Utils
def city = "ÂåÑô"
def destCity = "ÂåÑô"
def region = "¼ªÀû"
def destRegion = "ÃϽò"
def city = "ÖÐɽ"
def destCity = "ÖÐɽ"
def region = "»ð¾æ"
def destRegion = "ÖÐɽ¸Û"
@@ -1,6 +1,39 @@
package com.fangjia.script.wenyan
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\国信达级联价格-上海.xlsx",{List item->
// if(item["AB"]){
// return
// }
// String region = null
// String name = item["F"]
// if(name.endsWith("小区")){
// name = name.replaceAll("小区\$", "")
// }
// region = RegionData.formatRegion(item["B"], item["D"])
// def results = []
// DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(item["B"], region, name)).each {
// if(!results.contains(it)){
// results.add(it)
// }
// }
// DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(item["B"], region).append("alias", name)).each {
// if(!results.contains(it)){
// results.add(it)
// }
// }
// if(results.size() == 1){
// item["Z"] = results[0]._id.toString()
// item["AA"] = results[0].region
// item["AB"] = results[0].name
// println(item)
// }
//})
import com.fangjia.core.ProvinceData
import com.fangjia.core.RegionData
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
@@ -12,6 +45,9 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilders
//DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018)
//def count = 0
//List<List<String>> data = Collections.synchronizedList([])
@@ -76,7 +112,77 @@ import org.apache.commons.lang.StringUtils
// }, new BasicDBObject("township", 1))
//}
//println(count)
//def GXD_CCB = [:]
//def CCB_FJW = [:]
//ExcelUtils.read("D:\\Users\\wenyan\\Downloads\\4月级联待优化小区_3个城市.xlsx",sheetIndex = 1).each {List item->
// if(item["V"] == "级联"){
// if(item["Y"] == "国信达"){
// GXD_CCB[item['J']] = item["E"]
// }else if(item["Y"] == "房价网"){
// CCB_FJW[item['E']] = item["J"]
// }
// }
//}
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\4月级联待优化小区_3个城市.xlsx",{List item->
// if(item["V"] == "级联" && item["Y"] == "国信达"){
// def ccbId = GXD_CCB.get(item['J'], null)
// def fjwId = CCB_FJW.get(ccbId, null)
// if(fjwId != null){
// item["AB"] = fjwId
//// BasicDBObject seaweed = DbUtils.seaweed.findById(fjwId)
//// List results = DbUtils.findAroundDistrict(seaweed)
//// long total = 0
//// long count = 0
//// for(BasicDBObject s : results){
//// def avg = DbUtils.findDistrictAveragePrice(seaweed.city, seaweed.region, seaweed.name, Date.parse('yyyyMMdd', '20220619'), seaweed.estate_type2)
//// if(avg){
//// total += avg.avg_price
//// count += 1
//// }
//// }
//// if(count > 0){
//// item['Z'] = (int)(total/count)
//// println(item)
//// }
// }
// }
//},sheetIndex = 1)
ElasticUtils.executeSqlQuery("select count(*), p_date from seawater where p_date > 20210501 group by p_date",{
println(it)
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\小区字段补充.xlsx",{List item->
BasicDBObject seaweed = DbUtils.seaweed.findById(item["A"])
if(StringUtils.isBlank(seaweed.create_year)){
def names = []
names.add(seaweed.name)
if(seaweed.alias){
for(def it in seaweed.alias){
if(StringUtils.isNotBlank(it) && !names.contains(it)){
names.add(it)
}
}
}
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery()
boolQueryBuilder.must(QueryBuilders.termQuery("city", seaweed.city))
boolQueryBuilder.must(QueryBuilders.termQuery("region", seaweed.region))
boolQueryBuilder.must(QueryBuilders.termsQuery("name", names));
item["B"] = seaweed.city
item["C"] = seaweed.region
item["D"] = seaweed.name
List result = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", boolQueryBuilder)
def buildYears = []
result.each {
if(StringUtils.isNotBlank(it.build_year) && !StringUtils.containsAny(it.build_year,"无")){
String build_year = it.build_year
build_year = build_year.replaceAll("((|\\().*()|\\))", "")
build_year = build_year.replaceAll("[年建]", "")
// property_fee = property_fee.replace("元/㎡·月", "元/平米/月")
buildYears.add(build_year)
}
}
buildYears = buildYears.unique()
if(buildYears.size() > 0){
DbUtils.seaweed.update(new BasicDBObject("_id", seaweed._id), new BasicDBObject('$set', new BasicDBObject('create_year', buildYears[0])))
println(item)
println(buildYears)
}
}
})
@@ -0,0 +1,151 @@
package com.fangjia.script.wenyan
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.bson.Document
//def data = ExcelUtils.read("D:\\Users\\wenyan\\Downloads\\TC重新跑数据.xlsx")
//def addMap = [:]
//data.each {def list->
// if(list["C"] == "有效"){
// addMap[list["B"]] = list
// }
//}
//ExcelUtils.modify("D:\\Users\\wenyan\\Downloads\\data_gj20220616.xlsx",{List item->
// if(addMap.containsKey(item["B"])){
// List list = addMap.get(item["B"])
// item["C"] = list["D"]
// item["D"] = list["E"]
// item["E"] = list["F"]
// item["S"] = "有效"
// println(item)
// }
//})
def DISTRICT_ANALY_DATA = Mongo3Utils.getDbCollection("fangjia_stat", "district_analy_data")
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\data_gj20220616.xlsx", { List item ->
// if(item["S"] == null){
// if(item["Q"] != null || item["R"] != null){
// item["S"] = "有效"
// }
// }
// if(item["S"] == null){
// def source = null
// JSONObject resp = OpenApiClient.addressMatch(item["A"], item["B"], "property")
// source = "小区库"
// if(resp.credit == 0){
// resp = OpenApiClient.addressMatch(item["A"], item["B"], null)
// source = "小区库"
// }
// if(resp.credit == 0){
// resp = OpenApiClient.addressMatch(item["A"], item["B"], null, true)
// source = "地图"
// }
//
// if(resp.credit > 0){
// println(resp)
// item["C"] = resp.city
// item["D"] = resp.district
// item["E"] = resp.name
// item["F"] = source
// item["G"] = resp.lng
// item["H"] = resp.lat
// item["I"] = resp.propertyType
// }
// if(item["G"] != null && !["普通住宅", "商住", "别墅"].contains(item["I"])){
// List seaweeds = DbUtils.findAroundDistrict(item["G"], item["H"], 1000 ,1, 2)
// if(seaweeds.size() > 0){
// BasicDBObject s = seaweeds[0]
// item["C"] = s.city
// item["D"] = s.region
// item["E"] = s.name
// item["F"] = "周边"
// item["G"] = s.lng/10000000000d
// item["H"] = s.lat/10000000000d
// item["I"] = s.estate_type2
// }
// }
// }
// if(item["S"]|| item["T"]){
// if(!item["i"]){
// JSONObject result = GaoDeApiUtils.geo(item["A"], item["B"])
// if(result != null && result.getInt("status") == 1){
// JSONArray geocodes = result.getJSONArray("geocodes");
// if(geocodes != null && geocodes.size() > 0){
// JSONObject geocode = geocodes.getJSONObject(0);
// String location = geocode.getString("location")
// item["i"] = location.split(",")[0].toDouble()
// item["j"] = location.split(",")[1].toDouble()
// }
// }
// println(item)
// }
// }
// if(item["D"] == null && item["F"] != null){
// String kw = item["F"]
// if(item["E"]){
// kw = item["E"] + kw
// }
// JSONObject result = GaoDeApiUtils.geo(item["A"], kw)
// if(result != null && result.getInt("status") == 1){
// JSONArray geocodes = result.getJSONArray("geocodes");
// if(geocodes != null && geocodes.size() > 0){
// JSONObject geocode = geocodes.getJSONObject(0);
// String location = geocode.getString("location")
// item["i"] = location.split(",")[0].toDouble()
// item["j"] = location.split(",")[1].toDouble()
// }
// }
// println(item)
// }
// if(item["E"] != null && ["普通住宅", "商住", "别墅"].contains(item["I"])){
// Document doc = DISTRICT_ANALY_DATA.find(new Document("city", item["C"]).append("region", item["D"]).append("district_name", item["E"])).first()
// if(doc){
// item["N"] =doc["rank"]
// }
// }
// if (item["S"] != "有效" || item["F"] != null) {
// return
// }
// JSONObject resp = OpenApiClient.addressMatch(item["A"], item["E"], "property", false, item["D"], null, "strict")
// if(resp.credit > 0){
// println(resp)
// item["C"] = resp.city
// item["D"] = resp.district
// item["E"] = resp.name
// item["F"] = "小区库"
// item["G"] = resp.lng
// item["H"] = resp.lat
// item["I"] = resp.propertyType
// }
// println(resp)
// if (item["F"] == null) {
// String kw = item["E"]
// if (item["D"]) {
// kw = item["D"] + kw
// }
// JSONObject result = GaoDeApiUtils.geo(item["A"], kw)
// if (result != null && result.getInt("status") == 1) {
// JSONArray geocodes = result.getJSONArray("geocodes");
// if (geocodes != null && geocodes.size() > 0) {
// JSONObject geocode = geocodes.getJSONObject(0);
// String location = geocode.getString("location")
// item["G"] = location.split(",")[0].toDouble()
// item["H"] = location.split(",")[1].toDouble()
// item["F"] = "反查经纬度"
// }
// }
// println(item)
// }
})