Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into script_gj

# Conflicts:
#	projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy
This commit is contained in:
2024-03-08 10:39:06 +08:00
4 changed files with 65 additions and 118 deletions
@@ -87,14 +87,9 @@ class DistrictHouseBatch extends BaseBatch {
long rentCount = 0 long rentCount = 0
for(def name : names){ for(def name : names){
String key = name.toString().trim() + "_" + district.get("region") + "_" + district.get("city") String key = name.toString().trim() + "_" + district.get("region") + "_" + district.get("city")
if(!sellCount){ sellCount = sellCount + (sellCountMap.get(key) ? sellCountMap.get(key) : 0)
sellCount = sellCountMap.get(key) ? sellCountMap.get(key) : 0 rentCount = rentCount + (rentCountMap.get(key) ? rentCountMap.get(key) : 0)
}
if(!rentCount){
rentCount = rentCountMap.get(key) ? rentCountMap.get(key) : 0
}
} }
saveToDistrictHouse(district.get('city'), district.get("region"), district.get("name"), sellCount, rentCount, sn) saveToDistrictHouse(district.get('city'), district.get("region"), district.get("name"), sellCount, rentCount, sn)
} }
@@ -1,7 +1,16 @@
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeoUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils import com.fangjia.core.util.Mongo3Utils
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.client.MongoCollection import com.mongodb.client.MongoCollection
import net.sf.json.JSONObject
import org.bson.Document import org.bson.Document
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.EsDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection import java.sql.Connection
@@ -11,67 +20,43 @@ import java.util.function.Consumer
import static com.mongodb.client.model.Filters.and import static com.mongodb.client.model.Filters.and
import static com.mongodb.client.model.Filters.eq import static com.mongodb.client.model.Filters.eq
//REGION_CONN = MongoUtils.getCollectionByName("fangjia", "region_block", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
//TOWNSHIP_CONN = MongoUtils.getCollectionByName("fangjia", "township", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
//TOWNSHIP_CONN.find().each {BasicDBObject township->
// township.area_station = township.central
// township.removeField("central")
// TOWNSHIP_CONN.save(township)
//
//}
//REGION_CONN.find(new BasicDBObject("category", "region")).each {BasicDBObject regionBlock->
// def lng = regionBlock.lng/10000000000l
// def lat = regionBlock.lat/10000000000l
// HttpRequest request = new HttpRequest()
// JSONObject resp = request.getJsonContent("http://localhost:8080/FangJiaAddressService/geocode/township", ['location':"$lng,$lat".toString()], "UTF-8")
// if(resp.result){
// TOWNSHIP_CONN.update(new BasicDBObject("_id", new ObjectId(resp.result.id)), new BasicDBObject('$set', new BasicDBObject('central', 1)))
// println(resp.result)
// }
//}
def cityList = ["上海", "广州", "北京", "深圳", "成都", "杭州", "重庆", "武汉", "西安", "天津"]
def roots = ["lianjia.com", "zhuge.com", "fang.com", "anjuke.com", "58.com", "soufy.cn"]
MongoCollection<Document> DISTRICT_STAT = Mongo3Utils.getDbCollection("fangjia", "district_stat_rent")
MongoCollection<Document> DEAL_PRICE = Mongo3Utils.getDbCollection("fangjia", "deal_price")
def data = [["城市", "区域", "小区名", "版块", "装修", "朝向", "室", "厅", "卫", '标题', '总价', '面积', '单价', '发布时间', '来源']] //CommonUtils.findAllCity().each {def city->
cityList.each { def city -> // DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city, null, null, "district"),{BasicDBObject office->
def statMap = [:] // if(!office.township){
DISTRICT_STAT.find(and(eq("city", city), eq("end_month_date", Date.parse("yyyyMMdd", "20190228")))).forEach(new Consumer<Document>() { // def lng = office.lng/10000000000l
@Override // def lat = office.lat/10000000000l
void accept(Document t) { // HttpRequest request = new HttpRequest()
def key = "${t.city}_${t.region}_${t.district_name}_${t.house_type}".toString() // JSONObject resp = request.getJsonContent("http://localhost:8080/FangJiaAddressService/geocode/township", ['location':"$lng,$lat".toString()], "UTF-8")
statMap[key] = t.avg_price // if(resp.result){
} // println(office._id.toString() + "\t" + office.township + "\t-->\t" + resp.result.streetName)
}) // DbUtils.seaweed.update(new BasicDBObject("_id", office._id), new BasicDBObject('$set', new BasicDBObject('township', resp.result.streetName)))
roots.each { def root -> // }
EsDataSource dataSource = new EsDataSource(); // }
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" // })
dataSource.setUrl(address) //}
Properties connectionProperties = new Properties() ExcelUtils.multiThreadingModify("C:\\Users\\goojia\\Desktop\\地图查询数据导出-添加经纬度-兴趣点.xlsx",{List item->
connectionProperties.put("user", "developer") def lng = item["AB"].toString().split(",")[0].toDouble()
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") def lat = item["AB"].toString().split(",")[1].toDouble()
dataSource.setProperties(connectionProperties) item["AD"] = GeoUtils.calculateDistance(121.526672,31.142637,lng, lat)
Connection connection = dataSource.getConnection() println(item)
Statement statement = connection.createStatement() })
ResultSet rs = statement.executeQuery("""select * from "seawater-2019" where city='${city}' and cat = 'rent' order by p_date""")
int count = 0
while (rs.next()) {
String region = rs.getString("region")
String district_name = rs.getString("district_name")
int room = rs.getInt("room")
def key = "${city}_${region}_${district_name}_${room}".toString()
if (!statMap.containsKey(key)) {
continue
}
int total_price = rs.getInt("total_price")
double area = rs.getDouble("area")
int avg_price = (int) (total_price / area)
if (total_price * 1.3 || total_price < statMap[key] * 0.7) {
continue
}
String block = rs.getString("block")
String name = rs.getString("name")
String fitment = rs.getString("fitment")
String direction = rs.getString("direction")
int hall = rs.getInt("hall")
int toilet = rs.getInt("toilet")
int p_date = rs.getInt("p_date")
if (!block || !name || !fitment || !direction || room == 0 || hall == 0 || toilet == 0) {
continue
}
println(rs.getString('url'))
def item = [city, region, district_name, block, fitment, direction, room, hall, toilet, name, total_price, area, avg_price, p_date, root]
println(item)
data << item
count++
if(count > 10){
break
}
}
}
}
ExcelUtils.write("D:\\建设银行\\租金样本.xlsx", data, "结果")
@@ -1,51 +1,15 @@
package com.fangjia.script.wenyan.test package com.fangjia.script.wenyan.test
import com.aliyuncs.DefaultAcsClient import com.fangjia.datastore.driver.DSCollection
import com.aliyuncs.IAcsClient import com.fangjia.datastore.driver.DataStore
import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityRequest import com.fangjia.script.util.DbUtils
import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityResponse import com.mongodb.BasicDBObject
import com.aliyuncs.profile.DefaultProfile
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils
import net.sf.json.JSONObject
def strt(String addr){
}
ExcelUtils.modify("D:\\nextcloud\\数据\\地址类服务测试\\建行地址相似度评级测试.xlsx",{List item->
def city = item["A"]
def address1 = item["B"]
def address2 = item["C"]
HttpRequest http = new HttpRequest()
def params = new HashMap()
params["city"] = city
params["address_1"] = address1
params["address_2"] = address2
params["token"]= "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
JSONObject json = http.getJsonContent("http://open.fangjia.com/address/similarity", params, "UTF-8")
JSONObject result = json.result
item["D"] = result?.level
item["E"] = result?.formattedAddress_1
item["F"] = result?.formattedAddress_2
item["G"] = result?.isSameAddress
DefaultProfile profile = DefaultProfile.getProfile(
"cn-hangzhou",
"LTAIdaj4eSO79gzz",
"xw6EsvqydZ9HC6zAvIB3ruQqVHynxf");
IAcsClient client = new DefaultAcsClient(profile);
GetAddressSimilarityRequest request = new GetAddressSimilarityRequest();
request.setServiceCode("addrp")
request.setAppKey("Xcgtyhupey0h")
request.setText("${address1};${address2}")
GetAddressSimilarityResponse response = client.getAcsResponse(request)
String data = response.getData()
json = JSONObject.fromObject(data)
item["H"] = json["sim_level"]["level"]
item["I"] = json["sim_level"]["address_a"]
item["J"] = json["sim_level"]["address_b"]
item["K"] = json["sim_level"]["is_same"]
println(json)
})
//、DSCollection seabird = new DataStore("192.168.0.38", 27007).getCollection("fangjia.seaweed")
//seabird.find(DbUtils.buildSeaweedQuery("常州", "溧阳", "吾悦广场").append("if_sell", 1)).each {
// println(it)
//}
long l = 0
if(!l){
println(l)
}
@@ -165,8 +165,11 @@ public class SellTriggerServiceImpl implements TriggerService {
newObj.put("districtname_key", district.getName() + "_" + newObj.get("region") + "_" + object.get("city")); newObj.put("districtname_key", district.getName() + "_" + newObj.get("region") + "_" + object.get("city"));
newObj.put("district_name", district.getName()); newObj.put("district_name", district.getName());
} }
}else{
if(object.containsField("city") && object.containsField("district_name") && object.containsField("region")){
newObj.put("districtname_key", object.get("district_name") + "_" + object.get("region") + "_" + object.get("city"));
}
} }
// newObj.put("if_school", 0); // newObj.put("if_school", 0);
// if (districtSchoolMap != null && districtSchoolMap.containsKey(districtKey)) { // if (districtSchoolMap != null && districtSchoolMap.containsKey(districtKey)) {
// List<SchoolKey> skList = districtSchoolMap.get(districtKey); // List<SchoolKey> skList = districtSchoolMap.get(districtKey);