Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into script_gj

# Conflicts:
#	projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictHouseBatch.groovy
This commit is contained in:
2024-03-08 10:39:06 +08:00
4 changed files with 65 additions and 118 deletions
@@ -87,14 +87,9 @@ class DistrictHouseBatch extends BaseBatch {
long rentCount = 0
for(def name : names){
String key = name.toString().trim() + "_" + district.get("region") + "_" + district.get("city")
if(!sellCount){
sellCount = sellCountMap.get(key) ? sellCountMap.get(key) : 0
}
if(!rentCount){
rentCount = rentCountMap.get(key) ? rentCountMap.get(key) : 0
}
sellCount = sellCount + (sellCountMap.get(key) ? sellCountMap.get(key) : 0)
rentCount = rentCount + (rentCountMap.get(key) ? rentCountMap.get(key) : 0)
}
saveToDistrictHouse(district.get('city'), district.get("region"), district.get("name"), sellCount, rentCount, sn)
}
@@ -1,7 +1,16 @@
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeoUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.client.MongoCollection
import net.sf.json.JSONObject
import org.bson.Document
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
@@ -11,67 +20,43 @@ import java.util.function.Consumer
import static com.mongodb.client.model.Filters.and
import static com.mongodb.client.model.Filters.eq
//REGION_CONN = MongoUtils.getCollectionByName("fangjia", "region_block", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
//TOWNSHIP_CONN = MongoUtils.getCollectionByName("fangjia", "township", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
//TOWNSHIP_CONN.find().each {BasicDBObject township->
// township.area_station = township.central
// township.removeField("central")
// TOWNSHIP_CONN.save(township)
//
//}
//REGION_CONN.find(new BasicDBObject("category", "region")).each {BasicDBObject regionBlock->
// def lng = regionBlock.lng/10000000000l
// def lat = regionBlock.lat/10000000000l
// HttpRequest request = new HttpRequest()
// JSONObject resp = request.getJsonContent("http://localhost:8080/FangJiaAddressService/geocode/township", ['location':"$lng,$lat".toString()], "UTF-8")
// if(resp.result){
// TOWNSHIP_CONN.update(new BasicDBObject("_id", new ObjectId(resp.result.id)), new BasicDBObject('$set', new BasicDBObject('central', 1)))
// println(resp.result)
// }
//}
def cityList = ["上海", "广州", "北京", "深圳", "成都", "杭州", "重庆", "武汉", "西安", "天津"]
def roots = ["lianjia.com", "zhuge.com", "fang.com", "anjuke.com", "58.com", "soufy.cn"]
MongoCollection<Document> DISTRICT_STAT = Mongo3Utils.getDbCollection("fangjia", "district_stat_rent")
MongoCollection<Document> DEAL_PRICE = Mongo3Utils.getDbCollection("fangjia", "deal_price")
def data = [["城市", "区域", "小区名", "版块", "装修", "朝向", "室", "厅", "卫", '标题', '总价', '面积', '单价', '发布时间', '来源']]
cityList.each { def city ->
def statMap = [:]
DISTRICT_STAT.find(and(eq("city", city), eq("end_month_date", Date.parse("yyyyMMdd", "20190228")))).forEach(new Consumer<Document>() {
@Override
void accept(Document t) {
def key = "${t.city}_${t.region}_${t.district_name}_${t.house_type}".toString()
statMap[key] = t.avg_price
}
})
roots.each { def root ->
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
Connection connection = dataSource.getConnection()
Statement statement = connection.createStatement()
ResultSet rs = statement.executeQuery("""select * from "seawater-2019" where city='${city}' and cat = 'rent' order by p_date""")
int count = 0
while (rs.next()) {
String region = rs.getString("region")
String district_name = rs.getString("district_name")
int room = rs.getInt("room")
def key = "${city}_${region}_${district_name}_${room}".toString()
if (!statMap.containsKey(key)) {
continue
}
int total_price = rs.getInt("total_price")
double area = rs.getDouble("area")
int avg_price = (int) (total_price / area)
if (total_price * 1.3 || total_price < statMap[key] * 0.7) {
continue
}
String block = rs.getString("block")
String name = rs.getString("name")
String fitment = rs.getString("fitment")
String direction = rs.getString("direction")
int hall = rs.getInt("hall")
int toilet = rs.getInt("toilet")
int p_date = rs.getInt("p_date")
if (!block || !name || !fitment || !direction || room == 0 || hall == 0 || toilet == 0) {
continue
}
println(rs.getString('url'))
def item = [city, region, district_name, block, fitment, direction, room, hall, toilet, name, total_price, area, avg_price, p_date, root]
println(item)
data << item
count++
if(count > 10){
break
}
}
}
}
ExcelUtils.write("D:\\建设银行\\租金样本.xlsx", data, "结果")
//CommonUtils.findAllCity().each {def city->
// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city, null, null, "district"),{BasicDBObject office->
// if(!office.township){
// def lng = office.lng/10000000000l
// def lat = office.lat/10000000000l
// HttpRequest request = new HttpRequest()
// JSONObject resp = request.getJsonContent("http://localhost:8080/FangJiaAddressService/geocode/township", ['location':"$lng,$lat".toString()], "UTF-8")
// if(resp.result){
// println(office._id.toString() + "\t" + office.township + "\t-->\t" + resp.result.streetName)
// DbUtils.seaweed.update(new BasicDBObject("_id", office._id), new BasicDBObject('$set', new BasicDBObject('township', resp.result.streetName)))
// }
// }
// })
//}
ExcelUtils.multiThreadingModify("C:\\Users\\goojia\\Desktop\\地图查询数据导出-添加经纬度-兴趣点.xlsx",{List item->
def lng = item["AB"].toString().split(",")[0].toDouble()
def lat = item["AB"].toString().split(",")[1].toDouble()
item["AD"] = GeoUtils.calculateDistance(121.526672,31.142637,lng, lat)
println(item)
})
@@ -1,51 +1,15 @@
package com.fangjia.script.wenyan.test
import com.aliyuncs.DefaultAcsClient
import com.aliyuncs.IAcsClient
import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityRequest
import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityResponse
import com.aliyuncs.profile.DefaultProfile
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils
import net.sf.json.JSONObject
def strt(String addr){
}
ExcelUtils.modify("D:\\nextcloud\\数据\\地址类服务测试\\建行地址相似度评级测试.xlsx",{List item->
def city = item["A"]
def address1 = item["B"]
def address2 = item["C"]
HttpRequest http = new HttpRequest()
def params = new HashMap()
params["city"] = city
params["address_1"] = address1
params["address_2"] = address2
params["token"]= "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
JSONObject json = http.getJsonContent("http://open.fangjia.com/address/similarity", params, "UTF-8")
JSONObject result = json.result
item["D"] = result?.level
item["E"] = result?.formattedAddress_1
item["F"] = result?.formattedAddress_2
item["G"] = result?.isSameAddress
DefaultProfile profile = DefaultProfile.getProfile(
"cn-hangzhou",
"LTAIdaj4eSO79gzz",
"xw6EsvqydZ9HC6zAvIB3ruQqVHynxf");
IAcsClient client = new DefaultAcsClient(profile);
GetAddressSimilarityRequest request = new GetAddressSimilarityRequest();
request.setServiceCode("addrp")
request.setAppKey("Xcgtyhupey0h")
request.setText("${address1};${address2}")
GetAddressSimilarityResponse response = client.getAcsResponse(request)
String data = response.getData()
json = JSONObject.fromObject(data)
item["H"] = json["sim_level"]["level"]
item["I"] = json["sim_level"]["address_a"]
item["J"] = json["sim_level"]["address_b"]
item["K"] = json["sim_level"]["is_same"]
println(json)
})
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
//、DSCollection seabird = new DataStore("192.168.0.38", 27007).getCollection("fangjia.seaweed")
//seabird.find(DbUtils.buildSeaweedQuery("常州", "溧阳", "吾悦广场").append("if_sell", 1)).each {
// println(it)
//}
long l = 0
if(!l){
println(l)
}
@@ -165,8 +165,11 @@ public class SellTriggerServiceImpl implements TriggerService {
newObj.put("districtname_key", district.getName() + "_" + newObj.get("region") + "_" + object.get("city"));
newObj.put("district_name", district.getName());
}
}else{
if(object.containsField("city") && object.containsField("district_name") && object.containsField("region")){
newObj.put("districtname_key", object.get("district_name") + "_" + object.get("region") + "_" + object.get("city"));
}
}
// newObj.put("if_school", 0);
// if (districtSchoolMap != null && districtSchoolMap.containsKey(districtKey)) {
// List<SchoolKey> skList = districtSchoolMap.get(districtKey);