Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into script_gj
# Conflicts: # projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy
This commit is contained in:
+18
-18
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
@@ -18,25 +19,24 @@ class EsSearchUtils {
|
||||
|
||||
static def findPois(def city, def region, def name){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT cityname,adname,name,address,type,location.lon,location.lat,m_date,typecode FROM amap_poi WHERE 1=1")
|
||||
sqlBuffer.append(" and cityname = '" + city + "'")
|
||||
sqlBuffer.append(" and adname = '" + region + "'")
|
||||
sqlBuffer.append(" and name = '" + name + "'")
|
||||
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def sql = "select * from amap_poi where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') "
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sql.toString())
|
||||
def rets = []
|
||||
while (rs.next()) {
|
||||
def m = [:]
|
||||
m.city = rs.getString("cityname")
|
||||
m.region = rs.getString("adname")
|
||||
m.name = rs.getString("name")
|
||||
m.address = rs.getString("address")
|
||||
m.type = rs.getString("type")
|
||||
m.code = rs.getString("typecode")
|
||||
m.lng = rs.getString("location.lon")
|
||||
m.lat = rs.getString("location.lat")
|
||||
rets << m
|
||||
}
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
def m = [:]
|
||||
m.city = data.cityname
|
||||
m.region = data.adname
|
||||
m.name = data.name
|
||||
m.address = data.address
|
||||
m.type = data.type
|
||||
m.code = data.typecode
|
||||
|
||||
def location = data.location
|
||||
m.lng = location.lon
|
||||
m.lat = location.lat
|
||||
rets << m
|
||||
})
|
||||
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
+110
@@ -0,0 +1,110 @@
|
||||
package com.fangjia.script.gj.village
|
||||
|
||||
import com.fangjia.script.gj.EsSearchUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||
|
||||
def filePath = "D:\\数据\\涉农\\县.xlsx"
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
// if(!row['D']){
|
||||
// def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府")
|
||||
// if(rets){
|
||||
// def obj = findOneByName(rets, row['C']+"人民政府")
|
||||
// if(obj){
|
||||
// row['D'] = obj.lng
|
||||
// row['E'] = obj.lat
|
||||
// println row['C'] + " "+obj
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
// if(row['D'] && !row['F']){
|
||||
// def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL)
|
||||
// def map = ["lon":row['D'], "lat":row['E']]
|
||||
// def township = InsidePolygon.isInArea2(map.lon, map.lat, townMap)
|
||||
// row['F'] = township
|
||||
// }
|
||||
|
||||
if(row['F'] && !row['K']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_name", row['B'])
|
||||
q.put("area_name", row['C'])
|
||||
def streetName = row['I']
|
||||
q.put("street_name", streetName)
|
||||
def count = COMMUNITY_CONN.count(q);
|
||||
row['K'] = count
|
||||
// row['H'] = streetName
|
||||
}
|
||||
// def gaodeMap = getTownshipInfo(row['D']+","+row['E'])
|
||||
// if(gaodeMap && gaodeMap.township){
|
||||
// row['I'] = gaodeMap.township
|
||||
// }
|
||||
|
||||
})
|
||||
|
||||
|
||||
def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
}
|
||||
}
|
||||
}
|
||||
if(district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
Thread.sleep(3000)
|
||||
print "."
|
||||
}
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.district = district
|
||||
map.township = township
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
def findOneByName(def rets, name){
|
||||
def obj = null
|
||||
for(def r : rets){
|
||||
if(r.name == name){
|
||||
obj = r
|
||||
break;
|
||||
}
|
||||
}
|
||||
return obj
|
||||
}
|
||||
|
||||
|
||||
def findTownRange(city, TOWN_POLYLINECOLL){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
def dbcursor = TOWN_POLYLINECOLL.find(q)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def pointList = InsidePolygon.createPoint3(d.polyline);
|
||||
townMap.put(d.fjw_region +"##"+ d.town, pointList)
|
||||
}
|
||||
return townMap
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package com.fangjia.script.majintao
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
|
||||
ExcelUtils.removeEmptyRow("F:\\估家\\文件\\北交所\\testcopy.xlsx")
|
||||
ExcelUtils.singleThreadingModify("F:\\估家\\文件\\北交所\\testcopy.xlsx", { List items ->
|
||||
String excelFilePath = "F:\\估家\\文件\\北交所\\testcopy.xlsx";
|
||||
String comprehensiveUrl = "http://192.168.10.15:9180/comprehensiveQuery/houseValueInformation";
|
||||
String communityUrl = "http://192.168.10.15:9180/addrSvc/avgPrice";
|
||||
String valueUrl = "http://192.168.10.15:9180/property/addressEvaluate";
|
||||
String caseUrl = "http://192.168.10.15:9180/house/case";
|
||||
String poiUrl = "http://192.168.10.15:9180/district/addrFacilitys";
|
||||
|
||||
HttpRequest httpRequest = new HttpRequest();
|
||||
map = new HashMap<String, String>()
|
||||
|
||||
String city = items['A']
|
||||
String address = items['B']
|
||||
double size = items['C']
|
||||
|
||||
System.out.println(city + "---" + address + "----" + size);
|
||||
|
||||
map.put("city", city)
|
||||
map.put("address", address)
|
||||
map.put("size", size.toString())
|
||||
map.put("strctureArea", size.toString())
|
||||
map.put("token", "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB")
|
||||
|
||||
if (!address.contains("%")) {
|
||||
// 发送HTTP请求
|
||||
long start
|
||||
long end
|
||||
|
||||
// start = System.currentTimeMillis();
|
||||
// httpRequest.getJsonContent(communityUrl, map, "UTF-8")
|
||||
//
|
||||
//
|
||||
// end = System.currentTimeMillis();
|
||||
// items['D'] = String.valueOf(end - start);
|
||||
//
|
||||
// start = System.currentTimeMillis();
|
||||
// httpRequest.getJsonContent(valueUrl, map, "UTF-8")
|
||||
// end = System.currentTimeMillis();
|
||||
// items['E'] = String.valueOf(end - start);
|
||||
//
|
||||
// start = System.currentTimeMillis();
|
||||
// httpRequest.getJsonContent(caseUrl, map, "UTF-8")
|
||||
// end = System.currentTimeMillis();
|
||||
// items['F'] = String.valueOf(end - start);
|
||||
//
|
||||
// start = System.currentTimeMillis();
|
||||
// httpRequest.getJsonContent(poiUrl, map, "UTF-8")
|
||||
// end = System.currentTimeMillis();
|
||||
// items['G'] = String.valueOf(end - start);
|
||||
|
||||
start = System.currentTimeMillis();
|
||||
def content = httpRequest.getJsonContent(comprehensiveUrl, map, "UTF-8")
|
||||
end = System.currentTimeMillis();
|
||||
items['H'] = String.valueOf(end - start);
|
||||
println content.get("msg")
|
||||
if(content.get("msg").contains("该地址不支持估价!")) {
|
||||
items['I'] = "该地址不支持估价!"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
})
|
||||
|
||||
@@ -443,6 +443,45 @@ public class ExcelUtils{
|
||||
}
|
||||
|
||||
|
||||
public static singleThreadingModify(String fileName, Closure closure, Integer srcSheetIndex = 0, String destSheetName = null){
|
||||
if(!destSheetName){
|
||||
def inp = new FileInputStream(fileName);
|
||||
Workbook workbook = WorkbookFactory.create(inp)
|
||||
workbook.setForceFormulaRecalculation(true)
|
||||
Sheet sheet = workbook.getSheetAt(srcSheetIndex);
|
||||
destSheetName = sheet.getSheetName();
|
||||
}
|
||||
|
||||
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
|
||||
def body = data.subList(1, data.size())
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 3
|
||||
println nThread
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(3)
|
||||
final AtomicInteger counter = new AtomicInteger(0)
|
||||
final double total = body.size()
|
||||
for(int i = 0 ; i < body.size(); i++){
|
||||
final row = body[i]
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try{
|
||||
closure.call(row)
|
||||
int count = counter.incrementAndGet()
|
||||
if(count % 1000 == 0){
|
||||
println(DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+"执行进度:${com.fangjia.core.util.DataUtil.roundDouble((count * 100)/total, 2)}%")
|
||||
}
|
||||
}catch (Exception e){
|
||||
println(row)
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
executorService.shutdown()
|
||||
executorService.awaitTermination(30, TimeUnit.DAYS)
|
||||
ExcelUtils.write(fileName, data, destSheetName)
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static Object getCellValue(Cell cell){
|
||||
|
||||
Reference in New Issue
Block a user