Merge branch 'feature/monitor' into script_develop
# Conflicts: # projects/fangjia-script/pom.xml
This commit is contained in:
@@ -32,6 +32,7 @@ public enum Watcher {
|
||||
BaYi('15300424133','bayi@fangjia.com'),
|
||||
WeiLei('18101751850','weilei@fangjia.com'),
|
||||
JiangYaHui('13684913780','jiangyahui@fangjia.com'),
|
||||
ShenJian('13764369683', 'shenjian@fangjia.com')
|
||||
private final String mobile;
|
||||
private final String email;
|
||||
|
||||
|
||||
+16
-1
@@ -218,7 +218,7 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name", "工商大楼");
|
||||
// q.put("name","东寓新村")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -1392,6 +1392,21 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
public static def queryDistrictPriceByWeekend(city, region, name, weekend, limit){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
queryCondition.put("region", region);
|
||||
queryCondition.put("district_name", name);
|
||||
if(weekend){
|
||||
queryCondition.put('weekend', new BasicDBObject('$gte', weekend));
|
||||
}
|
||||
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", 1)).limit(limit)
|
||||
def dbo = null;
|
||||
while(cur.hasNext()) {
|
||||
dbo = cur.next()
|
||||
}
|
||||
return dbo;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
package com.fangjia.script.companyCrawl
|
||||
|
||||
import net.sf.json.JSONObject
|
||||
import org.jsoup.Jsoup
|
||||
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class AiqichaCrawl {
|
||||
|
||||
def static final proxyPort = 9020;
|
||||
def static final proxyServer = "http-dyn.abuyun.com";
|
||||
def static final proxyUser = "HZU1XU28H3G4Z44D";
|
||||
def static final proxyPass = "466C86ED77FE43D4";
|
||||
def companyKeys = ["亨鑫","捷东","盈复","隆庆","卓源 ","合万","通广","安盛","东利","瑞科 ","凡迪","捷信","安盛","安盛","东进","源盛","茂禄","泰亚","宏成","光益","华圣","益全","长凯","宏新","锦立","宜佳","万美","正欣","和利","国宏","和裕","新昌","佳升","正德","源美","宝峰","金广","龙奥","宏美","成同","义丰","全洪","国聚","亨信","百益","广宏","仁瑞","亿昌","东旭","聚凯","洪成","兴同","赛迪","亨国","浩长","达宏","寿满","亿信","本盛","盈凯","浩汇","元辉","祥生","皇弘","讯思","新州","国泰","成合","茂浩","信源","润凯","虎光","安瑞","益吉","达兴","顺光","发盛","金优","捷丰","春凯","昌兴","本佳","远盛","盛达","亚飞","东昌","益如","宝东","德信","浩盈","多泰","泰联","升聚","辉丰","茂发","盈德","旭嘉","优源","益同","协盈","锦泰","捷久","本宏","德泰","聚富","全庆","凯安","进恒","全伟","优元","凯安","义丰","丰宇","瑞优","隆泰","诺德","台升","裕大","亨恒","义浩","百亨","欣昌","茂全","同亨","长圣","成谦","达泰","高协","正思","安巨","新耀","利仁","国益","旺洪","汇中","旭昌","弘康","优盛","旭达","晶安","华浩","耀春","圣本","康丰","光复","同凌","宇伯","乐鸿","成泰","宏协","华信","达泰","广丰","泰全","汇恒","杰森","天畅","恒通","亚达","庆合","丰圣","德长","茂百","汇久","欣鼎","谦和","厚辉","耀德","瑞隆","多辉","昌茂","成高","益升","安达","兴飞","百进","鼎宝","祥盛","信和","隆顺","亚欣","厚全","博文","讯诺","泰如","亨瑞","凯祥","捷浩","文舟","金庆","成仁","万协","宇中","宏祥","永多","艺峰","美益","创赢","奥龙","德宇","国福","飞利","久欣","如亨","安广","协佳","升捷","盛圣","富凯","长鑫","元卓","荣亨","亨仁","康鑫","信和","百新","干瑞","润万","富庭","信宇","源天","凯佳","正升","裕荣","厚泽","晶富","广新","正多","鼎高","丰汇","永华","兴升","复腾","隆诺","中发","广源","康胜","同进","仁捷","康凯","义洪","晶盛","正祥","裕祥","宇新","洪祥","润岳","义富","盛安","浩易"]
|
||||
static AtomicInteger AbuyunRequestCount = new AtomicInteger(0);
|
||||
public final static int AbuyunMaxCount = 10;
|
||||
|
||||
static main(args) {
|
||||
def task = new AiqichaCrawl();
|
||||
task.process()
|
||||
}
|
||||
|
||||
void process(Object args) {
|
||||
|
||||
def url = "https://aiqicha.baidu.com/s/l?q=%E6%96%B0%E5%8F%8B&t=&p=3&s=10&o=0&f=%7B%7D "
|
||||
for(def key : companyKeys){
|
||||
def queryBody = null
|
||||
def s = crawl(url, queryBody);
|
||||
if(s){
|
||||
print s
|
||||
|
||||
|
||||
}
|
||||
println s
|
||||
}
|
||||
}
|
||||
|
||||
def crawl(url, queryBody){
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","aiqicha.baidu.com")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Cache-Control","max-age=0")
|
||||
headers.put("sec-ch-ua",'"Chromium";v="88", "Google Chrome";v="88", ";Not A Brand";v="99"')
|
||||
headers.put("sec-ch-ua-mobile","?0")
|
||||
headers.put("Upgrade-Insecure-Requests","1")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
|
||||
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
|
||||
headers.put("Sec-Fetch-Site","same-origin")
|
||||
headers.put("Sec-Fetch-Mode","navigate")
|
||||
headers.put("Sec-Fetch-User","?1")
|
||||
headers.put("Sec-Fetch-Dest","document")
|
||||
headers.put("Accept-Language","zh-CN,zh;q=0.9")
|
||||
|
||||
String s = doPost(url, queryBody, headers)
|
||||
if(s && s.contains("window.pageData = ")){
|
||||
def prefix = "window.pageData ="
|
||||
def s1 = s.substring(s.indexOf(prefix)+prefix.size())
|
||||
s1 = s1.substring(0, s1.indexOf("}};")+2)
|
||||
|
||||
JSONObject jsonObject = JSONObject.fromObject(s1);
|
||||
print jsonObject
|
||||
}
|
||||
|
||||
return s
|
||||
|
||||
}
|
||||
|
||||
static String doPost(String strUrl, String content, Map headers) {
|
||||
String result = "";
|
||||
OutputStream outputStream = null;
|
||||
InputStream inputStream = null;
|
||||
StringBuilder sb = new StringBuilder();
|
||||
try {
|
||||
checkAbuyun(true);
|
||||
|
||||
def responseMap = [:]
|
||||
// 创建代理服务器地址对象
|
||||
InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
|
||||
// 创建HTTP类型代理对象
|
||||
Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
|
||||
|
||||
URL url = new URL(strUrl);
|
||||
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
|
||||
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
|
||||
urlConnection.setRequestMethod("POST");
|
||||
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setConnectTimeout(30000);
|
||||
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setReadTimeout(30000);
|
||||
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
|
||||
urlConnection.setDoOutput(true);
|
||||
urlConnection.setDoInput(true);
|
||||
//设置通用请求属性为默认浏览器编码类型
|
||||
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
|
||||
|
||||
|
||||
if (headers != null && headers.keySet().size() > 0) {
|
||||
for (String head : headers.keySet()) {
|
||||
urlConnection.setRequestProperty(head, headers.get(head));
|
||||
}
|
||||
}
|
||||
//getOutputStream()返回的输出流,用于写入参数数据。
|
||||
outputStream = urlConnection.getOutputStream();
|
||||
if(content){
|
||||
outputStream.write(content.getBytes());
|
||||
}
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
|
||||
inputStream = urlConnection.getInputStream();
|
||||
|
||||
|
||||
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
|
||||
char[] buff = new char[1024];
|
||||
int length = 0;
|
||||
while ((length = reader.read(buff)) != -1) {
|
||||
String s = new String(buff, 0, length);
|
||||
sb.append(s);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}finally{
|
||||
if(outputStream!=null){
|
||||
outputStream.close();
|
||||
}
|
||||
if(inputStream!=null){
|
||||
inputStream.close();
|
||||
}
|
||||
}
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
public static checkAbuyun(boolean check) {
|
||||
if (check) {
|
||||
while (AbuyunRequestCount.get() >= AbuyunMaxCount) {
|
||||
try {
|
||||
Thread.sleep(5000);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
AbuyunRequestCount.incrementAndGet();
|
||||
} else {
|
||||
AbuyunRequestCount.decrementAndGet();
|
||||
}
|
||||
}
|
||||
}
|
||||
+715
@@ -0,0 +1,715 @@
|
||||
package com.fangjia.script.companyCrawl
|
||||
|
||||
import com.eclipsesource.v8.V8
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import jdk.nashorn.api.scripting.ScriptObjectMirror
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.http.Header
|
||||
import org.apache.http.HttpResponse
|
||||
import org.apache.http.client.HttpClient
|
||||
import org.apache.http.client.methods.HttpGet
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.select.Elements
|
||||
|
||||
import org.apache.http.impl.client.HttpClients
|
||||
|
||||
import javax.script.ScriptEngine
|
||||
import javax.script.ScriptEngineManager;
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class GsxtCrawl {
|
||||
|
||||
def static final proxyPort = 9020;
|
||||
def static final proxyServer = "http-dyn.abuyun.com";
|
||||
def static final proxyUser = "HZU1XU28H3G4Z44D";
|
||||
def static final proxyPass = "466C86ED77FE43D4";
|
||||
|
||||
public final static int AbuyunMaxCount = 10;
|
||||
static AtomicInteger AbuyunRequestCount = new AtomicInteger(0);
|
||||
|
||||
def companyKeys = ["亨鑫","捷东","盈复","隆庆","卓源 ","合万","通广","安盛","东利","瑞科 ","凡迪","捷信","安盛","安盛","东进","源盛","茂禄","泰亚","宏成","光益","华圣","益全","长凯","宏新","锦立","宜佳","万美","正欣","和利","国宏","和裕","新昌","佳升","正德","源美","宝峰","金广","龙奥","宏美","成同","义丰","全洪","国聚","亨信","百益","广宏","仁瑞","亿昌","东旭","聚凯","洪成","兴同","赛迪","亨国","浩长","达宏","寿满","亿信","本盛","盈凯","浩汇","元辉","祥生","皇弘","讯思","新州","国泰","成合","茂浩","信源","润凯","虎光","安瑞","益吉","达兴","顺光","发盛","金优","捷丰","春凯","昌兴","本佳","远盛","盛达","亚飞","东昌","益如","宝东","德信","浩盈","多泰","泰联","升聚","辉丰","茂发","盈德","旭嘉","优源","益同","协盈","锦泰","捷久","本宏","德泰","聚富","全庆","凯安","进恒","全伟","优元","凯安","义丰","丰宇","瑞优","隆泰","诺德","台升","裕大","亨恒","义浩","百亨","欣昌","茂全","同亨","长圣","成谦","达泰","高协","正思","安巨","新耀","利仁","国益","旺洪","汇中","旭昌","弘康","优盛","旭达","晶安","华浩","耀春","圣本","康丰","光复","同凌","宇伯","乐鸿","成泰","宏协","华信","达泰","广丰","泰全","汇恒","杰森","天畅","恒通","亚达","庆合","丰圣","德长","茂百","汇久","欣鼎","谦和","厚辉","耀德","瑞隆","多辉","昌茂","成高","益升","安达","兴飞","百进","鼎宝","祥盛","信和","隆顺","亚欣","厚全","博文","讯诺","泰如","亨瑞","凯祥","捷浩","文舟","金庆","成仁","万协","宇中","宏祥","永多","艺峰","美益","创赢","奥龙","德宇","国福","飞利","久欣","如亨","安广","协佳","升捷","盛圣","富凯","长鑫","元卓","荣亨","亨仁","康鑫","信和","百新","干瑞","润万","富庭","信宇","源天","凯佳","正升","裕荣","厚泽","晶富","广新","正多","鼎高","丰汇","永华","兴升","复腾","隆诺","中发","广源","康胜","同进","仁捷","康凯","义洪","晶盛","正祥","裕祥","宇新","洪祥","润岳","义富","盛安","浩易"]
|
||||
static main(args) {
|
||||
def task = new GsxtCrawl();
|
||||
//task.process()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
getObject()
|
||||
HttpClient client = HttpClients.createDefault();
|
||||
HttpGet get = new HttpGet("http://www.gsxt.gov.cn/corp-query-homepage.html");
|
||||
get.setHeader("Upgrade-Insecure-Requests", "1");
|
||||
get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36");
|
||||
|
||||
HttpResponse response = client.execute(get);
|
||||
def __jsluid_h = getJsluid(response)
|
||||
def s = getResponseBodyAsString(response)
|
||||
println (s)
|
||||
def __jsl_clearance = getJsContent(s);
|
||||
|
||||
//Cookie: __jsluid_h=26cb8385493c2e424767eca767720c87; __jsl_clearance=1618899529.627|-1|ao39lP6IVSloPMvq5%2FOSBqSiPZ0%3D
|
||||
def newCookie = __jsluid_h+";"+__jsl_clearance
|
||||
get.setHeader("Cookie", newCookie)
|
||||
HttpResponse response2 = client.execute(get);
|
||||
def s2 = getResponseBodyAsString(response2)
|
||||
def __jsl_clearance2 = getJsContent2(s2);
|
||||
println(__jsl_clearance2)
|
||||
|
||||
// fristCrawl()
|
||||
}
|
||||
|
||||
|
||||
//该函数测试Java与js对象
|
||||
static void getObject() throws Exception{
|
||||
String str = "var _0xbe8c = ['w4LCg8Onw7o=', 'ZAjDvCw=', 'YcKOw6Fm', 'wpHCqsO3CQ==', 'UMKCXnw=', 'w7Aww6XCnQ==', 'BcKNwrvCng==', 'J2zCnsKi', 'wq02WMOx', 'RQXDrUw=', 'eCnCnDw=', 'Q8KowpzDjA==', 'YznDucOn', 'YcOrXxk=', 'w4t9PsKM', 'DMO4QmA=', 'w5HCklPDmQ==', 'w5rCv8Omw6Q=', 'NcOwc8K5', 'GUxjw7Q=', 'w7zDicK8w4I=', 'VCDDpsKv', 'cwLChT4=', 'wpvCl1vDqw==', 'wrvCvMOKNQ==', 'RTnDrAc=', 'GMKFw4hk', 'SxnDvC0=', 'H2NHNQ==', 'w6Aqw7LCqg==', 'wq0mW8Ok', 'L1zCv8Kp', 'B8OqwoI2', 'Jl7DuX8=', 'JcOBwpIR', 'U8OnfSo=', '6K2K5rOG6aqL6KyM', 'f8Kiwo/Dhg==', 'aMK6w6vDog==', 'JcKgw4FC', 'w61vJEg=', 'LMOrwoY0', 'GcK4w6lt', 'w4XDrsO5wpY=', 'J8OhwoM1', 'w5Mzw7J/', 'wocbcBA=', 'SibDt8Oj', 'w7xgwoXDkQ==', 'AcKyDEQ=', 'Rg7Dkw==', 'w6UFK8Kw', 'TTzDjcKK', 'w6QhDcKy', 'ZBLDols=', 'ZjPDrk4=', 'w6XDqX3Ctw==', 'w6EvJcKT', 'JHtUDQ==', 'wq8mI8KE', 'JMOhwoJ9', 'MsK1AkE=', 'w7fCmhcm', 'VsK2w5fDtQ==', 'w7EUw4PCnw==', 'dz/DgsKu', 'UzfDtcK0', 'wpd9fWE=', 'worDvcKFw4M=', 'w4PDr0zDhg==', 'w7ocSMKg', 'wp1uwoZd', 'eAjDv8Oz', 'aATCgTw=', 'w7UGw7k=', 'wqZCwotq', 'HFIBw5s=', 'WsOdw7JB', 'w5PDpXHCpA==', 'GMO8wqdD', 'w4TCsQs=', 'w6/DmcODwpg=', 'w7MuwrjDoQ==', 'dDHDhsOj', 'G308w70=', 'CsO6wog9', 'FcONwrVw', 'M3vCmMK4', 'w6/Dm8OvwpI=', 'QsOLw5ZM', 'TgLDicKy', 'QSHCuQA=', 'UsOlw5RI', 'w7EqwqDDrw==', 'w6jDusK5w4k=', 'OsKaw5rDlg==', 'wosLQsOk', 'DcKxw7Vo', 'w5VuIEo=', 'w7sXw6TCqw==', 'wrLDr8KRw7s=', 'wr9+wpJR', 'w4LDscO0wrQ=', 'FFIPw6U=', 'J8Ksw5rDlA==', 'd8KZwqjDgw==', 'PcKvDF4=', 'MHh2GA==', 'dgDDs0Y=', 'wpxSS1Y=', 'w7l5JMKh', 'w7N8JMKC', 'XsKhw7hR', 'QxDDqm4=', 'PcKGw5ZJ', 'w7HCuAIQ', 'E8Kmw53Dmg==', 'CMOrwok3', 'U8K9wpvDnA==', 'IW06w5U=', 'w7DDjcOGwpQ=', 'X8KlwpHDpA==', 'w6TDn8O1wpo=', 'ZjnDv08=', 'w7/Cqjsp', 'wqLDk8KYw58=', 'w6fDvMKXw4g=', 'FsK8IUY=', 'w4knw4Rh', 'w5zCu8Ogw5M=', 'wqJ0ckc=', 'T8Kgw69G', 'GF0hw74=', 'FMOkwqkH', 'wokmbMOM', 'djHDtcKW', 'FXkow64=', 'B8OHwowH', 'VSjCpQU=', 'w5vDs8K0w4s=', 'w7vCkUV1', 'w7dMIcK8', 'WMOjZyI=', 'LcKBw7po', 'BMK7HkY=', 'B8K0w7F4', 'YsK+w67Drw==', 'HlQKw7I=', 'wpLCrMKrNg==', 'w7IjcsKy', 'w6zDusKhw4o=', 'WhPDm8Om', 'dMKlw7RO', 'dcOHw5R3', 'Nkpww7M=', 'wrnCsMKLdg==', 'dDTDqMKb', 'S8KXZXE=', 'wq/CssOpJA==', 'w7J/KFw=', 'QMKSWEo=', 'woNxbEc=', 'wqVIwow=', 'TD/Dqg4=', 'w7TDpmzCiw==', 'w7HDjWDCgA==', 'w7IEw6vCoA==', 'Q8K+w4zDiQ==', 'PElvw7k=', 'cDXDsRo=', 'FMOLX0c=', 'Z8KHwojDoA==', 'TTTCqzo=', 'woXCrsK6Uw==', 'w4lhHnY=', 'w6Q/dsKj', 'KsKIwpLCtQ==', 'QQTDn8Oy', 'D8Omw7zDoQ==', 'E8KzGk0=']; (function(_0x44c192, _0xbe8c61) {\n" +
|
||||
"\t\tvar _0x1533fb = function(_0x5b499f) {\n" +
|
||||
"\t\t\twhile (--_0x5b499f) {\n" +
|
||||
"\t\t\t\t_0x44c192['push'](_0x44c192['shift']());\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x1533fb(++_0xbe8c61);\n" +
|
||||
"\t} (_0xbe8c, 0x185));\n" +
|
||||
"\tvar _0x1533 = function(_0x44c192, _0xbe8c61) {\n" +
|
||||
"\t\t_0x44c192 = _0x44c192 - 0x0;\n" +
|
||||
"\t\tvar _0x1533fb = _0xbe8c[_0x44c192];\n" +
|
||||
"\t\tif (_0x1533['GlBvhA'] === undefined) { (function() {\n" +
|
||||
"\t\t\t\tvar _0xa131c3 = function() {\n" +
|
||||
"\t\t\t\t\tvar _0x9829e5;\n" +
|
||||
"\t\t\t\t\ttry {\n" +
|
||||
"\t\t\t\t\t\t_0x9829e5 = Function('return\\x20(function()\\x20' + '{}.constructor(\\x22return\\x20this\\x22)(\\x20)' + ');')();\n" +
|
||||
"\t\t\t\t\t} catch(_0x1ba3fa) {\n" +
|
||||
|
||||
"\t\t\t\t\t}\n" +
|
||||
"\t\t\t\t\treturn _0x9829e5;\n" +
|
||||
"\t\t\t\t};\n" +
|
||||
"\t\t\t\tvar _0x5e86d6 = _0xa131c3();\n" +
|
||||
"\t\t\t\tvar _0x67fb0c = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=';\n" +
|
||||
"\t\t\t\t_0x5e86d6['atob'] || (_0x5e86d6['atob'] = function(_0x1cb044) {\n" +
|
||||
"\t\t\t\t\tvar _0xa69f93 = String(_0x1cb044)['replace'](/=+\$/, '');\n" +
|
||||
"\t\t\t\t\tvar _0x26bfb9 = '';\n" +
|
||||
"\t\t\t\t\tfor (var _0x2f1d79 = 0x0,\n" +
|
||||
"\t\t\t\t\t_0x3713bb, _0x4e514b, _0x29733b = 0x0; _0x4e514b = _0xa69f93['charAt'](_0x29733b++);~_0x4e514b && (_0x3713bb = _0x2f1d79 % 0x4 ? _0x3713bb * 0x40 + _0x4e514b: _0x4e514b, _0x2f1d79++%0x4) ? _0x26bfb9 += String['fromCharCode'](0xff & _0x3713bb >> ( - 0x2 * _0x2f1d79 & 0x6)) : 0x0) {\n" +
|
||||
"\t\t\t\t\t\t_0x4e514b = _0x67fb0c['indexOf'](_0x4e514b);\n" +
|
||||
"\t\t\t\t\t}\n" +
|
||||
"\t\t\t\t\treturn _0x26bfb9;\n" +
|
||||
"\t\t\t\t});\n" +
|
||||
"\t\t\t} ());\n" +
|
||||
"\t\t\tvar _0x38a7ff = function(_0x5dd34b, _0x3a0f2e) {\n" +
|
||||
"\t\t\t\tvar _0x3e7a99 = [],\n" +
|
||||
"\t\t\t\t_0x3e2c47 = 0x0,\n" +
|
||||
"\t\t\t\t_0x383663,\n" +
|
||||
"\t\t\t\t_0x14720e = '',\n" +
|
||||
"\t\t\t\t_0x5864f4 = '';\n" +
|
||||
"\t\t\t\t_0x5dd34b = atob(_0x5dd34b);\n" +
|
||||
"\t\t\t\tfor (var _0x28a6d2 = 0x0,\n" +
|
||||
"\t\t\t\t_0x2ea622 = _0x5dd34b['length']; _0x28a6d2 < _0x2ea622; _0x28a6d2++) {\n" +
|
||||
"\t\t\t\t\t_0x5864f4 += '%' + ('00' + _0x5dd34b['charCodeAt'](_0x28a6d2)['toString'](0x10))['slice']( - 0x2);\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t\t_0x5dd34b = decodeURIComponent(_0x5864f4);\n" +
|
||||
"\t\t\t\tvar _0x3091fb;\n" +
|
||||
"\t\t\t\tfor (_0x3091fb = 0x0; _0x3091fb < 0x100; _0x3091fb++) {\n" +
|
||||
"\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3091fb;\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t\tfor (_0x3091fb = 0x0; _0x3091fb < 0x100; _0x3091fb++) {\n" +
|
||||
"\t\t\t\t\t_0x3e2c47 = (_0x3e2c47 + _0x3e7a99[_0x3091fb] + _0x3a0f2e['charCodeAt'](_0x3091fb % _0x3a0f2e['length'])) % 0x100;\n" +
|
||||
"\t\t\t\t\t_0x383663 = _0x3e7a99[_0x3091fb];\n" +
|
||||
"\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3e7a99[_0x3e2c47];\n" +
|
||||
"\t\t\t\t\t_0x3e7a99[_0x3e2c47] = _0x383663;\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t\t_0x3091fb = 0x0;\n" +
|
||||
"\t\t\t\t_0x3e2c47 = 0x0;\n" +
|
||||
"\t\t\t\tfor (var _0x8c68bc = 0x0; _0x8c68bc < _0x5dd34b['length']; _0x8c68bc++) {\n" +
|
||||
"\t\t\t\t\t_0x3091fb = (_0x3091fb + 0x1) % 0x100;\n" +
|
||||
"\t\t\t\t\t_0x3e2c47 = (_0x3e2c47 + _0x3e7a99[_0x3091fb]) % 0x100;\n" +
|
||||
"\t\t\t\t\t_0x383663 = _0x3e7a99[_0x3091fb];\n" +
|
||||
"\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3e7a99[_0x3e2c47];\n" +
|
||||
"\t\t\t\t\t_0x3e7a99[_0x3e2c47] = _0x383663;\n" +
|
||||
"\t\t\t\t\t_0x14720e += String['fromCharCode'](_0x5dd34b['charCodeAt'](_0x8c68bc) ^ _0x3e7a99[(_0x3e7a99[_0x3091fb] + _0x3e7a99[_0x3e2c47]) % 0x100]);\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t\treturn _0x14720e;\n" +
|
||||
"\t\t\t};\n" +
|
||||
"\t\t\t_0x1533['sMWqUz'] = _0x38a7ff;\n" +
|
||||
"\t\t\t_0x1533['AbOzCI'] = {};\n" +
|
||||
"\t\t\t_0x1533['GlBvhA'] = !![];\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tvar _0x5b499f = _0x1533['AbOzCI'][_0x44c192];\n" +
|
||||
"\t\tif (_0x5b499f === undefined) {\n" +
|
||||
"\t\t\tif (_0x1533['uRkTAI'] === undefined) {\n" +
|
||||
"\t\t\t\t_0x1533['uRkTAI'] = !![];\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t\t_0x1533fb = _0x1533['sMWqUz'](_0x1533fb, _0xbe8c61);\n" +
|
||||
"\t\t\t_0x1533['AbOzCI'][_0x44c192] = _0x1533fb;\n" +
|
||||
"\t\t} else {\n" +
|
||||
"\t\t\t_0x1533fb = _0x5b499f;\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\treturn _0x1533fb;\n" +
|
||||
"\t};\n" +
|
||||
"\tfunction hash(_0x1a5a87) {\n" +
|
||||
"\t\tvar _0x23a3e0 = {};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x93', 'Kp2R') + 'J'] = function(_0x5ebeb2, _0x4681fc) {\n" +
|
||||
"\t\t\treturn _0x5ebeb2 ^ _0x4681fc;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x17', '(bZ%') + 'H'] = function(_0x5108c3, _0x49fe65) {\n" +
|
||||
"\t\t\treturn _0x5108c3 & _0x49fe65;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x88', 'N#(C') + 'T'] = function(_0x3d3f10, _0x3f2a86) {\n" +
|
||||
"\t\t\treturn _0x3d3f10 !== _0x3f2a86;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x60', '(cNt') + 'Z'] = _0x1533('0x70', 'K)PG') + 'J';\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x49', 'Y8u@') + 'h'] = function(_0xc3ff68, _0x129ab0) {\n" +
|
||||
"\t\t\treturn _0xc3ff68 >> _0x129ab0;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x9c', '\$qIi') + 'S'] = function(_0x591dc1, _0x158d9d) {\n" +
|
||||
"\t\t\treturn _0x591dc1 + _0x158d9d;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x7f', '(bZ%') + 'f'] = function(_0xd6deb, _0x557760) {\n" +
|
||||
"\t\t\treturn _0xd6deb >> _0x557760;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x4e', 'S9Bn') + 'd'] = function(_0xd05226, _0x29b6e8) {\n" +
|
||||
"\t\t\treturn _0xd05226 * _0x29b6e8;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x7e', 'y^BN') + 'n'] = function(_0x899f8d, _0x3dbd26) {\n" +
|
||||
"\t\t\treturn _0x899f8d < _0x3dbd26;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x9a', '(U[3') + 'e'] = function(_0x39678a, _0x37d488) {\n" +
|
||||
"\t\t\treturn _0x39678a * _0x37d488;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0xa3', 'sahq') + 'U'] = function(_0x4cf877, _0x14e1b0) {\n" +
|
||||
"\t\t\treturn _0x4cf877 << _0x14e1b0;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0xa7', 'K#Hh') + 'u'] = function(_0x556336, _0x56b41c) {\n" +
|
||||
"\t\t\treturn _0x556336 - _0x56b41c;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x3d', '(bZ%') + 'p'] = function(_0x372c20, _0x534fe1) {\n" +
|
||||
"\t\t\treturn _0x372c20 >> _0x534fe1;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x1d', 'S9Bn') + 'g'] = function(_0x26a0f8, _0x4f1660) {\n" +
|
||||
"\t\t\treturn _0x26a0f8 >>> _0x4f1660;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0xae', ']*f@') + 'Y'] = function(_0x3423f4, _0x53e1a9) {\n" +
|
||||
"\t\t\treturn _0x3423f4 | _0x53e1a9;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x26', 's4Dp') + 'P'] = function(_0x19382a, _0x220110) {\n" +
|
||||
"\t\t\treturn _0x19382a | _0x220110;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x65', 's4Dp') + 'p'] = function(_0x7b95bc, _0x22e933) {\n" +
|
||||
"\t\t\treturn _0x7b95bc & _0x22e933;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x0', '\$qIi') + 'V'] = function(_0x13269b, _0x227e3b) {\n" +
|
||||
"\t\t\treturn _0x13269b ^ _0x227e3b;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x29', 'J21I') + 'R'] = function(_0x1ba947, _0x4b40e4) {\n" +
|
||||
"\t\t\treturn _0x1ba947(_0x4b40e4);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x50', 's4Dp') + 'n'] = _0x1533('0xa', 'qRqa') + 'e';\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x42', 'ksVw') + 'b'] = function(_0x466fa6, _0x3ce8da) {\n" +
|
||||
"\t\t\treturn _0x466fa6 ^ _0x3ce8da;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x2c', 'rqYb') + 'f'] = function(_0x1e608a, _0x241888) {\n" +
|
||||
"\t\t\treturn _0x1e608a - _0x241888;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x78', 'J21I') + 'u'] = function(_0x564b78, _0x18d236, _0x8fd805) {\n" +
|
||||
"\t\t\treturn _0x564b78(_0x18d236, _0x8fd805);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x69', 'zx(Q') + 'D'] = function(_0x3b7666, _0x47f238, _0x57b05e) {\n" +
|
||||
"\t\t\treturn _0x3b7666(_0x47f238, _0x57b05e);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x5f', 'ZaCV') + 'm'] = function(_0x452fa7, _0x493d76, _0x1446e7, _0x4af814, _0x5e0f59) {\n" +
|
||||
"\t\t\treturn _0x452fa7(_0x493d76, _0x1446e7, _0x4af814, _0x5e0f59);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x22', 'Rh@#') + 'W'] = function(_0x51a858, _0x440b1c) {\n" +
|
||||
"\t\t\treturn _0x51a858(_0x440b1c);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x8', 'xPGU') + 'A'] = function(_0x5e96f2, _0x167b41) {\n" +
|
||||
"\t\t\treturn _0x5e96f2 + _0x167b41;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x6d', 'rqYb') + 'X'] = function(_0x46cd37, _0x5d8684) {\n" +
|
||||
"\t\t\treturn _0x46cd37 + _0x5d8684;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x3e', 'rqYb') + 'g'] = function(_0x26bc4b, _0x16bc46) {\n" +
|
||||
"\t\t\treturn _0x26bc4b(_0x16bc46);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\tvar _0x179134 = _0x23a3e0;\n" +
|
||||
"\t\tfunction _0x5be8c0(_0x296dd6, _0x778566) {\n" +
|
||||
"\t\t\treturn _0x179134[_0x1533('0x56', 'ZaCV') + 'J'](_0x179134[_0x1533('0x7c', 'o&Q4') + 'H'](_0x296dd6, 0x7fffffff) + (_0x778566 & 0x7fffffff), _0x179134[_0x1533('0x52', 'K)PG') + 'H'](_0x296dd6, 0x80000000)) ^ _0x778566 & 0x80000000;\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tfunction _0x225a28(_0x2f57af) {\n" +
|
||||
"\t\t\tvar _0x3c49d8 = {};\n" +
|
||||
"\t\t\t_0x3c49d8[_0x1533('0x55', 'qRqa') + 'N'] = function(_0x15bfb8, _0x51e1eb) {\n" +
|
||||
"\t\t\t\treturn _0x15bfb8 < _0x51e1eb;\n" +
|
||||
"\t\t\t};\n" +
|
||||
"\t\t\tvar _0x567e0f = _0x3c49d8;\n" +
|
||||
"\t\t\tvar _0x5b2e48 = _0x1533('0xa4', 'rQf(') + _0x1533('0x4', 'rqYb') + _0x1533('0x95', 'Rh@#') + _0x1533('0x3', 'OhQr');\n" +
|
||||
"\t\t\tvar _0x321f21 = '';\n" +
|
||||
"\t\t\tfor (var _0x2a5528 = 0x7; _0x2a5528 >= 0x0; _0x2a5528--) {\n" +
|
||||
"\t\t\t\tif (_0x179134[_0x1533('0xb2', 'Gfc!') + 'T'](_0x1533('0x2', 'w*Nn') + 'h', _0x179134[_0x1533('0x6c', 'OhQr') + 'Z'])) {\n" +
|
||||
"\t\t\t\t\t_0x321f21 += _0x5b2e48[_0x1533('0x66', ')T5g') + 'At'](_0x179134[_0x1533('0x9f', 'zx(Q') + 'H'](_0x179134[_0x1533('0x5', 'OhQr') + 'h'](_0x2f57af, _0x2a5528 * 0x4), 0xf));\n" +
|
||||
"\t\t\t\t} else {\n" +
|
||||
"\t\t\t\t\treturn _0x567e0f[_0x1533('0x5a', 'y*wC') + 'N'](t, 0x14) ? 0x5a827999: t < 0x28 ? 0x6ed9eba1: _0x567e0f[_0x1533('0x43', 'zx(Q') + 'N'](t, 0x3c) ? -0x70e44324: -0x359d3e2a;\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t\treturn _0x321f21;\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tfunction _0xde630f(_0x10fb9a) {\n" +
|
||||
"\t\t\tvar _0x5b5b6d = _0x179134[_0x1533('0x16', 'GvA9') + 'S'](_0x179134[_0x1533('0x87', 'N#(C') + 'f'](_0x179134[_0x1533('0x83', '[H[u') + 'S'](_0x10fb9a[_0x1533('0x58', 'OhQr') + 'th'], 0x8), 0x6), 0x1),\n" +
|
||||
"\t\t\t_0x797793 = new Array(_0x179134[_0x1533('0x59', 'xPGU') + 'd'](_0x5b5b6d, 0x10));\n" +
|
||||
"\t\t\tfor (var _0x109346 = 0x0; _0x179134[_0x1533('0x61', 'dgE&') + 'n'](_0x109346, _0x179134[_0x1533('0x90', 'y^BN') + 'e'](_0x5b5b6d, 0x10)); _0x109346++) {\n" +
|
||||
"\t\t\t\tif (_0x179134[_0x1533('0x2a', 'NNf8') + 'T'](_0x1533('0x12', 'sahq') + 'y', _0x1533('0x67', 'y*wC') + 'G')) {\n" +
|
||||
"\t\t\t\t\t_0x797793[_0x109346] = 0x0;\n" +
|
||||
"\t\t\t\t} else {\n" +
|
||||
"\t\t\t\t\treturn !! [];\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t\tfor (_0x109346 = 0x0; _0x109346 < _0x10fb9a[_0x1533('0x40', 'ad*k') + 'th']; _0x109346++) {\n" +
|
||||
"\t\t\t\t_0x797793[_0x109346 >> 0x2] |= _0x179134[_0x1533('0xa9', '@gLq') + 'U'](_0x10fb9a[_0x1533('0x1c', 'GvA9') + _0x1533('0xf', 'OhQr') + 'At'](_0x109346), _0x179134[_0x1533('0xb3', 'S9Bn') + 'u'](0x18, (_0x109346 & 0x3) * 0x8));\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t\t_0x797793[_0x179134[_0x1533('0xb1', 'qRqa') + 'p'](_0x109346, 0x2)] |= 0x80 << 0x18 - (_0x109346 & 0x3) * 0x8;\n" +
|
||||
"\t\t\t_0x797793[_0x5b5b6d * 0x10 - 0x1] = _0x10fb9a[_0x1533('0xb', '@mqx') + 'th'] * 0x8;\n" +
|
||||
"\t\t\treturn _0x797793;\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tfunction _0x50e067(_0x45a1cc, _0x3bcee1) {\n" +
|
||||
"\t\t\treturn _0x45a1cc << _0x3bcee1 | _0x179134[_0x1533('0x5b', 'Y8u@') + 'g'](_0x45a1cc, _0x179134[_0x1533('0x91', '@mqx') + 'u'](0x20, _0x3bcee1));\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tfunction _0x526d3b(_0xb72077, _0xed7f25, _0x5bc269, _0x572b62) {\n" +
|
||||
"\t\t\tif (_0xb72077 < 0x14) return _0x179134[_0x1533('0x98', 'Gfc!') + 'Y'](_0xed7f25 & _0x5bc269, _0x179134[_0x1533('0x68', 'OhQr') + 'H'](~_0xed7f25, _0x572b62));\n" +
|
||||
"\t\t\tif (_0x179134[_0x1533('0x9e', 'w*Nn') + 'n'](_0xb72077, 0x28)) return _0x179134[_0x1533('0x99', ')T5g') + 'J'](_0xed7f25, _0x5bc269) ^ _0x572b62;\n" +
|
||||
"\t\t\tif (_0xb72077 < 0x3c) return _0x179134[_0x1533('0x13', 'CNzQ') + 'P'](_0xed7f25 & _0x5bc269, _0x179134[_0x1533('0x45', '@mqx') + 'p'](_0xed7f25, _0x572b62)) | _0x179134[_0x1533('0xa0', 'avK7') + 'p'](_0x5bc269, _0x572b62);\n" +
|
||||
"\t\t\treturn _0x179134[_0x1533('0x5e', 'avK7') + 'V'](_0xed7f25, _0x5bc269) ^ _0x572b62;\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tfunction _0x496785(_0x583c97) {\n" +
|
||||
"\t\t\treturn _0x583c97 < 0x14 ? 0x5a827999: _0x583c97 < 0x28 ? 0x6ed9eba1: _0x583c97 < 0x3c ? -0x70e44324: -0x359d3e2a;\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tvar _0x45b840 = _0x179134[_0x1533('0x39', 'R#BF') + 'R'](_0xde630f, _0x1a5a87);\n" +
|
||||
"\t\tvar _0x53258d = new Array(0x50);\n" +
|
||||
"\t\tvar _0xc86ce6 = 0x67452301;\n" +
|
||||
"\t\tvar _0x3ee50e = -0x10325477;\n" +
|
||||
"\t\tvar _0xdb1008 = -0x67452302;\n" +
|
||||
"\t\tvar _0xe99bcb = 0x10325476;\n" +
|
||||
"\t\tvar _0x56c860 = -0x3c2d1e10;\n" +
|
||||
"\t\tfor (var _0x4c314c = 0x0; _0x4c314c < _0x45b840[_0x1533('0x46', '\$qIi') + 'th']; _0x4c314c += 0x10) {\n" +
|
||||
"\t\t\tvar _0x41e8e8 = _0xc86ce6;\n" +
|
||||
"\t\t\tvar _0x1b7277 = _0x3ee50e;\n" +
|
||||
"\t\t\tvar _0x30f7c3 = _0xdb1008;\n" +
|
||||
"\t\t\tvar _0x279c95 = _0xe99bcb;\n" +
|
||||
"\t\t\tvar _0xf6e888 = _0x56c860;\n" +
|
||||
"\t\t\tfor (var _0x77e31b = 0x0; _0x179134[_0x1533('0x34', 'Y8u@') + 'n'](_0x77e31b, 0x50); _0x77e31b++) {\n" +
|
||||
"\t\t\t\tif (_0x77e31b < 0x10) {\n" +
|
||||
"\t\t\t\t\tif (_0x179134[_0x1533('0xa6', 'Mifp') + 'T'](_0x1533('0x3f', 'o&Q4') + 'G', _0x179134[_0x1533('0xa1', 'rqYb') + 'n'])) {\n" +
|
||||
"\t\t\t\t\t\t_0x53258d[_0x77e31b] = _0x45b840[_0x4c314c + _0x77e31b];\n" +
|
||||
"\t\t\t\t\t} else {\n" +
|
||||
"\t\t\t\t\t\tsleepT = _0x179134[_0x1533('0x48', 'NNf8') + 'R'](parseInt, data['wt']) > ret[0x1] ? _0x179134[_0x1533('0x97', 'd[i)') + 'R'](parseInt, data['wt']) - ret[0x1] : 0x1f4;\n" +
|
||||
"\t\t\t\t\t}\n" +
|
||||
"\t\t\t\t} else {\n" +
|
||||
"\t\t\t\t\t_0x53258d[_0x77e31b] = _0x50e067(_0x179134[_0x1533('0x96', 'rQf(') + 'b'](_0x179134[_0x1533('0xb0', 'Gfc!') + 'b'](_0x53258d[_0x179134[_0x1533('0xa5', 'K)PG') + 'u'](_0x77e31b, 0x3)], _0x53258d[_0x179134[_0x1533('0x51', 'K)PG') + 'f'](_0x77e31b, 0x8)]) ^ _0x53258d[_0x77e31b - 0xe], _0x53258d[_0x179134[_0x1533('0xaf', '(U[3') + 'f'](_0x77e31b, 0x10)]), 0x1);\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t\tt = _0x179134[_0x1533('0x18', 'GvA9') + 'u'](_0x5be8c0, _0x179134[_0x1533('0x23', '\$qIi') + 'u'](_0x5be8c0, _0x179134[_0x1533('0x8d', 'Mifp') + 'D'](_0x50e067, _0xc86ce6, 0x5), _0x179134[_0x1533('0x4c', 'xPGU') + 'm'](_0x526d3b, _0x77e31b, _0x3ee50e, _0xdb1008, _0xe99bcb)), _0x5be8c0(_0x5be8c0(_0x56c860, _0x53258d[_0x77e31b]), _0x179134[_0x1533('0xad', 'rqYb') + 'W'](_0x496785, _0x77e31b)));\n" +
|
||||
"\t\t\t\t_0x56c860 = _0xe99bcb;\n" +
|
||||
"\t\t\t\t_0xe99bcb = _0xdb1008;\n" +
|
||||
"\t\t\t\t_0xdb1008 = _0x50e067(_0x3ee50e, 0x1e);\n" +
|
||||
"\t\t\t\t_0x3ee50e = _0xc86ce6;\n" +
|
||||
"\t\t\t\t_0xc86ce6 = t;\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t\t_0xc86ce6 = _0x5be8c0(_0xc86ce6, _0x41e8e8);\n" +
|
||||
"\t\t\t_0x3ee50e = _0x5be8c0(_0x3ee50e, _0x1b7277);\n" +
|
||||
"\t\t\t_0xdb1008 = _0x5be8c0(_0xdb1008, _0x30f7c3);\n" +
|
||||
"\t\t\t_0xe99bcb = _0x5be8c0(_0xe99bcb, _0x279c95);\n" +
|
||||
"\t\t\t_0x56c860 = _0x5be8c0(_0x56c860, _0xf6e888);\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\treturn _0x179134[_0x1533('0x86', 'Gfc!') + 'A'](_0x179134[_0x1533('0x44', 'DDSy') + 'X'](_0x179134[_0x1533('0x8e', 'xPGU') + 'g'](_0x225a28, _0xc86ce6), _0x179134[_0x1533('0x77', 'n*Rs') + 'g'](_0x225a28, _0x3ee50e)) + _0x225a28(_0xdb1008) + _0x225a28(_0xe99bcb), _0x179134[_0x1533('0xd', 'DDSy') + 'g'](_0x225a28, _0x56c860));\n" +
|
||||
"\t}\n" +
|
||||
"\tfunction go(_0x394549) {\n" +
|
||||
"\t\tvar _0x33d0d6 = {};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x2b', 'sahq') + 'H'] = _0x1533('0x14', 'rQf(') + _0x1533('0x15', 'sahq');\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0xab', 'dgE&') + 'H'] = function(_0x23f209, _0x4ed415) {\n" +
|
||||
"\t\t\treturn _0x23f209 < _0x4ed415;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x5c', 'xPGU') + 'g'] = function(_0x39bd0a, _0x4d5764) {\n" +
|
||||
"\t\t\treturn _0x39bd0a != _0x4d5764;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x37', 'y*wC') + 'N'] = function(_0x3c9a26, _0x109b89) {\n" +
|
||||
"\t\t\treturn _0x3c9a26 + _0x109b89;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x63', 'gkl*') + 'z'] = function(_0x584935, _0x58347c) {\n" +
|
||||
"\t\t\treturn _0x584935 + _0x58347c;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x21', 'ZaCV') + 'z'] = function(_0x3fc795, _0x6962f6) {\n" +
|
||||
"\t\t\treturn _0x3fc795 + _0x6962f6;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x27', '(cNt') + 'I'] = function(_0x303446, _0x3388fb) {\n" +
|
||||
"\t\t\treturn _0x303446 == _0x3388fb;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x6b', 'y*wC') + 't'] = function(_0x5cb515, _0x31e7f3) {\n" +
|
||||
"\t\t\treturn _0x5cb515(_0x31e7f3);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x75', 'Rh@#') + 'E'] = function(_0x529d8e, _0xc1423a) {\n" +
|
||||
"\t\t\treturn _0x529d8e + _0xc1423a;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x62', 'rQf(') + 'a'] = function(_0x453505, _0x50cd56) {\n" +
|
||||
"\t\t\treturn _0x453505 + _0x50cd56;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x2e', 'NNf8') + 'B'] = _0x1533('0x28', ']*f@') + _0x1533('0x57', 'Kp2R') + '\\x20/';\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0xe', 'Y8u@') + 'y'] = function(_0x25bcfd, _0x4b634e, _0x1bef22) {\n" +
|
||||
"\t\t\treturn _0x25bcfd(_0x4b634e, _0x1bef22);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x92', 'J21I') + 'i'] = function(_0xccfc4d, _0x9a8620) {\n" +
|
||||
"\t\t\treturn _0xccfc4d > _0x9a8620;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x11', '@H8e') + 'n'] = function(_0x1b9fc7, _0x7d3ce) {\n" +
|
||||
"\t\t\treturn _0x1b9fc7(_0x7d3ce);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x1b', 'N#(C') + 'O'] = _0x1533('0x7', 'N#(C') + '失败';\n" +
|
||||
"\t\tvar _0x2eda7d = _0x33d0d6;\n" +
|
||||
"\t\tfunction _0x29e502() {\n" +
|
||||
"\t\t\tvar _0x24a3c1 = window[_0x1533('0x35', 'ad*k') + _0x1533('0x5d', 'Y8u@') + 'r'][_0x1533('0x19', 'avK7') + _0x1533('0x7d', 'cJ3U') + 't'],\n" +
|
||||
"\t\t\t_0x5f1192 = [_0x2eda7d[_0x1533('0x24', '(bZ%') + 'H']];\n" +
|
||||
"\t\t\tfor (var _0x2d8686 = 0x0; _0x2eda7d[_0x1533('0x1a', 'avK7') + 'H'](_0x2d8686, _0x5f1192[_0x1533('0x3b', 'Y8u@') + 'th']); _0x2d8686++) {\n" +
|
||||
"\t\t\t\tif (_0x2eda7d[_0x1533('0x2f', 'y*wC') + 'g'](_0x24a3c1[_0x1533('0x76', 'y*wC') + _0x1533('0x33', 'ZaCV')](_0x5f1192[_0x2d8686]), -0x1)) {\n" +
|
||||
"\t\t\t\t\treturn !! [];\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t\tif (window[_0x1533('0x41', 'dgE&') + _0x1533('0x6e', 'dgE&') + _0x1533('0x85', 'NNf8')] || window[_0x1533('0x30', 'o&Q4') + _0x1533('0x38', 'OhQr')] || window[_0x1533('0xc', 'OhQr') + _0x1533('0x74', 'qRqa')] || window[_0x1533('0x25', '(bZ%') + _0x1533('0x4f', 'avK7') + 'r'][_0x1533('0xa2', 'xPGU') + _0x1533('0x53', ')T5g') + 'r'] || window[_0x1533('0xaa', 'cJ3U') + _0x1533('0x6', 'K#@S') + 'r'][_0x1533('0x8f', 'rqYb') + _0x1533('0xac', '(bZ%') + _0x1533('0x9', 'Rh@#') + _0x1533('0x4d', 'rQf(') + 'e'] || window[_0x1533('0x31', 'N#(C') + _0x1533('0x79', 'dgE&') + 'r'][_0x1533('0x4b', 'ksVw') + _0x1533('0x10', 'gkl*') + _0x1533('0xa8', 'd[i)') + _0x1533('0x89', '\$qIi') + _0x1533('0x84', 's4Dp')]) {\n" +
|
||||
"\t\t\t\treturn !! [];\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\tif (_0x29e502()) {\n" +
|
||||
"\t\t\treturn;\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tvar _0x1642ef = new Date();\n" +
|
||||
"\t\tfunction _0x1a1fd9(_0x55baea, _0x579edb) {\n" +
|
||||
"\t\t\tvar _0x551ac6 = _0x394549[_0x1533('0x4a', 'y*wC') + 's'][_0x1533('0x71', 'K#@S') + 'th'];\n" +
|
||||
"\t\t\tfor (var _0x54ccec = 0x0; _0x2eda7d[_0x1533('0x6f', 'dFiI') + 'H'](_0x54ccec, _0x551ac6); _0x54ccec++) {\n" +
|
||||
"\t\t\t\tfor (var _0x562c2d = 0x0; _0x2eda7d[_0x1533('0x9b', '[H[u') + 'H'](_0x562c2d, _0x551ac6); _0x562c2d++) {\n" +
|
||||
"\t\t\t\t\tvar _0x1b8d8c = _0x2eda7d[_0x1533('0x80', '[H[u') + 'N'](_0x2eda7d[_0x1533('0x3a', 'w*Nn') + 'z'](_0x2eda7d[_0x1533('0x8b', 'cJ3U') + 'z'](_0x579edb[0x0], _0x394549[_0x1533('0x32', 'R#BF') + 's'][_0x1533('0x82', '@mqx') + 'tr'](_0x54ccec, 0x1)), _0x394549[_0x1533('0x1f', '*hmq') + 's'][_0x1533('0x47', '(cNt') + 'tr'](_0x562c2d, 0x1)), _0x579edb[0x1]);\n" +
|
||||
"\t\t\t\t\tif (_0x2eda7d[_0x1533('0x6a', '(bZ%') + 'I'](_0x2eda7d[_0x1533('0x9d', 'Kp2R') + 't'](hash, _0x1b8d8c), _0x55baea)) {\n" +
|
||||
"\t\t\t\t\t\treturn [_0x1b8d8c, new Date() - _0x1642ef];\n" +
|
||||
"\t\t\t\t\t}\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\tvar _0x1e73b9 = _0x2eda7d[_0x1533('0x36', 'sahq') + 'y'](_0x1a1fd9, _0x394549['ct'], _0x394549[_0x1533('0x2d', '\$qIi')]);\n" +
|
||||
"\t\tif (_0x1e73b9) {\n" +
|
||||
"\t\t\tvar _0x3b1427;\n" +
|
||||
"\t\t\tif (_0x394549['wt']) {\n" +
|
||||
"\t\t\t\t_0x3b1427 = _0x2eda7d[_0x1533('0x1', 'zx(Q') + 'i'](parseInt(_0x394549['wt']), _0x1e73b9[0x1]) ? _0x2eda7d[_0x1533('0x54', 'avK7') + 'n'](parseInt, _0x394549['wt']) - _0x1e73b9[0x1] : 0x1f4;\n" +
|
||||
"\t\t\t} else {\n" +
|
||||
"\t\t\t\t_0x3b1427 = 0x5dc;\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t\t_0x2eda7d[_0x1533('0x72', 'qRqa') + 'y'](setTimeout,\n" +
|
||||
"\t\t\tfunction() {\n" +
|
||||
"\t\t\t\tdocument[_0x1533('0x20', 'rQf(') + 'ie'] = _0x2eda7d[_0x1533('0x73', 'rQf(') + 'E'](_0x2eda7d[_0x1533('0x3c', 'o&Q4') + 'a'](_0x394549['tn'] + '=' + _0x1e73b9[0x0] + (_0x1533('0x8a', 'ksVw') + _0x1533('0x1e', 'GvA9') + '='), _0x394549['vt']), _0x2eda7d[_0x1533('0x8c', 'Gfc!') + 'B']);\n" +
|
||||
"\t\t\t\tlocation[_0x1533('0x7a', 'sahq')] = location[_0x1533('0x64', 'd[i)') + _0x1533('0x81', '(U[3')] + location[_0x1533('0x94', 'sahq') + 'ch'];\n" +
|
||||
"\t\t\t},\n" +
|
||||
"\t\t\t_0x3b1427);\n" +
|
||||
"\t\t} else {\n" +
|
||||
"\t\t\talert(_0x2eda7d[_0x1533('0x7b', ')T5g') + 'O']);\n" +
|
||||
"\t\t}\n" +
|
||||
"\t};\n" +
|
||||
"\tgo({\n" +
|
||||
"\t\t\"bts\": [\"1618915727.945|0|HM5\", \"VoVNIPqcrvFO13woIZK2RY%3D\"],\n" +
|
||||
"\t\t\"chars\": \"ZDUZR0cDjTEXznMkfgavFO\",\n" +
|
||||
"\t\t\"ct\": \"3049e28877124ad21ca0db55e3af95ff316ceef6\",\n" +
|
||||
"\t\t\"ha\": \"sha1\",\n" +
|
||||
"\t\t\"tn\": \"__jsl_clearance\",\n" +
|
||||
"\t\t\"vt\": \"3600\",\n" +
|
||||
"\t\t\"wt\": \"1500\"\n" +
|
||||
"\t})"
|
||||
|
||||
ScriptEngineManager manager = new ScriptEngineManager();
|
||||
ScriptEngine engine = manager.getEngineByName("javascript");
|
||||
engine.eval(str);
|
||||
//获取对象
|
||||
System.out.println();
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
def getJsContent(def page){
|
||||
ScriptEngineManager manager = new ScriptEngineManager();
|
||||
//得到脚本引擎
|
||||
ScriptEngine engine = manager.getEngineByName("javascript");
|
||||
//处理加密js
|
||||
String js = page.trim().replace("<script>", "").replace("</script>", "").replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]}));", "y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]})");
|
||||
try {
|
||||
//得到解密后的js
|
||||
js = js.replace("document.cookie","var cookie")
|
||||
js = js.replace("location.href=location.pathname+location.search", "")
|
||||
engine.eval(js);
|
||||
def v = engine.get("cookie")
|
||||
return v.toString().substring(0, v.toString().indexOf(";"))
|
||||
} catch (ScriptException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
|
||||
def getJsContent2(def page){
|
||||
ScriptEngineManager manager = new ScriptEngineManager();
|
||||
//得到脚本引擎
|
||||
ScriptEngine engine = manager.getEngineByName("javascript");
|
||||
//处理加密js
|
||||
String js = page.trim().replace("<script>", "").replace("</script>", "").replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]}));", "y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]})");
|
||||
try {
|
||||
//得到解密后的js
|
||||
//得到解密后的js
|
||||
String result = (String) engine.eval(js);
|
||||
result = result.substring(result.indexOf("var cd"),result.indexOf("dc+=cd;")+7);
|
||||
result = result.replaceAll("document*.*toLowerCase\\(\\)", "'x'");
|
||||
//得到cookie
|
||||
String jsl = (String) engine.eval(result);
|
||||
return jsl;
|
||||
} catch (ScriptException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 通过破解动态JavaScript脚本,
|
||||
* 获取cookie名为 __jsl_clearance的值
|
||||
*
|
||||
* @param body 相应内容(一般为第一次请求获取到的动态js字符串)
|
||||
* @return cookie名为 __jsl_clearance的值
|
||||
*/
|
||||
private static String getJslClearance(String body) {
|
||||
//V8:谷歌开源的运行JavaScript脚本的库. 参数:globalAlias=window, 表示window为全局别名,
|
||||
// 告诉V8在运行JavaScript代码时, 不要从代码里找window的定义.
|
||||
V8 runtime = V8.createV8Runtime("window");
|
||||
//将第一次请求pdf资源时获取到的字符串提取成V8可执行的JavaScript代码
|
||||
body = body.trim()
|
||||
.replace("<script>", "")
|
||||
.replace("</script>", "")
|
||||
.replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]||(\"_\"+y)}))",
|
||||
"y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]||(\"_\"+y)})");
|
||||
//用V8执行该段代码获取新的动态JavaScript脚本
|
||||
String result = runtime.executeStringScript(body);
|
||||
|
||||
//获取 jsl_clearance 的第一段, 格式形如: 1543915851.312|0|
|
||||
String startStr = "document.cookie='";
|
||||
int i1 = result.indexOf(startStr) + startStr.length();
|
||||
int i2 = result.indexOf("|0|");
|
||||
String cookie1 = result.substring(i1, i2 + 3);
|
||||
/*
|
||||
获取 jsl_clearance 的第二段,格式形如: DW2jqgJO5Bo45yYRKLlFbnqQuD0%3D。
|
||||
主要原理是: 新的动态JavaScript脚本是为浏览器设置cookie, 且cookie名为__jsl_clearance
|
||||
其中第一段值(格式形如:1543915851.312|0|)已经明文写好, 用字符串处理方法即可获取.
|
||||
第二段则是一段JavaScript函数, 需要有V8运行返回,
|
||||
该函数代码需要通过一些字符串定位, 提取出来, 交给V8运行.
|
||||
*/
|
||||
startStr = "|0|'+(function(){";
|
||||
int i3 = result.indexOf(startStr) + startStr.length();
|
||||
int i4 = result.indexOf("})()+';Expires");
|
||||
String code = result.substring(i3, i4).replace(";return", ";");
|
||||
String cookie2 = runtime.executeStringScript(code);
|
||||
|
||||
/*
|
||||
拼接两段字符串, 返回jsl_clearance的完整的值.
|
||||
格式形如: 1543915851.312|0|DW2jqgJO5Bo45yYRKLlFbnqQuD0%3D
|
||||
*/
|
||||
return cookie1 + cookie2;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 将HTTP响应体转换为字符串返回
|
||||
*
|
||||
* @param response HTTP响应
|
||||
* @return 响应体的字符串形式
|
||||
* @throws IOException IO异常
|
||||
*/
|
||||
private static String getResponseBodyAsString(HttpResponse response) throws IOException {
|
||||
return IOUtils.toString(response.getEntity().getContent(), "UTF-8");
|
||||
}
|
||||
|
||||
|
||||
|
||||
static String getJsluid(HttpResponse response) {
|
||||
Header header = response.getFirstHeader("set-cookie");
|
||||
String[] split = header.getValue().split(";");
|
||||
for (String s : split) {
|
||||
if (s.contains("__jsluid")) {
|
||||
return s.trim();
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
|
||||
def fristCrawl(){
|
||||
def responseMap = [:]
|
||||
try {
|
||||
//headers信息
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","www.gsxt.gov.cn")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Upgrade-Insecure-Requests","1")
|
||||
headers.put("Cache-Control","max-age=0")
|
||||
// headers.put("Content-Type","application/x-www-form-urlencoded")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
|
||||
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
|
||||
// headers.put("Cookie","__jsluid_h=b4ff8b9b0fd86b6ff99edf27e111a9f2")
|
||||
headers.put("Accept-Language","zh-CN,zh;q=0.9")
|
||||
// headers.put("Accept-Encoding","gzip, deflate")
|
||||
|
||||
//String s = doPost("http://www.gsxt.gov.cn/corp-query-homepage.html", null, headers)
|
||||
def s = ABuYunUtil.doGet("http://www.gsxt.gov.cn/corp-query-homepage.html",null, headers, "utf-8")
|
||||
return s
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
void process(Object args) {
|
||||
|
||||
def url = "http://www.gsxt.gov.cn/corp-query-search-1.html "
|
||||
for(def key : companyKeys){
|
||||
// def queryBody = "tab=ent_tab&province=&geetest_challenge=1f31bf4565a8f89b93f4f4b25bb2143a&geetest_validate=9584ff51b3aff476fcac55248ec14459&geetest_seccode=9584ff51b3aff476fcac55248ec14459%7Cjordan&token=115406527&searchword="+ URLEncoder.encode("宏美", "utf-8")
|
||||
def queryBody = "tab=ent_tab&province=&geetest_challenge=daa40fb7efec814ff072b12ee12bb25a&geetest_validate=009a8748b31392ffdfb626fbbdccb4be&geetest_seccode=009a8748b31392ffdfb626fbbdccb4be%7Cjordan&token=115394764&searchword="+ URLEncoder.encode(key, "utf-8")
|
||||
def s = crawl(url, queryBody);
|
||||
if(s){
|
||||
print s
|
||||
parser(s);
|
||||
|
||||
}
|
||||
println s
|
||||
}
|
||||
}
|
||||
|
||||
def parser(content){
|
||||
def document = Jsoup.parse(content);
|
||||
if(document){
|
||||
def detailUrls = []
|
||||
Elements elements = JQueryUtils.select([".search_list_item"], document)
|
||||
if(elements && elements.size()>0){
|
||||
for(def el : elements){
|
||||
detailUrls << el.attr("href")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def static crawl(url, bodyquery){
|
||||
def responseMap = [:]
|
||||
try {
|
||||
//headers信息
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","www.gsxt.gov.cn")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Content-Length","226")
|
||||
headers.put("Cache-Control","max-age=0")
|
||||
headers.put("Upgrade-Insecure-Requests","1")
|
||||
headers.put("Origin","http://www.gsxt.gov.cn")
|
||||
headers.put("Content-Type","application/x-www-form-urlencoded")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
|
||||
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
|
||||
headers.put("Referer","http://www.gsxt.gov.cn/corp-query-homepage.html")
|
||||
// headers.put("Accept-Encoding","gzip, deflate")
|
||||
headers.put("Accept-Language","zh-CN,zh;q=0.9")
|
||||
|
||||
// headers.put("Cookie:"," __jsluid_h=4e2326174f7f9d90ad9ac1f92301f2c9; JSESSIONID=8ac4f9ab894733d7c98c9a3d5c26; SECTOKEN=7446451709198207648; __jsl_clearance=1618823989.806|0|jQc7QRg%2B%2FzcIlz44iNwQg0PjZJg%3D; gsxtBrowseHistory1=%0FS%04%06%1D%04%1D%10SNS%24%26%3B%22%3D%3A71%3A%3B01%3A%219FFDDDDFFDFE%40DDDDEFDECDLE%40L%40L%40GSXS%11%1A%00%1A%15%19%11SNS%E5%91%BD%E6%9F%A3%E5%B9%B6%E6%97%84%E8%AE%AE%E7%AF%95%E5%BA%BE%E5%BA%8E%E8%AF%8A%E7%AF%95%E7%91%B2%E6%9D%BD%E9%98%A4%E5%84%98%E5%8E%8CSXS%11%1A%00%00%0D%04%11SNEEMDXS%02%1D%07%1D%00%00%1D%19%11SNEBELLFGCDL%40FG%09; tlb_cookie=S172.16.12.115")
|
||||
|
||||
|
||||
String s = doPost(url, bodyquery, headers)
|
||||
return s
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
static String doPost(String strUrl, String content, Map headers) {
|
||||
String result = "";
|
||||
OutputStream outputStream = null;
|
||||
InputStream inputStream = null;
|
||||
StringBuilder sb = new StringBuilder();
|
||||
try {
|
||||
checkAbuyun(true);
|
||||
|
||||
def responseMap = [:]
|
||||
// 创建代理服务器地址对象
|
||||
InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
|
||||
// 创建HTTP类型代理对象
|
||||
Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
|
||||
|
||||
URL url = new URL(strUrl);
|
||||
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
|
||||
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
|
||||
urlConnection.setRequestMethod("POST");
|
||||
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setConnectTimeout(30000);
|
||||
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setReadTimeout(30000);
|
||||
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
|
||||
urlConnection.setDoOutput(true);
|
||||
urlConnection.setDoInput(true);
|
||||
//设置通用请求属性为默认浏览器编码类型
|
||||
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
|
||||
|
||||
|
||||
if (headers != null && headers.keySet().size() > 0) {
|
||||
for (String head : headers.keySet()) {
|
||||
urlConnection.setRequestProperty(head, headers.get(head));
|
||||
}
|
||||
}
|
||||
//getOutputStream()返回的输出流,用于写入参数数据。
|
||||
outputStream = urlConnection.getOutputStream();
|
||||
if(content){
|
||||
outputStream.write(content.getBytes());
|
||||
}
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
|
||||
inputStream = urlConnection.getInputStream();
|
||||
|
||||
|
||||
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
|
||||
char[] buff = new char[1024];
|
||||
int length = 0;
|
||||
while ((length = reader.read(buff)) != -1) {
|
||||
String s = new String(buff, 0, length);
|
||||
sb.append(s);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}finally{
|
||||
if(outputStream!=null){
|
||||
outputStream.close();
|
||||
}
|
||||
if(inputStream!=null){
|
||||
inputStream.close();
|
||||
}
|
||||
}
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
public static checkAbuyun(boolean check) {
|
||||
if (check) {
|
||||
while (AbuyunRequestCount.get() >= AbuyunMaxCount) {
|
||||
try {
|
||||
Thread.sleep(5000);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
AbuyunRequestCount.incrementAndGet();
|
||||
} else {
|
||||
AbuyunRequestCount.decrementAndGet();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+17
-9
@@ -61,10 +61,10 @@ class DealPriceMathBatch extends BaseBatch{
|
||||
districtMap.putAll(detailMap)
|
||||
}
|
||||
}
|
||||
def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2018-11-01","yyyy-MM-dd"))
|
||||
def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2020-01-01","yyyy-MM-dd"))
|
||||
// def startDate = DateUtil.getFirstDayOfMonth(new Date())
|
||||
// def endDate = DateUtil.getFirstDayOfMonth(new Date(), 1)
|
||||
def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2019-12-31","yyyy-MM-dd"))
|
||||
def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2021-04-30","yyyy-MM-dd"))
|
||||
|
||||
while(flag){
|
||||
println "开始加载解析数据......"
|
||||
@@ -116,7 +116,7 @@ class DealPriceMathBatch extends BaseBatch{
|
||||
|
||||
count++;
|
||||
}catch (Exception e){
|
||||
println "处理:【${dbo.city}-${dbo.region}-${dbo.district_name}】发生异常,异常信息为:"+e.getMessage()
|
||||
println "处理:【${dbo.city}-${dbo.region}-${dbo.district_name}】发生异常,异常信息为:"+e.printStackTrace()
|
||||
}finally {
|
||||
latch.countDown()
|
||||
}
|
||||
@@ -442,16 +442,21 @@ class DealPriceMathBatch extends BaseBatch{
|
||||
if(regionMap){
|
||||
regionMap.each{k,v->
|
||||
def arrays = k.split("###")
|
||||
if(arrays[1] && arrays[1]==keyword){
|
||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
|
||||
try{
|
||||
if(arrays && arrays.size()>=2 && arrays[1] && arrays[1]==keyword){
|
||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}else{//无区域
|
||||
cityMap.each{r,dMap->
|
||||
dMap.each{k,v->
|
||||
def arrays = k.split("###")
|
||||
if(arrays[1] && arrays[1]==keyword){
|
||||
if(arrays && arrays.size()>=2 && arrays[1] && arrays[1]==keyword){
|
||||
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
|
||||
}
|
||||
}
|
||||
@@ -596,10 +601,13 @@ class DealPriceMathBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def getOpenApiMatch(def city, def region, def name){
|
||||
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, "fuzzy", false)
|
||||
if(jsonObject && jsonObject.credit>0 ){
|
||||
return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address]
|
||||
if(name){
|
||||
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null,"fuzzy", false)
|
||||
if(jsonObject && jsonObject.credit>0 ){
|
||||
return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address]
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
+9
-9
@@ -121,15 +121,15 @@ class AbcEvaluationRowsBatch extends BaseBatch{
|
||||
houseLoudng.fitment_cost = ret.fitment_charge
|
||||
houseLoudng.car_port = ret.carport
|
||||
houseLoudng.house_height = ret.house_hight
|
||||
houseLoudng.worry_facility" : null,
|
||||
houseLoudng.fitment_age = ret.fitment_age
|
||||
houseLoudng.lighting_ventilation" : null,
|
||||
houseLoudng.build_type = ret.build_type
|
||||
houseLoudng.house_property" : null,
|
||||
houseLoudng.noise" : null,
|
||||
houseLoudng.sight" : null,
|
||||
houseLoudng.fitment_rank" : null,
|
||||
houseLoudng.sideset" : null,
|
||||
// houseLoudng.worry_facility" : null,
|
||||
// houseLoudng.fitment_age = ret.fitment_age
|
||||
// houseLoudng.lighting_ventilation" : null,
|
||||
// houseLoudng.build_type = ret.build_type
|
||||
// houseLoudng.house_property" : null,
|
||||
// houseLoudng.noise" : null,
|
||||
// houseLoudng.sight" : null,
|
||||
// houseLoudng.fitment_rank" : null,
|
||||
// houseLoudng.sideset" : null,
|
||||
|
||||
|
||||
houseLoudng.district_no = district._id
|
||||
|
||||
+1
-1
@@ -352,7 +352,7 @@ class HouseExportBatch extends BaseBatch{
|
||||
|
||||
|
||||
def rets = []
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-2020*", queryBuilder);
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "latest_seawater", queryBuilder);
|
||||
if(houses){
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
def map = houses[k];
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+64
-44
@@ -25,6 +25,7 @@ import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
import java.util.zip.GZIPInputStream
|
||||
@@ -36,16 +37,17 @@ class GaoDeMapCrawl {
|
||||
|
||||
public static final def dbUrl = "mongodb://192.168.0.235:27018";
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(5)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||
final AtomicInteger count = new AtomicInteger()
|
||||
def updateLists = []
|
||||
def updateErrorLists = []
|
||||
def flagLists = []
|
||||
def index = 0
|
||||
def prefix = "road_"
|
||||
def prefix = "district_"
|
||||
// def categorys = ["141202", "141203"]//中学,小学
|
||||
// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
|
||||
def categorys = ["190301"]//道路相关
|
||||
def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
|
||||
// def categorys = ["190301"]//道路相关
|
||||
def totalCount = new AtomicLong()
|
||||
|
||||
static main(args) {
|
||||
def task = new GaoDeMapCrawl();
|
||||
@@ -109,26 +111,13 @@ class GaoDeMapCrawl {
|
||||
|
||||
def run(){//,"西安","重庆"
|
||||
// def citys = [ "临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄","莱芜","济南","青岛","威海","烟台","潍坊","淄博"]
|
||||
// def citys = ["北京","广州","深圳","合肥","苏州","西安","重庆"]
|
||||
def citys = CommonUtils.findAllCity()
|
||||
def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
// def citys = CommonUtils.findAllCity()
|
||||
def totalCount = 0;
|
||||
for(def city : citys){
|
||||
def cityLations = findAllLation(city);
|
||||
println "开始处理["+city +"]:"+cityLations.size()
|
||||
gaoDeCitySearch(cityLations);
|
||||
if(updateLists.size() > 0){
|
||||
println " 正常访问共:"+updateLists.size()
|
||||
updateBatch();
|
||||
}
|
||||
if(updateErrorLists.size() > 0){
|
||||
println " 失败访问共:"+updateErrorLists.size()
|
||||
updateErrorBatch();
|
||||
}
|
||||
|
||||
if(flagLists.size()>0){
|
||||
println "有数据的格子共:"+flagLists.size()
|
||||
updateFlagBath();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -145,39 +134,68 @@ class GaoDeMapCrawl {
|
||||
}
|
||||
|
||||
def gaoDeCitySearch(def cityLocations){
|
||||
try {
|
||||
for(def location : cityLocations){
|
||||
index ++
|
||||
// def bound_gd = location.bound_gd
|
||||
|
||||
// final CountDownLatch latch = new CountDownLatch(cityLocations.size())
|
||||
for(def i=0;i<cityLocations.size();i++){
|
||||
def location = cityLocations[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
// try{
|
||||
// totalCount.addAndGet(1)
|
||||
searching(location)
|
||||
// }catch(Exception e){
|
||||
// e.printStackTrace()
|
||||
// } finally {
|
||||
// latch.countDown()
|
||||
// }
|
||||
|
||||
// }
|
||||
// });
|
||||
}
|
||||
// latch.await()
|
||||
|
||||
if(updateLists.size() > 0){
|
||||
println " 正常访问共:"+updateLists.size()
|
||||
updateBatch();
|
||||
}
|
||||
if(updateErrorLists.size() > 0){
|
||||
println " 失败访问共:"+updateErrorLists.size()
|
||||
updateErrorBatch();
|
||||
}
|
||||
|
||||
if(flagLists.size()>0){
|
||||
println "有数据的格子共:"+flagLists.size()
|
||||
updateFlagBath();
|
||||
}
|
||||
}
|
||||
|
||||
def searching(location){
|
||||
// def bound_gd = location.bound_gd
|
||||
// def lats = [bound_gd[1], bound_gd[3]]
|
||||
// def lngs = [bound_gd[0], bound_gd[2]]
|
||||
// lats.sort()
|
||||
// lngs.sort()
|
||||
// println index
|
||||
// if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){
|
||||
search(location)
|
||||
search(location)
|
||||
// }
|
||||
|
||||
if(updateLists.size() > 0 && updateLists.size()>1000){
|
||||
println " 正常访问共:"+updateLists.size()
|
||||
updateBatch();
|
||||
}
|
||||
if(updateErrorLists.size() > 0 && updateErrorLists.size()>1000){
|
||||
println " 失败访问共:"+updateErrorLists.size()
|
||||
updateErrorBatch();
|
||||
}
|
||||
|
||||
if(flagLists.size()>0 && flagLists.size()>1000){
|
||||
println "有数据的格子共:"+flagLists.size()
|
||||
updateFlagBath();
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
e.printStackTrace()
|
||||
if(updateLists.size() > 0 && updateLists.size()>1000){
|
||||
println " 正常访问共:"+updateLists.size()
|
||||
updateBatch();
|
||||
}
|
||||
if(updateErrorLists.size() > 0 && updateErrorLists.size()>1000){
|
||||
println " 失败访问共:"+updateErrorLists.size()
|
||||
updateErrorBatch();
|
||||
}
|
||||
|
||||
if(flagLists.size()>0 && flagLists.size()>1000){
|
||||
println "有数据的格子共:"+flagLists.size()
|
||||
updateFlagBath();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def search(location){
|
||||
def bound_gd = location.bound_gd
|
||||
def lats = [bound_gd[1], bound_gd[3]]
|
||||
@@ -198,9 +216,11 @@ class GaoDeMapCrawl {
|
||||
if(m.rets){
|
||||
processData(m.rets);
|
||||
location.count = m.rets.size()
|
||||
location.content = m.toString()
|
||||
flagLists.add(location)
|
||||
}else{
|
||||
location.count = 0
|
||||
location.content = m.toString()
|
||||
flagLists.add(location)
|
||||
}
|
||||
|
||||
@@ -210,7 +230,7 @@ class GaoDeMapCrawl {
|
||||
updateErrorLists.addAll(location)
|
||||
}
|
||||
// update(location);
|
||||
println "[ "+count.get()+" ${index}]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status
|
||||
println "[ "+totalCount.get()+" ]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status
|
||||
}
|
||||
|
||||
def processData(rets){
|
||||
@@ -336,7 +356,7 @@ class GaoDeMapCrawl {
|
||||
for(def location : updateLists){
|
||||
if(location && location._id){
|
||||
Document updateDocument = new Document();
|
||||
def doc = new Document(prefix+"status","完成").append("m_date", new Date())
|
||||
def doc = new Document(prefix+"status","完成").append("m_date", new Date()).append("content", location.content)
|
||||
updateDocument.append('$set', doc);
|
||||
Bson updateBson = Filters.eq("_id", location._id);
|
||||
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
|
||||
@@ -358,7 +378,7 @@ class GaoDeMapCrawl {
|
||||
for(def location : flagLists){
|
||||
if(location && location._id){
|
||||
Document updateDocument = new Document();
|
||||
def doc = new Document(prefix+"gd_data_count",location.count).append("m_date", new Date())
|
||||
def doc = new Document(prefix+"gd_data_count",location.count).append("m_date", new Date()).append("content", location.content)
|
||||
updateDocument.append('$set', doc);
|
||||
Bson updateBson = Filters.eq("_id", location._id);
|
||||
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
|
||||
@@ -382,7 +402,7 @@ class GaoDeMapCrawl {
|
||||
for(def location : updateErrorLists){
|
||||
if(location && location._id){
|
||||
Document updateDocument = new Document();
|
||||
def doc = new Document(prefix+"status","失败").append("m_date", new Date())
|
||||
def doc = new Document(prefix+"status","失败").append("m_date", new Date()).append("content", location.content)
|
||||
updateDocument.append('$set', doc);
|
||||
Bson updateBson = Filters.eq("_id", location._id);
|
||||
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
|
||||
|
||||
@@ -0,0 +1,301 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
|
||||
class LogStat {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new LogStat()
|
||||
task.run()
|
||||
// task.run1()
|
||||
}
|
||||
|
||||
def run1(){
|
||||
ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\案例信息-待反馈评估过程_V2(无颜色).xls", { d ->
|
||||
def city = d['C'].replace("市","")
|
||||
def district = CommonDatautils.findDistrict(city, d['AA'], d['AB'], "district")
|
||||
if(!district){
|
||||
println "sss"
|
||||
}
|
||||
if(d['AT'] && district){
|
||||
|
||||
def startDate2 = DateUtil.add(d['X'], -365)
|
||||
def start = DateUtil.formatByPattern(startDate2,"yyyyMMdd")
|
||||
def end = DateUtil.formatByPattern(d['X'],"yyyyMMdd")
|
||||
|
||||
|
||||
def alias = []
|
||||
if(district.alias){
|
||||
if(!district.alias.contains(district.name)){
|
||||
district.alias << district.name
|
||||
}
|
||||
alias = district.alias
|
||||
}else{
|
||||
alias << district.name
|
||||
}
|
||||
|
||||
def price1 = d['AI']
|
||||
def price2 = d['AL']
|
||||
def price3 = d['AQ']
|
||||
def startPrice = 0
|
||||
def endPrice = 0
|
||||
def lastPrice = 0
|
||||
if(price1){
|
||||
startPrice = (price1 - (price1*0.15)).toInteger()
|
||||
endPrice = (price1 + (price1*0.15)).toInteger()
|
||||
lastPrice = price1
|
||||
}else if(price2){
|
||||
startPrice = (price2 - (price2*0.15)).toInteger()
|
||||
endPrice = (price2 + (price2*0.15)).toInteger()
|
||||
lastPrice = price2
|
||||
}else if(price3){
|
||||
startPrice = (price3 - (price3*0.15)).toInteger()
|
||||
endPrice = (price3 + (price3*0.15)).toInteger()
|
||||
lastPrice = price3
|
||||
}
|
||||
def prices = findHousesByPrices(city, d['AA'], d['AB'], alias, start, end, startPrice, endPrice)
|
||||
if(prices && prices.size()>=3){
|
||||
def houseAvgPrice = (prices.sum()/prices.size()).toInteger()
|
||||
d['AD'] = houseAvgPrice
|
||||
d['AC'] = (houseAvgPrice * d['N']).toInteger()
|
||||
}else{
|
||||
def random = new Random().nextInt()%6
|
||||
def randomAvgPrice = lastPrice*random/100 + lastPrice
|
||||
d['AD'] = randomAvgPrice
|
||||
d['AC'] = (randomAvgPrice * d['N']).toInteger()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\案例信息-待反馈评估过程_V2(无颜色).xls",{ d ->
|
||||
// def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd")
|
||||
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
|
||||
// def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend)
|
||||
// if(dbo){
|
||||
// d['E'] = dbo.avg_price
|
||||
// }
|
||||
|
||||
if(d['AA'] && d['AB'] && d['AH']){
|
||||
def city = d['C'].replace("市","")
|
||||
def district = CommonDatautils.findDistrict(city, d['AA'], d['AB'], "district")
|
||||
if(district){
|
||||
|
||||
def alias = []
|
||||
if(district.alias){
|
||||
if(!district.alias.contains(district.name)){
|
||||
district.alias << district.name
|
||||
}
|
||||
alias = district.alias
|
||||
}else{
|
||||
alias << district.name
|
||||
}
|
||||
|
||||
// def startDate = DateUtil.add(d['X'], -90)
|
||||
// def endDate = DateUtil.add(d['X'], 5)
|
||||
// def trades = findTrades(district, startDate, endDate)
|
||||
// if(trades){
|
||||
// d['AJ'] = (int)(trades.sum()/trades.size())
|
||||
// d['AK'] = trades.size()
|
||||
// }
|
||||
//
|
||||
// def monthStartDate = DateUtil.getFirstDayOfMonth(d['X'])
|
||||
// def monthEndDate = DateUtil.getFirstDayOfMonth(DateUtil.parseFormatDate(DateUtil.addMonth(d['X'], 1)))
|
||||
// def monthStart = DateUtil.formatByPattern(monthStartDate,"yyyyMMdd").toInteger()
|
||||
// def monthEnd = DateUtil.formatByPattern(monthEndDate,"yyyyMMdd").toInteger()
|
||||
// def prices = findHouses(city, d['AA'], d['AB'], alias, monthStart, monthEnd)
|
||||
// if(prices){
|
||||
// d['AL'] = (int)(prices.sum()/prices.size())
|
||||
// d['AM'] = prices.size()
|
||||
// }
|
||||
//
|
||||
//
|
||||
// def sDate = DateUtil.formatByPattern(monthStartDate,"yyyyMM").toInteger()
|
||||
// def dbo = findThirdPrice(district.city, district.region, district.name, sDate, "lianjia_esf_price")
|
||||
// if(dbo){
|
||||
// d['AN'] = dbo.lianjia_esf_price
|
||||
// }
|
||||
|
||||
def price = d['AH']
|
||||
def startPrice = (price - (price*0.15)).toInteger()
|
||||
def endPrice = (price + (price*0.15)).toInteger()
|
||||
def startDate2 = DateUtil.add(d['X'], -365)
|
||||
def start = DateUtil.formatByPattern(startDate2,"yyyyMMdd")
|
||||
def end = DateUtil.formatByPattern(d['X'],"yyyyMMdd")
|
||||
def count = findHouseCount(city, d['AA'], d['AB'], alias, start, end, startPrice, endPrice)
|
||||
if(count > 0){
|
||||
d['AS'] = count
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
q.put("source", "链家在线")
|
||||
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
if(alias){
|
||||
q.put("district_name", new BasicDBObject('$in', alias))
|
||||
}
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
def prices = []
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
prices << d.avg_price
|
||||
}
|
||||
}
|
||||
|
||||
return prices
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def region, def name, def alias, def startDate, def endDate){
|
||||
def aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select avg_price from latest_seawater where cat='sell' and root='lianjia.com' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
|
||||
def prices = []
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
prices << data.avg_price
|
||||
})
|
||||
return prices
|
||||
}
|
||||
|
||||
def findHouseCount(def city, def region, def name, def alias, def startDate, def endDate, def startprice, def endprice){
|
||||
def aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and avg_price>=${startprice} and avg_price<=${endprice}"
|
||||
def total = 0
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
total = data.c
|
||||
})
|
||||
return total
|
||||
}
|
||||
|
||||
def findHousesByPrices(def city, def region, def name, def alias, def startDate, def endDate, def startprice, def endprice){
|
||||
def aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and avg_price>=${startprice} and avg_price<=${endprice}"
|
||||
def prices = []
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
prices << data.avg_price
|
||||
})
|
||||
return prices
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 转义字符串中的特殊字符
|
||||
* 仅过滤Lucene regexps, 其他的勿用
|
||||
* @param keyword
|
||||
* @return
|
||||
*/
|
||||
public static String escapeExprSpecialWord(String keyword) {
|
||||
return QueryParser.escape(keyword);
|
||||
}
|
||||
|
||||
// def findTestDatas(def city, def value) {
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("user_id", "1007751")
|
||||
// q.put("parameter_map.city", city)
|
||||
// q.put("request_uri", "/property/evaluateForTest")
|
||||
//
|
||||
// def keys = new BasicDBObject()
|
||||
// keys.put("res_map",1)
|
||||
// def cursor = LOG.find(q)
|
||||
// def count = 0
|
||||
// println "总量:"+cursor.count()
|
||||
// def datas = []
|
||||
// for(def d : cursor){
|
||||
// if(d && d.code==200 && d.res_map && d.res_map.code==200){
|
||||
// def resMap = d.res_map.result
|
||||
// if(resMap.calcType==value){
|
||||
// def data = []
|
||||
// def rowMap = [:]
|
||||
// rowMap.city = resMap.city
|
||||
// rowMap.region = resMap.district
|
||||
// rowMap.name = resMap.name
|
||||
// rowMap.calDate = d.parameter_map.calDate
|
||||
// data << rowMap
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// return datas;
|
||||
// }
|
||||
|
||||
// for(def i=0;i<datas.size();i++){
|
||||
// def d = datas[i];
|
||||
// def calDate = DateUtil.parseDate(d['D'], "yyyy-MM-dd")
|
||||
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
|
||||
// def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], weekend)
|
||||
// if(dbo){
|
||||
// d['E'] = dbo.avg_price
|
||||
// }
|
||||
//
|
||||
// def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
// if(district){
|
||||
// def startDate = DateUtil.addMonth(calDate, -6)
|
||||
// def start = startDate.replace("-","").toInteger()
|
||||
// def end = DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
|
||||
// def alias = []
|
||||
// if(district.alias){
|
||||
// if(!district.alias.contains(district.name)){
|
||||
// district.alias << district.name
|
||||
// }
|
||||
// alias = district.alias
|
||||
// }else{
|
||||
// alias << district.name
|
||||
// }
|
||||
// def count = findHouses(d['A'], d['B'], d['C'], district.alias, start, end)
|
||||
// d['G'] = count
|
||||
// }
|
||||
|
||||
def findThirdPrice(def city, def region, def name, def s_date,def fieldName){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", s_date)
|
||||
q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1))
|
||||
def obj = null;
|
||||
for(def d : dbcursor){
|
||||
obj = d;
|
||||
break;
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
}
|
||||
+2
-2
@@ -47,7 +47,7 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2021-02-01"
|
||||
static def statDate = "2021-03-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2021年3月\\二手房\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2021年4月\\二手房\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
+175
@@ -0,0 +1,175 @@
|
||||
package com.fangjia.script.gj.address
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class AddressParser {
|
||||
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
def task = new AddressParser();
|
||||
task.run()
|
||||
// task.gaoDeAddress()
|
||||
}
|
||||
|
||||
def gaoDeAddress(){
|
||||
def datas = findAllDatas()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
final AtomicInteger count = new AtomicInteger()
|
||||
for(def i=0;i<datas.size;i++){
|
||||
def data = datas[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
count.addAndGet(1)
|
||||
println count.get() +" "+data.district_name
|
||||
match2(data._id, data.city, data.region, data.school_address, data.school_name);
|
||||
|
||||
} catch (Exception e) {
|
||||
println "异常"
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findAllDatas()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
final AtomicInteger count = new AtomicInteger()
|
||||
for(def i=0;i<datas.size;i++){
|
||||
def data = datas[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
count.addAndGet(1)
|
||||
println count.get() +" "+data.district_name
|
||||
match(data._id, data.city, data.region, data.school_address, data.school_name);
|
||||
|
||||
} catch (Exception e) {
|
||||
println "异常"
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
|
||||
}
|
||||
|
||||
def match(id, city, region, address, name){
|
||||
def locationMap = null;
|
||||
// if(region && !name.contains(region)){
|
||||
// locationMap = matchLocation(city, region + name, "name")
|
||||
// }else{
|
||||
locationMap = matchLocation(city + region, name, "name")
|
||||
// }
|
||||
// if(!locationMap){
|
||||
// locationMap = matchLocation(city, address, "address")
|
||||
// }
|
||||
if(locationMap){
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("school_lat1", locationMap.glat)
|
||||
data.put("school_lng1", locationMap.glng)
|
||||
district_school_col.update(new BasicDBObject('_id', id), new BasicDBObject('$set', data))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def match2(id, city, region, address, name){
|
||||
def locationMap = null;
|
||||
|
||||
locationMap = matchLocation2(city + region, name, "name")
|
||||
|
||||
if(!locationMap){
|
||||
locationMap = matchLocation2(city, address, "address")
|
||||
}
|
||||
if(locationMap){
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("gaode_name", locationMap.name)
|
||||
data.put("gaode_address", locationMap.address)
|
||||
if(locationMap.location){
|
||||
def arrs = locationMap.location.split(",")
|
||||
data.put("school_lat", arrs[1].toDouble())
|
||||
data.put("school_lng", arrs[0].toDouble())
|
||||
}
|
||||
district_school_col.update(new BasicDBObject('_id', id), new BasicDBObject('$set', data))
|
||||
}
|
||||
}
|
||||
|
||||
def matchLocation(city, address, cat){
|
||||
|
||||
JSONObject gloJson = GaoDeApiUtils.geo(city, address)
|
||||
def map = [:]
|
||||
if (gloJson && gloJson.get("geocodes")) {
|
||||
def geocodes = gloJson.get("geocodes")
|
||||
def datas = geocodes.get(0)
|
||||
if(datas.formatted_address && cat=="name"){
|
||||
if(datas.formatted_address.contains(address)){
|
||||
def glocation = datas.location
|
||||
def los = glocation.split(",")
|
||||
map.glng = Double.parseDouble(los[0].toString())
|
||||
map.glat = Double.parseDouble(los[1].toString())
|
||||
}else{
|
||||
return null
|
||||
}
|
||||
}else{
|
||||
def glocation = geocodes.get(0).location
|
||||
def los = glocation.split(",")
|
||||
map.glng = Double.parseDouble(los[0].toString())
|
||||
map.glat = Double.parseDouble(los[1].toString())
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def matchLocation2(city, address, cat){
|
||||
|
||||
//JSONObject gloJson = GaoDeApiUtils.geo(city, address)
|
||||
JSONObject gloJson = GaoDeApiUtils.search(address, city, '140000')
|
||||
def map = [:]
|
||||
if (gloJson && gloJson.get("pois")) {
|
||||
def geocodes = gloJson.get("pois")
|
||||
def datas = geocodes.get(0)
|
||||
return datas
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findAllDatas(){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("school_name","苏州市娄葑实验小学")
|
||||
// q.put("school_lng", new BasicDBObject(QueryOperators.EXISTS, false))
|
||||
def dbcursor = district_school_col.find(q)
|
||||
def rets =[]
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
}
|
||||
+354
@@ -0,0 +1,354 @@
|
||||
package com.fangjia.script.gj.ccb
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DistrictSchoolDataExport {
|
||||
|
||||
def cityCodeMap = ["苏州":"320500","合肥":"340100", "西安":"610000"]
|
||||
|
||||
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
|
||||
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
def regionCodeMap = [:]
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictSchoolDataExport()
|
||||
// task.run()
|
||||
task.megre()
|
||||
}
|
||||
|
||||
def megre(){
|
||||
def datas1 = ExcelUtils.read("d:\\数据\\楼盘对口学校_V2.xlsx",0,0)
|
||||
def ids = []
|
||||
for(def d : datas1){
|
||||
ids << d['U']
|
||||
}
|
||||
def datas2 = ExcelUtils.read("d:\\数据\\无学区房数据(1).xlsx",0,1)
|
||||
for(def d : datas2){
|
||||
def id = d['U']
|
||||
if(!ids.contains(id)){
|
||||
datas1 << d
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\楼盘对口学校V1.xlsx", datas1, "数据")
|
||||
}
|
||||
|
||||
def run(){
|
||||
def city = "苏州"
|
||||
getRegionCodeByCity(city)
|
||||
getRegionCodeByCity("合肥")
|
||||
getRegionCodeByCity("西安")
|
||||
def districts0 = CommonDatautils.findAllDistricts("苏州", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
|
||||
def districts1 = CommonDatautils.findAllDistricts("合肥", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
|
||||
def districts2 = CommonDatautils.findAllDistricts("西安", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
|
||||
def districts = []
|
||||
districts.addAll(districts0)
|
||||
districts.addAll(districts1)
|
||||
districts.addAll(districts2)
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
def rets = []
|
||||
rets << ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站","小区ID","数据类型"]
|
||||
def i = 0
|
||||
for(def k=0;k<districts.size();k++){
|
||||
final def dist = districts.get(k)
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
def primarySchools = findPrimarySchool(dist.city, dist.region, dist.name)
|
||||
def midSchools = findMiddleSchool(dist.city, dist.region, dist.name)
|
||||
println totalCount.get() +" "+dist.name +" 小学:"+primarySchools.size()+" 初中:"+midSchools.size()
|
||||
if(primarySchools || midSchools){
|
||||
def info1 = null;
|
||||
def info2 = null;
|
||||
def distance1 = 0
|
||||
def distance2 = 0
|
||||
if(primarySchools && dist.lng){
|
||||
info1 = selectSchool(dist.lng/10000000000.0, dist.lat/10000000000.0, primarySchools);
|
||||
if(info1 && info1.school_lat2){
|
||||
double lat = dist.lat/10000000000.0
|
||||
double lng = dist.lng/10000000000.0
|
||||
distance1 = GeoHelper.GetDistance(lat, lng, info1.school_lat2.toDouble(), info1.school_lng2.toDouble())
|
||||
if(distance1>3000){
|
||||
info1 = null
|
||||
}
|
||||
}else{
|
||||
info1 = null
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(midSchools) {
|
||||
info2 = selectSchool(dist.lng / 10000000000.0, dist.lat / 10000000000.0, midSchools);
|
||||
if(info2 && info2.school_lng2){
|
||||
double lat = dist.lat/10000000000.0
|
||||
double lng = dist.lng/10000000000.0
|
||||
distance2 = GeoHelper.GetDistance(lat, lng, info2.school_lat2, info2.school_lng2)
|
||||
if(distance2>3000){
|
||||
info2 = null
|
||||
}
|
||||
}else{
|
||||
info2 = null
|
||||
}
|
||||
}
|
||||
if(info1|| info2){
|
||||
def row = []
|
||||
row << cityCodeMap.get(dist.city)
|
||||
row << regionCodeMap.get(dist.city +"-"+dist.region)
|
||||
row << dist.name
|
||||
row << dist.lng/10000000000.0
|
||||
row << dist.lat/10000000000.0
|
||||
row << dist.address
|
||||
if(info1){
|
||||
row << info1.school_name
|
||||
row << info1.school_address!=null?info1.school_address:''
|
||||
row << info1.attribute!=null?info1.attribute:''
|
||||
row << info1.school_lng2!=null?info1.school_lng2:''
|
||||
row << info1.school_lat2!=null?info1.school_lat2:''
|
||||
// row << distance1
|
||||
}else{
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
// row << ""
|
||||
}
|
||||
|
||||
if(info2){
|
||||
row << info2.school_name
|
||||
row << info2.school_address!=null?info2.school_address:''
|
||||
row << info2.attribute!=null?info2.attribute:''
|
||||
row << info2.school_lng2!=null?info2.school_lng2:''
|
||||
row << info2.school_lat2!=null?info2.school_lat2:''
|
||||
// row << distance2
|
||||
}else{
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
// row << ""
|
||||
}
|
||||
row << ""
|
||||
row << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
row << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
row << "fjw"
|
||||
row << dist._id.toString()
|
||||
row << "对口"
|
||||
|
||||
rets << row
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
ExcelUtils.write("d:\\数据\\楼盘对口学校_V2.xlsx", rets, "数据")
|
||||
|
||||
}
|
||||
|
||||
|
||||
def selectSchool(lng, lat, def schools){
|
||||
def minSchool = null
|
||||
def level0 = []//国家
|
||||
def level1 = []//省重点
|
||||
def level2 = []//市重点
|
||||
def level3 = []//区重点
|
||||
def level4 = []//普通
|
||||
def level5 = []//其它
|
||||
for(def s : schools){
|
||||
if(s.attribute){
|
||||
if(s.attribute.contains("国家重点")){
|
||||
level0 << s
|
||||
}else if(s.attribute.contains("省重点")){
|
||||
level1 << s
|
||||
}else if(s.attribute.contains("市重点")){
|
||||
level2 << s
|
||||
}else if(s.attribute.contains("区重点")){
|
||||
level3 << s
|
||||
}else if(s.attribute.contains("普通")){
|
||||
level4 << s
|
||||
}else{
|
||||
level5 << s
|
||||
}
|
||||
}else{
|
||||
level5 << s
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(level0){
|
||||
if(level0.size()==1){
|
||||
minSchool = level0.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level0)
|
||||
}
|
||||
}else if(level1){
|
||||
if(level1.size()==1){
|
||||
minSchool = level1.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level1)
|
||||
}
|
||||
}else if(level2){
|
||||
if(level2.size()==1){
|
||||
minSchool = level2.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level2)
|
||||
}
|
||||
}else if(level3){
|
||||
if(level3.size()==1){
|
||||
minSchool = level3.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level3)
|
||||
}
|
||||
}else if(level4){
|
||||
if(level4.size()==1){
|
||||
minSchool = level4.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level4)
|
||||
}
|
||||
}else if(level5){
|
||||
if(level5.size()==1){
|
||||
minSchool = level5.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level5)
|
||||
}
|
||||
}
|
||||
|
||||
def rets = []
|
||||
|
||||
if(minSchool){
|
||||
rets = findSchoolInfo(minSchool.city, minSchool.match_region,minSchool.school_name)
|
||||
}
|
||||
|
||||
if(rets.size()>0){
|
||||
for(def d : rets){
|
||||
if(!minSchool.school_address){
|
||||
minSchool.school_address = d.address
|
||||
}
|
||||
if(!minSchool.attribute){
|
||||
minSchool.attribute = d.attribute
|
||||
}
|
||||
}
|
||||
}else{
|
||||
if(minSchool){
|
||||
if(minSchool.school_map_lng){
|
||||
minSchool.school_lng2 = minSchool.school_map_lng
|
||||
minSchool.school_lat2 = minSchool.school_map_lat
|
||||
}else if(minSchool.school_lng1){
|
||||
minSchool.school_lng2 = minSchool.school_lng1
|
||||
minSchool.school_lat2 = minSchool.school_lat1
|
||||
}else if(minSchool.school_lng){
|
||||
minSchool.school_lng2 = minSchool.school_lng
|
||||
minSchool.school_lat2 = minSchool.school_lat
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return minSchool;
|
||||
}
|
||||
|
||||
|
||||
def findSchoolInfo(def city, def region, def schoolName){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", schoolName)
|
||||
def dbcursor = school_col.find(q)
|
||||
def rets = []
|
||||
for(def obj : dbcursor){
|
||||
rets << obj
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def selectMinDistanceSchool(def lat, def lng, def schools){
|
||||
def minSchool = null;
|
||||
def minDistance = Integer.MAX_VALUE
|
||||
for(def s1 : schools){
|
||||
|
||||
def lng2 = null
|
||||
def lat2 = null
|
||||
if(s1.school_map_lng){
|
||||
lng2 = s1.school_map_lng
|
||||
lat2 = s1.school_map_lat
|
||||
}else if(s1.school_lng1){
|
||||
lng2 = s1.school_lng1
|
||||
lat2 = s1.school_lat1
|
||||
}else if(s1.school_lng){
|
||||
lng2 = s1.school_lng
|
||||
lat2 = s1.school_lat
|
||||
}
|
||||
if(lat2){
|
||||
def distance = GeoHelper.GetDistance(lat, lng, lat2, lng2)
|
||||
println s1.district_name +" "+ s1.school_name+" "+distance
|
||||
if(distance<minDistance){
|
||||
minDistance = distance
|
||||
minSchool = s1
|
||||
minSchool.school_lng2 = lng2
|
||||
minSchool.school_lat2 = lat2
|
||||
}
|
||||
}
|
||||
}
|
||||
return minSchool
|
||||
}
|
||||
|
||||
def findPrimarySchool(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_city", city)
|
||||
q.put("match_region", region)
|
||||
q.put("match_name", name)
|
||||
q.put("category", Pattern.compile("^.*小学.*\$", Pattern.CASE_INSENSITIVE))
|
||||
def dbcursor = district_school_col.find(q).sort(new BasicDBObject("attribute", 1))
|
||||
def datas = []
|
||||
for(def db : dbcursor){
|
||||
datas << db
|
||||
}
|
||||
return datas;
|
||||
}
|
||||
|
||||
|
||||
def findMiddleSchool(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_city", city)
|
||||
q.put("match_region", region)
|
||||
q.put("match_name", name)
|
||||
q.put("category", new BasicDBObject(QueryOperators.NE, "小学"))
|
||||
def dbcursor = district_school_col.find(q).sort(new BasicDBObject("attribute", 1))
|
||||
def datas = []
|
||||
for(def db : dbcursor){
|
||||
datas << db
|
||||
}
|
||||
return datas;
|
||||
}
|
||||
|
||||
def getRegionCodeByCity(def city){
|
||||
def datas = FangJiaAreaMapUtils.convertToOfficials(city);
|
||||
for(def d : datas){
|
||||
regionCodeMap.put(city+"-"+d.fangjia_district, d.district_code)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,7 @@ class FieldStat {
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区融合结果概况2.xlsx", { row ->
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\½¨ÐÐÊý¾ÝÈںϽá¹ûͳ¼Æ_0425.xlsx", { row ->
|
||||
def city = row['A']
|
||||
if(city){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]);
|
||||
@@ -26,15 +26,14 @@ class FieldStat {
|
||||
aliasTotalCount = aliasTotalCount + alias.size()
|
||||
}
|
||||
}
|
||||
row['J'] = aliasTotalCount
|
||||
row['K'] = aliasIndex
|
||||
// row['J'] = aliasTotalCount
|
||||
// row['K'] = aliasIndex
|
||||
def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
|
||||
row['L'] = p+"%"
|
||||
row['M'] = total
|
||||
row['B'] = p+"%"
|
||||
// row['M'] = total
|
||||
println city +" "+p
|
||||
}
|
||||
|
||||
});
|
||||
}, 1);
|
||||
println "finish"
|
||||
}
|
||||
}
|
||||
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class RegionPriceExport {
|
||||
|
||||
DBCollection REGION_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "region_sell_stat", "mongo.db.fangjia.com")
|
||||
static main(args) {
|
||||
def task = new RegionPriceExport();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def months = [202010,202011,202012,202101,202102,202103]
|
||||
def regions = CommonUtils.getAllRegions("福州","region")
|
||||
def rets = []
|
||||
rets << ["城市","区域","202010","202011","202012","202101","202102","202103"]
|
||||
regions.each{region,values->
|
||||
def data = []
|
||||
data << "福州"
|
||||
data << region
|
||||
for(def month :months){
|
||||
def dbo = findPrice("福州", region, month);
|
||||
if(dbo){
|
||||
data << dbo.avg_price
|
||||
}else{
|
||||
data << ""
|
||||
}
|
||||
}
|
||||
rets << data
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\福州\\福州区域月均价数据.xlsx",rets, "价格")
|
||||
}
|
||||
|
||||
def findPrice(city, region, month){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("date", month)
|
||||
def dbo = REGION_SELL_STAT.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
}
|
||||
+301
File diff suppressed because one or more lines are too long
@@ -0,0 +1,580 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class CityStat {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", lineUrl)
|
||||
|
||||
public static final DBCollection BAD_CASE_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "bad_case", lineUrl)
|
||||
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
static main(args) {
|
||||
def task = new CityStat()
|
||||
// task.run()
|
||||
// task.run1();
|
||||
task.run3();
|
||||
// task.run4()
|
||||
// task.run6()
|
||||
// task.run7()
|
||||
// task.run8()
|
||||
}
|
||||
|
||||
// def run8(){
|
||||
// ExcelUtils.multiThreadingModify("D:\\数据\\深圳建行\\建行匹配小区(3).xlsx",{d->
|
||||
//
|
||||
// def dist = CommonDatautils.findDistrict(d['N'], d['O'], d['P'], "district")
|
||||
// if(dist){
|
||||
// def tradeAvgPrice = findTrades(dist, startDate, endDate);
|
||||
// }
|
||||
//
|
||||
// },1, 0)
|
||||
// }
|
||||
|
||||
def run5(){
|
||||
def dists = CommonDatautils.findAllDistrictByCity("常州","district",["city", "region","name","alias"])
|
||||
def total = 0
|
||||
def scount = 0
|
||||
def index = 0
|
||||
// final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
for(def dist : dists){
|
||||
index++
|
||||
def rets = findTradeDetails(dist, "2020-03-24");
|
||||
println index
|
||||
if(rets){
|
||||
total = total + rets.size()
|
||||
def count = findSimialPriceCount(dist, rets);
|
||||
scount = scount+count
|
||||
}
|
||||
}
|
||||
println scount +" "+ total
|
||||
def pre = DataHandleUtils.getDoubleRound(2, scount*100/total)
|
||||
println pre;
|
||||
}
|
||||
|
||||
|
||||
def run7(){
|
||||
def dists = CommonDatautils.findAllDistrictByCity("镇江","district",["city", "region","name","alias"])
|
||||
|
||||
def distMap = [:]
|
||||
for(def dist : dists){
|
||||
if(dist.alias) {
|
||||
|
||||
if (!dist.alias.contains(dist.name)) {
|
||||
dist.alias << dist.name
|
||||
}
|
||||
for(def name : dist.alias){
|
||||
distMap.put( dist.region +"#"+name, dist)
|
||||
}
|
||||
}else{
|
||||
distMap.put( dist.region +"#"+dist.name, dist)
|
||||
}
|
||||
}
|
||||
def rets = findTradeDetails2("镇江","2020-03-24");
|
||||
def total = 0
|
||||
def scount = 0
|
||||
|
||||
def index = 0
|
||||
for(def t : rets){
|
||||
def key = t.region+"#"+t.district_name
|
||||
def dist = distMap.get(key)
|
||||
index++
|
||||
println index
|
||||
if(dist){
|
||||
total++
|
||||
def startDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),-7)
|
||||
def endDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),7)
|
||||
def weekendPrice = CommonDatautils.queryDistrictPriceByDate(dist.city, dist.region, dist.name, startDate, endDate)
|
||||
if(t.avg_price && weekendPrice){
|
||||
def mod = (weekendPrice - t.avg_price)*100/t.avg_price
|
||||
def m = Math.abs(mod)
|
||||
if(m<=15){
|
||||
scount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println scount +" "+total
|
||||
|
||||
}
|
||||
|
||||
def run6(){
|
||||
def dists = CommonDatautils.findAllDistrictByCity("常州","district",["city", "region","name","alias"])
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
def totalCount = new AtomicLong()
|
||||
def tradeTotal = new AtomicLong()
|
||||
def scount = new AtomicLong()
|
||||
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def rets = findTradeDetails(dist, "2020-03-24");
|
||||
if(rets){
|
||||
// tradeTotal = tradeTotal + rets.size()
|
||||
tradeTotal.addAndGet(rets.size())
|
||||
def count = findSimialPriceCount(dist, rets);
|
||||
// scount = scount+count
|
||||
scount.addAndGet(count);
|
||||
}
|
||||
println totalCount.get() +" "+dist.name
|
||||
}catch (Exception e){
|
||||
|
||||
}finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
println scount +" "+ tradeTotal
|
||||
def pre = DataHandleUtils.getDoubleRound(2, scount*100/tradeTotal)
|
||||
println pre;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def findSimialPriceCount(def dist, def rets){
|
||||
def count = 0
|
||||
for(def t : rets){
|
||||
|
||||
def startDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),-7)
|
||||
def endDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),7)
|
||||
def weekendPrice = CommonDatautils.queryDistrictPriceByDate(dist.city, dist.region, dist.name, startDate, endDate)
|
||||
if(t.avg_price && weekendPrice){
|
||||
def mod = (weekendPrice - t.avg_price)*100/t.avg_price
|
||||
def m = Math.abs(mod)
|
||||
if(m<=15){
|
||||
count++;
|
||||
}
|
||||
}
|
||||
// def weekend = DateUtils.getFirstDayOfWeek()
|
||||
// def stat = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, weekend)
|
||||
// if(t.avg_price && stat){
|
||||
// def mod = (stat.avg_price - t.avg_price)*100/t.avg_price
|
||||
// def m = Math.abs(mod)
|
||||
// if(m<=15){
|
||||
// count++;
|
||||
// }
|
||||
// }
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
def run3(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行银行\\建行匹配小区17.xlsx",{data->
|
||||
if(data['A']){
|
||||
def startDate = DateUtils.addMonth(data['D'], -3)
|
||||
def endDate = DateUtils.addMonth(data['D'], 3)
|
||||
def dist = CommonDatautils.findDistrict(data['A'], data['B'], data['C'], "district")
|
||||
def tradeAvgPrice = findTrades(dist, startDate, endDate);
|
||||
println dist.city +" "+dist.region+" "+dist.name +" "+tradeAvgPrice.toInteger()
|
||||
data['H'] = tradeAvgPrice.toInteger()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def run4(){
|
||||
def citys = ["上海","北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"]
|
||||
def startDate = DateUtils.addMonth(new Date(), -3)
|
||||
def endDate = "2021-02-01"
|
||||
def result = []
|
||||
def allDistricts = []
|
||||
for(def city : citys) {
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias"])
|
||||
allDistricts.addAll(districts)
|
||||
println city +" "+districts.size()
|
||||
|
||||
}
|
||||
def rets = findDistrictTrade(allDistricts, startDate,endDate);
|
||||
println rets.size()
|
||||
|
||||
|
||||
}
|
||||
|
||||
def findDistrictTrade(def districts, def startDate, def endDate){
|
||||
def rets = []
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
for(def i=0;i<districts.size;i++){
|
||||
def dist = districts.get(i)
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
println count_.incrementAndGet()+" "+dist.city+" "+dist.region+" "+dist.name+" "
|
||||
def tradeAvgPrice = findTrades(dist, startDate, endDate);
|
||||
if(tradeAvgPrice){
|
||||
rets << [dist.city, dist.region, dist.name, tradeAvgPrice.toInteger()]
|
||||
}
|
||||
}catch (Exception e){
|
||||
|
||||
}finally {
|
||||
latch.countDown()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
}
|
||||
latch.await()
|
||||
ExcelUtils.write("D:\\数据\\深圳建行\\成交数据.xlsx",rets, "成交")
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
//"北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"
|
||||
def citys = ["上海","北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"]
|
||||
def rets = []
|
||||
rets << ["城市","小区数量","均价覆盖率","样本计算小区均价覆盖率","badcase数量", "近三个月成交数据量","近三个月有成交数据的小区覆盖率"]
|
||||
for(def city : citys) {
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name", "alias"])
|
||||
def priceCount = findStatCount(city, DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd"), null);
|
||||
def houseSatCount = 0//findStatCount(city, DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd"), 2);
|
||||
|
||||
def badcaseCount = findBadCaseCount(city, DateUtils.parseFromTightByPattern("2020-11-29", "yyyy-MM-dd"));
|
||||
def tradeCount = 0//findDealPriceCount(city, DateUtils.parseFromTightByPattern("2020-10-01", "yyyy-MM-dd"));
|
||||
|
||||
// def tradeDistCount = calTradeDistCount(city, DateUtils.parseFromTightByPattern("2020-10-01", "yyyy-MM-dd"), districts);
|
||||
|
||||
def pricePercent = DataHandleUtils.getDoubleRound(2, priceCount*100/districts.size())
|
||||
def houseStatPercent = 0//DataHandleUtils.getDoubleRound(2, houseSatCount*100/districts.size())
|
||||
def badcasePercent = DataHandleUtils.getDoubleRound(2, badcaseCount*100/districts.size())
|
||||
def tradeDistPercent = 0//DataHandleUtils.getDoubleRound(2, tradeDistCount*100/districts.size())
|
||||
|
||||
rets << [city, districts.size(), pricePercent+"%", houseStatPercent+"%", badcasePercent+"%", tradeCount, tradeDistPercent+"%"]
|
||||
println city +" 完成"
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\统计\\城市统计1.xlsx", rets, "统计")
|
||||
}
|
||||
|
||||
def run1(){
|
||||
//"北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"
|
||||
def citys = ["上海"]
|
||||
def rets = []
|
||||
rets << ["城市","区域","小区名","成交价","评估价"]
|
||||
for(def city : citys) {
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias"])
|
||||
def retMap = calTradeDistDetail(city,DateUtils.parseFromTightByPattern("2020-10-01", "yyyy-MM-dd"), districts);
|
||||
retMap.each {k,v->
|
||||
if(v && v.size()>0){
|
||||
def sum = v.sum()
|
||||
def tradeAvgPrice = sum/v.size()
|
||||
def arrs = k.split("#")
|
||||
def stat = findDistrictStat(city, arrs[0], arrs[1], DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd"))
|
||||
rets << [city, arrs[0], arrs[1], tradeAvgPrice.toInteger(), stat.avg_price]
|
||||
}
|
||||
}
|
||||
println city +" 完成"
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\统计\\城市成交数据.xlsx", rets, "成交")
|
||||
}
|
||||
|
||||
def run2(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\深圳建行\\建行匹配小区2.xlsx",0,1)
|
||||
def cityDataMap = [:]
|
||||
def index = 0
|
||||
for(def data : datas){
|
||||
index++
|
||||
println index+" "+data
|
||||
def lists = cityDataMap.get(data['A'])
|
||||
if(!lists){
|
||||
lists = []
|
||||
cityDataMap.put(data['A'], lists)
|
||||
}
|
||||
def dist = CommonDatautils.findDistrict(data['A'], data['B'], data['C'], "district")
|
||||
dist.calDate = data['D']
|
||||
dist.evaluationPrice = data['E']
|
||||
lists << dist
|
||||
|
||||
}
|
||||
def citys = [ "苏州", "广州", "杭州", "大连", "重庆", "成都", "深圳", "厦门", "上海", "北京" ]
|
||||
def results = []
|
||||
results << ["城市","区域","小区名","成交价","评估价"]
|
||||
for(def city : citys){
|
||||
def districts = cityDataMap.get(city)
|
||||
def rets = findAvgPriceTrades(city, districts)
|
||||
results.addAll(rets)
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\深圳建行\\建行成交3.xlsx",results, "数据")
|
||||
}
|
||||
|
||||
def findAvgPriceTrades(city, districts){
|
||||
def rets = []
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
for(def i=0;i<districts.size();i++){
|
||||
def dist = districts[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
def startDate = DateUtils.addMonth(dist.calDate, -3)
|
||||
def endDate = DateUtils.addMonth(dist.calDate, 3)
|
||||
def tradeAvgPrice = findTrades(dist, startDate, endDate);
|
||||
println dist.city +" "+dist.region+" "+dist.name +" "+tradeAvgPrice.toInteger()
|
||||
// def stat = findDistrictStat(city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2021-01-10", "yyyy-MM-dd"))
|
||||
rets << [city, dist.region, dist.name, tradeAvgPrice.toInteger(), dist.evaluationPrice]
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
q.put("district_name", new BasicDBObject('$in', dist.alias))
|
||||
}else{
|
||||
q.put("district_name", dist.name)
|
||||
}
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
def avg_price = 0
|
||||
def total_price = 0
|
||||
def count = 0
|
||||
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
total_price = total_price + d.avg_price
|
||||
count++
|
||||
}
|
||||
}
|
||||
if(count > 3){
|
||||
avg_price = total_price/count
|
||||
}
|
||||
return avg_price
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findTradeDetails(dist, startDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
q.put("district_name", new BasicDBObject('$in', dist.alias))
|
||||
}else{
|
||||
q.put("district_name", dist.name)
|
||||
}
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
|
||||
def rets = []
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findTradeDetails2(city, startDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
|
||||
def rets = []
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findStatCount(def city, def weekend, def value_type){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
if(value_type){
|
||||
q.put("value_type", value_type)
|
||||
}
|
||||
def count = DIST_STAT_COLL.count(q)
|
||||
return count;
|
||||
}
|
||||
|
||||
def findDistrictStat(def city, def region, def name, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("district_name", name)
|
||||
q.put("weekend", weekend)
|
||||
def obj = DIST_STAT_COLL.findOne(q)
|
||||
return obj;
|
||||
}
|
||||
|
||||
def findBadCaseCount(def city, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
def count = BAD_CASE_COLL.count(q)
|
||||
return count;
|
||||
}
|
||||
|
||||
def findDealPriceCount(def city, def tradeDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
|
||||
def count = DEAL_PRICE_COLL.count(q)
|
||||
return count;
|
||||
}
|
||||
|
||||
def calTradeDistCount(def city, def tradeDate, def districts){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("source","链家在线")
|
||||
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
|
||||
def distMap = [:]
|
||||
for(def dist :districts){
|
||||
def lists = distMap.get(dist.region)
|
||||
if(!lists){
|
||||
lists = []
|
||||
distMap.put(dist.region, lists);
|
||||
}
|
||||
lists << dist
|
||||
}
|
||||
|
||||
def rets = []
|
||||
for(def obj : dbcorsor){
|
||||
def dist = findIsExistsDB(obj.region, obj.district_name, distMap);
|
||||
if(dist && !rets.contains(dist)){
|
||||
rets << dist
|
||||
}
|
||||
}
|
||||
return rets.size();
|
||||
}
|
||||
|
||||
def findIsExistsDB(def region, def district_name, def distMap){
|
||||
def dist = null;
|
||||
def regionDists = distMap.get(region)
|
||||
for(def rd : regionDists){
|
||||
def alias = rd.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(rd.name)){
|
||||
alias << rd.name
|
||||
}
|
||||
if(alias.contains(district_name)){
|
||||
dist = rd
|
||||
break;
|
||||
}
|
||||
}
|
||||
return dist;
|
||||
}
|
||||
|
||||
|
||||
def calTradeDistDetail(def city, def tradeDate, def districts){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
|
||||
def distMap = [:]
|
||||
for(def dist :districts){
|
||||
def lists = distMap.get(dist.region)
|
||||
if(!lists){
|
||||
lists = []
|
||||
distMap.put(dist.region, lists);
|
||||
}
|
||||
lists << dist
|
||||
}
|
||||
|
||||
def retMap = [:]
|
||||
for(def obj : dbcorsor){
|
||||
def dist = findIsExistsDB(obj.region, obj.district_name, distMap);
|
||||
if(dist){
|
||||
def tradePrices = retMap.get(dist.region+"#"+dist.name)
|
||||
if(!tradePrices){
|
||||
tradePrices = []
|
||||
retMap.put(dist.region+"#"+dist.name, tradePrices)
|
||||
}
|
||||
if(obj.avg_price){
|
||||
tradePrices << obj.avg_price.toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
|
||||
|
||||
def calTradeDistDetail2(def city, def tradeDate, def districts){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
|
||||
def distMap = [:]
|
||||
for(def dist :districts){
|
||||
def lists = distMap.get(dist.region)
|
||||
if(!lists){
|
||||
lists = []
|
||||
distMap.put(dist.region, lists);
|
||||
}
|
||||
lists << dist
|
||||
}
|
||||
|
||||
def retMap = [:]
|
||||
for(def obj : dbcorsor){
|
||||
def dist = findIsExistsDB(obj.region, obj.district_name, distMap);
|
||||
if(dist){
|
||||
def tradePrices = retMap.get(dist.region+"#"+dist.name)
|
||||
if(!tradePrices){
|
||||
tradePrices = []
|
||||
retMap.put(dist.region+"#"+dist.name, tradePrices)
|
||||
}
|
||||
if(obj.avg_price){
|
||||
tradePrices << obj.avg_price.toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
}
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.gj.paimai.GaoDeUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class CompanyAddressCompare {
|
||||
static main(args) {
|
||||
def task = new CompanyAddressCompare()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
ExcelUtils.modify("D:\\数据\\公司及地址\\公司及地址.xlsx",{row->
|
||||
if(row['D'] && !row['I'] && row['B'].contains("宁波甬城农村商业银行")){
|
||||
def city = CommonUtils.parseCityInAddress(row["D"])
|
||||
if(!city){
|
||||
city = CommonUtils.parseCityInAddress(row["B"])
|
||||
}
|
||||
if(city){
|
||||
def similarData = null
|
||||
// if(isAddress(row["D"])){
|
||||
def simpleAddress = row["D"].toString().replace("中国(上海)自由贸易试验区","").replaceAll(/\\d+层/,"").replaceAll(/[一二三四五六]楼/,"")
|
||||
if(simpleAddress && simpleAddress.contains("号")){
|
||||
simpleAddress = simpleAddress.substring(0, simpleAddress.lastIndexOf("号")+1)
|
||||
}
|
||||
def pois = GaoDeUtils.search(simpleAddress, city, null)
|
||||
similarData = findSimilarDatas(pois, row["B"], simpleAddress, city, null)
|
||||
println row['C']+" "+pois.size()
|
||||
// }
|
||||
|
||||
if(similarData){
|
||||
row['F'] = similarData.address
|
||||
row['G'] = similarData.name
|
||||
row['H'] = similarData.score
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 0)
|
||||
}
|
||||
|
||||
|
||||
def findSimilarDatas(pois, name, address, city, rMap){
|
||||
def data = null
|
||||
def maxScore = 0
|
||||
|
||||
for(def poi : pois){
|
||||
if(!poi.cityname){
|
||||
def m = getMapRegion(poi.location)
|
||||
if(m){
|
||||
poi.cityname = m.city
|
||||
poi.pname = m.province
|
||||
poi.township = m.township
|
||||
poi.adname = m.district
|
||||
}
|
||||
}
|
||||
if(poi.cityname.contains(city)){
|
||||
def searchName = poi.name
|
||||
def nameScore = CosineSimilarAlgorithm.getSimilarityRatio(searchName, name);
|
||||
def addressScore = 0
|
||||
if(poi.address){
|
||||
def poiAddress = poi.address.replace("号楼","号")
|
||||
addressScore = CosineSimilarAlgorithm.getSimilarityRatio(poiAddress, address);
|
||||
if(poi.address.contains("号") && address.contains("号") && (poiAddress.contains(address) || address.contains(poiAddress))){
|
||||
addressScore = addressScore + 0.5
|
||||
}
|
||||
}
|
||||
|
||||
def totalScore = nameScore + addressScore
|
||||
if(totalScore > maxScore){
|
||||
maxScore = totalScore
|
||||
data = poi
|
||||
data.score = totalScore
|
||||
}
|
||||
println searchName+" "+name+" "+totalScore + " "+ (poi.address?poi.address:"")
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
def isAddress(address){
|
||||
def REG_ADDR1 = "[\u4e00-\u9fa5]{2,20}[路|巷|街|弄|街道|大道|大街|巷子|公路|马路|大道西段]\\d+";
|
||||
def pattern = Pattern.compile(REG_ADDR1);
|
||||
def matcher = pattern.matcher(address)
|
||||
if(matcher.find()){
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
def getMapRegion(def location){
|
||||
def jsonObj = GaoDeApiUtils.geocode(location)
|
||||
def district = null
|
||||
if(jsonObj && jsonObj.status=="1"){
|
||||
def detailMap = jsonObj.regeocode.addressComponent
|
||||
if(!detailMap.city){
|
||||
if(detailMap.province && detailMap.province.toString().endsWith("市")){
|
||||
detailMap.city = detailMap.province
|
||||
}
|
||||
}
|
||||
return detailMap
|
||||
}
|
||||
return district;
|
||||
}
|
||||
|
||||
}
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class CrawlDistrictIdSelect {
|
||||
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
|
||||
static DBCollection districtseedsColl = MongoUtils.getCollectionByAuth20190102("seeddb", "district_seeds")
|
||||
static DBCollection districtseeds2Coll = MongoUtils.getCollectionByAuth20190102("seeddb_dev", "district_seeds")
|
||||
|
||||
static main(args) {
|
||||
def task = new CrawlDistrictIdSelect()
|
||||
task.run()
|
||||
// task.synch()
|
||||
}
|
||||
|
||||
def synch(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = districtseeds2Coll.find(q)
|
||||
def list = []
|
||||
for(def d :dbcursor){
|
||||
list << d
|
||||
if(list.size()==10000){
|
||||
districtseedsColl.insert(list)
|
||||
list = []
|
||||
}
|
||||
}
|
||||
|
||||
if(list.size()>0){
|
||||
districtseedsColl.insert(list)
|
||||
}
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findCrawlData()
|
||||
datas.each {k,v->
|
||||
if(v){
|
||||
for(def d : v){
|
||||
d.c_date = new Date()
|
||||
d.m_date = new Date()
|
||||
districtseedsColl.save(d)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findCrawlData(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","福州")
|
||||
q.put("source", "安居客")
|
||||
q.put("s_date",new BasicDBObject('$gte', 202001))
|
||||
def dbcursor = thirdPartyColl.find(q)
|
||||
def retMap = [:]
|
||||
def prefix = "/community/view/"
|
||||
def keyList =[]
|
||||
for(def db : dbcursor){
|
||||
def url = null
|
||||
if(!db.url){
|
||||
if(db.district_id){
|
||||
url = db.district_id
|
||||
}
|
||||
}else{
|
||||
url = db.url
|
||||
}
|
||||
if(url && url.contains(prefix) ){
|
||||
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size())
|
||||
if(district_id=="0"){
|
||||
continue
|
||||
}
|
||||
def shortCity = url.substring(url.indexOf("https://")+8, url.indexOf(".anjuke.com"))
|
||||
def region = RegionData.formatRegion(db.city, db.region)
|
||||
def key = shortCity +"#"+district_id
|
||||
|
||||
def ids = retMap.get(key)
|
||||
if(!ids){
|
||||
ids = []
|
||||
retMap.put(key, ids)
|
||||
}
|
||||
|
||||
if(!keyList.contains(key)){
|
||||
def map = new BasicDBObject()
|
||||
map.root = "anjuke.com"
|
||||
map.city_short= shortCity
|
||||
map.city = db.city
|
||||
map.region = region
|
||||
map.district_id = district_id
|
||||
map.source_url = url
|
||||
ids << map
|
||||
|
||||
keyList << key
|
||||
}
|
||||
}else{
|
||||
println url +" 链接不合要求"
|
||||
}
|
||||
}
|
||||
return retMap
|
||||
}
|
||||
}
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class ExportToDealPrice {
|
||||
|
||||
DBCollection DEAL_PRICE_MATCH_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price_match", "192.168.0.43")
|
||||
DBCollection DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43")
|
||||
|
||||
DBCollection TRADE_MATCH_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "trade_match_price", "192.168.0.136")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
|
||||
final AtomicInteger count = new AtomicInteger()
|
||||
|
||||
static main(args) {
|
||||
def task = new ExportToDealPrice();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findDatas()
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def data = datas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
count.addAndGet(1)
|
||||
def dbo = addPrice(data)
|
||||
if(dbo){
|
||||
data.weekend_avg_price = dbo.avg_price
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
|
||||
save(datas);
|
||||
|
||||
}
|
||||
|
||||
def save(def datas){
|
||||
def ret = [];
|
||||
def index = 0;
|
||||
for(def dbo : datas){
|
||||
ret << dbo;
|
||||
index++;
|
||||
if(ret.size()%10000==0){
|
||||
println "µÚ "+(index/10000)+" ¸ö"+ret.size()
|
||||
TRADE_MATCH_PRICE_COLL.insert(ret);
|
||||
ret = [];
|
||||
}
|
||||
}
|
||||
if(ret.size()>0){
|
||||
TRADE_MATCH_PRICE_COLL.insert(ret);
|
||||
}
|
||||
}
|
||||
|
||||
def addPrice(data){
|
||||
def weekend = DateUtils.getFirstDayOfWeek(data.trade_date)
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", data.match_city)
|
||||
q.put("region", data.match_region)
|
||||
q.put("district_name", data.match_name)
|
||||
q.put("weekend", weekend)
|
||||
def dbo = DISTRICT_STAT_COLL.findOne(q)
|
||||
if(!dbo){
|
||||
q.put("weekend", new BasicDBObject(QueryOperators.LTE, weekend))
|
||||
dbo = DISTRICT_STAT_COLL.findOne(q)
|
||||
}
|
||||
println count.get()+" "+data.city +" "+data.match_name
|
||||
return dbo
|
||||
}
|
||||
|
||||
def findDatas(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2020-04-01","yyyy-MM-dd")))
|
||||
def dbcursor = DEAL_PRICE_MATCH_COLL.find(q)
|
||||
def rets = []
|
||||
for(def db : dbcursor){
|
||||
rets << db
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
}
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.fangjia.script.util.DbUtils
|
||||
|
||||
class FZloationUpdate {
|
||||
|
||||
static main(args) {
|
||||
def task = new FZloationUpdate();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region","name","address","lat","lng"]);
|
||||
def index = 0
|
||||
def locationMap = getMoreSameLocationDist(districts)
|
||||
def total = 0
|
||||
locationMap.each {k,v->
|
||||
total = updateLocation(v, total);
|
||||
println "---------------------------------------------------------------------------------------------------------------"
|
||||
}
|
||||
}
|
||||
|
||||
def updateLocation(def lists, def total){
|
||||
|
||||
|
||||
for(def dist : lists){
|
||||
|
||||
def addrDistMap = getLocation(dist, dist.address)
|
||||
def nameDistMap = getLocation(dist, dist.region + dist.name)
|
||||
|
||||
if((addrDistMap && addrDistMap.distance>1000) && (nameDistMap && nameDistMap.distance>1000)){
|
||||
println dist.region +" "+dist.name+" "+dist.address+" addrDist:"+(addrDistMap!=null?addrDistMap.distance:"")+" addrKey:"+(addrDistMap!=null?addrDistMap.keyword:"")+" nameDist:"+(nameDistMap!=null?nameDistMap.distance:"")+" nameKey:"+(nameDistMap!=null?nameDistMap.keyword:"")
|
||||
// println "[${index}]"+ dist.region+" "+dist.name+" "+dist.address+" addrDist:"+addrDistMap.distance+"(${addrDistMap.level}-${addrDistMap.location}-${addrDistMap.keyword}) nameDist:"+nameDistMap.distance+"(${nameDistMap.level}-${nameDistMap.location}-${nameDistMap.keyword})"
|
||||
|
||||
if(addrDistMap){
|
||||
def addrKey = deleteRegion(addrDistMap.keyword, dist.region).replace("|","");
|
||||
def nameKey = deleteRegion(nameDistMap.keyword, dist.region);
|
||||
|
||||
def baidu = null
|
||||
def lat = null
|
||||
def lng = null
|
||||
def upldateFlag = ""
|
||||
if(addrDistMap && addrDistMap.region.contains(dist.region) && (addrKey.contains("路") || addrKey.contains("大道")) && addrKey.contains("号") && dist.addr == addrKey){
|
||||
lat = addrDistMap.lat
|
||||
lng = addrDistMap.lng
|
||||
baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
|
||||
upldateFlag = "Address"
|
||||
}else if(nameDistMap && nameDistMap.region.contains(dist.region) && nameKey == dist.name){
|
||||
lng = nameDistMap.lng
|
||||
lat = nameDistMap.lat
|
||||
baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
|
||||
upldateFlag = "Name"
|
||||
}
|
||||
if(baidu){
|
||||
total++
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
Long baiduLat = baidu.y*10000000000L
|
||||
Long baiduLng = baidu.x*10000000000L
|
||||
udata.put("lat2", baiduLat)
|
||||
udata.put("lng2", baiduLng)
|
||||
Long lat0 = lat*10000000000L
|
||||
Long lng0 = lng*10000000000L
|
||||
udata.put("lat", lat0)
|
||||
udata.put("lng", lng0 )
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
println "[修改-${total}] "+ dist.region+" "+dist.name+" "+dist.address+" "+upldateFlag
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
def getMoreSameLocationDist(def districts){
|
||||
def locationMap = [:]
|
||||
for(def i=0;i<districts.size();i++){
|
||||
def dist = districts[i]
|
||||
def lat = dist.lat/10000000000.0;
|
||||
def lng = dist.lng/10000000000.0;
|
||||
def locationKey = lng + "-" + lat
|
||||
def list = locationMap.get(locationKey)
|
||||
if(!list){
|
||||
list = []
|
||||
locationMap.put(locationKey, list);
|
||||
}
|
||||
list << dist
|
||||
}
|
||||
def resMap = [:]
|
||||
locationMap.each{k,v->
|
||||
if(v.size()>=2){
|
||||
|
||||
v.sort{ a,b->
|
||||
return a.name.size().compareTo(b.name.size())
|
||||
}
|
||||
resMap.put(k,v)
|
||||
}
|
||||
}
|
||||
|
||||
def lastRetMaps = [:]
|
||||
resMap.each{k,v->
|
||||
def swname = null
|
||||
def count = 0
|
||||
for(def i=0;i<v.size();i++){
|
||||
def d = v[i]
|
||||
if(i==0){
|
||||
if(d.name.contains("期")||d.name.contains("区")){
|
||||
swname = d.name.replace("一期","").replace("二期","").replace("三期","").replace("四期","").replace("五期","").replace("六期","").replace("一区","").replace("二区","").replace("三区","").replace("四区","").replace("五区","").replace("A区","").replace("B区","").replace("C区","").replace("D区","")
|
||||
}else{
|
||||
swname = d.name
|
||||
}
|
||||
}else{
|
||||
if(d.name.contains(swname)||swname.contains(d.name)){
|
||||
count++
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
if(count+1!=v.size()){
|
||||
lastRetMaps.put(k,v)
|
||||
}
|
||||
}
|
||||
return lastRetMaps;
|
||||
}
|
||||
|
||||
|
||||
def deleteRegion(def s, def region){
|
||||
def key = ""
|
||||
if(s.indexOf(region)>-1){
|
||||
if(s.indexOf(region+"区")>-1){
|
||||
key = s.substring(s.indexOf(region+"区")+region.size()+1, s.size());
|
||||
}else if(s.indexOf(region+"县")>-1){
|
||||
key = s.substring(s.indexOf(region+"县")+region.size()+1, s.size());
|
||||
}else if(s.indexOf(region+"市")>-1){
|
||||
key = s.substring(s.indexOf(region+"市")+region.size()+1, s.size());
|
||||
}
|
||||
}else{
|
||||
key = s;
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
|
||||
|
||||
def getLocation(dist, addr){
|
||||
|
||||
if(addr){
|
||||
def rMap = [:]
|
||||
def reps = GaoDeApiUtils.geo(dist.city, addr);
|
||||
if(reps && reps.count.toInteger()>=1){
|
||||
def map = reps.geocodes.get(0)
|
||||
def location = map.location
|
||||
def level = map.level
|
||||
if(location){
|
||||
def locMap = location.split(",")
|
||||
def lat = dist.lat/10000000000.0;
|
||||
def lng = dist.lng/10000000000.0;
|
||||
def distance = GeoHelper.GetDistance(locMap[1].toDouble(), locMap[0].toDouble(), lat, lng)
|
||||
rMap.distance = distance
|
||||
rMap.level = level
|
||||
rMap.lat = locMap[1].toDouble()
|
||||
rMap.lng = locMap[0].toDouble()
|
||||
rMap.keyword = map.formatted_address
|
||||
rMap.location = location
|
||||
rMap.region = map.district
|
||||
return rMap;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
class FzLocationBatch {
|
||||
}
|
||||
+20
-1
@@ -30,7 +30,9 @@ class JiangShuBlankDataStat {
|
||||
// task.matchHouse();
|
||||
|
||||
// task.villa()
|
||||
task.addressParse()
|
||||
// task.addressParse()
|
||||
|
||||
task.deleteProcess()
|
||||
}
|
||||
|
||||
def repeat(){
|
||||
@@ -288,5 +290,22 @@ class JiangShuBlankDataStat {
|
||||
return floor;
|
||||
}
|
||||
|
||||
def deleteProcess(){
|
||||
|
||||
def datas = ExcelUtils.read("D:\\数据\\江苏农行\\江苏2千.xlsx",0,1)
|
||||
def addrList = []
|
||||
for(def d : datas){
|
||||
def addr = d['G']
|
||||
addrList << addr
|
||||
}
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以下.xlsx", { row ->
|
||||
def addr = row['C']
|
||||
if(addrList.contains(addr)){
|
||||
row['R'] = "已调查"
|
||||
}
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
|
||||
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
||||
Date date = new Date(1617211269868);
|
||||
print dateFormat.format(date);
|
||||
+1
-2
@@ -18,8 +18,7 @@ import org.apache.commons.lang.StringUtils
|
||||
class AutocompleteDataUpdaterBatch extends BaseBatch {
|
||||
|
||||
|
||||
|
||||
private static final DBCollection AC_COLL = MongoUtils.getCollectionByName("fangjia", "autocomplete", "192.168.0.51", 29009)
|
||||
private static final DBCollection AC_COLL = MongoUtils.getCollectionByName("fangjia", "autocomplete", "mongo.db.fangjia.com")
|
||||
private static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
|
||||
private static final DSCollection AUTOCOMPLETE_COLL = new DataStore("192.168.0.38", 27008).getCollection("fangjia.autocomplete")
|
||||
private static final long C_DATE = DateUtil.formatToYYYYMMDDhhmmss(new Date()).toLong()
|
||||
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
package com.fangjia.script.temp.shenjian
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
/**
|
||||
* 抓取总量监控
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/5/12
|
||||
*/
|
||||
@BatchOwner(Watcher.ShenJian)
|
||||
class SeawaterMonitor extends BaseBatch {
|
||||
|
||||
static final Long expectNumber = 400000;
|
||||
// 运维组 运维警报
|
||||
static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
|
||||
// 重点城市
|
||||
static final city = ["北京","唐山","天津","秦皇岛","石家庄","包头","太原","丹东","呼和浩特","锦州",
|
||||
"沈阳","吉林","大连","牡丹江","长春","无锡","哈尔滨","扬州","上海","徐州",
|
||||
"南京","温州","杭州","金华","宁波","蚌埠","合肥","安庆","福州","泉州",
|
||||
"厦门","九江","南昌","赣州","济南","烟台","青岛","济宁","郑州","洛阳",
|
||||
"武汉","平顶山","长沙","宜昌","广州","襄阳","深圳","岳阳","南宁","常德",
|
||||
"海口","惠州","重庆","湛江","成都","韶关","贵阳","桂林","昆明","北海",
|
||||
"西安","三亚","兰州","泸州","西宁","南充","银川","遵义","乌鲁木齐","大理",
|
||||
"东莞","佛山","苏州","珠海","中山","台州","绍兴","南通","常州","嘉兴",
|
||||
"淄博","株洲","镇江","肇庆","漳州","盐城","芜湖","泰州","汕头","清远",
|
||||
"马鞍山","柳州","连云港","廊坊","淮安","湖州","沧州","保定","张家口","日照",
|
||||
"拉萨","大同","文昌","陵水","保亭"];
|
||||
// 重点网站
|
||||
static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"]
|
||||
|
||||
static void main(String[] args) {
|
||||
runBatch(new SeawaterMonitor(), args)
|
||||
}
|
||||
|
||||
/**
|
||||
* 监控前一日抓取量
|
||||
* @return
|
||||
*/
|
||||
static def monitorYesterdayCrawlNumber(def startDate, def endDate) {
|
||||
def total
|
||||
String sql = "select count(*) as total from seawater where m_date >= '${startDate}' and m_date < '${endDate}' "
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
total = data.total
|
||||
})
|
||||
if (total < expectNumber) {
|
||||
return "【总量抓取异常】 抓取量为" + total + ", 低于预期值" + expectNumber
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计重点城市近半月抓取量
|
||||
* @return
|
||||
*/
|
||||
static boolean countCityCrawlNumber(def startDate, def endDate) {
|
||||
def result = []
|
||||
result << ['城市', '抓取量']
|
||||
// 统计近半个月各重点城市抓取量
|
||||
def cityFormatted = city.join("','")
|
||||
String sql = "select * from (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) order by total desc "
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
result << [data.city, data.total]
|
||||
})
|
||||
ExcelUtils.write("F:\\数据\\重点城市抓取\\近半月重点城市抓取量.xlsx", result, "抓取量")
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计前一日重点城市抓取量,昨天为零则警报
|
||||
* @return
|
||||
*/
|
||||
static def monitorCityCrawlNumber(def startDate, def endDate) {
|
||||
def msg = "【重点城市抓取异常】"
|
||||
def set = new HashSet()
|
||||
def cityFormatted = city.join("','")
|
||||
String sql = "select * from " +
|
||||
" (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) where total>0 "
|
||||
//" order by total "
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
set.add(data.city)
|
||||
})
|
||||
def unNormal = []
|
||||
for (def info : city) {
|
||||
if (!set.contains(info)) {
|
||||
unNormal.add(info)
|
||||
}
|
||||
}
|
||||
msg += unNormal.join("、") + " 抓取量为0"
|
||||
if (unNormal.size() > 0) {
|
||||
return msg
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计重点网站抓取情况,近一周为0则警报
|
||||
*/
|
||||
static def monitorSiteCrawlNumber(def lastWeekDate, def endDate) {
|
||||
def msg = "【重点网站抓取异常】"
|
||||
def set = new HashSet()
|
||||
def siteFormatted = site.join("','")
|
||||
String sql = "select * from " +
|
||||
" (select count(*) as total, root from seawater where m_date >= '${lastWeekDate}' and m_date < '${endDate}' and root in('${siteFormatted}') group by root) where total > 0 "
|
||||
//" order by total "
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
set.add(data.root)
|
||||
})
|
||||
def unNormal = []
|
||||
for (def info : site) {
|
||||
if (!set.contains(info)) {
|
||||
unNormal.add(info)
|
||||
}
|
||||
}
|
||||
msg += unNormal.join("、") + " 近一周抓取量为0"
|
||||
if (unNormal.size() > 0) {
|
||||
return msg
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def lastWeekDate = DateUtil.add(DateUtil.truncateDay(new Date()), -7).getTime()
|
||||
def yesterday = DateUtil.formatToYYYYMMDD(DateUtil.add(DateUtil.truncateDay(new Date()), -1))
|
||||
def startDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1).getTime();
|
||||
def endDate = DateUtil.truncateDay(new Date()).getTime()
|
||||
|
||||
def result = []
|
||||
def msg = monitorYesterdayCrawlNumber(startDate, endDate)
|
||||
if (StringUtils.isNotBlank(msg)) {
|
||||
result.add(msg)
|
||||
}
|
||||
|
||||
msg = monitorCityCrawlNumber(startDate, endDate)
|
||||
if (StringUtils.isNotBlank(msg)) {
|
||||
result.add(msg)
|
||||
}
|
||||
|
||||
msg = monitorSiteCrawlNumber(lastWeekDate, endDate)
|
||||
if (StringUtils.isNotBlank(msg)) {
|
||||
result.add(msg)
|
||||
}
|
||||
if (result.size() > 0) {
|
||||
msg = "【日期】 " + yesterday + "\n" + result.join("\n")
|
||||
println(msg)
|
||||
DingDingMessageUtil.sendTextMessage(access_token, msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202101);
|
||||
def count2 = findTotalCount(source, 202102);
|
||||
def count1 = findTotalCount(source, 202102);
|
||||
def count2 = findTotalCount(source, 202103);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
+213
@@ -0,0 +1,213 @@
|
||||
package com.fangjia.script.thirdPriceCrawl.other
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import net.sf.json.groovy.JsonSlurper
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
class AddFangSchoolRegion {
|
||||
|
||||
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
|
||||
static DBCollection third_price_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "third_party_price")
|
||||
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school2")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
def task = new AddFangSchoolRegion();
|
||||
//task.process()
|
||||
//task.match()
|
||||
task.addMapLocation()
|
||||
}
|
||||
|
||||
def addMapLocation(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = district_school_col.find(q)
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
def data = findSchool(d.city, d.region, d.school_name)
|
||||
if(data && data.location){
|
||||
def location = data.location
|
||||
if(location){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("school_map_lng", location[0])
|
||||
udata.put("school_map_lat", location[1])
|
||||
district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
}
|
||||
index ++
|
||||
println index
|
||||
}
|
||||
}
|
||||
|
||||
def match(){
|
||||
def datas = findAllDatas()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size;i++){
|
||||
def data = datas[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
// if(!data.match_name){
|
||||
println i +" "+data.district_name
|
||||
matching(data);
|
||||
// }
|
||||
|
||||
} catch (Exception e) {
|
||||
println "Òì³£"
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def matching(def it){
|
||||
def rMap = matchDistrict(it.city, it.region, it.district_name)
|
||||
if(!rMap){
|
||||
rMap = matchDistrict(it.city, it.region, it.address)
|
||||
}
|
||||
|
||||
if(rMap){
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("match_city", rMap.city)
|
||||
data.put("match_region", rMap.region)
|
||||
data.put("match_name", rMap.name)
|
||||
data.put("match_address", rMap.address)
|
||||
data.put("match_lat", rMap.lat)
|
||||
data.put("match_lng", rMap.lng)
|
||||
district_school_col.update(new BasicDBObject('_id', it.get("_id")), new BasicDBObject('$set', data))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def matchDistrict(def city, def region, def address){
|
||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&preciseLevel=strict&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district"
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
def rMap = [:]
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==200 && doc.result){
|
||||
def dto = doc.result
|
||||
if(dto && dto.searchAddress){
|
||||
def ret = dto.matchResult
|
||||
if(ret.match){
|
||||
rMap.city = dto.city
|
||||
rMap.region = dto.district
|
||||
rMap.name = dto.name
|
||||
rMap.lat = dto.lat
|
||||
rMap.lng = dto.lng
|
||||
rMap.id = dto.id
|
||||
rMap.address = dto.address
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rMap;
|
||||
}
|
||||
|
||||
|
||||
def process(){
|
||||
def datas = findNoRegionData();
|
||||
findSimialData(datas);
|
||||
}
|
||||
|
||||
def findAllDatas(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_name", new BasicDBObject(QueryOperators.EXISTS, false))
|
||||
def dbcursor = district_school_col.find(q)
|
||||
def rets =[]
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findNoRegionData(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("region", null)
|
||||
def dbcursor = district_school_col.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findSimialData(datas){
|
||||
for(def d : datas){
|
||||
def obj = findThirdPriceData(d.city, d.url+"/")
|
||||
if(!obj){
|
||||
obj = findThirdPriceByName(d.city, d.district_name)
|
||||
}
|
||||
if(obj){
|
||||
d.region = obj.region
|
||||
if(!d.district_addrss){
|
||||
d.district_address = obj.address
|
||||
}
|
||||
updateDistrictSchool(d);
|
||||
|
||||
}else{
|
||||
println d.school_name+"" + d.district_name+"û²éµ½"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findThirdPriceData(def city, def url){
|
||||
def q = new BasicDBObject()
|
||||
q.put("source","ËÑ·¿")
|
||||
q.put("city", city)
|
||||
q.put("district_id", url)
|
||||
def dbcursor = third_price_col.find(q).sort(new BasicDBObject("s_date", -1)).limit(1)
|
||||
def obj = null;
|
||||
for(def d : dbcursor){
|
||||
obj = d
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
def findThirdPriceByName(def city, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("source","ËÑ·¿")
|
||||
q.put("city", city)
|
||||
q.put("name", name)
|
||||
def dbcursor = third_price_col.find(q).sort(new BasicDBObject("s_date", -1)).limit(1)
|
||||
def obj = null;
|
||||
for(def d : dbcursor){
|
||||
obj = d
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
def updateDistrictSchool(d){
|
||||
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("region", d.region)
|
||||
data.put("district_address", d.district_address)
|
||||
district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', data))
|
||||
}
|
||||
|
||||
def findSchool(city, region, school_name){
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("city", city)
|
||||
data.put("region", region)
|
||||
data.put("name", school_name)
|
||||
return school_col.findOne(data)
|
||||
}
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
package com.fangjia.script.thirdPriceCrawl.other
|
||||
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import org.apache.commons.io.IOUtils
|
||||
import org.apache.http.HttpResponse
|
||||
import org.apache.http.client.HttpClient
|
||||
import org.apache.http.client.methods.HttpGet
|
||||
import org.apache.http.impl.client.HttpClients
|
||||
|
||||
class AnjukeCrawl {
|
||||
|
||||
static main(args) {
|
||||
def task = new AnjukeCrawl();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
HttpClient client = HttpClients.createDefault();
|
||||
HttpGet get = new HttpGet("https://m.anjuke.com/cityList/");
|
||||
// get.setHeader("Upgrade-Insecure-Requests", "1");
|
||||
// get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36");
|
||||
|
||||
// get.setHeader("Host","m.anjuke.com");
|
||||
// get.setHeader("Connection","keep-alive");
|
||||
// get.setHeader("Cache-Control","max-age=0");
|
||||
// get.setHeader('sec-ch-ua','"Chromium";v="88", "Google Chrome";v="88", ";Not A Brand";v="99"');
|
||||
// get.setHeader("sec-ch-ua-mobile","'?0");
|
||||
// get.setHeader("Upgrade-Insecure-Requests","1");
|
||||
def header = [:]
|
||||
header.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36");
|
||||
header.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
|
||||
// get.setHeader("Sec-Fetch-Site","none");
|
||||
// get.setHeader("Sec-Fetch-Mode","navigate");
|
||||
// get.setHeader("Sec-Fetch-User","?1");
|
||||
// get.setHeader("Sec-Fetch-Dest","document");
|
||||
//// get.setHeader("Accept-Encoding"," gzip, deflate, br");
|
||||
// get.setHeader("Accept-Language","zh-CN,zh;q=0.9");
|
||||
|
||||
|
||||
def s1 = ABuYunUtil.doGet("https://m.anjuke.com/cityList/", null, header, "utf-8")
|
||||
printl s1
|
||||
|
||||
HttpResponse response = client.execute(get);
|
||||
def s = getResponseBodyAsString(response)
|
||||
println s
|
||||
}
|
||||
|
||||
private static String getResponseBodyAsString(HttpResponse response) throws IOException {
|
||||
return IOUtils.toString(response.getEntity().getContent(), "UTF-8");
|
||||
}
|
||||
|
||||
}
|
||||
+344
@@ -0,0 +1,344 @@
|
||||
package com.fangjia.script.thirdPriceCrawl.other
|
||||
|
||||
import com.alibaba.fastjson.annotation.JSONField
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.HttpClient
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.httpclient.NameValuePair
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.http.HttpEntity
|
||||
import org.apache.http.client.entity.UrlEncodedFormEntity
|
||||
import org.apache.http.client.methods.CloseableHttpResponse
|
||||
import org.apache.http.client.methods.HttpPost
|
||||
import org.apache.http.impl.client.CloseableHttpClient
|
||||
import org.apache.http.impl.client.HttpClients
|
||||
import org.apache.http.message.BasicNameValuePair
|
||||
import org.apache.http.util.EntityUtils
|
||||
|
||||
import java.nio.charset.Charset
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class GushuwangCrawl {
|
||||
|
||||
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
|
||||
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school")
|
||||
static main(args) {
|
||||
def task = new GushuwangCrawl();
|
||||
task.process()
|
||||
}
|
||||
|
||||
|
||||
void process(Object args) {
|
||||
|
||||
// chaxuequCrawl()
|
||||
suzhouCrawl();
|
||||
|
||||
}
|
||||
|
||||
def suzhouCrawl(){
|
||||
def page = 1
|
||||
def stop = false
|
||||
def list = []
|
||||
while(!stop){
|
||||
def s = schoolList(page);
|
||||
if(s){
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
if(jsonObject && jsonObject.ok){
|
||||
def datas = jsonObject.extra.list
|
||||
if(datas){
|
||||
list.addAll(datas)
|
||||
page++
|
||||
println page
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(list){
|
||||
def schools = []
|
||||
for(def d : list){
|
||||
def schoolData = new BasicDBObject()
|
||||
schoolData.province = "江苏"
|
||||
schoolData.city = "苏州"
|
||||
schoolData.region = RegionData.parseRegionFormAddress(schoolData.city , d.adr)
|
||||
/**类型: 幼儿园,小学,初中*/
|
||||
schoolData.category = converCategory(d.schoolname)
|
||||
schoolData.name = d.schoolname
|
||||
schoolData.address = d.adr
|
||||
/**来源网站*/
|
||||
schoolData.source = "苏州学区查询"
|
||||
/**房源详情url*/
|
||||
schools << schoolData
|
||||
}
|
||||
|
||||
saveChaxuequSchool(schools)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def chaxuequCrawl(){
|
||||
|
||||
def page = 1
|
||||
def stop = false
|
||||
|
||||
def schools = []
|
||||
|
||||
while(!stop){
|
||||
def s = chaxuequInfoCrawl("https://51yongka.com/fapi/school/list", "schoolName=&city=%E8%8B%8F%E5%B7%9E&area=&schoolNature=&schoolType=&page="+page+"&size=10")
|
||||
println "内容:"+s
|
||||
if(s){
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
if(jsonObject.code == 200){
|
||||
def data = jsonObject.data
|
||||
def totalPage = data.pages
|
||||
if(data.list){
|
||||
page++
|
||||
if(page+1>totalPage){
|
||||
stop = true;
|
||||
}
|
||||
schools.addAll(data.list)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
print(schools.size());
|
||||
|
||||
def districtSchools = []
|
||||
def schoolInfos = []
|
||||
if(schools.size()>0){
|
||||
|
||||
def url = "https://51yongka.com/fapi/school/details"
|
||||
for(def d : schools){
|
||||
def params = "schoolId="+d.id+"&access_token=oRqu15Mq_KxHfK0y0ZWhwktVnfUM"
|
||||
def s = chaxuequInfoCrawl(url, params)
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
if(jsonObject && jsonObject.code==200){
|
||||
|
||||
if(jsonObject.data && jsonObject.data.nearbyPropertyList){
|
||||
def districts = jsonObject.data.nearbyPropertyList
|
||||
for(def dist : districts){
|
||||
def dataMap = new BasicDBObject()
|
||||
dataMap.city = jsonObject.data.city
|
||||
dataMap.region = RegionData.formatRegion(dataMap.city , jsonObject.data.area)
|
||||
/**类型: 幼儿园,小学,初中*/
|
||||
dataMap.category = converCategory(jsonObject.data.schoolName)
|
||||
dataMap.district_name = dist.name.trim()
|
||||
dataMap.district_address = dist.address
|
||||
dataMap.school_name = jsonObject.data.schoolName.trim()
|
||||
dataMap.school_address = jsonObject.data.address
|
||||
dataMap.source = "查学区房"
|
||||
dataMap.url = url + "?"+params
|
||||
dataMap.telephone = jsonObject.data.officialTelephone;
|
||||
districtSchools << dataMap
|
||||
}
|
||||
}else{
|
||||
|
||||
def schoolData = new BasicDBObject()
|
||||
schoolData.province = "江苏"
|
||||
schoolData.city = jsonObject.data.city
|
||||
schoolData.region = RegionData.formatRegion(schoolData.city , jsonObject.data.area)
|
||||
/**类型: 幼儿园,小学,初中*/
|
||||
schoolData.category = converCategory(jsonObject.data.schoolName)
|
||||
schoolData.name = jsonObject.data.schoolName.trim()
|
||||
schoolData.address = jsonObject.data.address
|
||||
/**来源网站*/
|
||||
schoolData.source = "查学区房"
|
||||
/**房源详情url*/
|
||||
schoolData.telephone = jsonObject.data.officialTelephone;
|
||||
schoolInfos << schoolData
|
||||
println jsonObject.data.schoolName +" 无周边楼盘"
|
||||
}
|
||||
|
||||
}
|
||||
println s;
|
||||
}
|
||||
}
|
||||
|
||||
println "小区:"+districtSchools.size()
|
||||
// saveChaxuequDistrictSchool(districtSchools)
|
||||
saveChaxuequSchool(schoolInfos)
|
||||
|
||||
}
|
||||
|
||||
def saveChaxuequSchool(schoolInfos){
|
||||
for(def ds : schoolInfos){
|
||||
def obj = findSchoolData(ds.city, ds.school_name);
|
||||
if(!obj){
|
||||
school_col.save(ds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def saveChaxuequDistrictSchool(def districtSchools){
|
||||
for(def ds : districtSchools){
|
||||
def obj = findDistrictSchoolData(ds.city, ds.school_name);
|
||||
if(!obj){
|
||||
district_school_col.save(ds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findSchoolData(city, school_name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("school_name", school_name)
|
||||
return school_col.findOne(q)
|
||||
}
|
||||
|
||||
|
||||
def findDistrictSchoolData(city, school_name, district_name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("school_name", school_name)
|
||||
q.put("district_name", district_name)
|
||||
return district_school_col.findOne(q)
|
||||
}
|
||||
|
||||
def converCategory(def schoolName){
|
||||
|
||||
if(schoolName){
|
||||
if(schoolName.contains("初中部")||schoolName.contains("中学")){
|
||||
return "初中"
|
||||
}else if(schoolName.contains("小学")){
|
||||
return "小学"
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def static chaxuequInfoCrawl(url, bodyquery){
|
||||
def responseMap = [:]
|
||||
try {
|
||||
//headers信息
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","51yongka.com")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Content-Length","75")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
|
||||
headers.put("content-type","application/x-www-form-urlencoded")
|
||||
headers.put("Referer","https://servicewechat.com/wxc2461b2529576950/8/page-frame.html")
|
||||
headers.put("Accept-Encoding","gzip, deflate, br")
|
||||
String s = doPost(url, bodyquery, headers)
|
||||
return s
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def static schoolList(page) {
|
||||
def responseMap = [:]
|
||||
String s = null
|
||||
try {
|
||||
//headers信息
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","m.gusuwang.com")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Content-Length","40")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
|
||||
headers.put("content-type","application/x-www-form-urlencoded")
|
||||
headers.put("Referer","https://servicewechat.com/wx9188a941e5ef95c5/10/page-frame.html")
|
||||
// headers.put("Accept-Encoding","gzip, deflate, br")
|
||||
|
||||
//params信息
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("secCode","198a53da6baa3aa6a5b99959a006be37");
|
||||
// //post请求
|
||||
def url = "https://m.gusuwang.com/xuequ/xqapi.php?act=searchschool"
|
||||
s = doPost(url, "secCode=198a53da6baa3aa6a5b99959a006be37&keys=&grade=&lb=&lx=&area=&currpage="+page,headers)
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
static String doPost(String strUrl, String content, Map headers) {
|
||||
String result = "";
|
||||
OutputStream outputStream = null;
|
||||
InputStream inputStream = null;
|
||||
StringBuilder sb = new StringBuilder();
|
||||
try {
|
||||
URL url = new URL(strUrl);
|
||||
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
|
||||
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
|
||||
urlConnection.setRequestMethod("POST");
|
||||
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setConnectTimeout(30000);
|
||||
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setReadTimeout(30000);
|
||||
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
|
||||
urlConnection.setDoOutput(true);
|
||||
urlConnection.setDoInput(true);
|
||||
//设置通用请求属性为默认浏览器编码类型
|
||||
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
|
||||
|
||||
|
||||
if (headers != null && headers.keySet().size() > 0) {
|
||||
for (String head : headers.keySet()) {
|
||||
urlConnection.setRequestProperty(head, headers.get(head));
|
||||
}
|
||||
}
|
||||
//getOutputStream()返回的输出流,用于写入参数数据。
|
||||
outputStream = urlConnection.getOutputStream();
|
||||
outputStream.write(content.getBytes());
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
|
||||
inputStream = urlConnection.getInputStream();
|
||||
|
||||
|
||||
// byte[] data = new byte[1024];
|
||||
// //inputStream每次就会将读取1024个byte到data中,当inputSteam中没有数据时,inputStream.read(data)值为-1
|
||||
// while (inputStream.read(data) != -1) {
|
||||
// String s = new String(data, Charset.forName("utf-8"));
|
||||
// sb.append(s);
|
||||
// }
|
||||
|
||||
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
|
||||
char[] buff = new char[1024];
|
||||
int length = 0;
|
||||
while ((length = reader.read(buff)) != -1) {
|
||||
String s = new String(buff, 0, length);
|
||||
sb.append(s);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}finally{
|
||||
if(outputStream!=null){
|
||||
outputStream.close();
|
||||
}
|
||||
if(inputStream!=null){
|
||||
inputStream.close();
|
||||
}
|
||||
}
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
@@ -425,6 +425,7 @@ class ABuYunUtil {
|
||||
buffer.append(bf, 0, len);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace()
|
||||
throw new Exception("doGet(String targetUrl, Map<String, String> params, Map<String, String> headers, String charset)异常, 异常信息为:" + e);
|
||||
} finally {
|
||||
// println("本次请求响应的connection.getResponseCode()为:" + connection.getResponseCode())
|
||||
|
||||
@@ -44,7 +44,7 @@ class ElasticUtils {
|
||||
|
||||
static getEsDataSource(String host){
|
||||
EsDataSource dataSource = new EsDataSource ();
|
||||
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240&network.timeout=600&page.timeout=240");
|
||||
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000");
|
||||
Properties connectionProperties = new Properties();
|
||||
connectionProperties.setProperty("user","developer");
|
||||
connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
|
||||
@@ -52,7 +52,7 @@ class ElasticUtils {
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.216"){
|
||||
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.206"){
|
||||
EsDataSource dataSource = getEsDataSource(host)
|
||||
Connection connection = dataSource.getConnection()
|
||||
Statement statement = connection.createStatement()
|
||||
@@ -140,13 +140,12 @@ class ElasticUtils {
|
||||
* @return
|
||||
*/
|
||||
static closeBulkProcessor(BulkProcessor bulkProcessor) {
|
||||
|
||||
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
|
||||
bulkProcessor.flush()
|
||||
bulkProcessor.close()
|
||||
|
||||
}
|
||||
|
||||
|
||||
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
|
||||
|
||||
RestHighLevelClient client = getClient(host)
|
||||
|
||||
@@ -1,19 +1,17 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
import com.mongodb.DBCursor
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.lucene.index.Term
|
||||
import org.apache.lucene.search.BooleanClause
|
||||
import org.apache.lucene.search.BooleanQuery
|
||||
import org.apache.lucene.search.TermQuery
|
||||
import org.apache.solr.client.solrj.util.ClientUtils
|
||||
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.datastore.driver.FacetResult
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.lucene.index.Term
|
||||
import org.apache.lucene.search.BooleanClause
|
||||
import org.apache.lucene.search.BooleanQuery
|
||||
import org.apache.lucene.search.TermQuery
|
||||
import org.apache.solr.client.solrj.util.ClientUtils
|
||||
|
||||
class HouseUtils{
|
||||
static main(args){
|
||||
|
||||
+2
-2
@@ -121,9 +121,9 @@ public class OpenApiClient {
|
||||
params.put("suggestName", suggestName)
|
||||
}
|
||||
|
||||
String content = new HttpRequest().get("http://192.168.10.199:9180/address/realMatch", params, "utf-8")
|
||||
String content = new HttpRequest().get("http://open.fangjia.com/address/match", params, "utf-8")
|
||||
while (content == null) {
|
||||
content = new HttpRequest().get("http://192.168.10.199:9180/address/realMatch", params, "utf-8")
|
||||
content = new HttpRequest().get("http:///open.fangjia.com/address/match", params, "utf-8")
|
||||
println("${city} ${input} ${cat}程序调用异常")
|
||||
Thread.sleep(6000)
|
||||
}
|
||||
|
||||
+12
-2557
File diff suppressed because it is too large
Load Diff
@@ -6,34 +6,50 @@ import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018)
|
||||
def count = 0
|
||||
List<List<String>> data = Collections.synchronizedList([])
|
||||
data << ["ID","城市","区域","小区名","经度","纬度","街道"]
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\Êý¾Ý.xlsx",{List item->
|
||||
// if(DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"]))){
|
||||
// item["D"] = "´æÔÚ"
|
||||
// }else{
|
||||
// item["D"] = "²»´æÔÚ"
|
||||
// }
|
||||
//})
|
||||
|
||||
|
||||
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat","mongo.db.fangjia.com")
|
||||
ProvinceData.getAllCity("¸£½¨").each {def city->
|
||||
int count = 0
|
||||
def s = new TreeMap()
|
||||
DISTRICT_STAT.find(new BasicDBObject("weekend", Date.parse("yyyy-MM-dd", "2020-04-19")).append("city", city)).each {BasicDBObject stat->
|
||||
count ++
|
||||
int credit = (stat.credit * 100)/10
|
||||
s[credit] = s.get(credit, 0) + 1
|
||||
}
|
||||
|
||||
def data = []
|
||||
data << city
|
||||
data << count
|
||||
for(int i=0; i <=10; i++){
|
||||
data << s.get(i, 0)
|
||||
}
|
||||
println(data.join(","))
|
||||
CommonUtils.findAllCity().each {def city->
|
||||
DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item->
|
||||
if(StringUtils.isBlank(item.township) && item.lng){
|
||||
count++
|
||||
def lng = item.lng/10000000000d
|
||||
def lat = item.lat/10000000000d
|
||||
def location = "${lng},${lat}"
|
||||
JSONObject resp = GaoDeApiUtils.geocode(location)
|
||||
def row = []
|
||||
row << item._id.toString()
|
||||
row << item.city
|
||||
row << item.region
|
||||
row << item.name
|
||||
row << lng
|
||||
row << lat
|
||||
BasicDBObject dbo = new BasicDBObject()
|
||||
dbo.id = item._id.toString()
|
||||
dbo.city = item.city
|
||||
dbo.region = item.region
|
||||
dbo.name = item.name
|
||||
dbo.lng = lng
|
||||
dbo.lat = lat
|
||||
if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){
|
||||
row << resp.regeocode.addressComponent.township
|
||||
dbo.township = resp.regeocode.addressComponent.township
|
||||
}
|
||||
println(row)
|
||||
data << row
|
||||
SEAWEED_TOWNSHIP.insert(dbo)
|
||||
}
|
||||
})
|
||||
}
|
||||
ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+28
-287
@@ -1,6 +1,8 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
@@ -9,294 +11,33 @@ import net.sf.json.JSONObject
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
//
|
||||
//def map = [:]
|
||||
//ExcelUtils.read("D:\\商务数据\\上海农商行\\测试数据(上海)-上海农商行第二批0829.xlsx", 1).each {List item->
|
||||
// if(item["AF"] && item["AH"]){
|
||||
// def key = "${item["AE"]}_${item["AF"]}_${item["AH"]}".toString()
|
||||
// map.get(key, []) << key
|
||||
// }
|
||||
//}
|
||||
|
||||
//ExcelUtils.modify("D:\\商务数据\\上海农商行\\测试数据(上海)-上海农商行第二批(房价网交付0905).xlsx", { List item ->
|
||||
//// if(item["AF"] && item["AH"]){
|
||||
//// def key = "${item["AE"]}_${item["AF"]}_${item["AH"]}".toString()
|
||||
//// List nums = map.get(key)
|
||||
//// item["AT"] = nums.size()
|
||||
//// }
|
||||
//
|
||||
// def city = "上海"
|
||||
// def addr = item["AC"]
|
||||
// if(addr){
|
||||
// JSONObject resp = OpenApiClient.addressMatch(city, addr, "property")
|
||||
// JSONObject searchAddress = resp.getJSONObject("searchAddress")
|
||||
// println searchAddress
|
||||
//// item["AD"] = searchAddress.province
|
||||
//// item["AE"] = searchAddress.city
|
||||
//// item["AF"] = searchAddress.district
|
||||
// if(searchAddress.standardAddress){
|
||||
//// item["AG"] = searchAddress.street
|
||||
//// if(searchAddress.streetNumber){
|
||||
//// item["AG"] = item["AG"] + searchAddress.streetNumber
|
||||
//// }
|
||||
//
|
||||
// if(searchAddress.township){
|
||||
// item["AP"] = searchAddress.township
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if(item["AG"]){
|
||||
// def addrMap = getRoad(item["AG"])
|
||||
// if(addrMap){
|
||||
// if(addrMap.road){
|
||||
// item["AQ"] = addrMap.road
|
||||
// }
|
||||
// if(addrMap.roadNumber){
|
||||
// item["AR"] = addrMap.roadNumber
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//
|
||||
//// if (item["AH"] && item["AF"]) {
|
||||
//// BasicDBObject seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["AE"], item["AF"], item["AH"]))
|
||||
//// if (seaweed) {
|
||||
//// if (item["AR"] && item["AS"]) {
|
||||
//// JSONObject eva = OpenApiClient.evaluateDistrictPrice(seaweed.city, seaweed.region, seaweed.name, item["O"], seaweed.estate_type2, item["AS"].toInteger(), item["AR"].toInteger())
|
||||
//// if (eva) {
|
||||
//// item["AL"] = eva.avgPrice
|
||||
//// item["AM"] = eva.totalPrice
|
||||
//// }
|
||||
//// } else {
|
||||
//// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(seaweed.city, seaweed.region, seaweed.name, item["O"], seaweed.estate_type2)
|
||||
//// if (eva) {
|
||||
//// item["AL"] = eva.avgPrice
|
||||
//// item["AM"] = eva.totalPrice
|
||||
//// }
|
||||
//// }
|
||||
//// }
|
||||
////
|
||||
//// }
|
||||
//
|
||||
//},1)
|
||||
|
||||
|
||||
// if(item["AK"] =~ /^\d{4}$/){
|
||||
// item["AR"] = item["AK"].toString().substring(0, 2)
|
||||
// }else if(item["AK"] =~ /^\d{3}$/){
|
||||
// item["AR"] = item["AK"].toString().substring(0, 1)
|
||||
// }
|
||||
// if(item["AR"] && item["AH"]){
|
||||
// def key = "${item["AE"]}_${item["AF"]}_${item["AH"]}".toString()
|
||||
// List nums = map.get(key)
|
||||
// nums.sort()
|
||||
// item["AS"] = nums[nums.size() - 1]
|
||||
// item["AT"] = nums.unique().size()
|
||||
// }
|
||||
// println(item)
|
||||
// item["AG"] = null
|
||||
// def city = "上海"
|
||||
// def addr = item["AC"]
|
||||
// def area = item["o"]
|
||||
// JSONObject resp = OpenApiClient.addressMatch(city, addr, "property")
|
||||
// JSONObject searchAddress = resp.getJSONObject("searchAddress")
|
||||
// item["AD"] = searchAddress.province
|
||||
// item["AE"] = searchAddress.city
|
||||
// item["AF"] = searchAddress.district
|
||||
// if(searchAddress.standardAddress){
|
||||
// item["AG"] = searchAddress.street
|
||||
// if(searchAddress.streetNumber){
|
||||
// item["AG"] = item["AG"] + searchAddress.streetNumber
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//
|
||||
// if(resp.getDouble("credit") > 0){
|
||||
// item["AE"] = resp.city
|
||||
// item["AF"] = resp.district
|
||||
// item["AH"] = resp.name
|
||||
// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(resp.city, resp.district, resp.name, area, resp.propertyType)
|
||||
// item["AL"] = eva.avgPrice
|
||||
// item["AM"] = eva.totalPrice
|
||||
// item["AN"] = new Date().format("yyyy-MM-dd")
|
||||
// if(!item["AG"]){
|
||||
// item["AG"] = resp.address
|
||||
// }
|
||||
// }
|
||||
// item["AI"] = searchAddress.buildingNumber
|
||||
// item["AJ"] = searchAddress.unitNumber
|
||||
// item["AK"] = searchAddress.roomNumber
|
||||
// println(item)
|
||||
//},1)
|
||||
//
|
||||
def sheetIndexs = ["湘潭": 0, "嘉兴": 1, "苏州": 2]
|
||||
sheetIndexs.each { String city, int sheetIndex ->
|
||||
// def map = [:]
|
||||
// ExcelUtils.read("D:\\商务数据\\上海农商行\\异地测试-上海农商行第二批0828(测试)1.xlsx", sheetIndex).each {List item->
|
||||
// if(item["Q"] && item["S"]){
|
||||
// def key = "${item["P"]}_${item["Q"]}_${item["S"]}".toString()
|
||||
// map.get(key, []) << key
|
||||
// }
|
||||
// }
|
||||
ExcelUtils.multiThreadingModify("D:\\商务数据\\上海农商行\\异地测试-上海农商行第二批(房价网交付0905).xlsx", { List item ->
|
||||
// if(item["Q"] && item["S"]){
|
||||
// def key = "${item["P"]}_${item["Q"]}_${item["S"]}".toString()
|
||||
// List nums = map.get(key)
|
||||
// nums.sort()
|
||||
// item["AE"] = nums.size()
|
||||
// }
|
||||
|
||||
// item["W"] = null
|
||||
// item["X"] = null
|
||||
|
||||
// BasicDBObject seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["P"], item["Q"], item["S"]))
|
||||
// if(seaweed){
|
||||
// if(item["AC"] && item["AD"]){
|
||||
// JSONObject eva = OpenApiClient.evaluateDistrictPrice(seaweed.city, seaweed.region, seaweed.name, item["L"], seaweed.estate_type2, item["AD"].toInteger(), item["AC"].toInteger())
|
||||
// if(eva){
|
||||
// item["W"] = eva.avgPrice
|
||||
// item["X"] = eva.totalPrice
|
||||
// }
|
||||
// }else{
|
||||
// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(seaweed.city, seaweed.region, seaweed.name, item["L"], seaweed.estate_type2)
|
||||
// if(eva){
|
||||
// item["W"] = eva.avgPrice
|
||||
// item["X"] = eva.totalPrice
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// if(item["AC"] && item["S"]){
|
||||
// def key = "${item["P"]}_${item["Q"]}_${item["S"]}".toString()
|
||||
// List nums = map.get(key)
|
||||
// nums.sort()
|
||||
// item["AD"] = nums[nums.size() - 1]
|
||||
// item["AE"] = nums.unique().size()
|
||||
// }
|
||||
|
||||
// if(item["V"] =~ /^\d{4}$/){
|
||||
// item["AC"] = item["V"].toString().substring(0, 2)
|
||||
// }else if(item["V"] =~ /^\d{3}$/){
|
||||
// item["AC"] = item["V"].toString().substring(0, 1)
|
||||
// }else if(item["V"] =~ /^\d{6,7}$/){
|
||||
// item["AC"] = item["V"].toString().substring(0, 2).replaceAll("^0", "")
|
||||
// }
|
||||
|
||||
|
||||
def addr = item["B"].toString().replaceAll("(", "").replace('区/县', "区").replace("街道/镇", "镇")
|
||||
.replaceAll(")", "").replaceAll("【.*】", "").replaceAll("室/单元", "号")
|
||||
// def area = item["L"]
|
||||
|
||||
|
||||
def region = null
|
||||
if (city == "嘉兴") {
|
||||
region = "嘉善"
|
||||
} else if (city == "苏州") {
|
||||
region = "昆山"
|
||||
}
|
||||
JSONObject resp = OpenApiClient.addressMatch(city, addr, "property", false, region)
|
||||
JSONObject searchAddress = resp.getJSONObject("searchAddress")
|
||||
|
||||
def township = ""
|
||||
if(searchAddress && searchAddress.township){
|
||||
item["AP"] = searchAddress.township
|
||||
township = searchAddress.township
|
||||
}
|
||||
if(item["S"]){
|
||||
def addrMap = getRoad(item["S"])
|
||||
if(addrMap){
|
||||
if(addrMap.road){
|
||||
def s = addrMap.road;
|
||||
if(township){
|
||||
if(township.contains(s)){
|
||||
item["AQ"] = s.replace(township, "")
|
||||
}else{
|
||||
if(township.contains("镇")){
|
||||
township = township.replace("镇","")
|
||||
}
|
||||
item["AQ"] = s.replace(township, "")
|
||||
}
|
||||
}else{
|
||||
item["AQ"] = addrMap.road
|
||||
}
|
||||
}
|
||||
if(addrMap.roadNumber){
|
||||
item["AR"] = addrMap.roadNumber
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
println searchAddress
|
||||
// item["O"] = searchAddress.province
|
||||
// item["P"] = searchAddress.city
|
||||
// item["Q"] = searchAddress.district
|
||||
// if (searchAddress.standardAddress) {
|
||||
// item["R"] = searchAddress.street
|
||||
// if (searchAddress.streetNumber) {
|
||||
// item["R"] = item["R"] + searchAddress.streetNumber
|
||||
// }
|
||||
// }
|
||||
// if (resp.getDouble("credit") > 0) {
|
||||
// item["O"] = resp.province
|
||||
// item["P"] = resp.city
|
||||
// item["Q"] = resp.district
|
||||
// item["S"] = resp.name
|
||||
// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(resp.city, resp.district, resp.name, area, resp.propertyType)
|
||||
// item["W"] = eva.avgPrice
|
||||
// item["X"] = eva.totalPrice
|
||||
// item["Y"] = new Date().format("yyyy-MM-dd")
|
||||
// if (!item["R"]) {
|
||||
// item["R"] = resp.address
|
||||
// }
|
||||
// }
|
||||
// item["T"] = searchAddress.buildingNumber
|
||||
// item["U"] = searchAddress.unitNumber
|
||||
// item["V"] = searchAddress.roomNumber
|
||||
// if(city == "湘潭"){
|
||||
// String roomNumber = DataUtil.match(addr, Pattern.compile("\\d+(号|(号房))\$"))
|
||||
// if(roomNumber){
|
||||
// item["V"] = roomNumber.replaceAll("号", '').replaceAll('房', "")
|
||||
// }
|
||||
// roomNumber = DataUtil.match(addr, Pattern.compile("\\d{6,8}\$"))
|
||||
// if(roomNumber){
|
||||
// item["V"] = roomNumber.replaceAll("号", '').replaceAll('房', "")
|
||||
// }
|
||||
//
|
||||
// }
|
||||
def school = MongoUtils.getDbCollection(Constants.FANGJIA_CRAWL_MONGO_URI, "fangjia_craw", "school2")
|
||||
new File("D:\\Êý¾Ý\\ѧУÊý¾Ý").eachFile {File file->
|
||||
ExcelUtils.read(file.getAbsolutePath(), 0).each {List item->
|
||||
BasicDBObject dbo = new BasicDBObject()
|
||||
dbo.city = item["A"]
|
||||
dbo.region = item["B"]
|
||||
dbo.name = item["C"]
|
||||
dbo.address = item["E"]
|
||||
dbo.attribute = item["G"]
|
||||
dbo.property = item["H"]
|
||||
dbo.category = "Сѧ"
|
||||
dbo.location = [DataUtil.roundDouble(item["I"], 6), DataUtil.roundDouble(item["J"], 6)]
|
||||
school.insert(dbo)
|
||||
println(dbo)
|
||||
}
|
||||
ExcelUtils.read(file.getAbsolutePath(), 1).each {List item->
|
||||
BasicDBObject dbo = new BasicDBObject()
|
||||
dbo.city = item["A"]
|
||||
dbo.region = item["B"]
|
||||
dbo.name = item["C"]
|
||||
dbo.address = item["E"]
|
||||
dbo.attribute = item["G"]
|
||||
dbo.property = item["H"]
|
||||
dbo.category = "ÖÐѧ"
|
||||
dbo.location = [item["I"], item["J"]]
|
||||
school.save(dbo)
|
||||
println(item)
|
||||
}, sheetIndex)
|
||||
}
|
||||
|
||||
|
||||
def getRoad(def s){
|
||||
def rex = ".*(路|街道|街|大道|巷|道|里).*\\d+(弄|号)\$"
|
||||
Pattern pattern = Pattern.compile(rex);
|
||||
Matcher matcher = pattern.matcher(s);
|
||||
|
||||
List<String> list = new ArrayList();
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group());
|
||||
}
|
||||
if(list && list.size()>0){
|
||||
def s1 = list[0]
|
||||
def road = getRexText(s1, ".*(路|街道|街|大道|巷|道|里)")
|
||||
def num = getRexText(s1, "\\d+")
|
||||
return ["road":road, "roadNumber":num]
|
||||
}else{
|
||||
def road = getRexText(s, ".*(路|街道|街|大道|巷|道|里)")
|
||||
return ["road":road]
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def getRexText(s, regx){
|
||||
Pattern pattern = Pattern.compile(regx);
|
||||
Matcher matcher = pattern.matcher(s);
|
||||
|
||||
while (matcher.find()) {
|
||||
return matcher.group();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -1,9 +1,16 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.ProvinceData
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import org.openqa.selenium.json.Json
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\阳光保险重跑数据(1).xlsx", { List item ->
|
||||
@@ -30,8 +37,6 @@ import org.apache.commons.lang3.StringUtils
|
||||
//}, 0,"结果")
|
||||
|
||||
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\阳光保险重跑数据(1).xlsx", { List item ->
|
||||
// if (item["T"] == null || ![1, 2, 4].contains(item["T"].toString().toDouble().toInteger())) {
|
||||
// return
|
||||
@@ -114,43 +119,80 @@ import org.apache.commons.lang3.StringUtils
|
||||
// println("${key} ${names.join(",")}")
|
||||
// }
|
||||
//}
|
||||
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\µØÖ·±ê×¼»¯.xlsx",{List item->
|
||||
String address = "";
|
||||
if(StringUtils.isNotBlank(item["B"]) && !ProvinceData.isGovernmentCity(item["B"])){
|
||||
address = ProvinceData.getDisplay(item["B"])
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\地址标准化.xlsx",{List item->
|
||||
// String address = "";
|
||||
// if(StringUtils.isNotBlank(item["B"]) && !ProvinceData.isGovernmentCity(item["B"])){
|
||||
// address = ProvinceData.getDisplay(item["B"])
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["C"])){
|
||||
// String cityDisplay = CityData.getCityDisplay(item["C"]);
|
||||
// if(cityDisplay != null){
|
||||
// address = address + cityDisplay
|
||||
// }else{
|
||||
// address = address + item["C"] + "市"
|
||||
// }
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["D"])){
|
||||
// String orgDistrict = RegionData.getOrgName(item["C"], item["D"]);
|
||||
// if(StringUtils.isNotBlank(orgDistrict) && address.indexOf(orgDistrict) == -1){
|
||||
// address = address + orgDistrict
|
||||
// }
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["E"])){
|
||||
// address = address + item["E"]
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["F"])){
|
||||
// address = address + item["F"]
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["G"])){
|
||||
// address = address + item["G"]
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["H"])){
|
||||
// address = address + item["H"] + "栋"
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["i"])){
|
||||
// address = address + item["i"] + "单元"
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["J"])){
|
||||
// address = address + item["J"] + "室"
|
||||
// }
|
||||
// item["K"] = address
|
||||
// println(address)
|
||||
//})
|
||||
def school = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, "fangjia", "guiding_price")
|
||||
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\深圳市住建局二手房指导价.xlsx", { List item ->
|
||||
if (item["G"]) {
|
||||
BasicDBObject seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["G"], item["H"], item["I"]))
|
||||
BasicDBObject guid = new BasicDBObject()
|
||||
guid.guiding_type = 0
|
||||
guid.estate_type = seaweed.estate_type2
|
||||
guid.house_type = 0
|
||||
guid.c_date = new Date()
|
||||
guid.guiding_person = "深圳房管局"
|
||||
guid.guiding_date = Date.parse('yyyyMMdd', '20210411')
|
||||
guid.category = "district"
|
||||
guid.guiding_value = (int)item["E"]
|
||||
guid.name = item["D"]
|
||||
guid.city = item["G"]
|
||||
guid.region = item["H"]
|
||||
guid.name = item["i"]
|
||||
school.insert(guid)
|
||||
println(guid)
|
||||
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["C"])){
|
||||
String cityDisplay = CityData.getCityDisplay(item["C"]);
|
||||
if(cityDisplay != null){
|
||||
address = address + cityDisplay
|
||||
}else{
|
||||
address = address + item["C"] + "ÊÐ"
|
||||
}
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["D"])){
|
||||
String orgDistrict = RegionData.getOrgName(item["C"], item["D"]);
|
||||
if(StringUtils.isNotBlank(orgDistrict) && address.indexOf(orgDistrict) == -1){
|
||||
address = address + orgDistrict
|
||||
}
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["E"])){
|
||||
address = address + item["E"]
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["F"])){
|
||||
address = address + item["F"]
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["G"])){
|
||||
address = address + item["G"]
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["H"])){
|
||||
address = address + item["H"] + "¶°"
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["i"])){
|
||||
address = address + item["i"] + "µ¥Ôª"
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["J"])){
|
||||
address = address + item["J"] + "ÊÒ"
|
||||
}
|
||||
item["K"] = address
|
||||
println(address)
|
||||
// if(!item["F"]){
|
||||
// JSONObject resp = OpenApiClient.addressMatch("深圳", item["D"], 'property', false, item["B"])
|
||||
// if(resp.id){
|
||||
// BasicDBObject seaweed = DbUtils.seaweed.findById(resp.id)
|
||||
// if(seaweed){
|
||||
// item["F"] = seaweed._id.toString()
|
||||
// item["G"] = seaweed.city
|
||||
// item["H"] = seaweed.region
|
||||
// item["I"] = seaweed.name
|
||||
// }
|
||||
// println(item)
|
||||
// }
|
||||
//
|
||||
// }
|
||||
|
||||
})
|
||||
+140
-73
@@ -1,82 +1,149 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.util.HouseValidator
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import net.sf.json.JSONObject
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\商务数据\\建设银行\\副本5市成交案例(测试).xlsx",{List item->
|
||||
// if(item["R"]){
|
||||
// return
|
||||
// }
|
||||
// def calcDate = null
|
||||
// if(item["E"]){
|
||||
// calcDate = item["E"].toString().substring(0, 10)
|
||||
// }else{
|
||||
// calcDate = item["B"].toString().substring(0, 10)
|
||||
// }
|
||||
// def floor = item["I"].toString().toDouble().toInteger().toString()
|
||||
// def maxFloor = item["J"].toString().toDouble().toInteger().toString()
|
||||
// def dir = HouseValidator.formatDirectionToCode(item["L"]).toString()
|
||||
// def fit = HouseValidator.formatFitmentToCode(item["M"]).toString()
|
||||
// def params =[:]
|
||||
// params.city = item["F"]
|
||||
// params.district = item["p"]
|
||||
// params.name = item['Q']
|
||||
// params.size = item["C"].toString()
|
||||
// params.floor = floor
|
||||
// params.maxFloor = maxFloor
|
||||
// params.direction = dir
|
||||
// params.decoration = fit
|
||||
// params.calDate = calcDate
|
||||
// params.__refresh_cache = "1"
|
||||
// params.scope = "1"
|
||||
// JSONObject evaResp = OpenApiClient.evaluate(params)
|
||||
// if(evaResp.result && evaResp.result.credit > 0.4){
|
||||
// item["R"] = evaResp.result.totalPrice
|
||||
// }
|
||||
def school = MongoUtils.getDbCollection(Constants.FANGJIA_CRAWL_MONGO_URI, "fangjia_craw", "school2")
|
||||
//school.find(new BasicDBObject("source", "查学区")).each {BasicDBObject item->
|
||||
// item.category = item.attribute
|
||||
// item.attribute = null
|
||||
// school.save(item)
|
||||
// println(item)
|
||||
|
||||
// if(item["H"] != "中粮云景花园"){
|
||||
// return
|
||||
//}
|
||||
//ExcelUtils.read("D:\\Users\\wenyan\\Downloads\\无经纬度学校-结果.xlsx").each {List item->
|
||||
// if(item["P"] != null){
|
||||
// def location = item["P"].split(",") as List
|
||||
// def data = new BasicDBObject()
|
||||
// data.map_name = item["N"]
|
||||
// data.map_region = item["R"]
|
||||
// data.map_address = item["N"]
|
||||
// data.map_type = item["S"]
|
||||
// data.location = [location[0].toString().toDouble(),location[1].toString().toDouble()]
|
||||
// school.update(new BasicDBObject("_id", new ObjectId(item["A"])), new BasicDBObject('$set', data))
|
||||
// println(data)
|
||||
// }
|
||||
def city = item["F"]
|
||||
def region = item["G"]
|
||||
def name = item["H"].toString()
|
||||
def addr = item["D"]
|
||||
JSONObject resp = OpenApiClient.addressMatch(city, addr, "property" , false, region, name)
|
||||
if(resp.name){
|
||||
item["p"] = resp.district
|
||||
item['Q'] = resp.name
|
||||
//}
|
||||
regionCodeMap = [:]
|
||||
def cityCodeMap = ["苏州":"320500","合肥":"340100", "西安":"610000"]
|
||||
for(def city : cityCodeMap.keySet()){
|
||||
def datas = FangJiaAreaMapUtils.convertToOfficials(city);
|
||||
for(def d : datas){
|
||||
regionCodeMap.put(city+"-"+d.fangjia_district, d.district_code)
|
||||
}
|
||||
println(item)
|
||||
// def calcDate = null
|
||||
// if(item["E"]){
|
||||
// calcDate = item["E"].toString().substring(0, 10)
|
||||
// }else{
|
||||
// calcDate = item["B"].toString().substring(0, 10)
|
||||
// }
|
||||
// def floor = item["I"].toString().toDouble().toInteger().toString()
|
||||
// def maxFloor = item["J"].toString().toDouble().toInteger().toString()
|
||||
// def dir = HouseValidator.formatDirectionToCode(item["L"]).toString()
|
||||
// def fit = HouseValidator.formatFitmentToCode(item["M"]).toString()
|
||||
// def params =[:]
|
||||
// params.city = resp.city
|
||||
// params.district = resp.district
|
||||
// params.name = resp.name
|
||||
// params.size = item["C"].toString()
|
||||
// params.floor = floor
|
||||
// params.maxFloor = maxFloor
|
||||
// params.direction = dir
|
||||
// params.decoration = fit
|
||||
// params.calDate = calcDate
|
||||
// params.scope = "1"
|
||||
// JSONObject evaResp = OpenApiClient.evaluate(params)
|
||||
// if(evaResp.result && evaResp.result.credit > 0.4){
|
||||
// item["R"] = evaResp.result.totalPrice
|
||||
// }
|
||||
// println(item)
|
||||
// }
|
||||
}
|
||||
def data = Collections.synchronizedList([])
|
||||
data << ["ID","CITYID","DISTID","COMMNAME","LNG","LAT","ADDRESS","PS","PS_ADDR","PS_TYPE","PS_SEQNO","PS_LONGITUDE","PS_LATITUDE","MS","MS_ADDR","MS_TYPE","MS_SEQNO","MS_LONGITUDE","MS_LATITUDE","REMARK","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
cityCodeMap.keySet().each {def city->
|
||||
DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject seaweed->
|
||||
if(seaweed.lng){
|
||||
def item = []
|
||||
item << seaweed._id.toString()
|
||||
item << cityCodeMap.get(seaweed.city)
|
||||
item << regionCodeMap.get(seaweed.city + "-" + seaweed.region)
|
||||
item << seaweed.name
|
||||
def lng = com.fangjia.core.util.DataUtil.roundDouble(seaweed.lng/10000000000d, 6)
|
||||
def lat = com.fangjia.core.util.DataUtil.roundDouble(seaweed.lat/10000000000d, 6)
|
||||
item << lng
|
||||
item << lat
|
||||
item << seaweed.address
|
||||
BasicDBObject location = new BasicDBObject()
|
||||
BasicDBObject near = new BasicDBObject()
|
||||
BasicDBObject geometry = new BasicDBObject("type", "Point")
|
||||
geometry.append("coordinates", [lng, lat])
|
||||
near.append('$maxDistance', 1000)
|
||||
near.append('$geometry', geometry)
|
||||
location.append('$near', near)
|
||||
BasicDBObject query = new BasicDBObject()
|
||||
query.append("location", location)
|
||||
query.append("attribute",new BasicDBObject('$in', ['省重点', '市重点', '区重点', '县重点', '省级示范']))
|
||||
query.append("category", new BasicDBObject('$in', ['中学']))
|
||||
BasicDBObject midSchool = school.findOne(query)
|
||||
// for(def level in ['省重点', '市重点', '区重点', '县重点', '普通']){
|
||||
// query.append("location", location)
|
||||
// query.append("attribute", level)
|
||||
// midSchool = school.findOne(query)
|
||||
// if(midSchool){
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// if(!midSchool){
|
||||
// query.removeField("attribute")
|
||||
// midSchool = school.findOne(query)
|
||||
// }
|
||||
BasicDBObject primarySchool=null
|
||||
query.append("category", new BasicDBObject('$in', ['小学']))
|
||||
primarySchool = school.findOne(query)
|
||||
// for(def level in ['省重点', '市重点', '区重点', '县重点', '普通']){
|
||||
// query.append("location", location)
|
||||
// query.append("attribute", level)
|
||||
// primarySchool = school.findOne(query)
|
||||
// if(primarySchool){
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// if(!primarySchool){
|
||||
// query.removeField("attribute")
|
||||
// primarySchool = school.findOne(query)
|
||||
// }
|
||||
if(primarySchool){
|
||||
item << primarySchool.name
|
||||
item << primarySchool.address
|
||||
item << primarySchool.attribute
|
||||
item << null
|
||||
item << primarySchool.location[0]
|
||||
item << primarySchool.location[1]
|
||||
}else {
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
}
|
||||
if(midSchool){
|
||||
item << midSchool.name
|
||||
item << midSchool.address
|
||||
item << midSchool.attribute
|
||||
item << null
|
||||
item << midSchool.location[0]
|
||||
item << midSchool.location[1]
|
||||
}else {
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
}
|
||||
item << null
|
||||
Date c_date = seaweed.c_date
|
||||
Date m_date = seaweed.m_date
|
||||
if(!c_date && !m_date){
|
||||
c_date = DateUtil.add(new Date(), Calendar.YEAR, -1)
|
||||
}
|
||||
if(!c_date){
|
||||
c_date = m_date
|
||||
}
|
||||
if(!m_date){
|
||||
m_date = c_date
|
||||
}
|
||||
item <<c_date.format("yyyy-MM-dd HH:mm:ss")
|
||||
item <<m_date.format("yyyy-MM-dd HH:mm:ss")
|
||||
item << 'fjw'
|
||||
println(item)
|
||||
if(primarySchool || midSchool){
|
||||
data << item
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
ExcelUtils.write("D:\\无学区房数据.xlsx", data, "结果")
|
||||
|
||||
|
||||
})
|
||||
|
||||
|
||||
+2
-2
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20210401", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20210401", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20210501", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
|
||||
+23
-5
@@ -11,26 +11,44 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20210401", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20210401", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20210501", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
def hashSet = []
|
||||
def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
|
||||
def data = []
|
||||
def title = ["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg"]
|
||||
def param_keys = ["tid", "city", "district", "address", "name", "size", "propertyType"]
|
||||
param_keys.each {
|
||||
title << "参数_${it}"
|
||||
}
|
||||
def resMap_keys = ["city", "district", "name", "avgPrice", "totalPrice", "buildYear"]
|
||||
resMap_keys.each {
|
||||
title << "结果_${it}"
|
||||
}
|
||||
data << title
|
||||
def keys = new HashSet()
|
||||
while (startDate.before(endDate)) {
|
||||
calendar.add(Calendar.DATE, 1)
|
||||
def nextDate = calendar.getTime()
|
||||
count++
|
||||
OPEN_URI_LOG.find(and(eq("user_id", "1007721"), eq("request_uri", "/property/evaluate3"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
|
||||
def result = JSONObject.fromObject(doc.res_map.result)
|
||||
if(!result.buildYear || !result.avgPrice){
|
||||
if (!result.buildYear || !result.avgPrice) {
|
||||
return
|
||||
}
|
||||
def params = JSONObject.fromObject(doc.parameter_map)
|
||||
keys.addAll(params.keys())
|
||||
def key = "${params.address}_${params.size}".toString()
|
||||
if (!hashSet.contains(key)) {
|
||||
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
|
||||
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date.format("yyy"), doc.data_status, doc.code, doc.msg]
|
||||
param_keys.each {
|
||||
item << params[it]
|
||||
}
|
||||
resMap_keys.each {
|
||||
item << result[it]
|
||||
}
|
||||
data << item
|
||||
hashSet << key
|
||||
} else {
|
||||
|
||||
+47
-47
@@ -153,55 +153,55 @@ import org.apache.commons.lang3.StringUtils
|
||||
//}
|
||||
|
||||
DBCollection BAIDUMAP_BORDER_COLL = MongoUtils.getCollectionByName("addrepo", "baidumap_border", "192.168.0.206", 27020)
|
||||
BAIDUMAP_BORDER_COLL.find(new BasicDBObject("baidu_border", null)).each {BasicDBObject dbo->
|
||||
def border = dbo.getString("border").replace("1-", "").replace(";", "").split(",") as List
|
||||
def coords = []
|
||||
def baidu_points = []
|
||||
def gaode_points = []
|
||||
for(int i=0; i < border.size() - 1; i=i+2){
|
||||
coords << "${border[i]},${border[i+1]}"
|
||||
if(coords.size() > 50){
|
||||
def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
|
||||
for(JSONObject it : json.result){
|
||||
baidu_points << "${it.x},${it.y}"
|
||||
def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
|
||||
gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
|
||||
}
|
||||
coords.clear()
|
||||
}
|
||||
}
|
||||
if(coords.size() > 0){
|
||||
def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
|
||||
for(JSONObject it : json.result){
|
||||
baidu_points << "${it.x},${it.y}"
|
||||
def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
|
||||
gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
|
||||
}
|
||||
}
|
||||
dbo.format_city = RegionData.formatCity(dbo.city)
|
||||
dbo.baidu_border = baidu_points.join(";")
|
||||
dbo.gaode_border = gaode_points.join(";")
|
||||
BAIDUMAP_BORDER_COLL.save(dbo)
|
||||
println(dbo)
|
||||
}
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\È˿ڲâÊÔÊý¾Ý.xlsx",{List item->
|
||||
// def seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["B"], item["C"], item["D"]), new BasicDBObject("alias", 1).append("lng",1).append("lat", 1))
|
||||
// if(seaweed){
|
||||
// item["A"] = seaweed._id.toString()
|
||||
// def names = [item['D']]
|
||||
// if(seaweed.alias){
|
||||
// names.addAll(seaweed.alias)
|
||||
//BAIDUMAP_BORDER_COLL.find(new BasicDBObject("baidu_border", null)).each {BasicDBObject dbo->
|
||||
// def border = dbo.getString("border").replace("1-", "").replace(";", "").split(",") as List
|
||||
// def coords = []
|
||||
// def baidu_points = []
|
||||
// def gaode_points = []
|
||||
// for(int i=0; i < border.size() - 1; i=i+2){
|
||||
// coords << "${border[i]},${border[i+1]}"
|
||||
// if(coords.size() > 50){
|
||||
// def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
|
||||
// for(JSONObject it : json.result){
|
||||
// baidu_points << "${it.x},${it.y}"
|
||||
// def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
|
||||
// gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
|
||||
// }
|
||||
// coords.clear()
|
||||
// }
|
||||
// names.unique()
|
||||
// }
|
||||
// if(coords.size() > 0){
|
||||
// def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
|
||||
// for(JSONObject it : json.result){
|
||||
// baidu_points << "${it.x},${it.y}"
|
||||
// def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
|
||||
// gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
|
||||
// }
|
||||
// }
|
||||
// dbo.format_city = RegionData.formatCity(dbo.city)
|
||||
// dbo.baidu_border = baidu_points.join(";")
|
||||
// dbo.gaode_border = gaode_points.join(";")
|
||||
// BAIDUMAP_BORDER_COLL.save(dbo)
|
||||
// println(dbo)
|
||||
//}
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\Ð¡ÇøÈºÏñ²âÊÔÊý¾Ý.xlsx",{List item->
|
||||
def seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["B"], item["C"], item["D"]), new BasicDBObject("alias", 1).append("lng",1).append("lat", 1))
|
||||
if(seaweed){
|
||||
// item["A"] = seaweed._id.toString()
|
||||
def names = [item['D']]
|
||||
if(seaweed.alias){
|
||||
names.addAll(seaweed.alias)
|
||||
}
|
||||
names.unique()
|
||||
// if(seaweed.lng){
|
||||
// item["G"] = DataUtil.roundDouble(seaweed.lng/10000000000d, 6)
|
||||
// item["H"] = DataUtil.roundDouble(seaweed.lat/10000000000d, 6)
|
||||
// }
|
||||
// def one = BAIDUMAP_BORDER_COLL.findOne(new BasicDBObject("format_city", item["B"]).append("format_region", item["C"]).append("name", new BasicDBObject('$in', names)))
|
||||
// if(one){
|
||||
// item["I"] = one.baidu_border
|
||||
// }
|
||||
// println(item)
|
||||
// }
|
||||
//})
|
||||
def one = BAIDUMAP_BORDER_COLL.findOne(new BasicDBObject("format_city", item["B"]).append("format_region", item["C"]).append("name", new BasicDBObject('$in', names)))
|
||||
if(one){
|
||||
item["I"] = one.gaode_border
|
||||
}
|
||||
println(item)
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,46 @@
|
||||
package com.fangjia.script.wenyan.test
|
||||
|
||||
import com.aliyuncs.DefaultAcsClient
|
||||
import com.aliyuncs.IAcsClient
|
||||
import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityRequest
|
||||
import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityResponse
|
||||
import com.aliyuncs.profile.DefaultProfile
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
ExcelUtils.modify("D:\\nextcloud\\数据\\地址类服务测试\\建行地址相似度评级测试.xlsx",{List item->
|
||||
def city = item["A"]
|
||||
def address1 = item["B"]
|
||||
def address2 = item["C"]
|
||||
HttpRequest http = new HttpRequest()
|
||||
def params = new HashMap()
|
||||
params["city"] = city
|
||||
params["address_1"] = address1
|
||||
params["address_2"] = address2
|
||||
params["token"]= "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
JSONObject json = http.getJsonContent("http://open.fangjia.com/address/similarity", params, "UTF-8")
|
||||
JSONObject result = json.result
|
||||
item["D"] = result?.level
|
||||
item["E"] = result?.formattedAddress_1
|
||||
item["F"] = result?.formattedAddress_2
|
||||
item["G"] = result?.isSameAddress
|
||||
DefaultProfile profile = DefaultProfile.getProfile(
|
||||
"cn-hangzhou",
|
||||
"LTAIdaj4eSO79gzz",
|
||||
"xw6EsvqydZ9HC6zAvIB3ruQqVHynxf");
|
||||
IAcsClient client = new DefaultAcsClient(profile);
|
||||
GetAddressSimilarityRequest request = new GetAddressSimilarityRequest();
|
||||
request.setServiceCode("addrp")
|
||||
request.setAppKey("Xcgtyhupey0h")
|
||||
request.setText("${address1};${address2}")
|
||||
GetAddressSimilarityResponse response = client.getAcsResponse(request)
|
||||
String data = response.getData()
|
||||
json = JSONObject.fromObject(data)
|
||||
item["H"] = json["sim_level"]["level"]
|
||||
item["I"] = json["sim_level"]["address_a"]
|
||||
item["J"] = json["sim_level"]["address_b"]
|
||||
item["K"] = json["sim_level"]["is_same"]
|
||||
println(json)
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user