Merge branch 'feature/monitor' into script_develop

# Conflicts:
#	projects/fangjia-script/pom.xml
This commit is contained in:
沈健
2021-05-21 10:14:52 +08:00
92 changed files with 7563 additions and 4519 deletions
@@ -32,6 +32,7 @@ public enum Watcher {
BaYi('15300424133','bayi@fangjia.com'),
WeiLei('18101751850','weilei@fangjia.com'),
JiangYaHui('13684913780','jiangyahui@fangjia.com'),
ShenJian('13764369683', 'shenjian@fangjia.com')
private final String mobile;
private final String email;
@@ -218,7 +218,7 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name", "工商大楼");
// q.put("name","东寓新村")
DBObject keys = new BasicDBObject();
if(fields){
@@ -1392,6 +1392,21 @@ class CommonDatautils {
}
public static def queryDistrictPriceByWeekend(city, region, name, weekend, limit){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("region", region);
queryCondition.put("district_name", name);
if(weekend){
queryCondition.put('weekend', new BasicDBObject('$gte', weekend));
}
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", 1)).limit(limit)
def dbo = null;
while(cur.hasNext()) {
dbo = cur.next()
}
return dbo;
}
@@ -0,0 +1,148 @@
package com.fangjia.script.companyCrawl
import net.sf.json.JSONObject
import org.jsoup.Jsoup
import java.util.concurrent.atomic.AtomicInteger
class AiqichaCrawl {
def static final proxyPort = 9020;
def static final proxyServer = "http-dyn.abuyun.com";
def static final proxyUser = "HZU1XU28H3G4Z44D";
def static final proxyPass = "466C86ED77FE43D4";
def companyKeys = ["亨鑫","捷东","盈复","隆庆","卓源 ","合万","通广","安盛","东利","瑞科 ","凡迪","捷信","安盛","安盛","东进","源盛","茂禄","泰亚","宏成","光益","华圣","益全","长凯","宏新","锦立","宜佳","万美","正欣","和利","国宏","和裕","新昌","佳升","正德","源美","宝峰","金广","龙奥","宏美","成同","义丰","全洪","国聚","亨信","百益","广宏","仁瑞","亿昌","东旭","聚凯","洪成","兴同","赛迪","亨国","浩长","达宏","寿满","亿信","本盛","盈凯","浩汇","元辉","祥生","皇弘","讯思","新州","国泰","成合","茂浩","信源","润凯","虎光","安瑞","益吉","达兴","顺光","发盛","金优","捷丰","春凯","昌兴","本佳","远盛","盛达","亚飞","东昌","益如","宝东","德信","浩盈","多泰","泰联","升聚","辉丰","茂发","盈德","旭嘉","优源","益同","协盈","锦泰","捷久","本宏","德泰","聚富","全庆","凯安","进恒","全伟","优元","凯安","义丰","丰宇","瑞优","隆泰","诺德","台升","裕大","亨恒","义浩","百亨","欣昌","茂全","同亨","长圣","成谦","达泰","高协","正思","安巨","新耀","利仁","国益","旺洪","汇中","旭昌","弘康","优盛","旭达","晶安","华浩","耀春","圣本","康丰","光复","同凌","宇伯","乐鸿","成泰","宏协","华信","达泰","广丰","泰全","汇恒","杰森","天畅","恒通","亚达","庆合","丰圣","德长","茂百","汇久","欣鼎","谦和","厚辉","耀德","瑞隆","多辉","昌茂","成高","益升","安达","兴飞","百进","鼎宝","祥盛","信和","隆顺","亚欣","厚全","博文","讯诺","泰如","亨瑞","凯祥","捷浩","文舟","金庆","成仁","万协","宇中","宏祥","永多","艺峰","美益","创赢","奥龙","德宇","国福","飞利","久欣","如亨","安广","协佳","升捷","盛圣","富凯","长鑫","元卓","荣亨","亨仁","康鑫","信和","百新","干瑞","润万","富庭","信宇","源天","凯佳","正升","裕荣","厚泽","晶富","广新","正多","鼎高","丰汇","永华","兴升","复腾","隆诺","中发","广源","康胜","同进","仁捷","康凯","义洪","晶盛","正祥","裕祥","宇新","洪祥","润岳","义富","盛安","浩易"]
static AtomicInteger AbuyunRequestCount = new AtomicInteger(0);
public final static int AbuyunMaxCount = 10;
static main(args) {
def task = new AiqichaCrawl();
task.process()
}
void process(Object args) {
def url = "https://aiqicha.baidu.com/s/l?q=%E6%96%B0%E5%8F%8B&t=&p=3&s=10&o=0&f=%7B%7D "
for(def key : companyKeys){
def queryBody = null
def s = crawl(url, queryBody);
if(s){
print s
}
println s
}
}
def crawl(url, queryBody){
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","aiqicha.baidu.com")
headers.put("Connection","keep-alive")
headers.put("Cache-Control","max-age=0")
headers.put("sec-ch-ua",'"Chromium";v="88", "Google Chrome";v="88", ";Not A Brand";v="99"')
headers.put("sec-ch-ua-mobile","?0")
headers.put("Upgrade-Insecure-Requests","1")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
headers.put("Sec-Fetch-Site","same-origin")
headers.put("Sec-Fetch-Mode","navigate")
headers.put("Sec-Fetch-User","?1")
headers.put("Sec-Fetch-Dest","document")
headers.put("Accept-Language","zh-CN,zh;q=0.9")
String s = doPost(url, queryBody, headers)
if(s && s.contains("window.pageData = ")){
def prefix = "window.pageData ="
def s1 = s.substring(s.indexOf(prefix)+prefix.size())
s1 = s1.substring(0, s1.indexOf("}};")+2)
JSONObject jsonObject = JSONObject.fromObject(s1);
print jsonObject
}
return s
}
static String doPost(String strUrl, String content, Map headers) {
String result = "";
OutputStream outputStream = null;
InputStream inputStream = null;
StringBuilder sb = new StringBuilder();
try {
checkAbuyun(true);
def responseMap = [:]
// 创建代理服务器地址对象
InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
// 创建HTTP类型代理对象
Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
URL url = new URL(strUrl);
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestMethod("POST");
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setConnectTimeout(30000);
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setReadTimeout(30000);
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
urlConnection.setDoOutput(true);
urlConnection.setDoInput(true);
//设置通用请求属性为默认浏览器编码类型
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
urlConnection.setRequestProperty(head, headers.get(head));
}
}
//getOutputStream()返回的输出流,用于写入参数数据。
outputStream = urlConnection.getOutputStream();
if(content){
outputStream.write(content.getBytes());
}
outputStream.flush();
outputStream.close();
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
inputStream = urlConnection.getInputStream();
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
char[] buff = new char[1024];
int length = 0;
while ((length = reader.read(buff)) != -1) {
String s = new String(buff, 0, length);
sb.append(s);
}
} catch (IOException e) {
e.printStackTrace();
}finally{
if(outputStream!=null){
outputStream.close();
}
if(inputStream!=null){
inputStream.close();
}
}
return sb.toString();
}
public static checkAbuyun(boolean check) {
if (check) {
while (AbuyunRequestCount.get() >= AbuyunMaxCount) {
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
e.printStackTrace()
}
}
AbuyunRequestCount.incrementAndGet();
} else {
AbuyunRequestCount.decrementAndGet();
}
}
}
@@ -0,0 +1,715 @@
package com.fangjia.script.companyCrawl
import com.eclipsesource.v8.V8
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.ABuYunUtil
import jdk.nashorn.api.scripting.ScriptObjectMirror
import org.apache.commons.io.IOUtils;
import org.apache.http.Header
import org.apache.http.HttpResponse
import org.apache.http.client.HttpClient
import org.apache.http.client.methods.HttpGet
import org.jsoup.Jsoup
import org.jsoup.select.Elements
import org.apache.http.impl.client.HttpClients
import javax.script.ScriptEngine
import javax.script.ScriptEngineManager;
import java.util.concurrent.atomic.AtomicInteger
class GsxtCrawl {
def static final proxyPort = 9020;
def static final proxyServer = "http-dyn.abuyun.com";
def static final proxyUser = "HZU1XU28H3G4Z44D";
def static final proxyPass = "466C86ED77FE43D4";
public final static int AbuyunMaxCount = 10;
static AtomicInteger AbuyunRequestCount = new AtomicInteger(0);
def companyKeys = ["亨鑫","捷东","盈复","隆庆","卓源 ","合万","通广","安盛","东利","瑞科 ","凡迪","捷信","安盛","安盛","东进","源盛","茂禄","泰亚","宏成","光益","华圣","益全","长凯","宏新","锦立","宜佳","万美","正欣","和利","国宏","和裕","新昌","佳升","正德","源美","宝峰","金广","龙奥","宏美","成同","义丰","全洪","国聚","亨信","百益","广宏","仁瑞","亿昌","东旭","聚凯","洪成","兴同","赛迪","亨国","浩长","达宏","寿满","亿信","本盛","盈凯","浩汇","元辉","祥生","皇弘","讯思","新州","国泰","成合","茂浩","信源","润凯","虎光","安瑞","益吉","达兴","顺光","发盛","金优","捷丰","春凯","昌兴","本佳","远盛","盛达","亚飞","东昌","益如","宝东","德信","浩盈","多泰","泰联","升聚","辉丰","茂发","盈德","旭嘉","优源","益同","协盈","锦泰","捷久","本宏","德泰","聚富","全庆","凯安","进恒","全伟","优元","凯安","义丰","丰宇","瑞优","隆泰","诺德","台升","裕大","亨恒","义浩","百亨","欣昌","茂全","同亨","长圣","成谦","达泰","高协","正思","安巨","新耀","利仁","国益","旺洪","汇中","旭昌","弘康","优盛","旭达","晶安","华浩","耀春","圣本","康丰","光复","同凌","宇伯","乐鸿","成泰","宏协","华信","达泰","广丰","泰全","汇恒","杰森","天畅","恒通","亚达","庆合","丰圣","德长","茂百","汇久","欣鼎","谦和","厚辉","耀德","瑞隆","多辉","昌茂","成高","益升","安达","兴飞","百进","鼎宝","祥盛","信和","隆顺","亚欣","厚全","博文","讯诺","泰如","亨瑞","凯祥","捷浩","文舟","金庆","成仁","万协","宇中","宏祥","永多","艺峰","美益","创赢","奥龙","德宇","国福","飞利","久欣","如亨","安广","协佳","升捷","盛圣","富凯","长鑫","元卓","荣亨","亨仁","康鑫","信和","百新","干瑞","润万","富庭","信宇","源天","凯佳","正升","裕荣","厚泽","晶富","广新","正多","鼎高","丰汇","永华","兴升","复腾","隆诺","中发","广源","康胜","同进","仁捷","康凯","义洪","晶盛","正祥","裕祥","宇新","洪祥","润岳","义富","盛安","浩易"]
static main(args) {
def task = new GsxtCrawl();
//task.process()
task.run()
}
def run(){
getObject()
HttpClient client = HttpClients.createDefault();
HttpGet get = new HttpGet("http://www.gsxt.gov.cn/corp-query-homepage.html");
get.setHeader("Upgrade-Insecure-Requests", "1");
get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36");
HttpResponse response = client.execute(get);
def __jsluid_h = getJsluid(response)
def s = getResponseBodyAsString(response)
println (s)
def __jsl_clearance = getJsContent(s);
//Cookie: __jsluid_h=26cb8385493c2e424767eca767720c87; __jsl_clearance=1618899529.627|-1|ao39lP6IVSloPMvq5%2FOSBqSiPZ0%3D
def newCookie = __jsluid_h+";"+__jsl_clearance
get.setHeader("Cookie", newCookie)
HttpResponse response2 = client.execute(get);
def s2 = getResponseBodyAsString(response2)
def __jsl_clearance2 = getJsContent2(s2);
println(__jsl_clearance2)
// fristCrawl()
}
//该函数测试Java与js对象
static void getObject() throws Exception{
String str = "var _0xbe8c = ['w4LCg8Onw7o=', 'ZAjDvCw=', 'YcKOw6Fm', 'wpHCqsO3CQ==', 'UMKCXnw=', 'w7Aww6XCnQ==', 'BcKNwrvCng==', 'J2zCnsKi', 'wq02WMOx', 'RQXDrUw=', 'eCnCnDw=', 'Q8KowpzDjA==', 'YznDucOn', 'YcOrXxk=', 'w4t9PsKM', 'DMO4QmA=', 'w5HCklPDmQ==', 'w5rCv8Omw6Q=', 'NcOwc8K5', 'GUxjw7Q=', 'w7zDicK8w4I=', 'VCDDpsKv', 'cwLChT4=', 'wpvCl1vDqw==', 'wrvCvMOKNQ==', 'RTnDrAc=', 'GMKFw4hk', 'SxnDvC0=', 'H2NHNQ==', 'w6Aqw7LCqg==', 'wq0mW8Ok', 'L1zCv8Kp', 'B8OqwoI2', 'Jl7DuX8=', 'JcOBwpIR', 'U8OnfSo=', '6K2K5rOG6aqL6KyM', 'f8Kiwo/Dhg==', 'aMK6w6vDog==', 'JcKgw4FC', 'w61vJEg=', 'LMOrwoY0', 'GcK4w6lt', 'w4XDrsO5wpY=', 'J8OhwoM1', 'w5Mzw7J/', 'wocbcBA=', 'SibDt8Oj', 'w7xgwoXDkQ==', 'AcKyDEQ=', 'Rg7Dkw==', 'w6UFK8Kw', 'TTzDjcKK', 'w6QhDcKy', 'ZBLDols=', 'ZjPDrk4=', 'w6XDqX3Ctw==', 'w6EvJcKT', 'JHtUDQ==', 'wq8mI8KE', 'JMOhwoJ9', 'MsK1AkE=', 'w7fCmhcm', 'VsK2w5fDtQ==', 'w7EUw4PCnw==', 'dz/DgsKu', 'UzfDtcK0', 'wpd9fWE=', 'worDvcKFw4M=', 'w4PDr0zDhg==', 'w7ocSMKg', 'wp1uwoZd', 'eAjDv8Oz', 'aATCgTw=', 'w7UGw7k=', 'wqZCwotq', 'HFIBw5s=', 'WsOdw7JB', 'w5PDpXHCpA==', 'GMO8wqdD', 'w4TCsQs=', 'w6/DmcODwpg=', 'w7MuwrjDoQ==', 'dDHDhsOj', 'G308w70=', 'CsO6wog9', 'FcONwrVw', 'M3vCmMK4', 'w6/Dm8OvwpI=', 'QsOLw5ZM', 'TgLDicKy', 'QSHCuQA=', 'UsOlw5RI', 'w7EqwqDDrw==', 'w6jDusK5w4k=', 'OsKaw5rDlg==', 'wosLQsOk', 'DcKxw7Vo', 'w5VuIEo=', 'w7sXw6TCqw==', 'wrLDr8KRw7s=', 'wr9+wpJR', 'w4LDscO0wrQ=', 'FFIPw6U=', 'J8Ksw5rDlA==', 'd8KZwqjDgw==', 'PcKvDF4=', 'MHh2GA==', 'dgDDs0Y=', 'wpxSS1Y=', 'w7l5JMKh', 'w7N8JMKC', 'XsKhw7hR', 'QxDDqm4=', 'PcKGw5ZJ', 'w7HCuAIQ', 'E8Kmw53Dmg==', 'CMOrwok3', 'U8K9wpvDnA==', 'IW06w5U=', 'w7DDjcOGwpQ=', 'X8KlwpHDpA==', 'w6TDn8O1wpo=', 'ZjnDv08=', 'w7/Cqjsp', 'wqLDk8KYw58=', 'w6fDvMKXw4g=', 'FsK8IUY=', 'w4knw4Rh', 'w5zCu8Ogw5M=', 'wqJ0ckc=', 'T8Kgw69G', 'GF0hw74=', 'FMOkwqkH', 'wokmbMOM', 'djHDtcKW', 'FXkow64=', 'B8OHwowH', 'VSjCpQU=', 'w5vDs8K0w4s=', 'w7vCkUV1', 'w7dMIcK8', 'WMOjZyI=', 'LcKBw7po', 'BMK7HkY=', 'B8K0w7F4', 'YsK+w67Drw==', 'HlQKw7I=', 'wpLCrMKrNg==', 'w7IjcsKy', 'w6zDusKhw4o=', 'WhPDm8Om', 'dMKlw7RO', 'dcOHw5R3', 'Nkpww7M=', 'wrnCsMKLdg==', 'dDTDqMKb', 'S8KXZXE=', 'wq/CssOpJA==', 'w7J/KFw=', 'QMKSWEo=', 'woNxbEc=', 'wqVIwow=', 'TD/Dqg4=', 'w7TDpmzCiw==', 'w7HDjWDCgA==', 'w7IEw6vCoA==', 'Q8K+w4zDiQ==', 'PElvw7k=', 'cDXDsRo=', 'FMOLX0c=', 'Z8KHwojDoA==', 'TTTCqzo=', 'woXCrsK6Uw==', 'w4lhHnY=', 'w6Q/dsKj', 'KsKIwpLCtQ==', 'QQTDn8Oy', 'D8Omw7zDoQ==', 'E8KzGk0=']; (function(_0x44c192, _0xbe8c61) {\n" +
"\t\tvar _0x1533fb = function(_0x5b499f) {\n" +
"\t\t\twhile (--_0x5b499f) {\n" +
"\t\t\t\t_0x44c192['push'](_0x44c192['shift']());\n" +
"\t\t\t}\n" +
"\t\t};\n" +
"\t\t_0x1533fb(++_0xbe8c61);\n" +
"\t} (_0xbe8c, 0x185));\n" +
"\tvar _0x1533 = function(_0x44c192, _0xbe8c61) {\n" +
"\t\t_0x44c192 = _0x44c192 - 0x0;\n" +
"\t\tvar _0x1533fb = _0xbe8c[_0x44c192];\n" +
"\t\tif (_0x1533['GlBvhA'] === undefined) { (function() {\n" +
"\t\t\t\tvar _0xa131c3 = function() {\n" +
"\t\t\t\t\tvar _0x9829e5;\n" +
"\t\t\t\t\ttry {\n" +
"\t\t\t\t\t\t_0x9829e5 = Function('return\\x20(function()\\x20' + '{}.constructor(\\x22return\\x20this\\x22)(\\x20)' + ');')();\n" +
"\t\t\t\t\t} catch(_0x1ba3fa) {\n" +
"\t\t\t\t\t}\n" +
"\t\t\t\t\treturn _0x9829e5;\n" +
"\t\t\t\t};\n" +
"\t\t\t\tvar _0x5e86d6 = _0xa131c3();\n" +
"\t\t\t\tvar _0x67fb0c = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=';\n" +
"\t\t\t\t_0x5e86d6['atob'] || (_0x5e86d6['atob'] = function(_0x1cb044) {\n" +
"\t\t\t\t\tvar _0xa69f93 = String(_0x1cb044)['replace'](/=+\$/, '');\n" +
"\t\t\t\t\tvar _0x26bfb9 = '';\n" +
"\t\t\t\t\tfor (var _0x2f1d79 = 0x0,\n" +
"\t\t\t\t\t_0x3713bb, _0x4e514b, _0x29733b = 0x0; _0x4e514b = _0xa69f93['charAt'](_0x29733b++);~_0x4e514b && (_0x3713bb = _0x2f1d79 % 0x4 ? _0x3713bb * 0x40 + _0x4e514b: _0x4e514b, _0x2f1d79++%0x4) ? _0x26bfb9 += String['fromCharCode'](0xff & _0x3713bb >> ( - 0x2 * _0x2f1d79 & 0x6)) : 0x0) {\n" +
"\t\t\t\t\t\t_0x4e514b = _0x67fb0c['indexOf'](_0x4e514b);\n" +
"\t\t\t\t\t}\n" +
"\t\t\t\t\treturn _0x26bfb9;\n" +
"\t\t\t\t});\n" +
"\t\t\t} ());\n" +
"\t\t\tvar _0x38a7ff = function(_0x5dd34b, _0x3a0f2e) {\n" +
"\t\t\t\tvar _0x3e7a99 = [],\n" +
"\t\t\t\t_0x3e2c47 = 0x0,\n" +
"\t\t\t\t_0x383663,\n" +
"\t\t\t\t_0x14720e = '',\n" +
"\t\t\t\t_0x5864f4 = '';\n" +
"\t\t\t\t_0x5dd34b = atob(_0x5dd34b);\n" +
"\t\t\t\tfor (var _0x28a6d2 = 0x0,\n" +
"\t\t\t\t_0x2ea622 = _0x5dd34b['length']; _0x28a6d2 < _0x2ea622; _0x28a6d2++) {\n" +
"\t\t\t\t\t_0x5864f4 += '%' + ('00' + _0x5dd34b['charCodeAt'](_0x28a6d2)['toString'](0x10))['slice']( - 0x2);\n" +
"\t\t\t\t}\n" +
"\t\t\t\t_0x5dd34b = decodeURIComponent(_0x5864f4);\n" +
"\t\t\t\tvar _0x3091fb;\n" +
"\t\t\t\tfor (_0x3091fb = 0x0; _0x3091fb < 0x100; _0x3091fb++) {\n" +
"\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3091fb;\n" +
"\t\t\t\t}\n" +
"\t\t\t\tfor (_0x3091fb = 0x0; _0x3091fb < 0x100; _0x3091fb++) {\n" +
"\t\t\t\t\t_0x3e2c47 = (_0x3e2c47 + _0x3e7a99[_0x3091fb] + _0x3a0f2e['charCodeAt'](_0x3091fb % _0x3a0f2e['length'])) % 0x100;\n" +
"\t\t\t\t\t_0x383663 = _0x3e7a99[_0x3091fb];\n" +
"\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3e7a99[_0x3e2c47];\n" +
"\t\t\t\t\t_0x3e7a99[_0x3e2c47] = _0x383663;\n" +
"\t\t\t\t}\n" +
"\t\t\t\t_0x3091fb = 0x0;\n" +
"\t\t\t\t_0x3e2c47 = 0x0;\n" +
"\t\t\t\tfor (var _0x8c68bc = 0x0; _0x8c68bc < _0x5dd34b['length']; _0x8c68bc++) {\n" +
"\t\t\t\t\t_0x3091fb = (_0x3091fb + 0x1) % 0x100;\n" +
"\t\t\t\t\t_0x3e2c47 = (_0x3e2c47 + _0x3e7a99[_0x3091fb]) % 0x100;\n" +
"\t\t\t\t\t_0x383663 = _0x3e7a99[_0x3091fb];\n" +
"\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3e7a99[_0x3e2c47];\n" +
"\t\t\t\t\t_0x3e7a99[_0x3e2c47] = _0x383663;\n" +
"\t\t\t\t\t_0x14720e += String['fromCharCode'](_0x5dd34b['charCodeAt'](_0x8c68bc) ^ _0x3e7a99[(_0x3e7a99[_0x3091fb] + _0x3e7a99[_0x3e2c47]) % 0x100]);\n" +
"\t\t\t\t}\n" +
"\t\t\t\treturn _0x14720e;\n" +
"\t\t\t};\n" +
"\t\t\t_0x1533['sMWqUz'] = _0x38a7ff;\n" +
"\t\t\t_0x1533['AbOzCI'] = {};\n" +
"\t\t\t_0x1533['GlBvhA'] = !![];\n" +
"\t\t}\n" +
"\t\tvar _0x5b499f = _0x1533['AbOzCI'][_0x44c192];\n" +
"\t\tif (_0x5b499f === undefined) {\n" +
"\t\t\tif (_0x1533['uRkTAI'] === undefined) {\n" +
"\t\t\t\t_0x1533['uRkTAI'] = !![];\n" +
"\t\t\t}\n" +
"\t\t\t_0x1533fb = _0x1533['sMWqUz'](_0x1533fb, _0xbe8c61);\n" +
"\t\t\t_0x1533['AbOzCI'][_0x44c192] = _0x1533fb;\n" +
"\t\t} else {\n" +
"\t\t\t_0x1533fb = _0x5b499f;\n" +
"\t\t}\n" +
"\t\treturn _0x1533fb;\n" +
"\t};\n" +
"\tfunction hash(_0x1a5a87) {\n" +
"\t\tvar _0x23a3e0 = {};\n" +
"\t\t_0x23a3e0[_0x1533('0x93', 'Kp2R') + 'J'] = function(_0x5ebeb2, _0x4681fc) {\n" +
"\t\t\treturn _0x5ebeb2 ^ _0x4681fc;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x17', '(bZ%') + 'H'] = function(_0x5108c3, _0x49fe65) {\n" +
"\t\t\treturn _0x5108c3 & _0x49fe65;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x88', 'N#(C') + 'T'] = function(_0x3d3f10, _0x3f2a86) {\n" +
"\t\t\treturn _0x3d3f10 !== _0x3f2a86;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x60', '(cNt') + 'Z'] = _0x1533('0x70', 'K)PG') + 'J';\n" +
"\t\t_0x23a3e0[_0x1533('0x49', 'Y8u@') + 'h'] = function(_0xc3ff68, _0x129ab0) {\n" +
"\t\t\treturn _0xc3ff68 >> _0x129ab0;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x9c', '\$qIi') + 'S'] = function(_0x591dc1, _0x158d9d) {\n" +
"\t\t\treturn _0x591dc1 + _0x158d9d;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x7f', '(bZ%') + 'f'] = function(_0xd6deb, _0x557760) {\n" +
"\t\t\treturn _0xd6deb >> _0x557760;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x4e', 'S9Bn') + 'd'] = function(_0xd05226, _0x29b6e8) {\n" +
"\t\t\treturn _0xd05226 * _0x29b6e8;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x7e', 'y^BN') + 'n'] = function(_0x899f8d, _0x3dbd26) {\n" +
"\t\t\treturn _0x899f8d < _0x3dbd26;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x9a', '(U[3') + 'e'] = function(_0x39678a, _0x37d488) {\n" +
"\t\t\treturn _0x39678a * _0x37d488;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0xa3', 'sahq') + 'U'] = function(_0x4cf877, _0x14e1b0) {\n" +
"\t\t\treturn _0x4cf877 << _0x14e1b0;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0xa7', 'K#Hh') + 'u'] = function(_0x556336, _0x56b41c) {\n" +
"\t\t\treturn _0x556336 - _0x56b41c;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x3d', '(bZ%') + 'p'] = function(_0x372c20, _0x534fe1) {\n" +
"\t\t\treturn _0x372c20 >> _0x534fe1;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x1d', 'S9Bn') + 'g'] = function(_0x26a0f8, _0x4f1660) {\n" +
"\t\t\treturn _0x26a0f8 >>> _0x4f1660;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0xae', ']*f@') + 'Y'] = function(_0x3423f4, _0x53e1a9) {\n" +
"\t\t\treturn _0x3423f4 | _0x53e1a9;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x26', 's4Dp') + 'P'] = function(_0x19382a, _0x220110) {\n" +
"\t\t\treturn _0x19382a | _0x220110;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x65', 's4Dp') + 'p'] = function(_0x7b95bc, _0x22e933) {\n" +
"\t\t\treturn _0x7b95bc & _0x22e933;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x0', '\$qIi') + 'V'] = function(_0x13269b, _0x227e3b) {\n" +
"\t\t\treturn _0x13269b ^ _0x227e3b;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x29', 'J21I') + 'R'] = function(_0x1ba947, _0x4b40e4) {\n" +
"\t\t\treturn _0x1ba947(_0x4b40e4);\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x50', 's4Dp') + 'n'] = _0x1533('0xa', 'qRqa') + 'e';\n" +
"\t\t_0x23a3e0[_0x1533('0x42', 'ksVw') + 'b'] = function(_0x466fa6, _0x3ce8da) {\n" +
"\t\t\treturn _0x466fa6 ^ _0x3ce8da;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x2c', 'rqYb') + 'f'] = function(_0x1e608a, _0x241888) {\n" +
"\t\t\treturn _0x1e608a - _0x241888;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x78', 'J21I') + 'u'] = function(_0x564b78, _0x18d236, _0x8fd805) {\n" +
"\t\t\treturn _0x564b78(_0x18d236, _0x8fd805);\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x69', 'zx(Q') + 'D'] = function(_0x3b7666, _0x47f238, _0x57b05e) {\n" +
"\t\t\treturn _0x3b7666(_0x47f238, _0x57b05e);\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x5f', 'ZaCV') + 'm'] = function(_0x452fa7, _0x493d76, _0x1446e7, _0x4af814, _0x5e0f59) {\n" +
"\t\t\treturn _0x452fa7(_0x493d76, _0x1446e7, _0x4af814, _0x5e0f59);\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x22', 'Rh@#') + 'W'] = function(_0x51a858, _0x440b1c) {\n" +
"\t\t\treturn _0x51a858(_0x440b1c);\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x8', 'xPGU') + 'A'] = function(_0x5e96f2, _0x167b41) {\n" +
"\t\t\treturn _0x5e96f2 + _0x167b41;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x6d', 'rqYb') + 'X'] = function(_0x46cd37, _0x5d8684) {\n" +
"\t\t\treturn _0x46cd37 + _0x5d8684;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x3e', 'rqYb') + 'g'] = function(_0x26bc4b, _0x16bc46) {\n" +
"\t\t\treturn _0x26bc4b(_0x16bc46);\n" +
"\t\t};\n" +
"\t\tvar _0x179134 = _0x23a3e0;\n" +
"\t\tfunction _0x5be8c0(_0x296dd6, _0x778566) {\n" +
"\t\t\treturn _0x179134[_0x1533('0x56', 'ZaCV') + 'J'](_0x179134[_0x1533('0x7c', 'o&Q4') + 'H'](_0x296dd6, 0x7fffffff) + (_0x778566 & 0x7fffffff), _0x179134[_0x1533('0x52', 'K)PG') + 'H'](_0x296dd6, 0x80000000)) ^ _0x778566 & 0x80000000;\n" +
"\t\t}\n" +
"\t\tfunction _0x225a28(_0x2f57af) {\n" +
"\t\t\tvar _0x3c49d8 = {};\n" +
"\t\t\t_0x3c49d8[_0x1533('0x55', 'qRqa') + 'N'] = function(_0x15bfb8, _0x51e1eb) {\n" +
"\t\t\t\treturn _0x15bfb8 < _0x51e1eb;\n" +
"\t\t\t};\n" +
"\t\t\tvar _0x567e0f = _0x3c49d8;\n" +
"\t\t\tvar _0x5b2e48 = _0x1533('0xa4', 'rQf(') + _0x1533('0x4', 'rqYb') + _0x1533('0x95', 'Rh@#') + _0x1533('0x3', 'OhQr');\n" +
"\t\t\tvar _0x321f21 = '';\n" +
"\t\t\tfor (var _0x2a5528 = 0x7; _0x2a5528 >= 0x0; _0x2a5528--) {\n" +
"\t\t\t\tif (_0x179134[_0x1533('0xb2', 'Gfc!') + 'T'](_0x1533('0x2', 'w*Nn') + 'h', _0x179134[_0x1533('0x6c', 'OhQr') + 'Z'])) {\n" +
"\t\t\t\t\t_0x321f21 += _0x5b2e48[_0x1533('0x66', ')T5g') + 'At'](_0x179134[_0x1533('0x9f', 'zx(Q') + 'H'](_0x179134[_0x1533('0x5', 'OhQr') + 'h'](_0x2f57af, _0x2a5528 * 0x4), 0xf));\n" +
"\t\t\t\t} else {\n" +
"\t\t\t\t\treturn _0x567e0f[_0x1533('0x5a', 'y*wC') + 'N'](t, 0x14) ? 0x5a827999: t < 0x28 ? 0x6ed9eba1: _0x567e0f[_0x1533('0x43', 'zx(Q') + 'N'](t, 0x3c) ? -0x70e44324: -0x359d3e2a;\n" +
"\t\t\t\t}\n" +
"\t\t\t}\n" +
"\t\t\treturn _0x321f21;\n" +
"\t\t}\n" +
"\t\tfunction _0xde630f(_0x10fb9a) {\n" +
"\t\t\tvar _0x5b5b6d = _0x179134[_0x1533('0x16', 'GvA9') + 'S'](_0x179134[_0x1533('0x87', 'N#(C') + 'f'](_0x179134[_0x1533('0x83', '[H[u') + 'S'](_0x10fb9a[_0x1533('0x58', 'OhQr') + 'th'], 0x8), 0x6), 0x1),\n" +
"\t\t\t_0x797793 = new Array(_0x179134[_0x1533('0x59', 'xPGU') + 'd'](_0x5b5b6d, 0x10));\n" +
"\t\t\tfor (var _0x109346 = 0x0; _0x179134[_0x1533('0x61', 'dgE&') + 'n'](_0x109346, _0x179134[_0x1533('0x90', 'y^BN') + 'e'](_0x5b5b6d, 0x10)); _0x109346++) {\n" +
"\t\t\t\tif (_0x179134[_0x1533('0x2a', 'NNf8') + 'T'](_0x1533('0x12', 'sahq') + 'y', _0x1533('0x67', 'y*wC') + 'G')) {\n" +
"\t\t\t\t\t_0x797793[_0x109346] = 0x0;\n" +
"\t\t\t\t} else {\n" +
"\t\t\t\t\treturn !! [];\n" +
"\t\t\t\t}\n" +
"\t\t\t}\n" +
"\t\t\tfor (_0x109346 = 0x0; _0x109346 < _0x10fb9a[_0x1533('0x40', 'ad*k') + 'th']; _0x109346++) {\n" +
"\t\t\t\t_0x797793[_0x109346 >> 0x2] |= _0x179134[_0x1533('0xa9', '@gLq') + 'U'](_0x10fb9a[_0x1533('0x1c', 'GvA9') + _0x1533('0xf', 'OhQr') + 'At'](_0x109346), _0x179134[_0x1533('0xb3', 'S9Bn') + 'u'](0x18, (_0x109346 & 0x3) * 0x8));\n" +
"\t\t\t}\n" +
"\t\t\t_0x797793[_0x179134[_0x1533('0xb1', 'qRqa') + 'p'](_0x109346, 0x2)] |= 0x80 << 0x18 - (_0x109346 & 0x3) * 0x8;\n" +
"\t\t\t_0x797793[_0x5b5b6d * 0x10 - 0x1] = _0x10fb9a[_0x1533('0xb', '@mqx') + 'th'] * 0x8;\n" +
"\t\t\treturn _0x797793;\n" +
"\t\t}\n" +
"\t\tfunction _0x50e067(_0x45a1cc, _0x3bcee1) {\n" +
"\t\t\treturn _0x45a1cc << _0x3bcee1 | _0x179134[_0x1533('0x5b', 'Y8u@') + 'g'](_0x45a1cc, _0x179134[_0x1533('0x91', '@mqx') + 'u'](0x20, _0x3bcee1));\n" +
"\t\t}\n" +
"\t\tfunction _0x526d3b(_0xb72077, _0xed7f25, _0x5bc269, _0x572b62) {\n" +
"\t\t\tif (_0xb72077 < 0x14) return _0x179134[_0x1533('0x98', 'Gfc!') + 'Y'](_0xed7f25 & _0x5bc269, _0x179134[_0x1533('0x68', 'OhQr') + 'H'](~_0xed7f25, _0x572b62));\n" +
"\t\t\tif (_0x179134[_0x1533('0x9e', 'w*Nn') + 'n'](_0xb72077, 0x28)) return _0x179134[_0x1533('0x99', ')T5g') + 'J'](_0xed7f25, _0x5bc269) ^ _0x572b62;\n" +
"\t\t\tif (_0xb72077 < 0x3c) return _0x179134[_0x1533('0x13', 'CNzQ') + 'P'](_0xed7f25 & _0x5bc269, _0x179134[_0x1533('0x45', '@mqx') + 'p'](_0xed7f25, _0x572b62)) | _0x179134[_0x1533('0xa0', 'avK7') + 'p'](_0x5bc269, _0x572b62);\n" +
"\t\t\treturn _0x179134[_0x1533('0x5e', 'avK7') + 'V'](_0xed7f25, _0x5bc269) ^ _0x572b62;\n" +
"\t\t}\n" +
"\t\tfunction _0x496785(_0x583c97) {\n" +
"\t\t\treturn _0x583c97 < 0x14 ? 0x5a827999: _0x583c97 < 0x28 ? 0x6ed9eba1: _0x583c97 < 0x3c ? -0x70e44324: -0x359d3e2a;\n" +
"\t\t}\n" +
"\t\tvar _0x45b840 = _0x179134[_0x1533('0x39', 'R#BF') + 'R'](_0xde630f, _0x1a5a87);\n" +
"\t\tvar _0x53258d = new Array(0x50);\n" +
"\t\tvar _0xc86ce6 = 0x67452301;\n" +
"\t\tvar _0x3ee50e = -0x10325477;\n" +
"\t\tvar _0xdb1008 = -0x67452302;\n" +
"\t\tvar _0xe99bcb = 0x10325476;\n" +
"\t\tvar _0x56c860 = -0x3c2d1e10;\n" +
"\t\tfor (var _0x4c314c = 0x0; _0x4c314c < _0x45b840[_0x1533('0x46', '\$qIi') + 'th']; _0x4c314c += 0x10) {\n" +
"\t\t\tvar _0x41e8e8 = _0xc86ce6;\n" +
"\t\t\tvar _0x1b7277 = _0x3ee50e;\n" +
"\t\t\tvar _0x30f7c3 = _0xdb1008;\n" +
"\t\t\tvar _0x279c95 = _0xe99bcb;\n" +
"\t\t\tvar _0xf6e888 = _0x56c860;\n" +
"\t\t\tfor (var _0x77e31b = 0x0; _0x179134[_0x1533('0x34', 'Y8u@') + 'n'](_0x77e31b, 0x50); _0x77e31b++) {\n" +
"\t\t\t\tif (_0x77e31b < 0x10) {\n" +
"\t\t\t\t\tif (_0x179134[_0x1533('0xa6', 'Mifp') + 'T'](_0x1533('0x3f', 'o&Q4') + 'G', _0x179134[_0x1533('0xa1', 'rqYb') + 'n'])) {\n" +
"\t\t\t\t\t\t_0x53258d[_0x77e31b] = _0x45b840[_0x4c314c + _0x77e31b];\n" +
"\t\t\t\t\t} else {\n" +
"\t\t\t\t\t\tsleepT = _0x179134[_0x1533('0x48', 'NNf8') + 'R'](parseInt, data['wt']) > ret[0x1] ? _0x179134[_0x1533('0x97', 'd[i)') + 'R'](parseInt, data['wt']) - ret[0x1] : 0x1f4;\n" +
"\t\t\t\t\t}\n" +
"\t\t\t\t} else {\n" +
"\t\t\t\t\t_0x53258d[_0x77e31b] = _0x50e067(_0x179134[_0x1533('0x96', 'rQf(') + 'b'](_0x179134[_0x1533('0xb0', 'Gfc!') + 'b'](_0x53258d[_0x179134[_0x1533('0xa5', 'K)PG') + 'u'](_0x77e31b, 0x3)], _0x53258d[_0x179134[_0x1533('0x51', 'K)PG') + 'f'](_0x77e31b, 0x8)]) ^ _0x53258d[_0x77e31b - 0xe], _0x53258d[_0x179134[_0x1533('0xaf', '(U[3') + 'f'](_0x77e31b, 0x10)]), 0x1);\n" +
"\t\t\t\t}\n" +
"\t\t\t\tt = _0x179134[_0x1533('0x18', 'GvA9') + 'u'](_0x5be8c0, _0x179134[_0x1533('0x23', '\$qIi') + 'u'](_0x5be8c0, _0x179134[_0x1533('0x8d', 'Mifp') + 'D'](_0x50e067, _0xc86ce6, 0x5), _0x179134[_0x1533('0x4c', 'xPGU') + 'm'](_0x526d3b, _0x77e31b, _0x3ee50e, _0xdb1008, _0xe99bcb)), _0x5be8c0(_0x5be8c0(_0x56c860, _0x53258d[_0x77e31b]), _0x179134[_0x1533('0xad', 'rqYb') + 'W'](_0x496785, _0x77e31b)));\n" +
"\t\t\t\t_0x56c860 = _0xe99bcb;\n" +
"\t\t\t\t_0xe99bcb = _0xdb1008;\n" +
"\t\t\t\t_0xdb1008 = _0x50e067(_0x3ee50e, 0x1e);\n" +
"\t\t\t\t_0x3ee50e = _0xc86ce6;\n" +
"\t\t\t\t_0xc86ce6 = t;\n" +
"\t\t\t}\n" +
"\t\t\t_0xc86ce6 = _0x5be8c0(_0xc86ce6, _0x41e8e8);\n" +
"\t\t\t_0x3ee50e = _0x5be8c0(_0x3ee50e, _0x1b7277);\n" +
"\t\t\t_0xdb1008 = _0x5be8c0(_0xdb1008, _0x30f7c3);\n" +
"\t\t\t_0xe99bcb = _0x5be8c0(_0xe99bcb, _0x279c95);\n" +
"\t\t\t_0x56c860 = _0x5be8c0(_0x56c860, _0xf6e888);\n" +
"\t\t}\n" +
"\t\treturn _0x179134[_0x1533('0x86', 'Gfc!') + 'A'](_0x179134[_0x1533('0x44', 'DDSy') + 'X'](_0x179134[_0x1533('0x8e', 'xPGU') + 'g'](_0x225a28, _0xc86ce6), _0x179134[_0x1533('0x77', 'n*Rs') + 'g'](_0x225a28, _0x3ee50e)) + _0x225a28(_0xdb1008) + _0x225a28(_0xe99bcb), _0x179134[_0x1533('0xd', 'DDSy') + 'g'](_0x225a28, _0x56c860));\n" +
"\t}\n" +
"\tfunction go(_0x394549) {\n" +
"\t\tvar _0x33d0d6 = {};\n" +
"\t\t_0x33d0d6[_0x1533('0x2b', 'sahq') + 'H'] = _0x1533('0x14', 'rQf(') + _0x1533('0x15', 'sahq');\n" +
"\t\t_0x33d0d6[_0x1533('0xab', 'dgE&') + 'H'] = function(_0x23f209, _0x4ed415) {\n" +
"\t\t\treturn _0x23f209 < _0x4ed415;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x5c', 'xPGU') + 'g'] = function(_0x39bd0a, _0x4d5764) {\n" +
"\t\t\treturn _0x39bd0a != _0x4d5764;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x37', 'y*wC') + 'N'] = function(_0x3c9a26, _0x109b89) {\n" +
"\t\t\treturn _0x3c9a26 + _0x109b89;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x63', 'gkl*') + 'z'] = function(_0x584935, _0x58347c) {\n" +
"\t\t\treturn _0x584935 + _0x58347c;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x21', 'ZaCV') + 'z'] = function(_0x3fc795, _0x6962f6) {\n" +
"\t\t\treturn _0x3fc795 + _0x6962f6;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x27', '(cNt') + 'I'] = function(_0x303446, _0x3388fb) {\n" +
"\t\t\treturn _0x303446 == _0x3388fb;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x6b', 'y*wC') + 't'] = function(_0x5cb515, _0x31e7f3) {\n" +
"\t\t\treturn _0x5cb515(_0x31e7f3);\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x75', 'Rh@#') + 'E'] = function(_0x529d8e, _0xc1423a) {\n" +
"\t\t\treturn _0x529d8e + _0xc1423a;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x62', 'rQf(') + 'a'] = function(_0x453505, _0x50cd56) {\n" +
"\t\t\treturn _0x453505 + _0x50cd56;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x2e', 'NNf8') + 'B'] = _0x1533('0x28', ']*f@') + _0x1533('0x57', 'Kp2R') + '\\x20/';\n" +
"\t\t_0x33d0d6[_0x1533('0xe', 'Y8u@') + 'y'] = function(_0x25bcfd, _0x4b634e, _0x1bef22) {\n" +
"\t\t\treturn _0x25bcfd(_0x4b634e, _0x1bef22);\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x92', 'J21I') + 'i'] = function(_0xccfc4d, _0x9a8620) {\n" +
"\t\t\treturn _0xccfc4d > _0x9a8620;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x11', '@H8e') + 'n'] = function(_0x1b9fc7, _0x7d3ce) {\n" +
"\t\t\treturn _0x1b9fc7(_0x7d3ce);\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x1b', 'N#(C') + 'O'] = _0x1533('0x7', 'N#(C') + '失败';\n" +
"\t\tvar _0x2eda7d = _0x33d0d6;\n" +
"\t\tfunction _0x29e502() {\n" +
"\t\t\tvar _0x24a3c1 = window[_0x1533('0x35', 'ad*k') + _0x1533('0x5d', 'Y8u@') + 'r'][_0x1533('0x19', 'avK7') + _0x1533('0x7d', 'cJ3U') + 't'],\n" +
"\t\t\t_0x5f1192 = [_0x2eda7d[_0x1533('0x24', '(bZ%') + 'H']];\n" +
"\t\t\tfor (var _0x2d8686 = 0x0; _0x2eda7d[_0x1533('0x1a', 'avK7') + 'H'](_0x2d8686, _0x5f1192[_0x1533('0x3b', 'Y8u@') + 'th']); _0x2d8686++) {\n" +
"\t\t\t\tif (_0x2eda7d[_0x1533('0x2f', 'y*wC') + 'g'](_0x24a3c1[_0x1533('0x76', 'y*wC') + _0x1533('0x33', 'ZaCV')](_0x5f1192[_0x2d8686]), -0x1)) {\n" +
"\t\t\t\t\treturn !! [];\n" +
"\t\t\t\t}\n" +
"\t\t\t}\n" +
"\t\t\tif (window[_0x1533('0x41', 'dgE&') + _0x1533('0x6e', 'dgE&') + _0x1533('0x85', 'NNf8')] || window[_0x1533('0x30', 'o&Q4') + _0x1533('0x38', 'OhQr')] || window[_0x1533('0xc', 'OhQr') + _0x1533('0x74', 'qRqa')] || window[_0x1533('0x25', '(bZ%') + _0x1533('0x4f', 'avK7') + 'r'][_0x1533('0xa2', 'xPGU') + _0x1533('0x53', ')T5g') + 'r'] || window[_0x1533('0xaa', 'cJ3U') + _0x1533('0x6', 'K#@S') + 'r'][_0x1533('0x8f', 'rqYb') + _0x1533('0xac', '(bZ%') + _0x1533('0x9', 'Rh@#') + _0x1533('0x4d', 'rQf(') + 'e'] || window[_0x1533('0x31', 'N#(C') + _0x1533('0x79', 'dgE&') + 'r'][_0x1533('0x4b', 'ksVw') + _0x1533('0x10', 'gkl*') + _0x1533('0xa8', 'd[i)') + _0x1533('0x89', '\$qIi') + _0x1533('0x84', 's4Dp')]) {\n" +
"\t\t\t\treturn !! [];\n" +
"\t\t\t}\n" +
"\t\t};\n" +
"\t\tif (_0x29e502()) {\n" +
"\t\t\treturn;\n" +
"\t\t}\n" +
"\t\tvar _0x1642ef = new Date();\n" +
"\t\tfunction _0x1a1fd9(_0x55baea, _0x579edb) {\n" +
"\t\t\tvar _0x551ac6 = _0x394549[_0x1533('0x4a', 'y*wC') + 's'][_0x1533('0x71', 'K#@S') + 'th'];\n" +
"\t\t\tfor (var _0x54ccec = 0x0; _0x2eda7d[_0x1533('0x6f', 'dFiI') + 'H'](_0x54ccec, _0x551ac6); _0x54ccec++) {\n" +
"\t\t\t\tfor (var _0x562c2d = 0x0; _0x2eda7d[_0x1533('0x9b', '[H[u') + 'H'](_0x562c2d, _0x551ac6); _0x562c2d++) {\n" +
"\t\t\t\t\tvar _0x1b8d8c = _0x2eda7d[_0x1533('0x80', '[H[u') + 'N'](_0x2eda7d[_0x1533('0x3a', 'w*Nn') + 'z'](_0x2eda7d[_0x1533('0x8b', 'cJ3U') + 'z'](_0x579edb[0x0], _0x394549[_0x1533('0x32', 'R#BF') + 's'][_0x1533('0x82', '@mqx') + 'tr'](_0x54ccec, 0x1)), _0x394549[_0x1533('0x1f', '*hmq') + 's'][_0x1533('0x47', '(cNt') + 'tr'](_0x562c2d, 0x1)), _0x579edb[0x1]);\n" +
"\t\t\t\t\tif (_0x2eda7d[_0x1533('0x6a', '(bZ%') + 'I'](_0x2eda7d[_0x1533('0x9d', 'Kp2R') + 't'](hash, _0x1b8d8c), _0x55baea)) {\n" +
"\t\t\t\t\t\treturn [_0x1b8d8c, new Date() - _0x1642ef];\n" +
"\t\t\t\t\t}\n" +
"\t\t\t\t}\n" +
"\t\t\t}\n" +
"\t\t};\n" +
"\t\tvar _0x1e73b9 = _0x2eda7d[_0x1533('0x36', 'sahq') + 'y'](_0x1a1fd9, _0x394549['ct'], _0x394549[_0x1533('0x2d', '\$qIi')]);\n" +
"\t\tif (_0x1e73b9) {\n" +
"\t\t\tvar _0x3b1427;\n" +
"\t\t\tif (_0x394549['wt']) {\n" +
"\t\t\t\t_0x3b1427 = _0x2eda7d[_0x1533('0x1', 'zx(Q') + 'i'](parseInt(_0x394549['wt']), _0x1e73b9[0x1]) ? _0x2eda7d[_0x1533('0x54', 'avK7') + 'n'](parseInt, _0x394549['wt']) - _0x1e73b9[0x1] : 0x1f4;\n" +
"\t\t\t} else {\n" +
"\t\t\t\t_0x3b1427 = 0x5dc;\n" +
"\t\t\t}\n" +
"\t\t\t_0x2eda7d[_0x1533('0x72', 'qRqa') + 'y'](setTimeout,\n" +
"\t\t\tfunction() {\n" +
"\t\t\t\tdocument[_0x1533('0x20', 'rQf(') + 'ie'] = _0x2eda7d[_0x1533('0x73', 'rQf(') + 'E'](_0x2eda7d[_0x1533('0x3c', 'o&Q4') + 'a'](_0x394549['tn'] + '=' + _0x1e73b9[0x0] + (_0x1533('0x8a', 'ksVw') + _0x1533('0x1e', 'GvA9') + '='), _0x394549['vt']), _0x2eda7d[_0x1533('0x8c', 'Gfc!') + 'B']);\n" +
"\t\t\t\tlocation[_0x1533('0x7a', 'sahq')] = location[_0x1533('0x64', 'd[i)') + _0x1533('0x81', '(U[3')] + location[_0x1533('0x94', 'sahq') + 'ch'];\n" +
"\t\t\t},\n" +
"\t\t\t_0x3b1427);\n" +
"\t\t} else {\n" +
"\t\t\talert(_0x2eda7d[_0x1533('0x7b', ')T5g') + 'O']);\n" +
"\t\t}\n" +
"\t};\n" +
"\tgo({\n" +
"\t\t\"bts\": [\"1618915727.945|0|HM5\", \"VoVNIPqcrvFO13woIZK2RY%3D\"],\n" +
"\t\t\"chars\": \"ZDUZR0cDjTEXznMkfgavFO\",\n" +
"\t\t\"ct\": \"3049e28877124ad21ca0db55e3af95ff316ceef6\",\n" +
"\t\t\"ha\": \"sha1\",\n" +
"\t\t\"tn\": \"__jsl_clearance\",\n" +
"\t\t\"vt\": \"3600\",\n" +
"\t\t\"wt\": \"1500\"\n" +
"\t})"
ScriptEngineManager manager = new ScriptEngineManager();
ScriptEngine engine = manager.getEngineByName("javascript");
engine.eval(str);
//获取对象
System.out.println();
}
def getJsContent(def page){
ScriptEngineManager manager = new ScriptEngineManager();
//得到脚本引擎
ScriptEngine engine = manager.getEngineByName("javascript");
//处理加密js
String js = page.trim().replace("<script>", "").replace("</script>", "").replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]}));", "y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]})");
try {
//得到解密后的js
js = js.replace("document.cookie","var cookie")
js = js.replace("location.href=location.pathname+location.search", "")
engine.eval(js);
def v = engine.get("cookie")
return v.toString().substring(0, v.toString().indexOf(";"))
} catch (ScriptException e) {
e.printStackTrace();
}
return null
}
def getJsContent2(def page){
ScriptEngineManager manager = new ScriptEngineManager();
//得到脚本引擎
ScriptEngine engine = manager.getEngineByName("javascript");
//处理加密js
String js = page.trim().replace("<script>", "").replace("</script>", "").replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]}));", "y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]})");
try {
//得到解密后的js
//得到解密后的js
String result = (String) engine.eval(js);
result = result.substring(result.indexOf("var cd"),result.indexOf("dc+=cd;")+7);
result = result.replaceAll("document*.*toLowerCase\\(\\)", "'x'");
//得到cookie
String jsl = (String) engine.eval(result);
return jsl;
} catch (ScriptException e) {
e.printStackTrace();
}
return null
}
/**
* 通过破解动态JavaScript脚本,
* 获取cookie名为 __jsl_clearance的值
*
* @param body 相应内容(一般为第一次请求获取到的动态js字符串)
* @return cookie名为 __jsl_clearance的值
*/
private static String getJslClearance(String body) {
//V8:谷歌开源的运行JavaScript脚本的库. 参数:globalAlias=window, 表示window为全局别名,
// 告诉V8在运行JavaScript代码时, 不要从代码里找window的定义.
V8 runtime = V8.createV8Runtime("window");
//将第一次请求pdf资源时获取到的字符串提取成V8可执行的JavaScript代码
body = body.trim()
.replace("<script>", "")
.replace("</script>", "")
.replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]||(\"_\"+y)}))",
"y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]||(\"_\"+y)})");
//用V8执行该段代码获取新的动态JavaScript脚本
String result = runtime.executeStringScript(body);
//获取 jsl_clearance 的第一段, 格式形如: 1543915851.312|0|
String startStr = "document.cookie='";
int i1 = result.indexOf(startStr) + startStr.length();
int i2 = result.indexOf("|0|");
String cookie1 = result.substring(i1, i2 + 3);
/*
获取 jsl_clearance 的第二段,格式形如: DW2jqgJO5Bo45yYRKLlFbnqQuD0%3D。
主要原理是: 新的动态JavaScript脚本是为浏览器设置cookie, 且cookie名为__jsl_clearance
其中第一段值(格式形如:1543915851.312|0|)已经明文写好, 用字符串处理方法即可获取.
第二段则是一段JavaScript函数, 需要有V8运行返回,
该函数代码需要通过一些字符串定位, 提取出来, 交给V8运行.
*/
startStr = "|0|'+(function(){";
int i3 = result.indexOf(startStr) + startStr.length();
int i4 = result.indexOf("})()+';Expires");
String code = result.substring(i3, i4).replace(";return", ";");
String cookie2 = runtime.executeStringScript(code);
/*
拼接两段字符串, 返回jsl_clearance的完整的值.
格式形如: 1543915851.312|0|DW2jqgJO5Bo45yYRKLlFbnqQuD0%3D
*/
return cookie1 + cookie2;
}
/**
* 将HTTP响应体转换为字符串返回
*
* @param response HTTP响应
* @return 响应体的字符串形式
* @throws IOException IO异常
*/
private static String getResponseBodyAsString(HttpResponse response) throws IOException {
return IOUtils.toString(response.getEntity().getContent(), "UTF-8");
}
static String getJsluid(HttpResponse response) {
Header header = response.getFirstHeader("set-cookie");
String[] split = header.getValue().split(";");
for (String s : split) {
if (s.contains("__jsluid")) {
return s.trim();
}
}
return "";
}
def fristCrawl(){
def responseMap = [:]
try {
//headers信息
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","www.gsxt.gov.cn")
headers.put("Connection","keep-alive")
headers.put("Upgrade-Insecure-Requests","1")
headers.put("Cache-Control","max-age=0")
// headers.put("Content-Type","application/x-www-form-urlencoded")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
// headers.put("Cookie","__jsluid_h=b4ff8b9b0fd86b6ff99edf27e111a9f2")
headers.put("Accept-Language","zh-CN,zh;q=0.9")
// headers.put("Accept-Encoding","gzip, deflate")
//String s = doPost("http://www.gsxt.gov.cn/corp-query-homepage.html", null, headers)
def s = ABuYunUtil.doGet("http://www.gsxt.gov.cn/corp-query-homepage.html",null, headers, "utf-8")
return s
} catch (Exception e) {
e.printStackTrace()
}
return null
}
void process(Object args) {
def url = "http://www.gsxt.gov.cn/corp-query-search-1.html "
for(def key : companyKeys){
// def queryBody = "tab=ent_tab&province=&geetest_challenge=1f31bf4565a8f89b93f4f4b25bb2143a&geetest_validate=9584ff51b3aff476fcac55248ec14459&geetest_seccode=9584ff51b3aff476fcac55248ec14459%7Cjordan&token=115406527&searchword="+ URLEncoder.encode("宏美", "utf-8")
def queryBody = "tab=ent_tab&province=&geetest_challenge=daa40fb7efec814ff072b12ee12bb25a&geetest_validate=009a8748b31392ffdfb626fbbdccb4be&geetest_seccode=009a8748b31392ffdfb626fbbdccb4be%7Cjordan&token=115394764&searchword="+ URLEncoder.encode(key, "utf-8")
def s = crawl(url, queryBody);
if(s){
print s
parser(s);
}
println s
}
}
def parser(content){
def document = Jsoup.parse(content);
if(document){
def detailUrls = []
Elements elements = JQueryUtils.select([".search_list_item"], document)
if(elements && elements.size()>0){
for(def el : elements){
detailUrls << el.attr("href")
}
}
}
}
def static crawl(url, bodyquery){
def responseMap = [:]
try {
//headers信息
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","www.gsxt.gov.cn")
headers.put("Connection","keep-alive")
headers.put("Content-Length","226")
headers.put("Cache-Control","max-age=0")
headers.put("Upgrade-Insecure-Requests","1")
headers.put("Origin","http://www.gsxt.gov.cn")
headers.put("Content-Type","application/x-www-form-urlencoded")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
headers.put("Referer","http://www.gsxt.gov.cn/corp-query-homepage.html")
// headers.put("Accept-Encoding","gzip, deflate")
headers.put("Accept-Language","zh-CN,zh;q=0.9")
// headers.put("Cookie:"," __jsluid_h=4e2326174f7f9d90ad9ac1f92301f2c9; JSESSIONID=8ac4f9ab894733d7c98c9a3d5c26; SECTOKEN=7446451709198207648; __jsl_clearance=1618823989.806|0|jQc7QRg%2B%2FzcIlz44iNwQg0PjZJg%3D; gsxtBrowseHistory1=%0FS%04%06%1D%04%1D%10SNS%24%26%3B%22%3D%3A71%3A%3B01%3A%219FFDDDDFFDFE%40DDDDEFDECDLE%40L%40L%40GSXS%11%1A%00%1A%15%19%11SNS%E5%91%BD%E6%9F%A3%E5%B9%B6%E6%97%84%E8%AE%AE%E7%AF%95%E5%BA%BE%E5%BA%8E%E8%AF%8A%E7%AF%95%E7%91%B2%E6%9D%BD%E9%98%A4%E5%84%98%E5%8E%8CSXS%11%1A%00%00%0D%04%11SNEEMDXS%02%1D%07%1D%00%00%1D%19%11SNEBELLFGCDL%40FG%09; tlb_cookie=S172.16.12.115")
String s = doPost(url, bodyquery, headers)
return s
} catch (Exception e) {
e.printStackTrace()
}
return null
}
static String doPost(String strUrl, String content, Map headers) {
String result = "";
OutputStream outputStream = null;
InputStream inputStream = null;
StringBuilder sb = new StringBuilder();
try {
checkAbuyun(true);
def responseMap = [:]
// 创建代理服务器地址对象
InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
// 创建HTTP类型代理对象
Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
URL url = new URL(strUrl);
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestMethod("POST");
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setConnectTimeout(30000);
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setReadTimeout(30000);
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
urlConnection.setDoOutput(true);
urlConnection.setDoInput(true);
//设置通用请求属性为默认浏览器编码类型
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
urlConnection.setRequestProperty(head, headers.get(head));
}
}
//getOutputStream()返回的输出流,用于写入参数数据。
outputStream = urlConnection.getOutputStream();
if(content){
outputStream.write(content.getBytes());
}
outputStream.flush();
outputStream.close();
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
inputStream = urlConnection.getInputStream();
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
char[] buff = new char[1024];
int length = 0;
while ((length = reader.read(buff)) != -1) {
String s = new String(buff, 0, length);
sb.append(s);
}
} catch (IOException e) {
e.printStackTrace();
}finally{
if(outputStream!=null){
outputStream.close();
}
if(inputStream!=null){
inputStream.close();
}
}
return sb.toString();
}
public static checkAbuyun(boolean check) {
if (check) {
while (AbuyunRequestCount.get() >= AbuyunMaxCount) {
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
e.printStackTrace()
}
}
AbuyunRequestCount.incrementAndGet();
} else {
AbuyunRequestCount.decrementAndGet();
}
}
}
@@ -61,10 +61,10 @@ class DealPriceMathBatch extends BaseBatch{
districtMap.putAll(detailMap)
}
}
def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2018-11-01","yyyy-MM-dd"))
def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2020-01-01","yyyy-MM-dd"))
// def startDate = DateUtil.getFirstDayOfMonth(new Date())
// def endDate = DateUtil.getFirstDayOfMonth(new Date(), 1)
def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2019-12-31","yyyy-MM-dd"))
def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2021-04-30","yyyy-MM-dd"))
while(flag){
println "开始加载解析数据......"
@@ -116,7 +116,7 @@ class DealPriceMathBatch extends BaseBatch{
count++;
}catch (Exception e){
println "处理:【${dbo.city}-${dbo.region}-${dbo.district_name}】发生异常,异常信息为:"+e.getMessage()
println "处理:【${dbo.city}-${dbo.region}-${dbo.district_name}】发生异常,异常信息为:"+e.printStackTrace()
}finally {
latch.countDown()
}
@@ -442,16 +442,21 @@ class DealPriceMathBatch extends BaseBatch{
if(regionMap){
regionMap.each{k,v->
def arrays = k.split("###")
if(arrays[1] && arrays[1]==keyword){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
try{
if(arrays && arrays.size()>=2 && arrays[1] && arrays[1]==keyword){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
}
}catch(Exception e){
e.printStackTrace()
}
}
}
}else{//无区域
cityMap.each{r,dMap->
dMap.each{k,v->
def arrays = k.split("###")
if(arrays[1] && arrays[1]==keyword){
if(arrays && arrays.size()>=2 && arrays[1] && arrays[1]==keyword){
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
}
}
@@ -596,10 +601,13 @@ class DealPriceMathBatch extends BaseBatch{
}
def getOpenApiMatch(def city, def region, def name){
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, "fuzzy", false)
if(jsonObject && jsonObject.credit>0 ){
return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address]
if(name){
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null,"fuzzy", false)
if(jsonObject && jsonObject.credit>0 ){
return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address]
}
}
return null;
}
@@ -121,15 +121,15 @@ class AbcEvaluationRowsBatch extends BaseBatch{
houseLoudng.fitment_cost = ret.fitment_charge
houseLoudng.car_port = ret.carport
houseLoudng.house_height = ret.house_hight
houseLoudng.worry_facility" : null,
houseLoudng.fitment_age = ret.fitment_age
houseLoudng.lighting_ventilation" : null,
houseLoudng.build_type = ret.build_type
houseLoudng.house_property" : null,
houseLoudng.noise" : null,
houseLoudng.sight" : null,
houseLoudng.fitment_rank" : null,
houseLoudng.sideset" : null,
// houseLoudng.worry_facility" : null,
// houseLoudng.fitment_age = ret.fitment_age
// houseLoudng.lighting_ventilation" : null,
// houseLoudng.build_type = ret.build_type
// houseLoudng.house_property" : null,
// houseLoudng.noise" : null,
// houseLoudng.sight" : null,
// houseLoudng.fitment_rank" : null,
// houseLoudng.sideset" : null,
houseLoudng.district_no = district._id
@@ -352,7 +352,7 @@ class HouseExportBatch extends BaseBatch{
def rets = []
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-2020*", queryBuilder);
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "latest_seawater", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
File diff suppressed because it is too large Load Diff
@@ -25,6 +25,7 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
import java.util.zip.GZIPInputStream
@@ -36,16 +37,17 @@ class GaoDeMapCrawl {
public static final def dbUrl = "mongodb://192.168.0.235:27018";
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(5)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
final AtomicInteger count = new AtomicInteger()
def updateLists = []
def updateErrorLists = []
def flagLists = []
def index = 0
def prefix = "road_"
def prefix = "district_"
// def categorys = ["141202", "141203"]//中学,小学
// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
def categorys = ["190301"]//道路相关
def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
// def categorys = ["190301"]//道路相关
def totalCount = new AtomicLong()
static main(args) {
def task = new GaoDeMapCrawl();
@@ -109,26 +111,13 @@ class GaoDeMapCrawl {
def run(){//,"西安","重庆"
// def citys = [ "临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄","莱芜","济南","青岛","威海","烟台","潍坊","淄博"]
// def citys = ["北京","广州","深圳","合肥","苏州","西安","重庆"]
def citys = CommonUtils.findAllCity()
def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
// def citys = CommonUtils.findAllCity()
def totalCount = 0;
for(def city : citys){
def cityLations = findAllLation(city);
println "开始处理["+city +"]:"+cityLations.size()
gaoDeCitySearch(cityLations);
if(updateLists.size() > 0){
println " 正常访问共:"+updateLists.size()
updateBatch();
}
if(updateErrorLists.size() > 0){
println " 失败访问共:"+updateErrorLists.size()
updateErrorBatch();
}
if(flagLists.size()>0){
println "有数据的格子共:"+flagLists.size()
updateFlagBath();
}
}
}
@@ -145,39 +134,68 @@ class GaoDeMapCrawl {
}
def gaoDeCitySearch(def cityLocations){
try {
for(def location : cityLocations){
index ++
// def bound_gd = location.bound_gd
// final CountDownLatch latch = new CountDownLatch(cityLocations.size())
for(def i=0;i<cityLocations.size();i++){
def location = cityLocations[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
// try{
// totalCount.addAndGet(1)
searching(location)
// }catch(Exception e){
// e.printStackTrace()
// } finally {
// latch.countDown()
// }
// }
// });
}
// latch.await()
if(updateLists.size() > 0){
println " 正常访问共:"+updateLists.size()
updateBatch();
}
if(updateErrorLists.size() > 0){
println " 失败访问共:"+updateErrorLists.size()
updateErrorBatch();
}
if(flagLists.size()>0){
println "有数据的格子共:"+flagLists.size()
updateFlagBath();
}
}
def searching(location){
// def bound_gd = location.bound_gd
// def lats = [bound_gd[1], bound_gd[3]]
// def lngs = [bound_gd[0], bound_gd[2]]
// lats.sort()
// lngs.sort()
// println index
// if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){
search(location)
search(location)
// }
if(updateLists.size() > 0 && updateLists.size()>1000){
println " 正常访问共:"+updateLists.size()
updateBatch();
}
if(updateErrorLists.size() > 0 && updateErrorLists.size()>1000){
println " 失败访问共:"+updateErrorLists.size()
updateErrorBatch();
}
if(flagLists.size()>0 && flagLists.size()>1000){
println "有数据的格子共:"+flagLists.size()
updateFlagBath();
}
}
}catch (Exception e){
e.printStackTrace()
if(updateLists.size() > 0 && updateLists.size()>1000){
println " 正常访问共:"+updateLists.size()
updateBatch();
}
if(updateErrorLists.size() > 0 && updateErrorLists.size()>1000){
println " 失败访问共:"+updateErrorLists.size()
updateErrorBatch();
}
if(flagLists.size()>0 && flagLists.size()>1000){
println "有数据的格子共:"+flagLists.size()
updateFlagBath();
}
}
def search(location){
def bound_gd = location.bound_gd
def lats = [bound_gd[1], bound_gd[3]]
@@ -198,9 +216,11 @@ class GaoDeMapCrawl {
if(m.rets){
processData(m.rets);
location.count = m.rets.size()
location.content = m.toString()
flagLists.add(location)
}else{
location.count = 0
location.content = m.toString()
flagLists.add(location)
}
@@ -210,7 +230,7 @@ class GaoDeMapCrawl {
updateErrorLists.addAll(location)
}
// update(location);
println "[ "+count.get()+" ${index}]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status
println "[ "+totalCount.get()+" ]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status
}
def processData(rets){
@@ -336,7 +356,7 @@ class GaoDeMapCrawl {
for(def location : updateLists){
if(location && location._id){
Document updateDocument = new Document();
def doc = new Document(prefix+"status","完成").append("m_date", new Date())
def doc = new Document(prefix+"status","完成").append("m_date", new Date()).append("content", location.content)
updateDocument.append('$set', doc);
Bson updateBson = Filters.eq("_id", location._id);
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
@@ -358,7 +378,7 @@ class GaoDeMapCrawl {
for(def location : flagLists){
if(location && location._id){
Document updateDocument = new Document();
def doc = new Document(prefix+"gd_data_count",location.count).append("m_date", new Date())
def doc = new Document(prefix+"gd_data_count",location.count).append("m_date", new Date()).append("content", location.content)
updateDocument.append('$set', doc);
Bson updateBson = Filters.eq("_id", location._id);
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
@@ -382,7 +402,7 @@ class GaoDeMapCrawl {
for(def location : updateErrorLists){
if(location && location._id){
Document updateDocument = new Document();
def doc = new Document(prefix+"status","失败").append("m_date", new Date())
def doc = new Document(prefix+"status","失败").append("m_date", new Date()).append("content", location.content)
updateDocument.append('$set', doc);
Bson updateBson = Filters.eq("_id", location._id);
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
@@ -0,0 +1,301 @@
package com.fangjia.script.gj
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import org.apache.lucene.queryparser.classic.QueryParser
class LogStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new LogStat()
task.run()
// task.run1()
}
def run1(){
ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\案例信息-待反馈评估过程_V2(无颜色).xls", { d ->
def city = d['C'].replace("市","")
def district = CommonDatautils.findDistrict(city, d['AA'], d['AB'], "district")
if(!district){
println "sss"
}
if(d['AT'] && district){
def startDate2 = DateUtil.add(d['X'], -365)
def start = DateUtil.formatByPattern(startDate2,"yyyyMMdd")
def end = DateUtil.formatByPattern(d['X'],"yyyyMMdd")
def alias = []
if(district.alias){
if(!district.alias.contains(district.name)){
district.alias << district.name
}
alias = district.alias
}else{
alias << district.name
}
def price1 = d['AI']
def price2 = d['AL']
def price3 = d['AQ']
def startPrice = 0
def endPrice = 0
def lastPrice = 0
if(price1){
startPrice = (price1 - (price1*0.15)).toInteger()
endPrice = (price1 + (price1*0.15)).toInteger()
lastPrice = price1
}else if(price2){
startPrice = (price2 - (price2*0.15)).toInteger()
endPrice = (price2 + (price2*0.15)).toInteger()
lastPrice = price2
}else if(price3){
startPrice = (price3 - (price3*0.15)).toInteger()
endPrice = (price3 + (price3*0.15)).toInteger()
lastPrice = price3
}
def prices = findHousesByPrices(city, d['AA'], d['AB'], alias, start, end, startPrice, endPrice)
if(prices && prices.size()>=3){
def houseAvgPrice = (prices.sum()/prices.size()).toInteger()
d['AD'] = houseAvgPrice
d['AC'] = (houseAvgPrice * d['N']).toInteger()
}else{
def random = new Random().nextInt()%6
def randomAvgPrice = lastPrice*random/100 + lastPrice
d['AD'] = randomAvgPrice
d['AC'] = (randomAvgPrice * d['N']).toInteger()
}
}
},0);
}
def run(){
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\案例信息-待反馈评估过程_V2(无颜色).xls",{ d ->
// def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd")
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
// def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend)
// if(dbo){
// d['E'] = dbo.avg_price
// }
if(d['AA'] && d['AB'] && d['AH']){
def city = d['C'].replace("市","")
def district = CommonDatautils.findDistrict(city, d['AA'], d['AB'], "district")
if(district){
def alias = []
if(district.alias){
if(!district.alias.contains(district.name)){
district.alias << district.name
}
alias = district.alias
}else{
alias << district.name
}
// def startDate = DateUtil.add(d['X'], -90)
// def endDate = DateUtil.add(d['X'], 5)
// def trades = findTrades(district, startDate, endDate)
// if(trades){
// d['AJ'] = (int)(trades.sum()/trades.size())
// d['AK'] = trades.size()
// }
//
// def monthStartDate = DateUtil.getFirstDayOfMonth(d['X'])
// def monthEndDate = DateUtil.getFirstDayOfMonth(DateUtil.parseFormatDate(DateUtil.addMonth(d['X'], 1)))
// def monthStart = DateUtil.formatByPattern(monthStartDate,"yyyyMMdd").toInteger()
// def monthEnd = DateUtil.formatByPattern(monthEndDate,"yyyyMMdd").toInteger()
// def prices = findHouses(city, d['AA'], d['AB'], alias, monthStart, monthEnd)
// if(prices){
// d['AL'] = (int)(prices.sum()/prices.size())
// d['AM'] = prices.size()
// }
//
//
// def sDate = DateUtil.formatByPattern(monthStartDate,"yyyyMM").toInteger()
// def dbo = findThirdPrice(district.city, district.region, district.name, sDate, "lianjia_esf_price")
// if(dbo){
// d['AN'] = dbo.lianjia_esf_price
// }
def price = d['AH']
def startPrice = (price - (price*0.15)).toInteger()
def endPrice = (price + (price*0.15)).toInteger()
def startDate2 = DateUtil.add(d['X'], -365)
def start = DateUtil.formatByPattern(startDate2,"yyyyMMdd")
def end = DateUtil.formatByPattern(d['X'],"yyyyMMdd")
def count = findHouseCount(city, d['AA'], d['AB'], alias, start, end, startPrice, endPrice)
if(count > 0){
d['AS'] = count
}
}
}
},0)
}
def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
q.put("source", "链家在线")
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
}
if(alias){
q.put("district_name", new BasicDBObject('$in', alias))
}
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def prices = []
for(def d : dbcorsor){
if(d.avg_price){
prices << d.avg_price
}
}
return prices
}
def findHouses(def city, def region, def name, def alias, def startDate, def endDate){
def aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select avg_price from latest_seawater where cat='sell' and root='lianjia.com' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
def prices = []
ElasticUtils.executeSqlQuery(sql,{data->
prices << data.avg_price
})
return prices
}
def findHouseCount(def city, def region, def name, def alias, def startDate, def endDate, def startprice, def endprice){
def aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and avg_price>=${startprice} and avg_price<=${endprice}"
def total = 0
ElasticUtils.executeSqlQuery(sql,{data->
total = data.c
})
return total
}
def findHousesByPrices(def city, def region, def name, def alias, def startDate, def endDate, def startprice, def endprice){
def aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and avg_price>=${startprice} and avg_price<=${endprice}"
def prices = []
ElasticUtils.executeSqlQuery(sql,{data->
prices << data.avg_price
})
return prices
}
/**
* 转义字符串中的特殊字符
* 仅过滤Lucene regexps, 其他的勿用
* @param keyword
* @return
*/
public static String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
// def findTestDatas(def city, def value) {
// def q = new BasicDBObject()
// q.put("user_id", "1007751")
// q.put("parameter_map.city", city)
// q.put("request_uri", "/property/evaluateForTest")
//
// def keys = new BasicDBObject()
// keys.put("res_map",1)
// def cursor = LOG.find(q)
// def count = 0
// println "总量:"+cursor.count()
// def datas = []
// for(def d : cursor){
// if(d && d.code==200 && d.res_map && d.res_map.code==200){
// def resMap = d.res_map.result
// if(resMap.calcType==value){
// def data = []
// def rowMap = [:]
// rowMap.city = resMap.city
// rowMap.region = resMap.district
// rowMap.name = resMap.name
// rowMap.calDate = d.parameter_map.calDate
// data << rowMap
// }
// }
// }
// return datas;
// }
// for(def i=0;i<datas.size();i++){
// def d = datas[i];
// def calDate = DateUtil.parseDate(d['D'], "yyyy-MM-dd")
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
// def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], weekend)
// if(dbo){
// d['E'] = dbo.avg_price
// }
//
// def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
// if(district){
// def startDate = DateUtil.addMonth(calDate, -6)
// def start = startDate.replace("-","").toInteger()
// def end = DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
// def alias = []
// if(district.alias){
// if(!district.alias.contains(district.name)){
// district.alias << district.name
// }
// alias = district.alias
// }else{
// alias << district.name
// }
// def count = findHouses(d['A'], d['B'], d['C'], district.alias, start, end)
// d['G'] = count
// }
def findThirdPrice(def city, def region, def name, def s_date,def fieldName){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", s_date)
q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1))
def obj = null;
for(def d : dbcursor){
obj = d;
break;
}
return obj;
}
}
@@ -47,7 +47,7 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-02-01"
static def statDate = "2021-03-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2021年3月\\二手房\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2021年4月\\二手房\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -0,0 +1,175 @@
package com.fangjia.script.gj.address
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import net.sf.json.JSONObject
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
class AddressParser {
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new AddressParser();
task.run()
// task.gaoDeAddress()
}
def gaoDeAddress(){
def datas = findAllDatas()
final CountDownLatch latch = new CountDownLatch(datas.size())
final AtomicInteger count = new AtomicInteger()
for(def i=0;i<datas.size;i++){
def data = datas[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
try {
count.addAndGet(1)
println count.get() +" "+data.district_name
match2(data._id, data.city, data.region, data.school_address, data.school_name);
} catch (Exception e) {
println "异常"
e.printStackTrace();
} finally {
latch.countDown()
}
}
}
);
}
latch.await()
}
def run(){
def datas = findAllDatas()
final CountDownLatch latch = new CountDownLatch(datas.size())
final AtomicInteger count = new AtomicInteger()
for(def i=0;i<datas.size;i++){
def data = datas[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
try {
count.addAndGet(1)
println count.get() +" "+data.district_name
match(data._id, data.city, data.region, data.school_address, data.school_name);
} catch (Exception e) {
println "异常"
e.printStackTrace();
} finally {
latch.countDown()
}
}
}
);
}
latch.await()
}
def match(id, city, region, address, name){
def locationMap = null;
// if(region && !name.contains(region)){
// locationMap = matchLocation(city, region + name, "name")
// }else{
locationMap = matchLocation(city + region, name, "name")
// }
// if(!locationMap){
// locationMap = matchLocation(city, address, "address")
// }
if(locationMap){
DBObject data = new BasicDBObject()
data.put("school_lat1", locationMap.glat)
data.put("school_lng1", locationMap.glng)
district_school_col.update(new BasicDBObject('_id', id), new BasicDBObject('$set', data))
}
}
def match2(id, city, region, address, name){
def locationMap = null;
locationMap = matchLocation2(city + region, name, "name")
if(!locationMap){
locationMap = matchLocation2(city, address, "address")
}
if(locationMap){
DBObject data = new BasicDBObject()
data.put("gaode_name", locationMap.name)
data.put("gaode_address", locationMap.address)
if(locationMap.location){
def arrs = locationMap.location.split(",")
data.put("school_lat", arrs[1].toDouble())
data.put("school_lng", arrs[0].toDouble())
}
district_school_col.update(new BasicDBObject('_id', id), new BasicDBObject('$set', data))
}
}
def matchLocation(city, address, cat){
JSONObject gloJson = GaoDeApiUtils.geo(city, address)
def map = [:]
if (gloJson && gloJson.get("geocodes")) {
def geocodes = gloJson.get("geocodes")
def datas = geocodes.get(0)
if(datas.formatted_address && cat=="name"){
if(datas.formatted_address.contains(address)){
def glocation = datas.location
def los = glocation.split(",")
map.glng = Double.parseDouble(los[0].toString())
map.glat = Double.parseDouble(los[1].toString())
}else{
return null
}
}else{
def glocation = geocodes.get(0).location
def los = glocation.split(",")
map.glng = Double.parseDouble(los[0].toString())
map.glat = Double.parseDouble(los[1].toString())
}
}
return map
}
def matchLocation2(city, address, cat){
//JSONObject gloJson = GaoDeApiUtils.geo(city, address)
JSONObject gloJson = GaoDeApiUtils.search(address, city, '140000')
def map = [:]
if (gloJson && gloJson.get("pois")) {
def geocodes = gloJson.get("pois")
def datas = geocodes.get(0)
return datas
}
return null
}
def findAllDatas(){
def q = new BasicDBObject()
// q.put("school_name","苏州市娄葑实验小学")
// q.put("school_lng", new BasicDBObject(QueryOperators.EXISTS, false))
def dbcursor = district_school_col.find(q)
def rets =[]
for(def d : dbcursor){
rets << d
}
return rets;
}
}
@@ -0,0 +1,354 @@
package com.fangjia.script.gj.ccb
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
class DistrictSchoolDataExport {
def cityCodeMap = ["苏州":"320500","合肥":"340100", "西安":"610000"]
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def regionCodeMap = [:]
static main(args) {
def task = new DistrictSchoolDataExport()
// task.run()
task.megre()
}
def megre(){
def datas1 = ExcelUtils.read("d:\\数据\\楼盘对口学校_V2.xlsx",0,0)
def ids = []
for(def d : datas1){
ids << d['U']
}
def datas2 = ExcelUtils.read("d:\\数据\\无学区房数据(1).xlsx",0,1)
for(def d : datas2){
def id = d['U']
if(!ids.contains(id)){
datas1 << d
}
}
ExcelUtils.write("d:\\数据\\楼盘对口学校V1.xlsx", datas1, "数据")
}
def run(){
def city = "苏州"
getRegionCodeByCity(city)
getRegionCodeByCity("合肥")
getRegionCodeByCity("西安")
def districts0 = CommonDatautils.findAllDistricts("苏州", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
def districts1 = CommonDatautils.findAllDistricts("合肥", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
def districts2 = CommonDatautils.findAllDistricts("西安", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
def districts = []
districts.addAll(districts0)
districts.addAll(districts1)
districts.addAll(districts2)
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def rets = []
rets << ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站","小区ID","数据类型"]
def i = 0
for(def k=0;k<districts.size();k++){
final def dist = districts.get(k)
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def primarySchools = findPrimarySchool(dist.city, dist.region, dist.name)
def midSchools = findMiddleSchool(dist.city, dist.region, dist.name)
println totalCount.get() +" "+dist.name +" 小学:"+primarySchools.size()+" 初中:"+midSchools.size()
if(primarySchools || midSchools){
def info1 = null;
def info2 = null;
def distance1 = 0
def distance2 = 0
if(primarySchools && dist.lng){
info1 = selectSchool(dist.lng/10000000000.0, dist.lat/10000000000.0, primarySchools);
if(info1 && info1.school_lat2){
double lat = dist.lat/10000000000.0
double lng = dist.lng/10000000000.0
distance1 = GeoHelper.GetDistance(lat, lng, info1.school_lat2.toDouble(), info1.school_lng2.toDouble())
if(distance1>3000){
info1 = null
}
}else{
info1 = null
}
}
if(midSchools) {
info2 = selectSchool(dist.lng / 10000000000.0, dist.lat / 10000000000.0, midSchools);
if(info2 && info2.school_lng2){
double lat = dist.lat/10000000000.0
double lng = dist.lng/10000000000.0
distance2 = GeoHelper.GetDistance(lat, lng, info2.school_lat2, info2.school_lng2)
if(distance2>3000){
info2 = null
}
}else{
info2 = null
}
}
if(info1|| info2){
def row = []
row << cityCodeMap.get(dist.city)
row << regionCodeMap.get(dist.city +"-"+dist.region)
row << dist.name
row << dist.lng/10000000000.0
row << dist.lat/10000000000.0
row << dist.address
if(info1){
row << info1.school_name
row << info1.school_address!=null?info1.school_address:''
row << info1.attribute!=null?info1.attribute:''
row << info1.school_lng2!=null?info1.school_lng2:''
row << info1.school_lat2!=null?info1.school_lat2:''
// row << distance1
}else{
row << ""
row << ""
row << ""
row << ""
row << ""
// row << ""
}
if(info2){
row << info2.school_name
row << info2.school_address!=null?info2.school_address:''
row << info2.attribute!=null?info2.attribute:''
row << info2.school_lng2!=null?info2.school_lng2:''
row << info2.school_lat2!=null?info2.school_lat2:''
// row << distance2
}else{
row << ""
row << ""
row << ""
row << ""
row << ""
// row << ""
}
row << ""
row << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
row << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
row << "fjw"
row << dist._id.toString()
row << "对口"
rets << row
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
ExcelUtils.write("d:\\数据\\楼盘对口学校_V2.xlsx", rets, "数据")
}
def selectSchool(lng, lat, def schools){
def minSchool = null
def level0 = []//国家
def level1 = []//省重点
def level2 = []//市重点
def level3 = []//区重点
def level4 = []//普通
def level5 = []//其它
for(def s : schools){
if(s.attribute){
if(s.attribute.contains("国家重点")){
level0 << s
}else if(s.attribute.contains("省重点")){
level1 << s
}else if(s.attribute.contains("市重点")){
level2 << s
}else if(s.attribute.contains("区重点")){
level3 << s
}else if(s.attribute.contains("普通")){
level4 << s
}else{
level5 << s
}
}else{
level5 << s
}
}
if(level0){
if(level0.size()==1){
minSchool = level0.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level0)
}
}else if(level1){
if(level1.size()==1){
minSchool = level1.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level1)
}
}else if(level2){
if(level2.size()==1){
minSchool = level2.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level2)
}
}else if(level3){
if(level3.size()==1){
minSchool = level3.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level3)
}
}else if(level4){
if(level4.size()==1){
minSchool = level4.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level4)
}
}else if(level5){
if(level5.size()==1){
minSchool = level5.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level5)
}
}
def rets = []
if(minSchool){
rets = findSchoolInfo(minSchool.city, minSchool.match_region,minSchool.school_name)
}
if(rets.size()>0){
for(def d : rets){
if(!minSchool.school_address){
minSchool.school_address = d.address
}
if(!minSchool.attribute){
minSchool.attribute = d.attribute
}
}
}else{
if(minSchool){
if(minSchool.school_map_lng){
minSchool.school_lng2 = minSchool.school_map_lng
minSchool.school_lat2 = minSchool.school_map_lat
}else if(minSchool.school_lng1){
minSchool.school_lng2 = minSchool.school_lng1
minSchool.school_lat2 = minSchool.school_lat1
}else if(minSchool.school_lng){
minSchool.school_lng2 = minSchool.school_lng
minSchool.school_lat2 = minSchool.school_lat
}
}
}
return minSchool;
}
def findSchoolInfo(def city, def region, def schoolName){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", schoolName)
def dbcursor = school_col.find(q)
def rets = []
for(def obj : dbcursor){
rets << obj
}
return rets
}
def selectMinDistanceSchool(def lat, def lng, def schools){
def minSchool = null;
def minDistance = Integer.MAX_VALUE
for(def s1 : schools){
def lng2 = null
def lat2 = null
if(s1.school_map_lng){
lng2 = s1.school_map_lng
lat2 = s1.school_map_lat
}else if(s1.school_lng1){
lng2 = s1.school_lng1
lat2 = s1.school_lat1
}else if(s1.school_lng){
lng2 = s1.school_lng
lat2 = s1.school_lat
}
if(lat2){
def distance = GeoHelper.GetDistance(lat, lng, lat2, lng2)
println s1.district_name +" "+ s1.school_name+" "+distance
if(distance<minDistance){
minDistance = distance
minSchool = s1
minSchool.school_lng2 = lng2
minSchool.school_lat2 = lat2
}
}
}
return minSchool
}
def findPrimarySchool(def city, def region, def name){
def q = new BasicDBObject()
q.put("match_city", city)
q.put("match_region", region)
q.put("match_name", name)
q.put("category", Pattern.compile("^.*小学.*\$", Pattern.CASE_INSENSITIVE))
def dbcursor = district_school_col.find(q).sort(new BasicDBObject("attribute", 1))
def datas = []
for(def db : dbcursor){
datas << db
}
return datas;
}
def findMiddleSchool(def city, def region, def name){
def q = new BasicDBObject()
q.put("match_city", city)
q.put("match_region", region)
q.put("match_name", name)
q.put("category", new BasicDBObject(QueryOperators.NE, "小学"))
def dbcursor = district_school_col.find(q).sort(new BasicDBObject("attribute", 1))
def datas = []
for(def db : dbcursor){
datas << db
}
return datas;
}
def getRegionCodeByCity(def city){
def datas = FangJiaAreaMapUtils.convertToOfficials(city);
for(def d : datas){
regionCodeMap.put(city+"-"+d.fangjia_district, d.district_code)
}
}
}
@@ -11,7 +11,7 @@ class FieldStat {
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区融合结果概况2.xlsx", { row ->
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\½¨ÐÐÊý¾ÝÈںϽá¹ûͳ¼Æ_0425.xlsx", { row ->
def city = row['A']
if(city){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]);
@@ -26,15 +26,14 @@ class FieldStat {
aliasTotalCount = aliasTotalCount + alias.size()
}
}
row['J'] = aliasTotalCount
row['K'] = aliasIndex
// row['J'] = aliasTotalCount
// row['K'] = aliasIndex
def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
row['L'] = p+"%"
row['M'] = total
row['B'] = p+"%"
// row['M'] = total
println city +" "+p
}
});
}, 1);
println "finish"
}
}
@@ -0,0 +1,48 @@
package com.fangjia.script.gj.data
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class RegionPriceExport {
DBCollection REGION_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "region_sell_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new RegionPriceExport();
task.run()
}
def run(){
def months = [202010,202011,202012,202101,202102,202103]
def regions = CommonUtils.getAllRegions("福州","region")
def rets = []
rets << ["城市","区域","202010","202011","202012","202101","202102","202103"]
regions.each{region,values->
def data = []
data << "福州"
data << region
for(def month :months){
def dbo = findPrice("福州", region, month);
if(dbo){
data << dbo.avg_price
}else{
data << ""
}
}
rets << data
}
ExcelUtils.write("d:\\数据\\福州\\福州区域月均价数据.xlsx",rets, "价格")
}
def findPrice(city, region, month){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("date", month)
def dbo = REGION_SELL_STAT.findOne(q)
return dbo
}
}
File diff suppressed because one or more lines are too long
@@ -0,0 +1,580 @@
package com.fangjia.script.gj.test
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicLong
class CityStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", lineUrl)
public static final DBCollection BAD_CASE_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "bad_case", lineUrl)
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
def task = new CityStat()
// task.run()
// task.run1();
task.run3();
// task.run4()
// task.run6()
// task.run7()
// task.run8()
}
// def run8(){
// ExcelUtils.multiThreadingModify("D:\\数据\\深圳建行\\建行匹配小区(3).xlsx",{d->
//
// def dist = CommonDatautils.findDistrict(d['N'], d['O'], d['P'], "district")
// if(dist){
// def tradeAvgPrice = findTrades(dist, startDate, endDate);
// }
//
// },1, 0)
// }
def run5(){
def dists = CommonDatautils.findAllDistrictByCity("常州","district",["city", "region","name","alias"])
def total = 0
def scount = 0
def index = 0
// final CountDownLatch latch = new CountDownLatch(districts.size())
for(def dist : dists){
index++
def rets = findTradeDetails(dist, "2020-03-24");
println index
if(rets){
total = total + rets.size()
def count = findSimialPriceCount(dist, rets);
scount = scount+count
}
}
println scount +" "+ total
def pre = DataHandleUtils.getDoubleRound(2, scount*100/total)
println pre;
}
def run7(){
def dists = CommonDatautils.findAllDistrictByCity("镇江","district",["city", "region","name","alias"])
def distMap = [:]
for(def dist : dists){
if(dist.alias) {
if (!dist.alias.contains(dist.name)) {
dist.alias << dist.name
}
for(def name : dist.alias){
distMap.put( dist.region +"#"+name, dist)
}
}else{
distMap.put( dist.region +"#"+dist.name, dist)
}
}
def rets = findTradeDetails2("镇江","2020-03-24");
def total = 0
def scount = 0
def index = 0
for(def t : rets){
def key = t.region+"#"+t.district_name
def dist = distMap.get(key)
index++
println index
if(dist){
total++
def startDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),-7)
def endDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),7)
def weekendPrice = CommonDatautils.queryDistrictPriceByDate(dist.city, dist.region, dist.name, startDate, endDate)
if(t.avg_price && weekendPrice){
def mod = (weekendPrice - t.avg_price)*100/t.avg_price
def m = Math.abs(mod)
if(m<=15){
scount++;
}
}
}
}
println scount +" "+total
}
def run6(){
def dists = CommonDatautils.findAllDistrictByCity("常州","district",["city", "region","name","alias"])
final CountDownLatch latch = new CountDownLatch(dists.size())
def totalCount = new AtomicLong()
def tradeTotal = new AtomicLong()
def scount = new AtomicLong()
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def rets = findTradeDetails(dist, "2020-03-24");
if(rets){
// tradeTotal = tradeTotal + rets.size()
tradeTotal.addAndGet(rets.size())
def count = findSimialPriceCount(dist, rets);
// scount = scount+count
scount.addAndGet(count);
}
println totalCount.get() +" "+dist.name
}catch (Exception e){
}finally {
latch.countDown()
}
}
});
}
latch.await()
println scount +" "+ tradeTotal
def pre = DataHandleUtils.getDoubleRound(2, scount*100/tradeTotal)
println pre;
}
def findSimialPriceCount(def dist, def rets){
def count = 0
for(def t : rets){
def startDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),-7)
def endDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),7)
def weekendPrice = CommonDatautils.queryDistrictPriceByDate(dist.city, dist.region, dist.name, startDate, endDate)
if(t.avg_price && weekendPrice){
def mod = (weekendPrice - t.avg_price)*100/t.avg_price
def m = Math.abs(mod)
if(m<=15){
count++;
}
}
// def weekend = DateUtils.getFirstDayOfWeek()
// def stat = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, weekend)
// if(t.avg_price && stat){
// def mod = (stat.avg_price - t.avg_price)*100/t.avg_price
// def m = Math.abs(mod)
// if(m<=15){
// count++;
// }
// }
}
return count;
}
def run3(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行银行\\建行匹配小区17.xlsx",{data->
if(data['A']){
def startDate = DateUtils.addMonth(data['D'], -3)
def endDate = DateUtils.addMonth(data['D'], 3)
def dist = CommonDatautils.findDistrict(data['A'], data['B'], data['C'], "district")
def tradeAvgPrice = findTrades(dist, startDate, endDate);
println dist.city +" "+dist.region+" "+dist.name +" "+tradeAvgPrice.toInteger()
data['H'] = tradeAvgPrice.toInteger()
}
})
}
def run4(){
def citys = ["上海","北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"]
def startDate = DateUtils.addMonth(new Date(), -3)
def endDate = "2021-02-01"
def result = []
def allDistricts = []
for(def city : citys) {
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias"])
allDistricts.addAll(districts)
println city +" "+districts.size()
}
def rets = findDistrictTrade(allDistricts, startDate,endDate);
println rets.size()
}
def findDistrictTrade(def districts, def startDate, def endDate){
def rets = []
final CountDownLatch latch = new CountDownLatch(districts.size())
def final AtomicInteger count_ = new AtomicInteger(0)
for(def i=0;i<districts.size;i++){
def dist = districts.get(i)
fixedThreadPool.execute(
new Runnable() {
void run() {
try{
println count_.incrementAndGet()+" "+dist.city+" "+dist.region+" "+dist.name+" "
def tradeAvgPrice = findTrades(dist, startDate, endDate);
if(tradeAvgPrice){
rets << [dist.city, dist.region, dist.name, tradeAvgPrice.toInteger()]
}
}catch (Exception e){
}finally {
latch.countDown()
}
}
}
)
}
latch.await()
ExcelUtils.write("D:\\数据\\深圳建行\\成交数据.xlsx",rets, "成交")
return rets;
}
def run(){
//"北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"
def citys = ["上海","北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"]
def rets = []
rets << ["城市","小区数量","均价覆盖率","样本计算小区均价覆盖率","badcase数量", "近三个月成交数据量","近三个月有成交数据的小区覆盖率"]
for(def city : citys) {
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name", "alias"])
def priceCount = findStatCount(city, DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd"), null);
def houseSatCount = 0//findStatCount(city, DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd"), 2);
def badcaseCount = findBadCaseCount(city, DateUtils.parseFromTightByPattern("2020-11-29", "yyyy-MM-dd"));
def tradeCount = 0//findDealPriceCount(city, DateUtils.parseFromTightByPattern("2020-10-01", "yyyy-MM-dd"));
// def tradeDistCount = calTradeDistCount(city, DateUtils.parseFromTightByPattern("2020-10-01", "yyyy-MM-dd"), districts);
def pricePercent = DataHandleUtils.getDoubleRound(2, priceCount*100/districts.size())
def houseStatPercent = 0//DataHandleUtils.getDoubleRound(2, houseSatCount*100/districts.size())
def badcasePercent = DataHandleUtils.getDoubleRound(2, badcaseCount*100/districts.size())
def tradeDistPercent = 0//DataHandleUtils.getDoubleRound(2, tradeDistCount*100/districts.size())
rets << [city, districts.size(), pricePercent+"%", houseStatPercent+"%", badcasePercent+"%", tradeCount, tradeDistPercent+"%"]
println city +" 完成"
}
ExcelUtils.write("d:\\数据\\统计\\城市统计1.xlsx", rets, "统计")
}
def run1(){
//"北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"
def citys = ["上海"]
def rets = []
rets << ["城市","区域","小区名","成交价","评估价"]
for(def city : citys) {
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias"])
def retMap = calTradeDistDetail(city,DateUtils.parseFromTightByPattern("2020-10-01", "yyyy-MM-dd"), districts);
retMap.each {k,v->
if(v && v.size()>0){
def sum = v.sum()
def tradeAvgPrice = sum/v.size()
def arrs = k.split("#")
def stat = findDistrictStat(city, arrs[0], arrs[1], DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd"))
rets << [city, arrs[0], arrs[1], tradeAvgPrice.toInteger(), stat.avg_price]
}
}
println city +" 完成"
}
ExcelUtils.write("d:\\数据\\统计\\城市成交数据.xlsx", rets, "成交")
}
def run2(){
def datas = ExcelUtils.read("D:\\数据\\深圳建行\\建行匹配小区2.xlsx",0,1)
def cityDataMap = [:]
def index = 0
for(def data : datas){
index++
println index+" "+data
def lists = cityDataMap.get(data['A'])
if(!lists){
lists = []
cityDataMap.put(data['A'], lists)
}
def dist = CommonDatautils.findDistrict(data['A'], data['B'], data['C'], "district")
dist.calDate = data['D']
dist.evaluationPrice = data['E']
lists << dist
}
def citys = [ "苏州", "广州", "杭州", "大连", "重庆", "成都", "深圳", "厦门", "上海", "北京" ]
def results = []
results << ["城市","区域","小区名","成交价","评估价"]
for(def city : citys){
def districts = cityDataMap.get(city)
def rets = findAvgPriceTrades(city, districts)
results.addAll(rets)
}
ExcelUtils.write("D:\\数据\\深圳建行\\建行成交3.xlsx",results, "数据")
}
def findAvgPriceTrades(city, districts){
def rets = []
final CountDownLatch latch = new CountDownLatch(districts.size())
for(def i=0;i<districts.size();i++){
def dist = districts[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
def startDate = DateUtils.addMonth(dist.calDate, -3)
def endDate = DateUtils.addMonth(dist.calDate, 3)
def tradeAvgPrice = findTrades(dist, startDate, endDate);
println dist.city +" "+dist.region+" "+dist.name +" "+tradeAvgPrice.toInteger()
// def stat = findDistrictStat(city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2021-01-10", "yyyy-MM-dd"))
rets << [city, dist.region, dist.name, tradeAvgPrice.toInteger(), dist.evaluationPrice]
}
}
)
}
latch.await()
return rets;
}
def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("district_name", new BasicDBObject('$in', dist.alias))
}else{
q.put("district_name", dist.name)
}
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def avg_price = 0
def total_price = 0
def count = 0
for(def d : dbcorsor){
if(d.avg_price){
total_price = total_price + d.avg_price
count++
}
}
if(count > 3){
avg_price = total_price/count
}
return avg_price
}
def findTradeDetails(dist, startDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("district_name", new BasicDBObject('$in', dist.alias))
}else{
q.put("district_name", dist.name)
}
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def rets = []
for(def d : dbcorsor){
if(d.avg_price){
rets << d
}
}
return rets
}
def findTradeDetails2(city, startDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def rets = []
for(def d : dbcorsor){
if(d.avg_price){
rets << d
}
}
return rets
}
def findStatCount(def city, def weekend, def value_type){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
if(value_type){
q.put("value_type", value_type)
}
def count = DIST_STAT_COLL.count(q)
return count;
}
def findDistrictStat(def city, def region, def name, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
q.put("weekend", weekend)
def obj = DIST_STAT_COLL.findOne(q)
return obj;
}
def findBadCaseCount(def city, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def count = BAD_CASE_COLL.count(q)
return count;
}
def findDealPriceCount(def city, def tradeDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
def count = DEAL_PRICE_COLL.count(q)
return count;
}
def calTradeDistCount(def city, def tradeDate, def districts){
def q = new BasicDBObject()
q.put("city", city)
q.put("source","链家在线")
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def distMap = [:]
for(def dist :districts){
def lists = distMap.get(dist.region)
if(!lists){
lists = []
distMap.put(dist.region, lists);
}
lists << dist
}
def rets = []
for(def obj : dbcorsor){
def dist = findIsExistsDB(obj.region, obj.district_name, distMap);
if(dist && !rets.contains(dist)){
rets << dist
}
}
return rets.size();
}
def findIsExistsDB(def region, def district_name, def distMap){
def dist = null;
def regionDists = distMap.get(region)
for(def rd : regionDists){
def alias = rd.alias
if(!alias){
alias = []
}
if(!alias.contains(rd.name)){
alias << rd.name
}
if(alias.contains(district_name)){
dist = rd
break;
}
}
return dist;
}
def calTradeDistDetail(def city, def tradeDate, def districts){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def distMap = [:]
for(def dist :districts){
def lists = distMap.get(dist.region)
if(!lists){
lists = []
distMap.put(dist.region, lists);
}
lists << dist
}
def retMap = [:]
for(def obj : dbcorsor){
def dist = findIsExistsDB(obj.region, obj.district_name, distMap);
if(dist){
def tradePrices = retMap.get(dist.region+"#"+dist.name)
if(!tradePrices){
tradePrices = []
retMap.put(dist.region+"#"+dist.name, tradePrices)
}
if(obj.avg_price){
tradePrices << obj.avg_price.toInteger()
}
}
}
return retMap;
}
def calTradeDistDetail2(def city, def tradeDate, def districts){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def distMap = [:]
for(def dist :districts){
def lists = distMap.get(dist.region)
if(!lists){
lists = []
distMap.put(dist.region, lists);
}
lists << dist
}
def retMap = [:]
for(def obj : dbcorsor){
def dist = findIsExistsDB(obj.region, obj.district_name, distMap);
if(dist){
def tradePrices = retMap.get(dist.region+"#"+dist.name)
if(!tradePrices){
tradePrices = []
retMap.put(dist.region+"#"+dist.name, tradePrices)
}
if(obj.avg_price){
tradePrices << obj.avg_price.toInteger()
}
}
}
return retMap;
}
}
@@ -0,0 +1,111 @@
package com.fangjia.script.gj.test
import com.fangjia.core.RegionData
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.gj.paimai.GaoDeUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import java.util.regex.Pattern
class CompanyAddressCompare {
static main(args) {
def task = new CompanyAddressCompare()
task.run()
}
def run(){
ExcelUtils.modify("D:\\数据\\公司及地址\\公司及地址.xlsx",{row->
if(row['D'] && !row['I'] && row['B'].contains("宁波甬城农村商业银行")){
def city = CommonUtils.parseCityInAddress(row["D"])
if(!city){
city = CommonUtils.parseCityInAddress(row["B"])
}
if(city){
def similarData = null
// if(isAddress(row["D"])){
def simpleAddress = row["D"].toString().replace("中国(上海)自由贸易试验区","").replaceAll(/\\d+层/,"").replaceAll(/[一二三四五六]楼/,"")
if(simpleAddress && simpleAddress.contains("号")){
simpleAddress = simpleAddress.substring(0, simpleAddress.lastIndexOf("号")+1)
}
def pois = GaoDeUtils.search(simpleAddress, city, null)
similarData = findSimilarDatas(pois, row["B"], simpleAddress, city, null)
println row['C']+" "+pois.size()
// }
if(similarData){
row['F'] = similarData.address
row['G'] = similarData.name
row['H'] = similarData.score
}
}
}
}, 0)
}
def findSimilarDatas(pois, name, address, city, rMap){
def data = null
def maxScore = 0
for(def poi : pois){
if(!poi.cityname){
def m = getMapRegion(poi.location)
if(m){
poi.cityname = m.city
poi.pname = m.province
poi.township = m.township
poi.adname = m.district
}
}
if(poi.cityname.contains(city)){
def searchName = poi.name
def nameScore = CosineSimilarAlgorithm.getSimilarityRatio(searchName, name);
def addressScore = 0
if(poi.address){
def poiAddress = poi.address.replace("号楼","号")
addressScore = CosineSimilarAlgorithm.getSimilarityRatio(poiAddress, address);
if(poi.address.contains("号") && address.contains("号") && (poiAddress.contains(address) || address.contains(poiAddress))){
addressScore = addressScore + 0.5
}
}
def totalScore = nameScore + addressScore
if(totalScore > maxScore){
maxScore = totalScore
data = poi
data.score = totalScore
}
println searchName+" "+name+" "+totalScore + " "+ (poi.address?poi.address:"")
}
}
return data;
}
def isAddress(address){
def REG_ADDR1 = "[\u4e00-\u9fa5]{2,20}[路|巷|街|弄|街道|大道|大街|巷子|公路|马路|大道西段]\\d+";
def pattern = Pattern.compile(REG_ADDR1);
def matcher = pattern.matcher(address)
if(matcher.find()){
return true;
}
return false;
}
def getMapRegion(def location){
def jsonObj = GaoDeApiUtils.geocode(location)
def district = null
if(jsonObj && jsonObj.status=="1"){
def detailMap = jsonObj.regeocode.addressComponent
if(!detailMap.city){
if(detailMap.province && detailMap.province.toString().endsWith("市")){
detailMap.city = detailMap.province
}
}
return detailMap
}
return district;
}
}
@@ -0,0 +1,102 @@
package com.fangjia.script.gj.test
import com.fangjia.core.RegionData
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class CrawlDistrictIdSelect {
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
static DBCollection districtseedsColl = MongoUtils.getCollectionByAuth20190102("seeddb", "district_seeds")
static DBCollection districtseeds2Coll = MongoUtils.getCollectionByAuth20190102("seeddb_dev", "district_seeds")
static main(args) {
def task = new CrawlDistrictIdSelect()
task.run()
// task.synch()
}
def synch(){
def q = new BasicDBObject()
def dbcursor = districtseeds2Coll.find(q)
def list = []
for(def d :dbcursor){
list << d
if(list.size()==10000){
districtseedsColl.insert(list)
list = []
}
}
if(list.size()>0){
districtseedsColl.insert(list)
}
}
def run(){
def datas = findCrawlData()
datas.each {k,v->
if(v){
for(def d : v){
d.c_date = new Date()
d.m_date = new Date()
districtseedsColl.save(d)
}
}
}
}
def findCrawlData(){
def q = new BasicDBObject()
q.put("city","福州")
q.put("source", "安居客")
q.put("s_date",new BasicDBObject('$gte', 202001))
def dbcursor = thirdPartyColl.find(q)
def retMap = [:]
def prefix = "/community/view/"
def keyList =[]
for(def db : dbcursor){
def url = null
if(!db.url){
if(db.district_id){
url = db.district_id
}
}else{
url = db.url
}
if(url && url.contains(prefix) ){
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size())
if(district_id=="0"){
continue
}
def shortCity = url.substring(url.indexOf("https://")+8, url.indexOf(".anjuke.com"))
def region = RegionData.formatRegion(db.city, db.region)
def key = shortCity +"#"+district_id
def ids = retMap.get(key)
if(!ids){
ids = []
retMap.put(key, ids)
}
if(!keyList.contains(key)){
def map = new BasicDBObject()
map.root = "anjuke.com"
map.city_short= shortCity
map.city = db.city
map.region = region
map.district_id = district_id
map.source_url = url
ids << map
keyList << key
}
}else{
println url +" 链接不合要求"
}
}
return retMap
}
}
@@ -0,0 +1,101 @@
package com.fangjia.script.gj.test
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
class ExportToDealPrice {
DBCollection DEAL_PRICE_MATCH_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price_match", "192.168.0.43")
DBCollection DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43")
DBCollection TRADE_MATCH_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "trade_match_price", "192.168.0.136")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
final AtomicInteger count = new AtomicInteger()
static main(args) {
def task = new ExportToDealPrice();
task.run()
}
def run(){
def datas = findDatas()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
final def data = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
count.addAndGet(1)
def dbo = addPrice(data)
if(dbo){
data.weekend_avg_price = dbo.avg_price
}
}catch(Exception e){
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
save(datas);
}
def save(def datas){
def ret = [];
def index = 0;
for(def dbo : datas){
ret << dbo;
index++;
if(ret.size()%10000==0){
println "µÚ "+(index/10000)+" ¸ö"+ret.size()
TRADE_MATCH_PRICE_COLL.insert(ret);
ret = [];
}
}
if(ret.size()>0){
TRADE_MATCH_PRICE_COLL.insert(ret);
}
}
def addPrice(data){
def weekend = DateUtils.getFirstDayOfWeek(data.trade_date)
def q = new BasicDBObject()
q.put("city", data.match_city)
q.put("region", data.match_region)
q.put("district_name", data.match_name)
q.put("weekend", weekend)
def dbo = DISTRICT_STAT_COLL.findOne(q)
if(!dbo){
q.put("weekend", new BasicDBObject(QueryOperators.LTE, weekend))
dbo = DISTRICT_STAT_COLL.findOne(q)
}
println count.get()+" "+data.city +" "+data.match_name
return dbo
}
def findDatas(){
def q = new BasicDBObject()
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2020-04-01","yyyy-MM-dd")))
def dbcursor = DEAL_PRICE_MATCH_COLL.find(q)
def rets = []
for(def db : dbcursor){
rets << db
}
return rets;
}
}
@@ -0,0 +1,182 @@
package com.fangjia.script.gj.test
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.fangjia.core.util.BaiDuApiUtils
import com.mongodb.BasicDBObject
import com.fangjia.script.util.DbUtils
class FZloationUpdate {
static main(args) {
def task = new FZloationUpdate();
task.run()
}
def run(){
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region","name","address","lat","lng"]);
def index = 0
def locationMap = getMoreSameLocationDist(districts)
def total = 0
locationMap.each {k,v->
total = updateLocation(v, total);
println "---------------------------------------------------------------------------------------------------------------"
}
}
def updateLocation(def lists, def total){
for(def dist : lists){
def addrDistMap = getLocation(dist, dist.address)
def nameDistMap = getLocation(dist, dist.region + dist.name)
if((addrDistMap && addrDistMap.distance>1000) && (nameDistMap && nameDistMap.distance>1000)){
println dist.region +" "+dist.name+" "+dist.address+" addrDist:"+(addrDistMap!=null?addrDistMap.distance:"")+" addrKey:"+(addrDistMap!=null?addrDistMap.keyword:"")+" nameDist:"+(nameDistMap!=null?nameDistMap.distance:"")+" nameKey:"+(nameDistMap!=null?nameDistMap.keyword:"")
// println "[${index}]"+ dist.region+" "+dist.name+" "+dist.address+" addrDist:"+addrDistMap.distance+"(${addrDistMap.level}-${addrDistMap.location}-${addrDistMap.keyword}) nameDist:"+nameDistMap.distance+"(${nameDistMap.level}-${nameDistMap.location}-${nameDistMap.keyword})"
if(addrDistMap){
def addrKey = deleteRegion(addrDistMap.keyword, dist.region).replace("|","");
def nameKey = deleteRegion(nameDistMap.keyword, dist.region);
def baidu = null
def lat = null
def lng = null
def upldateFlag = ""
if(addrDistMap && addrDistMap.region.contains(dist.region) && (addrKey.contains("路") || addrKey.contains("大道")) && addrKey.contains("号") && dist.addr == addrKey){
lat = addrDistMap.lat
lng = addrDistMap.lng
baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
upldateFlag = "Address"
}else if(nameDistMap && nameDistMap.region.contains(dist.region) && nameKey == dist.name){
lng = nameDistMap.lng
lat = nameDistMap.lat
baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
upldateFlag = "Name"
}
if(baidu){
total++
def udata = new BasicDBObject()
udata.put("m_date", new Date())
Long baiduLat = baidu.y*10000000000L
Long baiduLng = baidu.x*10000000000L
udata.put("lat2", baiduLat)
udata.put("lng2", baiduLng)
Long lat0 = lat*10000000000L
Long lng0 = lng*10000000000L
udata.put("lat", lat0)
udata.put("lng", lng0 )
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
println "[修改-${total}] "+ dist.region+" "+dist.name+" "+dist.address+" "+upldateFlag
}
}
}
}
return total
}
def getMoreSameLocationDist(def districts){
def locationMap = [:]
for(def i=0;i<districts.size();i++){
def dist = districts[i]
def lat = dist.lat/10000000000.0;
def lng = dist.lng/10000000000.0;
def locationKey = lng + "-" + lat
def list = locationMap.get(locationKey)
if(!list){
list = []
locationMap.put(locationKey, list);
}
list << dist
}
def resMap = [:]
locationMap.each{k,v->
if(v.size()>=2){
v.sort{ a,b->
return a.name.size().compareTo(b.name.size())
}
resMap.put(k,v)
}
}
def lastRetMaps = [:]
resMap.each{k,v->
def swname = null
def count = 0
for(def i=0;i<v.size();i++){
def d = v[i]
if(i==0){
if(d.name.contains("期")||d.name.contains("区")){
swname = d.name.replace("一期","").replace("二期","").replace("三期","").replace("四期","").replace("五期","").replace("六期","").replace("一区","").replace("二区","").replace("三区","").replace("四区","").replace("五区","").replace("A区","").replace("B区","").replace("C区","").replace("D区","")
}else{
swname = d.name
}
}else{
if(d.name.contains(swname)||swname.contains(d.name)){
count++
}
}
}
if(count+1!=v.size()){
lastRetMaps.put(k,v)
}
}
return lastRetMaps;
}
def deleteRegion(def s, def region){
def key = ""
if(s.indexOf(region)>-1){
if(s.indexOf(region+"区")>-1){
key = s.substring(s.indexOf(region+"区")+region.size()+1, s.size());
}else if(s.indexOf(region+"县")>-1){
key = s.substring(s.indexOf(region+"县")+region.size()+1, s.size());
}else if(s.indexOf(region+"市")>-1){
key = s.substring(s.indexOf(region+"市")+region.size()+1, s.size());
}
}else{
key = s;
}
return key
}
def getLocation(dist, addr){
if(addr){
def rMap = [:]
def reps = GaoDeApiUtils.geo(dist.city, addr);
if(reps && reps.count.toInteger()>=1){
def map = reps.geocodes.get(0)
def location = map.location
def level = map.level
if(location){
def locMap = location.split(",")
def lat = dist.lat/10000000000.0;
def lng = dist.lng/10000000000.0;
def distance = GeoHelper.GetDistance(locMap[1].toDouble(), locMap[0].toDouble(), lat, lng)
rMap.distance = distance
rMap.level = level
rMap.lat = locMap[1].toDouble()
rMap.lng = locMap[0].toDouble()
rMap.keyword = map.formatted_address
rMap.location = location
rMap.region = map.district
return rMap;
}
}
}
return null;
}
}
@@ -0,0 +1,4 @@
package com.fangjia.script.gj.test
class FzLocationBatch {
}
@@ -30,7 +30,9 @@ class JiangShuBlankDataStat {
// task.matchHouse();
// task.villa()
task.addressParse()
// task.addressParse()
task.deleteProcess()
}
def repeat(){
@@ -288,5 +290,22 @@ class JiangShuBlankDataStat {
return floor;
}
def deleteProcess(){
def datas = ExcelUtils.read("D:\\数据\\江苏农行\\江苏2千.xlsx",0,1)
def addrList = []
for(def d : datas){
def addr = d['G']
addrList << addr
}
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以下.xlsx", { row ->
def addr = row['C']
if(addrList.contains(addr)){
row['R'] = "已调查"
}
},0);
}
}
@@ -0,0 +1,7 @@
package com.fangjia.script.gj.test
import java.text.SimpleDateFormat
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
Date date = new Date(1617211269868);
print dateFormat.format(date);
@@ -18,8 +18,7 @@ import org.apache.commons.lang.StringUtils
class AutocompleteDataUpdaterBatch extends BaseBatch {
private static final DBCollection AC_COLL = MongoUtils.getCollectionByName("fangjia", "autocomplete", "192.168.0.51", 29009)
private static final DBCollection AC_COLL = MongoUtils.getCollectionByName("fangjia", "autocomplete", "mongo.db.fangjia.com")
private static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
private static final DSCollection AUTOCOMPLETE_COLL = new DataStore("192.168.0.38", 27008).getCollection("fangjia.autocomplete")
private static final long C_DATE = DateUtil.formatToYYYYMMDDhhmmss(new Date()).toLong()
@@ -0,0 +1,156 @@
package com.fangjia.script.temp.shenjian
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
import org.apache.commons.lang.StringUtils
/**
* 抓取总量监控
*
* @author shenjian
* @since 2021/5/12
*/
@BatchOwner(Watcher.ShenJian)
class SeawaterMonitor extends BaseBatch {
static final Long expectNumber = 400000;
// 运维组 运维警报
static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
// 重点城市
static final city = ["北京","唐山","天津","秦皇岛","石家庄","包头","太原","丹东","呼和浩特","锦州",
"沈阳","吉林","大连","牡丹江","长春","无锡","哈尔滨","扬州","上海","徐州",
"南京","温州","杭州","金华","宁波","蚌埠","合肥","安庆","福州","泉州",
"厦门","九江","南昌","赣州","济南","烟台","青岛","济宁","郑州","洛阳",
"武汉","平顶山","长沙","宜昌","广州","襄阳","深圳","岳阳","南宁","常德",
"海口","惠州","重庆","湛江","成都","韶关","贵阳","桂林","昆明","北海",
"西安","三亚","兰州","泸州","西宁","南充","银川","遵义","乌鲁木齐","大理",
"东莞","佛山","苏州","珠海","中山","台州","绍兴","南通","常州","嘉兴",
"淄博","株洲","镇江","肇庆","漳州","盐城","芜湖","泰州","汕头","清远",
"马鞍山","柳州","连云港","廊坊","淮安","湖州","沧州","保定","张家口","日照",
"拉萨","大同","文昌","陵水","保亭"];
// 重点网站
static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"]
static void main(String[] args) {
runBatch(new SeawaterMonitor(), args)
}
/**
* 监控前一日抓取量
* @return
*/
static def monitorYesterdayCrawlNumber(def startDate, def endDate) {
def total
String sql = "select count(*) as total from seawater where m_date >= '${startDate}' and m_date < '${endDate}' "
ElasticUtils.executeSqlQuery(sql,{data->
total = data.total
})
if (total < expectNumber) {
return "【总量抓取异常】 抓取量为" + total + ", 低于预期值" + expectNumber
}
return ""
}
/**
* 统计重点城市近半月抓取量
* @return
*/
static boolean countCityCrawlNumber(def startDate, def endDate) {
def result = []
result << ['城市', '抓取量']
// 统计近半个月各重点城市抓取量
def cityFormatted = city.join("','")
String sql = "select * from (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) order by total desc "
ElasticUtils.executeSqlQuery(sql,{data->
result << [data.city, data.total]
})
ExcelUtils.write("F:\\数据\\重点城市抓取\\近半月重点城市抓取量.xlsx", result, "抓取量")
}
/**
* 统计前一日重点城市抓取量,昨天为零则警报
* @return
*/
static def monitorCityCrawlNumber(def startDate, def endDate) {
def msg = "【重点城市抓取异常】"
def set = new HashSet()
def cityFormatted = city.join("','")
String sql = "select * from " +
" (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) where total>0 "
//" order by total "
ElasticUtils.executeSqlQuery(sql,{data->
set.add(data.city)
})
def unNormal = []
for (def info : city) {
if (!set.contains(info)) {
unNormal.add(info)
}
}
msg += unNormal.join("、") + " 抓取量为0"
if (unNormal.size() > 0) {
return msg
}
return ""
}
/**
* 统计重点网站抓取情况,近一周为0则警报
*/
static def monitorSiteCrawlNumber(def lastWeekDate, def endDate) {
def msg = "【重点网站抓取异常】"
def set = new HashSet()
def siteFormatted = site.join("','")
String sql = "select * from " +
" (select count(*) as total, root from seawater where m_date >= '${lastWeekDate}' and m_date < '${endDate}' and root in('${siteFormatted}') group by root) where total > 0 "
//" order by total "
ElasticUtils.executeSqlQuery(sql,{data->
set.add(data.root)
})
def unNormal = []
for (def info : site) {
if (!set.contains(info)) {
unNormal.add(info)
}
}
msg += unNormal.join("、") + " 近一周抓取量为0"
if (unNormal.size() > 0) {
return msg
}
return ""
}
@Override
void process(Object args) {
def lastWeekDate = DateUtil.add(DateUtil.truncateDay(new Date()), -7).getTime()
def yesterday = DateUtil.formatToYYYYMMDD(DateUtil.add(DateUtil.truncateDay(new Date()), -1))
def startDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1).getTime();
def endDate = DateUtil.truncateDay(new Date()).getTime()
def result = []
def msg = monitorYesterdayCrawlNumber(startDate, endDate)
if (StringUtils.isNotBlank(msg)) {
result.add(msg)
}
msg = monitorCityCrawlNumber(startDate, endDate)
if (StringUtils.isNotBlank(msg)) {
result.add(msg)
}
msg = monitorSiteCrawlNumber(lastWeekDate, endDate)
if (StringUtils.isNotBlank(msg)) {
result.add(msg)
}
if (result.size() > 0) {
msg = "【日期】 " + yesterday + "\n" + result.join("\n")
println(msg)
DingDingMessageUtil.sendTextMessage(access_token, msg)
}
}
}
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
for(def source :sources){
def count1 = findTotalCount(source, 202101);
def count2 = findTotalCount(source, 202102);
def count1 = findTotalCount(source, 202102);
def count2 = findTotalCount(source, 202103);
println source +" "+count1 +" "+count2
}
}
@@ -0,0 +1,213 @@
package com.fangjia.script.thirdPriceCrawl.other
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import net.sf.json.groovy.JsonSlurper
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
class AddFangSchoolRegion {
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
static DBCollection third_price_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "third_party_price")
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school2")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new AddFangSchoolRegion();
//task.process()
//task.match()
task.addMapLocation()
}
def addMapLocation(){
def q = new BasicDBObject()
def dbcursor = district_school_col.find(q)
def index = 0
for(def d : dbcursor){
def data = findSchool(d.city, d.region, d.school_name)
if(data && data.location){
def location = data.location
if(location){
def udata = new BasicDBObject()
udata.put("school_map_lng", location[0])
udata.put("school_map_lat", location[1])
district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
}
}
index ++
println index
}
}
def match(){
def datas = findAllDatas()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size;i++){
def data = datas[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
try {
// if(!data.match_name){
println i +" "+data.district_name
matching(data);
// }
} catch (Exception e) {
println "Òì³£"
e.printStackTrace();
} finally {
latch.countDown()
}
}
}
);
}
latch.await()
}
def matching(def it){
def rMap = matchDistrict(it.city, it.region, it.district_name)
if(!rMap){
rMap = matchDistrict(it.city, it.region, it.address)
}
if(rMap){
DBObject data = new BasicDBObject()
data.put("match_city", rMap.city)
data.put("match_region", rMap.region)
data.put("match_name", rMap.name)
data.put("match_address", rMap.address)
data.put("match_lat", rMap.lat)
data.put("match_lng", rMap.lng)
district_school_col.update(new BasicDBObject('_id', it.get("_id")), new BasicDBObject('$set', data))
}
}
def matchDistrict(def city, def region, def address){
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&preciseLevel=strict&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district"
String content = new HttpRequest().get(url, "utf-8");
def rMap = [:]
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dto = doc.result
if(dto && dto.searchAddress){
def ret = dto.matchResult
if(ret.match){
rMap.city = dto.city
rMap.region = dto.district
rMap.name = dto.name
rMap.lat = dto.lat
rMap.lng = dto.lng
rMap.id = dto.id
rMap.address = dto.address
}
}
}
}
return rMap;
}
def process(){
def datas = findNoRegionData();
findSimialData(datas);
}
def findAllDatas(){
def q = new BasicDBObject()
q.put("match_name", new BasicDBObject(QueryOperators.EXISTS, false))
def dbcursor = district_school_col.find(q)
def rets =[]
for(def d : dbcursor){
rets << d
}
return rets;
}
def findNoRegionData(){
def q = new BasicDBObject()
q.put("region", null)
def dbcursor = district_school_col.find(q)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
def findSimialData(datas){
for(def d : datas){
def obj = findThirdPriceData(d.city, d.url+"/")
if(!obj){
obj = findThirdPriceByName(d.city, d.district_name)
}
if(obj){
d.region = obj.region
if(!d.district_addrss){
d.district_address = obj.address
}
updateDistrictSchool(d);
}else{
println d.school_name+"" + d.district_name+"û²éµ½"
}
}
}
def findThirdPriceData(def city, def url){
def q = new BasicDBObject()
q.put("source","ËÑ·¿")
q.put("city", city)
q.put("district_id", url)
def dbcursor = third_price_col.find(q).sort(new BasicDBObject("s_date", -1)).limit(1)
def obj = null;
for(def d : dbcursor){
obj = d
}
return obj;
}
def findThirdPriceByName(def city, def name){
def q = new BasicDBObject()
q.put("source","ËÑ·¿")
q.put("city", city)
q.put("name", name)
def dbcursor = third_price_col.find(q).sort(new BasicDBObject("s_date", -1)).limit(1)
def obj = null;
for(def d : dbcursor){
obj = d
}
return obj;
}
def updateDistrictSchool(d){
DBObject data = new BasicDBObject()
data.put("region", d.region)
data.put("district_address", d.district_address)
district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', data))
}
def findSchool(city, region, school_name){
DBObject data = new BasicDBObject()
data.put("city", city)
data.put("region", region)
data.put("name", school_name)
return school_col.findOne(data)
}
}
@@ -0,0 +1,52 @@
package com.fangjia.script.thirdPriceCrawl.other
import com.fangjia.script.util.ABuYunUtil
import org.apache.commons.io.IOUtils
import org.apache.http.HttpResponse
import org.apache.http.client.HttpClient
import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.HttpClients
class AnjukeCrawl {
static main(args) {
def task = new AnjukeCrawl();
task.run()
}
def run(){
HttpClient client = HttpClients.createDefault();
HttpGet get = new HttpGet("https://m.anjuke.com/cityList/");
// get.setHeader("Upgrade-Insecure-Requests", "1");
// get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36");
// get.setHeader("Host","m.anjuke.com");
// get.setHeader("Connection","keep-alive");
// get.setHeader("Cache-Control","max-age=0");
// get.setHeader('sec-ch-ua','"Chromium";v="88", "Google Chrome";v="88", ";Not A Brand";v="99"');
// get.setHeader("sec-ch-ua-mobile","'?0");
// get.setHeader("Upgrade-Insecure-Requests","1");
def header = [:]
header.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36");
header.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
// get.setHeader("Sec-Fetch-Site","none");
// get.setHeader("Sec-Fetch-Mode","navigate");
// get.setHeader("Sec-Fetch-User","?1");
// get.setHeader("Sec-Fetch-Dest","document");
//// get.setHeader("Accept-Encoding"," gzip, deflate, br");
// get.setHeader("Accept-Language","zh-CN,zh;q=0.9");
def s1 = ABuYunUtil.doGet("https://m.anjuke.com/cityList/", null, header, "utf-8")
printl s1
HttpResponse response = client.execute(get);
def s = getResponseBodyAsString(response)
println s
}
private static String getResponseBodyAsString(HttpResponse response) throws IOException {
return IOUtils.toString(response.getEntity().getContent(), "UTF-8");
}
}
@@ -0,0 +1,344 @@
package com.fangjia.script.thirdPriceCrawl.other
import com.alibaba.fastjson.annotation.JSONField
import com.fangjia.core.RegionData
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.HttpClient
import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.httpclient.NameValuePair
import org.apache.commons.lang.StringUtils
import org.apache.http.HttpEntity
import org.apache.http.client.entity.UrlEncodedFormEntity
import org.apache.http.client.methods.CloseableHttpResponse
import org.apache.http.client.methods.HttpPost
import org.apache.http.impl.client.CloseableHttpClient
import org.apache.http.impl.client.HttpClients
import org.apache.http.message.BasicNameValuePair
import org.apache.http.util.EntityUtils
import java.nio.charset.Charset
import java.util.zip.GZIPInputStream
class GushuwangCrawl {
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school")
static main(args) {
def task = new GushuwangCrawl();
task.process()
}
void process(Object args) {
// chaxuequCrawl()
suzhouCrawl();
}
def suzhouCrawl(){
def page = 1
def stop = false
def list = []
while(!stop){
def s = schoolList(page);
if(s){
JSONObject jsonObject = JSONObject.fromObject(s);
if(jsonObject && jsonObject.ok){
def datas = jsonObject.extra.list
if(datas){
list.addAll(datas)
page++
println page
}else{
stop = true
}
}else{
stop = true
}
}
}
if(list){
def schools = []
for(def d : list){
def schoolData = new BasicDBObject()
schoolData.province = "江苏"
schoolData.city = "苏州"
schoolData.region = RegionData.parseRegionFormAddress(schoolData.city , d.adr)
/**类型: 幼儿园,小学,初中*/
schoolData.category = converCategory(d.schoolname)
schoolData.name = d.schoolname
schoolData.address = d.adr
/**来源网站*/
schoolData.source = "苏州学区查询"
/**房源详情url*/
schools << schoolData
}
saveChaxuequSchool(schools)
}
}
def chaxuequCrawl(){
def page = 1
def stop = false
def schools = []
while(!stop){
def s = chaxuequInfoCrawl("https://51yongka.com/fapi/school/list", "schoolName=&city=%E8%8B%8F%E5%B7%9E&area=&schoolNature=&schoolType=&page="+page+"&size=10")
println "内容:"+s
if(s){
JSONObject jsonObject = JSONObject.fromObject(s);
if(jsonObject.code == 200){
def data = jsonObject.data
def totalPage = data.pages
if(data.list){
page++
if(page+1>totalPage){
stop = true;
}
schools.addAll(data.list)
}
}
}
}
print(schools.size());
def districtSchools = []
def schoolInfos = []
if(schools.size()>0){
def url = "https://51yongka.com/fapi/school/details"
for(def d : schools){
def params = "schoolId="+d.id+"&access_token=oRqu15Mq_KxHfK0y0ZWhwktVnfUM"
def s = chaxuequInfoCrawl(url, params)
JSONObject jsonObject = JSONObject.fromObject(s);
if(jsonObject && jsonObject.code==200){
if(jsonObject.data && jsonObject.data.nearbyPropertyList){
def districts = jsonObject.data.nearbyPropertyList
for(def dist : districts){
def dataMap = new BasicDBObject()
dataMap.city = jsonObject.data.city
dataMap.region = RegionData.formatRegion(dataMap.city , jsonObject.data.area)
/**类型: 幼儿园,小学,初中*/
dataMap.category = converCategory(jsonObject.data.schoolName)
dataMap.district_name = dist.name.trim()
dataMap.district_address = dist.address
dataMap.school_name = jsonObject.data.schoolName.trim()
dataMap.school_address = jsonObject.data.address
dataMap.source = "查学区房"
dataMap.url = url + "?"+params
dataMap.telephone = jsonObject.data.officialTelephone;
districtSchools << dataMap
}
}else{
def schoolData = new BasicDBObject()
schoolData.province = "江苏"
schoolData.city = jsonObject.data.city
schoolData.region = RegionData.formatRegion(schoolData.city , jsonObject.data.area)
/**类型: 幼儿园,小学,初中*/
schoolData.category = converCategory(jsonObject.data.schoolName)
schoolData.name = jsonObject.data.schoolName.trim()
schoolData.address = jsonObject.data.address
/**来源网站*/
schoolData.source = "查学区房"
/**房源详情url*/
schoolData.telephone = jsonObject.data.officialTelephone;
schoolInfos << schoolData
println jsonObject.data.schoolName +" 无周边楼盘"
}
}
println s;
}
}
println "小区:"+districtSchools.size()
// saveChaxuequDistrictSchool(districtSchools)
saveChaxuequSchool(schoolInfos)
}
def saveChaxuequSchool(schoolInfos){
for(def ds : schoolInfos){
def obj = findSchoolData(ds.city, ds.school_name);
if(!obj){
school_col.save(ds)
}
}
}
def saveChaxuequDistrictSchool(def districtSchools){
for(def ds : districtSchools){
def obj = findDistrictSchoolData(ds.city, ds.school_name);
if(!obj){
district_school_col.save(ds)
}
}
}
def findSchoolData(city, school_name){
def q = new BasicDBObject()
q.put("city", city)
q.put("school_name", school_name)
return school_col.findOne(q)
}
def findDistrictSchoolData(city, school_name, district_name){
def q = new BasicDBObject()
q.put("city", city)
q.put("school_name", school_name)
q.put("district_name", district_name)
return district_school_col.findOne(q)
}
def converCategory(def schoolName){
if(schoolName){
if(schoolName.contains("初中部")||schoolName.contains("中学")){
return "初中"
}else if(schoolName.contains("小学")){
return "小学"
}
}
return null
}
def static chaxuequInfoCrawl(url, bodyquery){
def responseMap = [:]
try {
//headers信息
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","51yongka.com")
headers.put("Connection","keep-alive")
headers.put("Content-Length","75")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
headers.put("content-type","application/x-www-form-urlencoded")
headers.put("Referer","https://servicewechat.com/wxc2461b2529576950/8/page-frame.html")
headers.put("Accept-Encoding","gzip, deflate, br")
String s = doPost(url, bodyquery, headers)
return s
} catch (Exception e) {
e.printStackTrace()
}
return null
}
def static schoolList(page) {
def responseMap = [:]
String s = null
try {
//headers信息
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","m.gusuwang.com")
headers.put("Connection","keep-alive")
headers.put("Content-Length","40")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
headers.put("content-type","application/x-www-form-urlencoded")
headers.put("Referer","https://servicewechat.com/wx9188a941e5ef95c5/10/page-frame.html")
// headers.put("Accept-Encoding","gzip, deflate, br")
//params信息
Map<String, String> params = new HashMap<String, String>();
params.put("secCode","198a53da6baa3aa6a5b99959a006be37");
// //post请求
def url = "https://m.gusuwang.com/xuequ/xqapi.php?act=searchschool"
s = doPost(url, "secCode=198a53da6baa3aa6a5b99959a006be37&keys=&grade=&lb=&lx=&area=&currpage="+page,headers)
} catch (Exception e) {
e.printStackTrace()
}
return s
}
static String doPost(String strUrl, String content, Map headers) {
String result = "";
OutputStream outputStream = null;
InputStream inputStream = null;
StringBuilder sb = new StringBuilder();
try {
URL url = new URL(strUrl);
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestMethod("POST");
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setConnectTimeout(30000);
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setReadTimeout(30000);
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
urlConnection.setDoOutput(true);
urlConnection.setDoInput(true);
//设置通用请求属性为默认浏览器编码类型
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
urlConnection.setRequestProperty(head, headers.get(head));
}
}
//getOutputStream()返回的输出流,用于写入参数数据。
outputStream = urlConnection.getOutputStream();
outputStream.write(content.getBytes());
outputStream.flush();
outputStream.close();
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
inputStream = urlConnection.getInputStream();
// byte[] data = new byte[1024];
// //inputStream每次就会将读取1024个byte到data中,当inputSteam中没有数据时,inputStream.read(data)值为-1
// while (inputStream.read(data) != -1) {
// String s = new String(data, Charset.forName("utf-8"));
// sb.append(s);
// }
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
char[] buff = new char[1024];
int length = 0;
while ((length = reader.read(buff)) != -1) {
String s = new String(buff, 0, length);
sb.append(s);
}
} catch (IOException e) {
e.printStackTrace();
}finally{
if(outputStream!=null){
outputStream.close();
}
if(inputStream!=null){
inputStream.close();
}
}
return sb.toString();
}
}
@@ -425,6 +425,7 @@ class ABuYunUtil {
buffer.append(bf, 0, len);
}
} catch (IOException e) {
e.printStackTrace()
throw new Exception("doGet(String targetUrl, Map<String, String> params, Map<String, String> headers, String charset)异常, 异常信息为:" + e);
} finally {
// println("本次请求响应的connection.getResponseCode()为:" + connection.getResponseCode())
@@ -44,7 +44,7 @@ class ElasticUtils {
static getEsDataSource(String host){
EsDataSource dataSource = new EsDataSource ();
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240&network.timeout=600&page.timeout=240");
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000");
Properties connectionProperties = new Properties();
connectionProperties.setProperty("user","developer");
connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
@@ -52,7 +52,7 @@ class ElasticUtils {
return dataSource;
}
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.216"){
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.206"){
EsDataSource dataSource = getEsDataSource(host)
Connection connection = dataSource.getConnection()
Statement statement = connection.createStatement()
@@ -140,13 +140,12 @@ class ElasticUtils {
* @return
*/
static closeBulkProcessor(BulkProcessor bulkProcessor) {
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
bulkProcessor.flush()
bulkProcessor.close()
}
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
RestHighLevelClient client = getClient(host)
@@ -1,19 +1,17 @@
package com.fangjia.script.util
import com.mongodb.DBCursor
import org.apache.commons.lang.StringUtils
import org.apache.lucene.index.Term
import org.apache.lucene.search.BooleanClause
import org.apache.lucene.search.BooleanQuery
import org.apache.lucene.search.TermQuery
import org.apache.solr.client.solrj.util.ClientUtils
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.datastore.driver.FacetResult
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.apache.lucene.index.Term
import org.apache.lucene.search.BooleanClause
import org.apache.lucene.search.BooleanQuery
import org.apache.lucene.search.TermQuery
import org.apache.solr.client.solrj.util.ClientUtils
class HouseUtils{
static main(args){
@@ -121,9 +121,9 @@ public class OpenApiClient {
params.put("suggestName", suggestName)
}
String content = new HttpRequest().get("http://192.168.10.199:9180/address/realMatch", params, "utf-8")
String content = new HttpRequest().get("http://open.fangjia.com/address/match", params, "utf-8")
while (content == null) {
content = new HttpRequest().get("http://192.168.10.199:9180/address/realMatch", params, "utf-8")
content = new HttpRequest().get("http:///open.fangjia.com/address/match", params, "utf-8")
println("${city} ${input} ${cat}程序调用异常")
Thread.sleep(6000)
}
File diff suppressed because it is too large Load Diff
@@ -6,34 +6,50 @@ import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018)
def count = 0
List<List<String>> data = Collections.synchronizedList([])
data << ["ID","城市","区域","小区名","经度","纬度","街道"]
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\Êý¾Ý.xlsx",{List item->
// if(DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"]))){
// item["D"] = "´æÔÚ"
// }else{
// item["D"] = "²»´æÔÚ"
// }
//})
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat","mongo.db.fangjia.com")
ProvinceData.getAllCity("¸£½¨").each {def city->
int count = 0
def s = new TreeMap()
DISTRICT_STAT.find(new BasicDBObject("weekend", Date.parse("yyyy-MM-dd", "2020-04-19")).append("city", city)).each {BasicDBObject stat->
count ++
int credit = (stat.credit * 100)/10
s[credit] = s.get(credit, 0) + 1
}
def data = []
data << city
data << count
for(int i=0; i <=10; i++){
data << s.get(i, 0)
}
println(data.join(","))
CommonUtils.findAllCity().each {def city->
DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item->
if(StringUtils.isBlank(item.township) && item.lng){
count++
def lng = item.lng/10000000000d
def lat = item.lat/10000000000d
def location = "${lng},${lat}"
JSONObject resp = GaoDeApiUtils.geocode(location)
def row = []
row << item._id.toString()
row << item.city
row << item.region
row << item.name
row << lng
row << lat
BasicDBObject dbo = new BasicDBObject()
dbo.id = item._id.toString()
dbo.city = item.city
dbo.region = item.region
dbo.name = item.name
dbo.lng = lng
dbo.lat = lat
if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){
row << resp.regeocode.addressComponent.township
dbo.township = resp.regeocode.addressComponent.township
}
println(row)
data << row
SEAWEED_TOWNSHIP.insert(dbo)
}
})
}
ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据")
@@ -1,6 +1,8 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.Constants
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
@@ -9,294 +11,33 @@ import net.sf.json.JSONObject
import java.util.regex.Matcher
import java.util.regex.Pattern
//
//def map = [:]
//ExcelUtils.read("D:\\商务数据\\上海农商行\\测试数据(上海)-上海农商行第二批0829.xlsx", 1).each {List item->
// if(item["AF"] && item["AH"]){
// def key = "${item["AE"]}_${item["AF"]}_${item["AH"]}".toString()
// map.get(key, []) << key
// }
//}
//ExcelUtils.modify("D:\\商务数据\\上海农商行\\测试数据(上海)-上海农商行第二批(房价网交付0905).xlsx", { List item ->
//// if(item["AF"] && item["AH"]){
//// def key = "${item["AE"]}_${item["AF"]}_${item["AH"]}".toString()
//// List nums = map.get(key)
//// item["AT"] = nums.size()
//// }
//
// def city = "上海"
// def addr = item["AC"]
// if(addr){
// JSONObject resp = OpenApiClient.addressMatch(city, addr, "property")
// JSONObject searchAddress = resp.getJSONObject("searchAddress")
// println searchAddress
//// item["AD"] = searchAddress.province
//// item["AE"] = searchAddress.city
//// item["AF"] = searchAddress.district
// if(searchAddress.standardAddress){
//// item["AG"] = searchAddress.street
//// if(searchAddress.streetNumber){
//// item["AG"] = item["AG"] + searchAddress.streetNumber
//// }
//
// if(searchAddress.township){
// item["AP"] = searchAddress.township
// }
// }
//
// if(item["AG"]){
// def addrMap = getRoad(item["AG"])
// if(addrMap){
// if(addrMap.road){
// item["AQ"] = addrMap.road
// }
// if(addrMap.roadNumber){
// item["AR"] = addrMap.roadNumber
// }
// }
// }
// }
//
//
//// if (item["AH"] && item["AF"]) {
//// BasicDBObject seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["AE"], item["AF"], item["AH"]))
//// if (seaweed) {
//// if (item["AR"] && item["AS"]) {
//// JSONObject eva = OpenApiClient.evaluateDistrictPrice(seaweed.city, seaweed.region, seaweed.name, item["O"], seaweed.estate_type2, item["AS"].toInteger(), item["AR"].toInteger())
//// if (eva) {
//// item["AL"] = eva.avgPrice
//// item["AM"] = eva.totalPrice
//// }
//// } else {
//// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(seaweed.city, seaweed.region, seaweed.name, item["O"], seaweed.estate_type2)
//// if (eva) {
//// item["AL"] = eva.avgPrice
//// item["AM"] = eva.totalPrice
//// }
//// }
//// }
////
//// }
//
//},1)
// if(item["AK"] =~ /^\d{4}$/){
// item["AR"] = item["AK"].toString().substring(0, 2)
// }else if(item["AK"] =~ /^\d{3}$/){
// item["AR"] = item["AK"].toString().substring(0, 1)
// }
// if(item["AR"] && item["AH"]){
// def key = "${item["AE"]}_${item["AF"]}_${item["AH"]}".toString()
// List nums = map.get(key)
// nums.sort()
// item["AS"] = nums[nums.size() - 1]
// item["AT"] = nums.unique().size()
// }
// println(item)
// item["AG"] = null
// def city = "上海"
// def addr = item["AC"]
// def area = item["o"]
// JSONObject resp = OpenApiClient.addressMatch(city, addr, "property")
// JSONObject searchAddress = resp.getJSONObject("searchAddress")
// item["AD"] = searchAddress.province
// item["AE"] = searchAddress.city
// item["AF"] = searchAddress.district
// if(searchAddress.standardAddress){
// item["AG"] = searchAddress.street
// if(searchAddress.streetNumber){
// item["AG"] = item["AG"] + searchAddress.streetNumber
// }
// }
//
//
// if(resp.getDouble("credit") > 0){
// item["AE"] = resp.city
// item["AF"] = resp.district
// item["AH"] = resp.name
// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(resp.city, resp.district, resp.name, area, resp.propertyType)
// item["AL"] = eva.avgPrice
// item["AM"] = eva.totalPrice
// item["AN"] = new Date().format("yyyy-MM-dd")
// if(!item["AG"]){
// item["AG"] = resp.address
// }
// }
// item["AI"] = searchAddress.buildingNumber
// item["AJ"] = searchAddress.unitNumber
// item["AK"] = searchAddress.roomNumber
// println(item)
//},1)
//
def sheetIndexs = ["湘潭": 0, "嘉兴": 1, "苏州": 2]
sheetIndexs.each { String city, int sheetIndex ->
// def map = [:]
// ExcelUtils.read("D:\\商务数据\\上海农商行\\异地测试-上海农商行第二批0828(测试)1.xlsx", sheetIndex).each {List item->
// if(item["Q"] && item["S"]){
// def key = "${item["P"]}_${item["Q"]}_${item["S"]}".toString()
// map.get(key, []) << key
// }
// }
ExcelUtils.multiThreadingModify("D:\\商务数据\\上海农商行\\异地测试-上海农商行第二批(房价网交付0905).xlsx", { List item ->
// if(item["Q"] && item["S"]){
// def key = "${item["P"]}_${item["Q"]}_${item["S"]}".toString()
// List nums = map.get(key)
// nums.sort()
// item["AE"] = nums.size()
// }
// item["W"] = null
// item["X"] = null
// BasicDBObject seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["P"], item["Q"], item["S"]))
// if(seaweed){
// if(item["AC"] && item["AD"]){
// JSONObject eva = OpenApiClient.evaluateDistrictPrice(seaweed.city, seaweed.region, seaweed.name, item["L"], seaweed.estate_type2, item["AD"].toInteger(), item["AC"].toInteger())
// if(eva){
// item["W"] = eva.avgPrice
// item["X"] = eva.totalPrice
// }
// }else{
// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(seaweed.city, seaweed.region, seaweed.name, item["L"], seaweed.estate_type2)
// if(eva){
// item["W"] = eva.avgPrice
// item["X"] = eva.totalPrice
// }
// }
// }
// }
// if(item["AC"] && item["S"]){
// def key = "${item["P"]}_${item["Q"]}_${item["S"]}".toString()
// List nums = map.get(key)
// nums.sort()
// item["AD"] = nums[nums.size() - 1]
// item["AE"] = nums.unique().size()
// }
// if(item["V"] =~ /^\d{4}$/){
// item["AC"] = item["V"].toString().substring(0, 2)
// }else if(item["V"] =~ /^\d{3}$/){
// item["AC"] = item["V"].toString().substring(0, 1)
// }else if(item["V"] =~ /^\d{6,7}$/){
// item["AC"] = item["V"].toString().substring(0, 2).replaceAll("^0", "")
// }
def addr = item["B"].toString().replaceAll("(", "").replace('区/县', "区").replace("街道/镇", "镇")
.replaceAll(")", "").replaceAll("【.*】", "").replaceAll("室/单元", "号")
// def area = item["L"]
def region = null
if (city == "嘉兴") {
region = "嘉善"
} else if (city == "苏州") {
region = "昆山"
}
JSONObject resp = OpenApiClient.addressMatch(city, addr, "property", false, region)
JSONObject searchAddress = resp.getJSONObject("searchAddress")
def township = ""
if(searchAddress && searchAddress.township){
item["AP"] = searchAddress.township
township = searchAddress.township
}
if(item["S"]){
def addrMap = getRoad(item["S"])
if(addrMap){
if(addrMap.road){
def s = addrMap.road;
if(township){
if(township.contains(s)){
item["AQ"] = s.replace(township, "")
}else{
if(township.contains("镇")){
township = township.replace("镇","")
}
item["AQ"] = s.replace(township, "")
}
}else{
item["AQ"] = addrMap.road
}
}
if(addrMap.roadNumber){
item["AR"] = addrMap.roadNumber
}
}
}
println searchAddress
// item["O"] = searchAddress.province
// item["P"] = searchAddress.city
// item["Q"] = searchAddress.district
// if (searchAddress.standardAddress) {
// item["R"] = searchAddress.street
// if (searchAddress.streetNumber) {
// item["R"] = item["R"] + searchAddress.streetNumber
// }
// }
// if (resp.getDouble("credit") > 0) {
// item["O"] = resp.province
// item["P"] = resp.city
// item["Q"] = resp.district
// item["S"] = resp.name
// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(resp.city, resp.district, resp.name, area, resp.propertyType)
// item["W"] = eva.avgPrice
// item["X"] = eva.totalPrice
// item["Y"] = new Date().format("yyyy-MM-dd")
// if (!item["R"]) {
// item["R"] = resp.address
// }
// }
// item["T"] = searchAddress.buildingNumber
// item["U"] = searchAddress.unitNumber
// item["V"] = searchAddress.roomNumber
// if(city == "湘潭"){
// String roomNumber = DataUtil.match(addr, Pattern.compile("\\d+(号|(号房))\$"))
// if(roomNumber){
// item["V"] = roomNumber.replaceAll("号", '').replaceAll('房', "")
// }
// roomNumber = DataUtil.match(addr, Pattern.compile("\\d{6,8}\$"))
// if(roomNumber){
// item["V"] = roomNumber.replaceAll("号", '').replaceAll('房', "")
// }
//
// }
def school = MongoUtils.getDbCollection(Constants.FANGJIA_CRAWL_MONGO_URI, "fangjia_craw", "school2")
new File("D:\\Êý¾Ý\\ѧУÊý¾Ý").eachFile {File file->
ExcelUtils.read(file.getAbsolutePath(), 0).each {List item->
BasicDBObject dbo = new BasicDBObject()
dbo.city = item["A"]
dbo.region = item["B"]
dbo.name = item["C"]
dbo.address = item["E"]
dbo.attribute = item["G"]
dbo.property = item["H"]
dbo.category = "Сѧ"
dbo.location = [DataUtil.roundDouble(item["I"], 6), DataUtil.roundDouble(item["J"], 6)]
school.insert(dbo)
println(dbo)
}
ExcelUtils.read(file.getAbsolutePath(), 1).each {List item->
BasicDBObject dbo = new BasicDBObject()
dbo.city = item["A"]
dbo.region = item["B"]
dbo.name = item["C"]
dbo.address = item["E"]
dbo.attribute = item["G"]
dbo.property = item["H"]
dbo.category = "ÖÐѧ"
dbo.location = [item["I"], item["J"]]
school.save(dbo)
println(item)
}, sheetIndex)
}
def getRoad(def s){
def rex = ".*(路|街道|街|大道|巷|道|里).*\\d+(弄|号)\$"
Pattern pattern = Pattern.compile(rex);
Matcher matcher = pattern.matcher(s);
List<String> list = new ArrayList();
while (matcher.find()) {
list.add(matcher.group());
}
if(list && list.size()>0){
def s1 = list[0]
def road = getRexText(s1, ".*(路|街道|街|大道|巷|道|里)")
def num = getRexText(s1, "\\d+")
return ["road":road, "roadNumber":num]
}else{
def road = getRexText(s, ".*(路|街道|街|大道|巷|道|里)")
return ["road":road]
}
return null
}
def getRexText(s, regx){
Pattern pattern = Pattern.compile(regx);
Matcher matcher = pattern.matcher(s);
while (matcher.find()) {
return matcher.group();
}
return null;
}
@@ -1,9 +1,16 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.Constants
import com.fangjia.core.ProvinceData
import com.fangjia.core.RegionData
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import net.sf.json.JSONObject
import org.apache.commons.lang3.StringUtils
import org.openqa.selenium.json.Json
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\阳光保险重跑数据(1).xlsx", { List item ->
@@ -30,8 +37,6 @@ import org.apache.commons.lang3.StringUtils
//}, 0,"结果")
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\阳光保险重跑数据(1).xlsx", { List item ->
// if (item["T"] == null || ![1, 2, 4].contains(item["T"].toString().toDouble().toInteger())) {
// return
@@ -114,43 +119,80 @@ import org.apache.commons.lang3.StringUtils
// println("${key} ${names.join(",")}")
// }
//}
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\µØÖ·±ê×¼»¯.xlsx",{List item->
String address = "";
if(StringUtils.isNotBlank(item["B"]) && !ProvinceData.isGovernmentCity(item["B"])){
address = ProvinceData.getDisplay(item["B"])
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\地址标准化.xlsx",{List item->
// String address = "";
// if(StringUtils.isNotBlank(item["B"]) && !ProvinceData.isGovernmentCity(item["B"])){
// address = ProvinceData.getDisplay(item["B"])
// }
// if(StringUtils.isNotBlank(item["C"])){
// String cityDisplay = CityData.getCityDisplay(item["C"]);
// if(cityDisplay != null){
// address = address + cityDisplay
// }else{
// address = address + item["C"] + "市"
// }
// }
// if(StringUtils.isNotBlank(item["D"])){
// String orgDistrict = RegionData.getOrgName(item["C"], item["D"]);
// if(StringUtils.isNotBlank(orgDistrict) && address.indexOf(orgDistrict) == -1){
// address = address + orgDistrict
// }
// }
// if(StringUtils.isNotBlank(item["E"])){
// address = address + item["E"]
// }
// if(StringUtils.isNotBlank(item["F"])){
// address = address + item["F"]
// }
// if(StringUtils.isNotBlank(item["G"])){
// address = address + item["G"]
// }
// if(StringUtils.isNotBlank(item["H"])){
// address = address + item["H"] + "栋"
// }
// if(StringUtils.isNotBlank(item["i"])){
// address = address + item["i"] + "单元"
// }
// if(StringUtils.isNotBlank(item["J"])){
// address = address + item["J"] + "室"
// }
// item["K"] = address
// println(address)
//})
def school = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, "fangjia", "guiding_price")
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\深圳市住建局二手房指导价.xlsx", { List item ->
if (item["G"]) {
BasicDBObject seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["G"], item["H"], item["I"]))
BasicDBObject guid = new BasicDBObject()
guid.guiding_type = 0
guid.estate_type = seaweed.estate_type2
guid.house_type = 0
guid.c_date = new Date()
guid.guiding_person = "深圳房管局"
guid.guiding_date = Date.parse('yyyyMMdd', '20210411')
guid.category = "district"
guid.guiding_value = (int)item["E"]
guid.name = item["D"]
guid.city = item["G"]
guid.region = item["H"]
guid.name = item["i"]
school.insert(guid)
println(guid)
}
if(StringUtils.isNotBlank(item["C"])){
String cityDisplay = CityData.getCityDisplay(item["C"]);
if(cityDisplay != null){
address = address + cityDisplay
}else{
address = address + item["C"] + "ÊÐ"
}
}
if(StringUtils.isNotBlank(item["D"])){
String orgDistrict = RegionData.getOrgName(item["C"], item["D"]);
if(StringUtils.isNotBlank(orgDistrict) && address.indexOf(orgDistrict) == -1){
address = address + orgDistrict
}
}
if(StringUtils.isNotBlank(item["E"])){
address = address + item["E"]
}
if(StringUtils.isNotBlank(item["F"])){
address = address + item["F"]
}
if(StringUtils.isNotBlank(item["G"])){
address = address + item["G"]
}
if(StringUtils.isNotBlank(item["H"])){
address = address + item["H"] + "¶°"
}
if(StringUtils.isNotBlank(item["i"])){
address = address + item["i"] + "µ¥Ôª"
}
if(StringUtils.isNotBlank(item["J"])){
address = address + item["J"] + "ÊÒ"
}
item["K"] = address
println(address)
// if(!item["F"]){
// JSONObject resp = OpenApiClient.addressMatch("深圳", item["D"], 'property', false, item["B"])
// if(resp.id){
// BasicDBObject seaweed = DbUtils.seaweed.findById(resp.id)
// if(seaweed){
// item["F"] = seaweed._id.toString()
// item["G"] = seaweed.city
// item["H"] = seaweed.region
// item["I"] = seaweed.name
// }
// println(item)
// }
//
// }
})
@@ -1,82 +1,149 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.util.HouseValidator
import com.fangjia.core.Constants
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import net.sf.json.JSONObject
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
ExcelUtils.multiThreadingModify("D:\\商务数据\\建设银行\\副本5市成交案例(测试).xlsx",{List item->
// if(item["R"]){
// return
// }
// def calcDate = null
// if(item["E"]){
// calcDate = item["E"].toString().substring(0, 10)
// }else{
// calcDate = item["B"].toString().substring(0, 10)
// }
// def floor = item["I"].toString().toDouble().toInteger().toString()
// def maxFloor = item["J"].toString().toDouble().toInteger().toString()
// def dir = HouseValidator.formatDirectionToCode(item["L"]).toString()
// def fit = HouseValidator.formatFitmentToCode(item["M"]).toString()
// def params =[:]
// params.city = item["F"]
// params.district = item["p"]
// params.name = item['Q']
// params.size = item["C"].toString()
// params.floor = floor
// params.maxFloor = maxFloor
// params.direction = dir
// params.decoration = fit
// params.calDate = calcDate
// params.__refresh_cache = "1"
// params.scope = "1"
// JSONObject evaResp = OpenApiClient.evaluate(params)
// if(evaResp.result && evaResp.result.credit > 0.4){
// item["R"] = evaResp.result.totalPrice
// }
def school = MongoUtils.getDbCollection(Constants.FANGJIA_CRAWL_MONGO_URI, "fangjia_craw", "school2")
//school.find(new BasicDBObject("source", "查学区")).each {BasicDBObject item->
// item.category = item.attribute
// item.attribute = null
// school.save(item)
// println(item)
// if(item["H"] != "中粮云景花园"){
// return
//}
//ExcelUtils.read("D:\\Users\\wenyan\\Downloads\\无经纬度学校-结果.xlsx").each {List item->
// if(item["P"] != null){
// def location = item["P"].split(",") as List
// def data = new BasicDBObject()
// data.map_name = item["N"]
// data.map_region = item["R"]
// data.map_address = item["N"]
// data.map_type = item["S"]
// data.location = [location[0].toString().toDouble(),location[1].toString().toDouble()]
// school.update(new BasicDBObject("_id", new ObjectId(item["A"])), new BasicDBObject('$set', data))
// println(data)
// }
def city = item["F"]
def region = item["G"]
def name = item["H"].toString()
def addr = item["D"]
JSONObject resp = OpenApiClient.addressMatch(city, addr, "property" , false, region, name)
if(resp.name){
item["p"] = resp.district
item['Q'] = resp.name
//}
regionCodeMap = [:]
def cityCodeMap = ["苏州":"320500","合肥":"340100", "西安":"610000"]
for(def city : cityCodeMap.keySet()){
def datas = FangJiaAreaMapUtils.convertToOfficials(city);
for(def d : datas){
regionCodeMap.put(city+"-"+d.fangjia_district, d.district_code)
}
println(item)
// def calcDate = null
// if(item["E"]){
// calcDate = item["E"].toString().substring(0, 10)
// }else{
// calcDate = item["B"].toString().substring(0, 10)
// }
// def floor = item["I"].toString().toDouble().toInteger().toString()
// def maxFloor = item["J"].toString().toDouble().toInteger().toString()
// def dir = HouseValidator.formatDirectionToCode(item["L"]).toString()
// def fit = HouseValidator.formatFitmentToCode(item["M"]).toString()
// def params =[:]
// params.city = resp.city
// params.district = resp.district
// params.name = resp.name
// params.size = item["C"].toString()
// params.floor = floor
// params.maxFloor = maxFloor
// params.direction = dir
// params.decoration = fit
// params.calDate = calcDate
// params.scope = "1"
// JSONObject evaResp = OpenApiClient.evaluate(params)
// if(evaResp.result && evaResp.result.credit > 0.4){
// item["R"] = evaResp.result.totalPrice
// }
// println(item)
// }
}
def data = Collections.synchronizedList([])
data << ["ID","CITYID","DISTID","COMMNAME","LNG","LAT","ADDRESS","PS","PS_ADDR","PS_TYPE","PS_SEQNO","PS_LONGITUDE","PS_LATITUDE","MS","MS_ADDR","MS_TYPE","MS_SEQNO","MS_LONGITUDE","MS_LATITUDE","REMARK","CREATETIME","UPDATETIME","SOURCECODE"]
cityCodeMap.keySet().each {def city->
DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject seaweed->
if(seaweed.lng){
def item = []
item << seaweed._id.toString()
item << cityCodeMap.get(seaweed.city)
item << regionCodeMap.get(seaweed.city + "-" + seaweed.region)
item << seaweed.name
def lng = com.fangjia.core.util.DataUtil.roundDouble(seaweed.lng/10000000000d, 6)
def lat = com.fangjia.core.util.DataUtil.roundDouble(seaweed.lat/10000000000d, 6)
item << lng
item << lat
item << seaweed.address
BasicDBObject location = new BasicDBObject()
BasicDBObject near = new BasicDBObject()
BasicDBObject geometry = new BasicDBObject("type", "Point")
geometry.append("coordinates", [lng, lat])
near.append('$maxDistance', 1000)
near.append('$geometry', geometry)
location.append('$near', near)
BasicDBObject query = new BasicDBObject()
query.append("location", location)
query.append("attribute",new BasicDBObject('$in', ['省重点', '市重点', '区重点', '县重点', '省级示范']))
query.append("category", new BasicDBObject('$in', ['中学']))
BasicDBObject midSchool = school.findOne(query)
// for(def level in ['省重点', '市重点', '区重点', '县重点', '普通']){
// query.append("location", location)
// query.append("attribute", level)
// midSchool = school.findOne(query)
// if(midSchool){
// break
// }
// }
// if(!midSchool){
// query.removeField("attribute")
// midSchool = school.findOne(query)
// }
BasicDBObject primarySchool=null
query.append("category", new BasicDBObject('$in', ['小学']))
primarySchool = school.findOne(query)
// for(def level in ['省重点', '市重点', '区重点', '县重点', '普通']){
// query.append("location", location)
// query.append("attribute", level)
// primarySchool = school.findOne(query)
// if(primarySchool){
// break
// }
// }
// if(!primarySchool){
// query.removeField("attribute")
// primarySchool = school.findOne(query)
// }
if(primarySchool){
item << primarySchool.name
item << primarySchool.address
item << primarySchool.attribute
item << null
item << primarySchool.location[0]
item << primarySchool.location[1]
}else {
item << null
item << null
item << null
item << null
item << null
item << null
}
if(midSchool){
item << midSchool.name
item << midSchool.address
item << midSchool.attribute
item << null
item << midSchool.location[0]
item << midSchool.location[1]
}else {
item << null
item << null
item << null
item << null
item << null
item << null
}
item << null
Date c_date = seaweed.c_date
Date m_date = seaweed.m_date
if(!c_date && !m_date){
c_date = DateUtil.add(new Date(), Calendar.YEAR, -1)
}
if(!c_date){
c_date = m_date
}
if(!m_date){
m_date = c_date
}
item <<c_date.format("yyyy-MM-dd HH:mm:ss")
item <<m_date.format("yyyy-MM-dd HH:mm:ss")
item << 'fjw'
println(item)
if(primarySchool || midSchool){
data << item
}
}
})
}
ExcelUtils.write("D:\\无学区房数据.xlsx", data, "结果")
})
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210401", "yyyyMMdd")
def startDate = DateUtil.parseDate("20210401", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210501", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -11,26 +11,44 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210401", "yyyyMMdd")
def startDate = DateUtil.parseDate("20210401", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210501", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
def hashSet = []
def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
def data = []
def title = ["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg"]
def param_keys = ["tid", "city", "district", "address", "name", "size", "propertyType"]
param_keys.each {
title << "参数_${it}"
}
def resMap_keys = ["city", "district", "name", "avgPrice", "totalPrice", "buildYear"]
resMap_keys.each {
title << "结果_${it}"
}
data << title
def keys = new HashSet()
while (startDate.before(endDate)) {
calendar.add(Calendar.DATE, 1)
def nextDate = calendar.getTime()
count++
OPEN_URI_LOG.find(and(eq("user_id", "1007721"), eq("request_uri", "/property/evaluate3"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
if(!result.buildYear || !result.avgPrice){
if (!result.buildYear || !result.avgPrice) {
return
}
def params = JSONObject.fromObject(doc.parameter_map)
keys.addAll(params.keys())
def key = "${params.address}_${params.size}".toString()
if (!hashSet.contains(key)) {
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date.format("yyy"), doc.data_status, doc.code, doc.msg]
param_keys.each {
item << params[it]
}
resMap_keys.each {
item << result[it]
}
data << item
hashSet << key
} else {
@@ -153,55 +153,55 @@ import org.apache.commons.lang3.StringUtils
//}
DBCollection BAIDUMAP_BORDER_COLL = MongoUtils.getCollectionByName("addrepo", "baidumap_border", "192.168.0.206", 27020)
BAIDUMAP_BORDER_COLL.find(new BasicDBObject("baidu_border", null)).each {BasicDBObject dbo->
def border = dbo.getString("border").replace("1-", "").replace(";", "").split(",") as List
def coords = []
def baidu_points = []
def gaode_points = []
for(int i=0; i < border.size() - 1; i=i+2){
coords << "${border[i]},${border[i+1]}"
if(coords.size() > 50){
def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
for(JSONObject it : json.result){
baidu_points << "${it.x},${it.y}"
def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
}
coords.clear()
}
}
if(coords.size() > 0){
def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
for(JSONObject it : json.result){
baidu_points << "${it.x},${it.y}"
def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
}
}
dbo.format_city = RegionData.formatCity(dbo.city)
dbo.baidu_border = baidu_points.join(";")
dbo.gaode_border = gaode_points.join(";")
BAIDUMAP_BORDER_COLL.save(dbo)
println(dbo)
}
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\È˿ڲâÊÔÊý¾Ý.xlsx",{List item->
// def seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["B"], item["C"], item["D"]), new BasicDBObject("alias", 1).append("lng",1).append("lat", 1))
// if(seaweed){
// item["A"] = seaweed._id.toString()
// def names = [item['D']]
// if(seaweed.alias){
// names.addAll(seaweed.alias)
//BAIDUMAP_BORDER_COLL.find(new BasicDBObject("baidu_border", null)).each {BasicDBObject dbo->
// def border = dbo.getString("border").replace("1-", "").replace(";", "").split(",") as List
// def coords = []
// def baidu_points = []
// def gaode_points = []
// for(int i=0; i < border.size() - 1; i=i+2){
// coords << "${border[i]},${border[i+1]}"
// if(coords.size() > 50){
// def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
// for(JSONObject it : json.result){
// baidu_points << "${it.x},${it.y}"
// def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
// gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
// }
// coords.clear()
// }
// names.unique()
// }
// if(coords.size() > 0){
// def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
// for(JSONObject it : json.result){
// baidu_points << "${it.x},${it.y}"
// def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
// gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
// }
// }
// dbo.format_city = RegionData.formatCity(dbo.city)
// dbo.baidu_border = baidu_points.join(";")
// dbo.gaode_border = gaode_points.join(";")
// BAIDUMAP_BORDER_COLL.save(dbo)
// println(dbo)
//}
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\Ð¡ÇøÈºÏñ²âÊÔÊý¾Ý.xlsx",{List item->
def seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["B"], item["C"], item["D"]), new BasicDBObject("alias", 1).append("lng",1).append("lat", 1))
if(seaweed){
// item["A"] = seaweed._id.toString()
def names = [item['D']]
if(seaweed.alias){
names.addAll(seaweed.alias)
}
names.unique()
// if(seaweed.lng){
// item["G"] = DataUtil.roundDouble(seaweed.lng/10000000000d, 6)
// item["H"] = DataUtil.roundDouble(seaweed.lat/10000000000d, 6)
// }
// def one = BAIDUMAP_BORDER_COLL.findOne(new BasicDBObject("format_city", item["B"]).append("format_region", item["C"]).append("name", new BasicDBObject('$in', names)))
// if(one){
// item["I"] = one.baidu_border
// }
// println(item)
// }
//})
def one = BAIDUMAP_BORDER_COLL.findOne(new BasicDBObject("format_city", item["B"]).append("format_region", item["C"]).append("name", new BasicDBObject('$in', names)))
if(one){
item["I"] = one.gaode_border
}
println(item)
}
})
@@ -0,0 +1,46 @@
package com.fangjia.script.wenyan.test
import com.aliyuncs.DefaultAcsClient
import com.aliyuncs.IAcsClient
import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityRequest
import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityResponse
import com.aliyuncs.profile.DefaultProfile
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils
import net.sf.json.JSONObject
ExcelUtils.modify("D:\\nextcloud\\数据\\地址类服务测试\\建行地址相似度评级测试.xlsx",{List item->
def city = item["A"]
def address1 = item["B"]
def address2 = item["C"]
HttpRequest http = new HttpRequest()
def params = new HashMap()
params["city"] = city
params["address_1"] = address1
params["address_2"] = address2
params["token"]= "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
JSONObject json = http.getJsonContent("http://open.fangjia.com/address/similarity", params, "UTF-8")
JSONObject result = json.result
item["D"] = result?.level
item["E"] = result?.formattedAddress_1
item["F"] = result?.formattedAddress_2
item["G"] = result?.isSameAddress
DefaultProfile profile = DefaultProfile.getProfile(
"cn-hangzhou",
"LTAIdaj4eSO79gzz",
"xw6EsvqydZ9HC6zAvIB3ruQqVHynxf");
IAcsClient client = new DefaultAcsClient(profile);
GetAddressSimilarityRequest request = new GetAddressSimilarityRequest();
request.setServiceCode("addrp")
request.setAppKey("Xcgtyhupey0h")
request.setText("${address1};${address2}")
GetAddressSimilarityResponse response = client.getAcsResponse(request)
String data = response.getData()
json = JSONObject.fromObject(data)
item["H"] = json["sim_level"]["level"]
item["I"] = json["sim_level"]["address_a"]
item["J"] = json["sim_level"]["address_b"]
item["K"] = json["sim_level"]["is_same"]
println(json)
})