Merge branch 'script_gj' into 'master'

Script gj

See merge request !20
This commit is contained in:
2021-04-22 12:58:35 +08:00
11 changed files with 985 additions and 53 deletions
+7
View File
@@ -476,6 +476,13 @@
<artifactId>aliyun-java-sdk-address-purification</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>com.eclipsesource.j2v8</groupId>
<artifactId>j2v8_win32_x86_64</artifactId>
<version>4.6.0</version>
</dependency>
</dependencies>
<repositories>
<repository>
@@ -0,0 +1,148 @@
package com.fangjia.script.companyCrawl
import net.sf.json.JSONObject
import org.jsoup.Jsoup
import java.util.concurrent.atomic.AtomicInteger
class AiqichaCrawl {
def static final proxyPort = 9020;
def static final proxyServer = "http-dyn.abuyun.com";
def static final proxyUser = "HZU1XU28H3G4Z44D";
def static final proxyPass = "466C86ED77FE43D4";
def companyKeys = ["亨鑫","捷东","盈复","隆庆","卓源 ","合万","通广","安盛","东利","瑞科 ","凡迪","捷信","安盛","安盛","东进","源盛","茂禄","泰亚","宏成","光益","华圣","益全","长凯","宏新","锦立","宜佳","万美","正欣","和利","国宏","和裕","新昌","佳升","正德","源美","宝峰","金广","龙奥","宏美","成同","义丰","全洪","国聚","亨信","百益","广宏","仁瑞","亿昌","东旭","聚凯","洪成","兴同","赛迪","亨国","浩长","达宏","寿满","亿信","本盛","盈凯","浩汇","元辉","祥生","皇弘","讯思","新州","国泰","成合","茂浩","信源","润凯","虎光","安瑞","益吉","达兴","顺光","发盛","金优","捷丰","春凯","昌兴","本佳","远盛","盛达","亚飞","东昌","益如","宝东","德信","浩盈","多泰","泰联","升聚","辉丰","茂发","盈德","旭嘉","优源","益同","协盈","锦泰","捷久","本宏","德泰","聚富","全庆","凯安","进恒","全伟","优元","凯安","义丰","丰宇","瑞优","隆泰","诺德","台升","裕大","亨恒","义浩","百亨","欣昌","茂全","同亨","长圣","成谦","达泰","高协","正思","安巨","新耀","利仁","国益","旺洪","汇中","旭昌","弘康","优盛","旭达","晶安","华浩","耀春","圣本","康丰","光复","同凌","宇伯","乐鸿","成泰","宏协","华信","达泰","广丰","泰全","汇恒","杰森","天畅","恒通","亚达","庆合","丰圣","德长","茂百","汇久","欣鼎","谦和","厚辉","耀德","瑞隆","多辉","昌茂","成高","益升","安达","兴飞","百进","鼎宝","祥盛","信和","隆顺","亚欣","厚全","博文","讯诺","泰如","亨瑞","凯祥","捷浩","文舟","金庆","成仁","万协","宇中","宏祥","永多","艺峰","美益","创赢","奥龙","德宇","国福","飞利","久欣","如亨","安广","协佳","升捷","盛圣","富凯","长鑫","元卓","荣亨","亨仁","康鑫","信和","百新","干瑞","润万","富庭","信宇","源天","凯佳","正升","裕荣","厚泽","晶富","广新","正多","鼎高","丰汇","永华","兴升","复腾","隆诺","中发","广源","康胜","同进","仁捷","康凯","义洪","晶盛","正祥","裕祥","宇新","洪祥","润岳","义富","盛安","浩易"]
static AtomicInteger AbuyunRequestCount = new AtomicInteger(0);
public final static int AbuyunMaxCount = 10;
static main(args) {
def task = new AiqichaCrawl();
task.process()
}
void process(Object args) {
def url = "https://aiqicha.baidu.com/s/l?q=%E6%96%B0%E5%8F%8B&t=&p=3&s=10&o=0&f=%7B%7D "
for(def key : companyKeys){
def queryBody = null
def s = crawl(url, queryBody);
if(s){
print s
}
println s
}
}
def crawl(url, queryBody){
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","aiqicha.baidu.com")
headers.put("Connection","keep-alive")
headers.put("Cache-Control","max-age=0")
headers.put("sec-ch-ua",'"Chromium";v="88", "Google Chrome";v="88", ";Not A Brand";v="99"')
headers.put("sec-ch-ua-mobile","?0")
headers.put("Upgrade-Insecure-Requests","1")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
headers.put("Sec-Fetch-Site","same-origin")
headers.put("Sec-Fetch-Mode","navigate")
headers.put("Sec-Fetch-User","?1")
headers.put("Sec-Fetch-Dest","document")
headers.put("Accept-Language","zh-CN,zh;q=0.9")
String s = doPost(url, queryBody, headers)
if(s && s.contains("window.pageData = ")){
def prefix = "window.pageData ="
def s1 = s.substring(s.indexOf(prefix)+prefix.size())
s1 = s1.substring(0, s1.indexOf("}};")+2)
JSONObject jsonObject = JSONObject.fromObject(s1);
print jsonObject
}
return s
}
static String doPost(String strUrl, String content, Map headers) {
String result = "";
OutputStream outputStream = null;
InputStream inputStream = null;
StringBuilder sb = new StringBuilder();
try {
checkAbuyun(true);
def responseMap = [:]
// 创建代理服务器地址对象
InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
// 创建HTTP类型代理对象
Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
URL url = new URL(strUrl);
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestMethod("POST");
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setConnectTimeout(30000);
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setReadTimeout(30000);
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
urlConnection.setDoOutput(true);
urlConnection.setDoInput(true);
//设置通用请求属性为默认浏览器编码类型
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
urlConnection.setRequestProperty(head, headers.get(head));
}
}
//getOutputStream()返回的输出流,用于写入参数数据。
outputStream = urlConnection.getOutputStream();
if(content){
outputStream.write(content.getBytes());
}
outputStream.flush();
outputStream.close();
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
inputStream = urlConnection.getInputStream();
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
char[] buff = new char[1024];
int length = 0;
while ((length = reader.read(buff)) != -1) {
String s = new String(buff, 0, length);
sb.append(s);
}
} catch (IOException e) {
e.printStackTrace();
}finally{
if(outputStream!=null){
outputStream.close();
}
if(inputStream!=null){
inputStream.close();
}
}
return sb.toString();
}
public static checkAbuyun(boolean check) {
if (check) {
while (AbuyunRequestCount.get() >= AbuyunMaxCount) {
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
e.printStackTrace()
}
}
AbuyunRequestCount.incrementAndGet();
} else {
AbuyunRequestCount.decrementAndGet();
}
}
}
@@ -0,0 +1,715 @@
package com.fangjia.script.companyCrawl
import com.eclipsesource.v8.V8
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.ABuYunUtil
import jdk.nashorn.api.scripting.ScriptObjectMirror
import org.apache.commons.io.IOUtils;
import org.apache.http.Header
import org.apache.http.HttpResponse
import org.apache.http.client.HttpClient
import org.apache.http.client.methods.HttpGet
import org.jsoup.Jsoup
import org.jsoup.select.Elements
import org.apache.http.impl.client.HttpClients
import javax.script.ScriptEngine
import javax.script.ScriptEngineManager;
import java.util.concurrent.atomic.AtomicInteger
class GsxtCrawl {
def static final proxyPort = 9020;
def static final proxyServer = "http-dyn.abuyun.com";
def static final proxyUser = "HZU1XU28H3G4Z44D";
def static final proxyPass = "466C86ED77FE43D4";
public final static int AbuyunMaxCount = 10;
static AtomicInteger AbuyunRequestCount = new AtomicInteger(0);
def companyKeys = ["亨鑫","捷东","盈复","隆庆","卓源 ","合万","通广","安盛","东利","瑞科 ","凡迪","捷信","安盛","安盛","东进","源盛","茂禄","泰亚","宏成","光益","华圣","益全","长凯","宏新","锦立","宜佳","万美","正欣","和利","国宏","和裕","新昌","佳升","正德","源美","宝峰","金广","龙奥","宏美","成同","义丰","全洪","国聚","亨信","百益","广宏","仁瑞","亿昌","东旭","聚凯","洪成","兴同","赛迪","亨国","浩长","达宏","寿满","亿信","本盛","盈凯","浩汇","元辉","祥生","皇弘","讯思","新州","国泰","成合","茂浩","信源","润凯","虎光","安瑞","益吉","达兴","顺光","发盛","金优","捷丰","春凯","昌兴","本佳","远盛","盛达","亚飞","东昌","益如","宝东","德信","浩盈","多泰","泰联","升聚","辉丰","茂发","盈德","旭嘉","优源","益同","协盈","锦泰","捷久","本宏","德泰","聚富","全庆","凯安","进恒","全伟","优元","凯安","义丰","丰宇","瑞优","隆泰","诺德","台升","裕大","亨恒","义浩","百亨","欣昌","茂全","同亨","长圣","成谦","达泰","高协","正思","安巨","新耀","利仁","国益","旺洪","汇中","旭昌","弘康","优盛","旭达","晶安","华浩","耀春","圣本","康丰","光复","同凌","宇伯","乐鸿","成泰","宏协","华信","达泰","广丰","泰全","汇恒","杰森","天畅","恒通","亚达","庆合","丰圣","德长","茂百","汇久","欣鼎","谦和","厚辉","耀德","瑞隆","多辉","昌茂","成高","益升","安达","兴飞","百进","鼎宝","祥盛","信和","隆顺","亚欣","厚全","博文","讯诺","泰如","亨瑞","凯祥","捷浩","文舟","金庆","成仁","万协","宇中","宏祥","永多","艺峰","美益","创赢","奥龙","德宇","国福","飞利","久欣","如亨","安广","协佳","升捷","盛圣","富凯","长鑫","元卓","荣亨","亨仁","康鑫","信和","百新","干瑞","润万","富庭","信宇","源天","凯佳","正升","裕荣","厚泽","晶富","广新","正多","鼎高","丰汇","永华","兴升","复腾","隆诺","中发","广源","康胜","同进","仁捷","康凯","义洪","晶盛","正祥","裕祥","宇新","洪祥","润岳","义富","盛安","浩易"]
static main(args) {
def task = new GsxtCrawl();
//task.process()
task.run()
}
def run(){
getObject()
HttpClient client = HttpClients.createDefault();
HttpGet get = new HttpGet("http://www.gsxt.gov.cn/corp-query-homepage.html");
get.setHeader("Upgrade-Insecure-Requests", "1");
get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36");
HttpResponse response = client.execute(get);
def __jsluid_h = getJsluid(response)
def s = getResponseBodyAsString(response)
println (s)
def __jsl_clearance = getJsContent(s);
//Cookie: __jsluid_h=26cb8385493c2e424767eca767720c87; __jsl_clearance=1618899529.627|-1|ao39lP6IVSloPMvq5%2FOSBqSiPZ0%3D
def newCookie = __jsluid_h+";"+__jsl_clearance
get.setHeader("Cookie", newCookie)
HttpResponse response2 = client.execute(get);
def s2 = getResponseBodyAsString(response2)
def __jsl_clearance2 = getJsContent2(s2);
println(__jsl_clearance2)
// fristCrawl()
}
//该函数测试Java与js对象
static void getObject() throws Exception{
String str = "var _0xbe8c = ['w4LCg8Onw7o=', 'ZAjDvCw=', 'YcKOw6Fm', 'wpHCqsO3CQ==', 'UMKCXnw=', 'w7Aww6XCnQ==', 'BcKNwrvCng==', 'J2zCnsKi', 'wq02WMOx', 'RQXDrUw=', 'eCnCnDw=', 'Q8KowpzDjA==', 'YznDucOn', 'YcOrXxk=', 'w4t9PsKM', 'DMO4QmA=', 'w5HCklPDmQ==', 'w5rCv8Omw6Q=', 'NcOwc8K5', 'GUxjw7Q=', 'w7zDicK8w4I=', 'VCDDpsKv', 'cwLChT4=', 'wpvCl1vDqw==', 'wrvCvMOKNQ==', 'RTnDrAc=', 'GMKFw4hk', 'SxnDvC0=', 'H2NHNQ==', 'w6Aqw7LCqg==', 'wq0mW8Ok', 'L1zCv8Kp', 'B8OqwoI2', 'Jl7DuX8=', 'JcOBwpIR', 'U8OnfSo=', '6K2K5rOG6aqL6KyM', 'f8Kiwo/Dhg==', 'aMK6w6vDog==', 'JcKgw4FC', 'w61vJEg=', 'LMOrwoY0', 'GcK4w6lt', 'w4XDrsO5wpY=', 'J8OhwoM1', 'w5Mzw7J/', 'wocbcBA=', 'SibDt8Oj', 'w7xgwoXDkQ==', 'AcKyDEQ=', 'Rg7Dkw==', 'w6UFK8Kw', 'TTzDjcKK', 'w6QhDcKy', 'ZBLDols=', 'ZjPDrk4=', 'w6XDqX3Ctw==', 'w6EvJcKT', 'JHtUDQ==', 'wq8mI8KE', 'JMOhwoJ9', 'MsK1AkE=', 'w7fCmhcm', 'VsK2w5fDtQ==', 'w7EUw4PCnw==', 'dz/DgsKu', 'UzfDtcK0', 'wpd9fWE=', 'worDvcKFw4M=', 'w4PDr0zDhg==', 'w7ocSMKg', 'wp1uwoZd', 'eAjDv8Oz', 'aATCgTw=', 'w7UGw7k=', 'wqZCwotq', 'HFIBw5s=', 'WsOdw7JB', 'w5PDpXHCpA==', 'GMO8wqdD', 'w4TCsQs=', 'w6/DmcODwpg=', 'w7MuwrjDoQ==', 'dDHDhsOj', 'G308w70=', 'CsO6wog9', 'FcONwrVw', 'M3vCmMK4', 'w6/Dm8OvwpI=', 'QsOLw5ZM', 'TgLDicKy', 'QSHCuQA=', 'UsOlw5RI', 'w7EqwqDDrw==', 'w6jDusK5w4k=', 'OsKaw5rDlg==', 'wosLQsOk', 'DcKxw7Vo', 'w5VuIEo=', 'w7sXw6TCqw==', 'wrLDr8KRw7s=', 'wr9+wpJR', 'w4LDscO0wrQ=', 'FFIPw6U=', 'J8Ksw5rDlA==', 'd8KZwqjDgw==', 'PcKvDF4=', 'MHh2GA==', 'dgDDs0Y=', 'wpxSS1Y=', 'w7l5JMKh', 'w7N8JMKC', 'XsKhw7hR', 'QxDDqm4=', 'PcKGw5ZJ', 'w7HCuAIQ', 'E8Kmw53Dmg==', 'CMOrwok3', 'U8K9wpvDnA==', 'IW06w5U=', 'w7DDjcOGwpQ=', 'X8KlwpHDpA==', 'w6TDn8O1wpo=', 'ZjnDv08=', 'w7/Cqjsp', 'wqLDk8KYw58=', 'w6fDvMKXw4g=', 'FsK8IUY=', 'w4knw4Rh', 'w5zCu8Ogw5M=', 'wqJ0ckc=', 'T8Kgw69G', 'GF0hw74=', 'FMOkwqkH', 'wokmbMOM', 'djHDtcKW', 'FXkow64=', 'B8OHwowH', 'VSjCpQU=', 'w5vDs8K0w4s=', 'w7vCkUV1', 'w7dMIcK8', 'WMOjZyI=', 'LcKBw7po', 'BMK7HkY=', 'B8K0w7F4', 'YsK+w67Drw==', 'HlQKw7I=', 'wpLCrMKrNg==', 'w7IjcsKy', 'w6zDusKhw4o=', 'WhPDm8Om', 'dMKlw7RO', 'dcOHw5R3', 'Nkpww7M=', 'wrnCsMKLdg==', 'dDTDqMKb', 'S8KXZXE=', 'wq/CssOpJA==', 'w7J/KFw=', 'QMKSWEo=', 'woNxbEc=', 'wqVIwow=', 'TD/Dqg4=', 'w7TDpmzCiw==', 'w7HDjWDCgA==', 'w7IEw6vCoA==', 'Q8K+w4zDiQ==', 'PElvw7k=', 'cDXDsRo=', 'FMOLX0c=', 'Z8KHwojDoA==', 'TTTCqzo=', 'woXCrsK6Uw==', 'w4lhHnY=', 'w6Q/dsKj', 'KsKIwpLCtQ==', 'QQTDn8Oy', 'D8Omw7zDoQ==', 'E8KzGk0=']; (function(_0x44c192, _0xbe8c61) {\n" +
"\t\tvar _0x1533fb = function(_0x5b499f) {\n" +
"\t\t\twhile (--_0x5b499f) {\n" +
"\t\t\t\t_0x44c192['push'](_0x44c192['shift']());\n" +
"\t\t\t}\n" +
"\t\t};\n" +
"\t\t_0x1533fb(++_0xbe8c61);\n" +
"\t} (_0xbe8c, 0x185));\n" +
"\tvar _0x1533 = function(_0x44c192, _0xbe8c61) {\n" +
"\t\t_0x44c192 = _0x44c192 - 0x0;\n" +
"\t\tvar _0x1533fb = _0xbe8c[_0x44c192];\n" +
"\t\tif (_0x1533['GlBvhA'] === undefined) { (function() {\n" +
"\t\t\t\tvar _0xa131c3 = function() {\n" +
"\t\t\t\t\tvar _0x9829e5;\n" +
"\t\t\t\t\ttry {\n" +
"\t\t\t\t\t\t_0x9829e5 = Function('return\\x20(function()\\x20' + '{}.constructor(\\x22return\\x20this\\x22)(\\x20)' + ');')();\n" +
"\t\t\t\t\t} catch(_0x1ba3fa) {\n" +
"\t\t\t\t\t}\n" +
"\t\t\t\t\treturn _0x9829e5;\n" +
"\t\t\t\t};\n" +
"\t\t\t\tvar _0x5e86d6 = _0xa131c3();\n" +
"\t\t\t\tvar _0x67fb0c = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=';\n" +
"\t\t\t\t_0x5e86d6['atob'] || (_0x5e86d6['atob'] = function(_0x1cb044) {\n" +
"\t\t\t\t\tvar _0xa69f93 = String(_0x1cb044)['replace'](/=+\$/, '');\n" +
"\t\t\t\t\tvar _0x26bfb9 = '';\n" +
"\t\t\t\t\tfor (var _0x2f1d79 = 0x0,\n" +
"\t\t\t\t\t_0x3713bb, _0x4e514b, _0x29733b = 0x0; _0x4e514b = _0xa69f93['charAt'](_0x29733b++);~_0x4e514b && (_0x3713bb = _0x2f1d79 % 0x4 ? _0x3713bb * 0x40 + _0x4e514b: _0x4e514b, _0x2f1d79++%0x4) ? _0x26bfb9 += String['fromCharCode'](0xff & _0x3713bb >> ( - 0x2 * _0x2f1d79 & 0x6)) : 0x0) {\n" +
"\t\t\t\t\t\t_0x4e514b = _0x67fb0c['indexOf'](_0x4e514b);\n" +
"\t\t\t\t\t}\n" +
"\t\t\t\t\treturn _0x26bfb9;\n" +
"\t\t\t\t});\n" +
"\t\t\t} ());\n" +
"\t\t\tvar _0x38a7ff = function(_0x5dd34b, _0x3a0f2e) {\n" +
"\t\t\t\tvar _0x3e7a99 = [],\n" +
"\t\t\t\t_0x3e2c47 = 0x0,\n" +
"\t\t\t\t_0x383663,\n" +
"\t\t\t\t_0x14720e = '',\n" +
"\t\t\t\t_0x5864f4 = '';\n" +
"\t\t\t\t_0x5dd34b = atob(_0x5dd34b);\n" +
"\t\t\t\tfor (var _0x28a6d2 = 0x0,\n" +
"\t\t\t\t_0x2ea622 = _0x5dd34b['length']; _0x28a6d2 < _0x2ea622; _0x28a6d2++) {\n" +
"\t\t\t\t\t_0x5864f4 += '%' + ('00' + _0x5dd34b['charCodeAt'](_0x28a6d2)['toString'](0x10))['slice']( - 0x2);\n" +
"\t\t\t\t}\n" +
"\t\t\t\t_0x5dd34b = decodeURIComponent(_0x5864f4);\n" +
"\t\t\t\tvar _0x3091fb;\n" +
"\t\t\t\tfor (_0x3091fb = 0x0; _0x3091fb < 0x100; _0x3091fb++) {\n" +
"\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3091fb;\n" +
"\t\t\t\t}\n" +
"\t\t\t\tfor (_0x3091fb = 0x0; _0x3091fb < 0x100; _0x3091fb++) {\n" +
"\t\t\t\t\t_0x3e2c47 = (_0x3e2c47 + _0x3e7a99[_0x3091fb] + _0x3a0f2e['charCodeAt'](_0x3091fb % _0x3a0f2e['length'])) % 0x100;\n" +
"\t\t\t\t\t_0x383663 = _0x3e7a99[_0x3091fb];\n" +
"\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3e7a99[_0x3e2c47];\n" +
"\t\t\t\t\t_0x3e7a99[_0x3e2c47] = _0x383663;\n" +
"\t\t\t\t}\n" +
"\t\t\t\t_0x3091fb = 0x0;\n" +
"\t\t\t\t_0x3e2c47 = 0x0;\n" +
"\t\t\t\tfor (var _0x8c68bc = 0x0; _0x8c68bc < _0x5dd34b['length']; _0x8c68bc++) {\n" +
"\t\t\t\t\t_0x3091fb = (_0x3091fb + 0x1) % 0x100;\n" +
"\t\t\t\t\t_0x3e2c47 = (_0x3e2c47 + _0x3e7a99[_0x3091fb]) % 0x100;\n" +
"\t\t\t\t\t_0x383663 = _0x3e7a99[_0x3091fb];\n" +
"\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3e7a99[_0x3e2c47];\n" +
"\t\t\t\t\t_0x3e7a99[_0x3e2c47] = _0x383663;\n" +
"\t\t\t\t\t_0x14720e += String['fromCharCode'](_0x5dd34b['charCodeAt'](_0x8c68bc) ^ _0x3e7a99[(_0x3e7a99[_0x3091fb] + _0x3e7a99[_0x3e2c47]) % 0x100]);\n" +
"\t\t\t\t}\n" +
"\t\t\t\treturn _0x14720e;\n" +
"\t\t\t};\n" +
"\t\t\t_0x1533['sMWqUz'] = _0x38a7ff;\n" +
"\t\t\t_0x1533['AbOzCI'] = {};\n" +
"\t\t\t_0x1533['GlBvhA'] = !![];\n" +
"\t\t}\n" +
"\t\tvar _0x5b499f = _0x1533['AbOzCI'][_0x44c192];\n" +
"\t\tif (_0x5b499f === undefined) {\n" +
"\t\t\tif (_0x1533['uRkTAI'] === undefined) {\n" +
"\t\t\t\t_0x1533['uRkTAI'] = !![];\n" +
"\t\t\t}\n" +
"\t\t\t_0x1533fb = _0x1533['sMWqUz'](_0x1533fb, _0xbe8c61);\n" +
"\t\t\t_0x1533['AbOzCI'][_0x44c192] = _0x1533fb;\n" +
"\t\t} else {\n" +
"\t\t\t_0x1533fb = _0x5b499f;\n" +
"\t\t}\n" +
"\t\treturn _0x1533fb;\n" +
"\t};\n" +
"\tfunction hash(_0x1a5a87) {\n" +
"\t\tvar _0x23a3e0 = {};\n" +
"\t\t_0x23a3e0[_0x1533('0x93', 'Kp2R') + 'J'] = function(_0x5ebeb2, _0x4681fc) {\n" +
"\t\t\treturn _0x5ebeb2 ^ _0x4681fc;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x17', '(bZ%') + 'H'] = function(_0x5108c3, _0x49fe65) {\n" +
"\t\t\treturn _0x5108c3 & _0x49fe65;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x88', 'N#(C') + 'T'] = function(_0x3d3f10, _0x3f2a86) {\n" +
"\t\t\treturn _0x3d3f10 !== _0x3f2a86;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x60', '(cNt') + 'Z'] = _0x1533('0x70', 'K)PG') + 'J';\n" +
"\t\t_0x23a3e0[_0x1533('0x49', 'Y8u@') + 'h'] = function(_0xc3ff68, _0x129ab0) {\n" +
"\t\t\treturn _0xc3ff68 >> _0x129ab0;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x9c', '\$qIi') + 'S'] = function(_0x591dc1, _0x158d9d) {\n" +
"\t\t\treturn _0x591dc1 + _0x158d9d;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x7f', '(bZ%') + 'f'] = function(_0xd6deb, _0x557760) {\n" +
"\t\t\treturn _0xd6deb >> _0x557760;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x4e', 'S9Bn') + 'd'] = function(_0xd05226, _0x29b6e8) {\n" +
"\t\t\treturn _0xd05226 * _0x29b6e8;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x7e', 'y^BN') + 'n'] = function(_0x899f8d, _0x3dbd26) {\n" +
"\t\t\treturn _0x899f8d < _0x3dbd26;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x9a', '(U[3') + 'e'] = function(_0x39678a, _0x37d488) {\n" +
"\t\t\treturn _0x39678a * _0x37d488;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0xa3', 'sahq') + 'U'] = function(_0x4cf877, _0x14e1b0) {\n" +
"\t\t\treturn _0x4cf877 << _0x14e1b0;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0xa7', 'K#Hh') + 'u'] = function(_0x556336, _0x56b41c) {\n" +
"\t\t\treturn _0x556336 - _0x56b41c;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x3d', '(bZ%') + 'p'] = function(_0x372c20, _0x534fe1) {\n" +
"\t\t\treturn _0x372c20 >> _0x534fe1;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x1d', 'S9Bn') + 'g'] = function(_0x26a0f8, _0x4f1660) {\n" +
"\t\t\treturn _0x26a0f8 >>> _0x4f1660;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0xae', ']*f@') + 'Y'] = function(_0x3423f4, _0x53e1a9) {\n" +
"\t\t\treturn _0x3423f4 | _0x53e1a9;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x26', 's4Dp') + 'P'] = function(_0x19382a, _0x220110) {\n" +
"\t\t\treturn _0x19382a | _0x220110;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x65', 's4Dp') + 'p'] = function(_0x7b95bc, _0x22e933) {\n" +
"\t\t\treturn _0x7b95bc & _0x22e933;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x0', '\$qIi') + 'V'] = function(_0x13269b, _0x227e3b) {\n" +
"\t\t\treturn _0x13269b ^ _0x227e3b;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x29', 'J21I') + 'R'] = function(_0x1ba947, _0x4b40e4) {\n" +
"\t\t\treturn _0x1ba947(_0x4b40e4);\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x50', 's4Dp') + 'n'] = _0x1533('0xa', 'qRqa') + 'e';\n" +
"\t\t_0x23a3e0[_0x1533('0x42', 'ksVw') + 'b'] = function(_0x466fa6, _0x3ce8da) {\n" +
"\t\t\treturn _0x466fa6 ^ _0x3ce8da;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x2c', 'rqYb') + 'f'] = function(_0x1e608a, _0x241888) {\n" +
"\t\t\treturn _0x1e608a - _0x241888;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x78', 'J21I') + 'u'] = function(_0x564b78, _0x18d236, _0x8fd805) {\n" +
"\t\t\treturn _0x564b78(_0x18d236, _0x8fd805);\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x69', 'zx(Q') + 'D'] = function(_0x3b7666, _0x47f238, _0x57b05e) {\n" +
"\t\t\treturn _0x3b7666(_0x47f238, _0x57b05e);\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x5f', 'ZaCV') + 'm'] = function(_0x452fa7, _0x493d76, _0x1446e7, _0x4af814, _0x5e0f59) {\n" +
"\t\t\treturn _0x452fa7(_0x493d76, _0x1446e7, _0x4af814, _0x5e0f59);\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x22', 'Rh@#') + 'W'] = function(_0x51a858, _0x440b1c) {\n" +
"\t\t\treturn _0x51a858(_0x440b1c);\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x8', 'xPGU') + 'A'] = function(_0x5e96f2, _0x167b41) {\n" +
"\t\t\treturn _0x5e96f2 + _0x167b41;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x6d', 'rqYb') + 'X'] = function(_0x46cd37, _0x5d8684) {\n" +
"\t\t\treturn _0x46cd37 + _0x5d8684;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x3e', 'rqYb') + 'g'] = function(_0x26bc4b, _0x16bc46) {\n" +
"\t\t\treturn _0x26bc4b(_0x16bc46);\n" +
"\t\t};\n" +
"\t\tvar _0x179134 = _0x23a3e0;\n" +
"\t\tfunction _0x5be8c0(_0x296dd6, _0x778566) {\n" +
"\t\t\treturn _0x179134[_0x1533('0x56', 'ZaCV') + 'J'](_0x179134[_0x1533('0x7c', 'o&Q4') + 'H'](_0x296dd6, 0x7fffffff) + (_0x778566 & 0x7fffffff), _0x179134[_0x1533('0x52', 'K)PG') + 'H'](_0x296dd6, 0x80000000)) ^ _0x778566 & 0x80000000;\n" +
"\t\t}\n" +
"\t\tfunction _0x225a28(_0x2f57af) {\n" +
"\t\t\tvar _0x3c49d8 = {};\n" +
"\t\t\t_0x3c49d8[_0x1533('0x55', 'qRqa') + 'N'] = function(_0x15bfb8, _0x51e1eb) {\n" +
"\t\t\t\treturn _0x15bfb8 < _0x51e1eb;\n" +
"\t\t\t};\n" +
"\t\t\tvar _0x567e0f = _0x3c49d8;\n" +
"\t\t\tvar _0x5b2e48 = _0x1533('0xa4', 'rQf(') + _0x1533('0x4', 'rqYb') + _0x1533('0x95', 'Rh@#') + _0x1533('0x3', 'OhQr');\n" +
"\t\t\tvar _0x321f21 = '';\n" +
"\t\t\tfor (var _0x2a5528 = 0x7; _0x2a5528 >= 0x0; _0x2a5528--) {\n" +
"\t\t\t\tif (_0x179134[_0x1533('0xb2', 'Gfc!') + 'T'](_0x1533('0x2', 'w*Nn') + 'h', _0x179134[_0x1533('0x6c', 'OhQr') + 'Z'])) {\n" +
"\t\t\t\t\t_0x321f21 += _0x5b2e48[_0x1533('0x66', ')T5g') + 'At'](_0x179134[_0x1533('0x9f', 'zx(Q') + 'H'](_0x179134[_0x1533('0x5', 'OhQr') + 'h'](_0x2f57af, _0x2a5528 * 0x4), 0xf));\n" +
"\t\t\t\t} else {\n" +
"\t\t\t\t\treturn _0x567e0f[_0x1533('0x5a', 'y*wC') + 'N'](t, 0x14) ? 0x5a827999: t < 0x28 ? 0x6ed9eba1: _0x567e0f[_0x1533('0x43', 'zx(Q') + 'N'](t, 0x3c) ? -0x70e44324: -0x359d3e2a;\n" +
"\t\t\t\t}\n" +
"\t\t\t}\n" +
"\t\t\treturn _0x321f21;\n" +
"\t\t}\n" +
"\t\tfunction _0xde630f(_0x10fb9a) {\n" +
"\t\t\tvar _0x5b5b6d = _0x179134[_0x1533('0x16', 'GvA9') + 'S'](_0x179134[_0x1533('0x87', 'N#(C') + 'f'](_0x179134[_0x1533('0x83', '[H[u') + 'S'](_0x10fb9a[_0x1533('0x58', 'OhQr') + 'th'], 0x8), 0x6), 0x1),\n" +
"\t\t\t_0x797793 = new Array(_0x179134[_0x1533('0x59', 'xPGU') + 'd'](_0x5b5b6d, 0x10));\n" +
"\t\t\tfor (var _0x109346 = 0x0; _0x179134[_0x1533('0x61', 'dgE&') + 'n'](_0x109346, _0x179134[_0x1533('0x90', 'y^BN') + 'e'](_0x5b5b6d, 0x10)); _0x109346++) {\n" +
"\t\t\t\tif (_0x179134[_0x1533('0x2a', 'NNf8') + 'T'](_0x1533('0x12', 'sahq') + 'y', _0x1533('0x67', 'y*wC') + 'G')) {\n" +
"\t\t\t\t\t_0x797793[_0x109346] = 0x0;\n" +
"\t\t\t\t} else {\n" +
"\t\t\t\t\treturn !! [];\n" +
"\t\t\t\t}\n" +
"\t\t\t}\n" +
"\t\t\tfor (_0x109346 = 0x0; _0x109346 < _0x10fb9a[_0x1533('0x40', 'ad*k') + 'th']; _0x109346++) {\n" +
"\t\t\t\t_0x797793[_0x109346 >> 0x2] |= _0x179134[_0x1533('0xa9', '@gLq') + 'U'](_0x10fb9a[_0x1533('0x1c', 'GvA9') + _0x1533('0xf', 'OhQr') + 'At'](_0x109346), _0x179134[_0x1533('0xb3', 'S9Bn') + 'u'](0x18, (_0x109346 & 0x3) * 0x8));\n" +
"\t\t\t}\n" +
"\t\t\t_0x797793[_0x179134[_0x1533('0xb1', 'qRqa') + 'p'](_0x109346, 0x2)] |= 0x80 << 0x18 - (_0x109346 & 0x3) * 0x8;\n" +
"\t\t\t_0x797793[_0x5b5b6d * 0x10 - 0x1] = _0x10fb9a[_0x1533('0xb', '@mqx') + 'th'] * 0x8;\n" +
"\t\t\treturn _0x797793;\n" +
"\t\t}\n" +
"\t\tfunction _0x50e067(_0x45a1cc, _0x3bcee1) {\n" +
"\t\t\treturn _0x45a1cc << _0x3bcee1 | _0x179134[_0x1533('0x5b', 'Y8u@') + 'g'](_0x45a1cc, _0x179134[_0x1533('0x91', '@mqx') + 'u'](0x20, _0x3bcee1));\n" +
"\t\t}\n" +
"\t\tfunction _0x526d3b(_0xb72077, _0xed7f25, _0x5bc269, _0x572b62) {\n" +
"\t\t\tif (_0xb72077 < 0x14) return _0x179134[_0x1533('0x98', 'Gfc!') + 'Y'](_0xed7f25 & _0x5bc269, _0x179134[_0x1533('0x68', 'OhQr') + 'H'](~_0xed7f25, _0x572b62));\n" +
"\t\t\tif (_0x179134[_0x1533('0x9e', 'w*Nn') + 'n'](_0xb72077, 0x28)) return _0x179134[_0x1533('0x99', ')T5g') + 'J'](_0xed7f25, _0x5bc269) ^ _0x572b62;\n" +
"\t\t\tif (_0xb72077 < 0x3c) return _0x179134[_0x1533('0x13', 'CNzQ') + 'P'](_0xed7f25 & _0x5bc269, _0x179134[_0x1533('0x45', '@mqx') + 'p'](_0xed7f25, _0x572b62)) | _0x179134[_0x1533('0xa0', 'avK7') + 'p'](_0x5bc269, _0x572b62);\n" +
"\t\t\treturn _0x179134[_0x1533('0x5e', 'avK7') + 'V'](_0xed7f25, _0x5bc269) ^ _0x572b62;\n" +
"\t\t}\n" +
"\t\tfunction _0x496785(_0x583c97) {\n" +
"\t\t\treturn _0x583c97 < 0x14 ? 0x5a827999: _0x583c97 < 0x28 ? 0x6ed9eba1: _0x583c97 < 0x3c ? -0x70e44324: -0x359d3e2a;\n" +
"\t\t}\n" +
"\t\tvar _0x45b840 = _0x179134[_0x1533('0x39', 'R#BF') + 'R'](_0xde630f, _0x1a5a87);\n" +
"\t\tvar _0x53258d = new Array(0x50);\n" +
"\t\tvar _0xc86ce6 = 0x67452301;\n" +
"\t\tvar _0x3ee50e = -0x10325477;\n" +
"\t\tvar _0xdb1008 = -0x67452302;\n" +
"\t\tvar _0xe99bcb = 0x10325476;\n" +
"\t\tvar _0x56c860 = -0x3c2d1e10;\n" +
"\t\tfor (var _0x4c314c = 0x0; _0x4c314c < _0x45b840[_0x1533('0x46', '\$qIi') + 'th']; _0x4c314c += 0x10) {\n" +
"\t\t\tvar _0x41e8e8 = _0xc86ce6;\n" +
"\t\t\tvar _0x1b7277 = _0x3ee50e;\n" +
"\t\t\tvar _0x30f7c3 = _0xdb1008;\n" +
"\t\t\tvar _0x279c95 = _0xe99bcb;\n" +
"\t\t\tvar _0xf6e888 = _0x56c860;\n" +
"\t\t\tfor (var _0x77e31b = 0x0; _0x179134[_0x1533('0x34', 'Y8u@') + 'n'](_0x77e31b, 0x50); _0x77e31b++) {\n" +
"\t\t\t\tif (_0x77e31b < 0x10) {\n" +
"\t\t\t\t\tif (_0x179134[_0x1533('0xa6', 'Mifp') + 'T'](_0x1533('0x3f', 'o&Q4') + 'G', _0x179134[_0x1533('0xa1', 'rqYb') + 'n'])) {\n" +
"\t\t\t\t\t\t_0x53258d[_0x77e31b] = _0x45b840[_0x4c314c + _0x77e31b];\n" +
"\t\t\t\t\t} else {\n" +
"\t\t\t\t\t\tsleepT = _0x179134[_0x1533('0x48', 'NNf8') + 'R'](parseInt, data['wt']) > ret[0x1] ? _0x179134[_0x1533('0x97', 'd[i)') + 'R'](parseInt, data['wt']) - ret[0x1] : 0x1f4;\n" +
"\t\t\t\t\t}\n" +
"\t\t\t\t} else {\n" +
"\t\t\t\t\t_0x53258d[_0x77e31b] = _0x50e067(_0x179134[_0x1533('0x96', 'rQf(') + 'b'](_0x179134[_0x1533('0xb0', 'Gfc!') + 'b'](_0x53258d[_0x179134[_0x1533('0xa5', 'K)PG') + 'u'](_0x77e31b, 0x3)], _0x53258d[_0x179134[_0x1533('0x51', 'K)PG') + 'f'](_0x77e31b, 0x8)]) ^ _0x53258d[_0x77e31b - 0xe], _0x53258d[_0x179134[_0x1533('0xaf', '(U[3') + 'f'](_0x77e31b, 0x10)]), 0x1);\n" +
"\t\t\t\t}\n" +
"\t\t\t\tt = _0x179134[_0x1533('0x18', 'GvA9') + 'u'](_0x5be8c0, _0x179134[_0x1533('0x23', '\$qIi') + 'u'](_0x5be8c0, _0x179134[_0x1533('0x8d', 'Mifp') + 'D'](_0x50e067, _0xc86ce6, 0x5), _0x179134[_0x1533('0x4c', 'xPGU') + 'm'](_0x526d3b, _0x77e31b, _0x3ee50e, _0xdb1008, _0xe99bcb)), _0x5be8c0(_0x5be8c0(_0x56c860, _0x53258d[_0x77e31b]), _0x179134[_0x1533('0xad', 'rqYb') + 'W'](_0x496785, _0x77e31b)));\n" +
"\t\t\t\t_0x56c860 = _0xe99bcb;\n" +
"\t\t\t\t_0xe99bcb = _0xdb1008;\n" +
"\t\t\t\t_0xdb1008 = _0x50e067(_0x3ee50e, 0x1e);\n" +
"\t\t\t\t_0x3ee50e = _0xc86ce6;\n" +
"\t\t\t\t_0xc86ce6 = t;\n" +
"\t\t\t}\n" +
"\t\t\t_0xc86ce6 = _0x5be8c0(_0xc86ce6, _0x41e8e8);\n" +
"\t\t\t_0x3ee50e = _0x5be8c0(_0x3ee50e, _0x1b7277);\n" +
"\t\t\t_0xdb1008 = _0x5be8c0(_0xdb1008, _0x30f7c3);\n" +
"\t\t\t_0xe99bcb = _0x5be8c0(_0xe99bcb, _0x279c95);\n" +
"\t\t\t_0x56c860 = _0x5be8c0(_0x56c860, _0xf6e888);\n" +
"\t\t}\n" +
"\t\treturn _0x179134[_0x1533('0x86', 'Gfc!') + 'A'](_0x179134[_0x1533('0x44', 'DDSy') + 'X'](_0x179134[_0x1533('0x8e', 'xPGU') + 'g'](_0x225a28, _0xc86ce6), _0x179134[_0x1533('0x77', 'n*Rs') + 'g'](_0x225a28, _0x3ee50e)) + _0x225a28(_0xdb1008) + _0x225a28(_0xe99bcb), _0x179134[_0x1533('0xd', 'DDSy') + 'g'](_0x225a28, _0x56c860));\n" +
"\t}\n" +
"\tfunction go(_0x394549) {\n" +
"\t\tvar _0x33d0d6 = {};\n" +
"\t\t_0x33d0d6[_0x1533('0x2b', 'sahq') + 'H'] = _0x1533('0x14', 'rQf(') + _0x1533('0x15', 'sahq');\n" +
"\t\t_0x33d0d6[_0x1533('0xab', 'dgE&') + 'H'] = function(_0x23f209, _0x4ed415) {\n" +
"\t\t\treturn _0x23f209 < _0x4ed415;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x5c', 'xPGU') + 'g'] = function(_0x39bd0a, _0x4d5764) {\n" +
"\t\t\treturn _0x39bd0a != _0x4d5764;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x37', 'y*wC') + 'N'] = function(_0x3c9a26, _0x109b89) {\n" +
"\t\t\treturn _0x3c9a26 + _0x109b89;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x63', 'gkl*') + 'z'] = function(_0x584935, _0x58347c) {\n" +
"\t\t\treturn _0x584935 + _0x58347c;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x21', 'ZaCV') + 'z'] = function(_0x3fc795, _0x6962f6) {\n" +
"\t\t\treturn _0x3fc795 + _0x6962f6;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x27', '(cNt') + 'I'] = function(_0x303446, _0x3388fb) {\n" +
"\t\t\treturn _0x303446 == _0x3388fb;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x6b', 'y*wC') + 't'] = function(_0x5cb515, _0x31e7f3) {\n" +
"\t\t\treturn _0x5cb515(_0x31e7f3);\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x75', 'Rh@#') + 'E'] = function(_0x529d8e, _0xc1423a) {\n" +
"\t\t\treturn _0x529d8e + _0xc1423a;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x62', 'rQf(') + 'a'] = function(_0x453505, _0x50cd56) {\n" +
"\t\t\treturn _0x453505 + _0x50cd56;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x2e', 'NNf8') + 'B'] = _0x1533('0x28', ']*f@') + _0x1533('0x57', 'Kp2R') + '\\x20/';\n" +
"\t\t_0x33d0d6[_0x1533('0xe', 'Y8u@') + 'y'] = function(_0x25bcfd, _0x4b634e, _0x1bef22) {\n" +
"\t\t\treturn _0x25bcfd(_0x4b634e, _0x1bef22);\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x92', 'J21I') + 'i'] = function(_0xccfc4d, _0x9a8620) {\n" +
"\t\t\treturn _0xccfc4d > _0x9a8620;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x11', '@H8e') + 'n'] = function(_0x1b9fc7, _0x7d3ce) {\n" +
"\t\t\treturn _0x1b9fc7(_0x7d3ce);\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x1b', 'N#(C') + 'O'] = _0x1533('0x7', 'N#(C') + '失败';\n" +
"\t\tvar _0x2eda7d = _0x33d0d6;\n" +
"\t\tfunction _0x29e502() {\n" +
"\t\t\tvar _0x24a3c1 = window[_0x1533('0x35', 'ad*k') + _0x1533('0x5d', 'Y8u@') + 'r'][_0x1533('0x19', 'avK7') + _0x1533('0x7d', 'cJ3U') + 't'],\n" +
"\t\t\t_0x5f1192 = [_0x2eda7d[_0x1533('0x24', '(bZ%') + 'H']];\n" +
"\t\t\tfor (var _0x2d8686 = 0x0; _0x2eda7d[_0x1533('0x1a', 'avK7') + 'H'](_0x2d8686, _0x5f1192[_0x1533('0x3b', 'Y8u@') + 'th']); _0x2d8686++) {\n" +
"\t\t\t\tif (_0x2eda7d[_0x1533('0x2f', 'y*wC') + 'g'](_0x24a3c1[_0x1533('0x76', 'y*wC') + _0x1533('0x33', 'ZaCV')](_0x5f1192[_0x2d8686]), -0x1)) {\n" +
"\t\t\t\t\treturn !! [];\n" +
"\t\t\t\t}\n" +
"\t\t\t}\n" +
"\t\t\tif (window[_0x1533('0x41', 'dgE&') + _0x1533('0x6e', 'dgE&') + _0x1533('0x85', 'NNf8')] || window[_0x1533('0x30', 'o&Q4') + _0x1533('0x38', 'OhQr')] || window[_0x1533('0xc', 'OhQr') + _0x1533('0x74', 'qRqa')] || window[_0x1533('0x25', '(bZ%') + _0x1533('0x4f', 'avK7') + 'r'][_0x1533('0xa2', 'xPGU') + _0x1533('0x53', ')T5g') + 'r'] || window[_0x1533('0xaa', 'cJ3U') + _0x1533('0x6', 'K#@S') + 'r'][_0x1533('0x8f', 'rqYb') + _0x1533('0xac', '(bZ%') + _0x1533('0x9', 'Rh@#') + _0x1533('0x4d', 'rQf(') + 'e'] || window[_0x1533('0x31', 'N#(C') + _0x1533('0x79', 'dgE&') + 'r'][_0x1533('0x4b', 'ksVw') + _0x1533('0x10', 'gkl*') + _0x1533('0xa8', 'd[i)') + _0x1533('0x89', '\$qIi') + _0x1533('0x84', 's4Dp')]) {\n" +
"\t\t\t\treturn !! [];\n" +
"\t\t\t}\n" +
"\t\t};\n" +
"\t\tif (_0x29e502()) {\n" +
"\t\t\treturn;\n" +
"\t\t}\n" +
"\t\tvar _0x1642ef = new Date();\n" +
"\t\tfunction _0x1a1fd9(_0x55baea, _0x579edb) {\n" +
"\t\t\tvar _0x551ac6 = _0x394549[_0x1533('0x4a', 'y*wC') + 's'][_0x1533('0x71', 'K#@S') + 'th'];\n" +
"\t\t\tfor (var _0x54ccec = 0x0; _0x2eda7d[_0x1533('0x6f', 'dFiI') + 'H'](_0x54ccec, _0x551ac6); _0x54ccec++) {\n" +
"\t\t\t\tfor (var _0x562c2d = 0x0; _0x2eda7d[_0x1533('0x9b', '[H[u') + 'H'](_0x562c2d, _0x551ac6); _0x562c2d++) {\n" +
"\t\t\t\t\tvar _0x1b8d8c = _0x2eda7d[_0x1533('0x80', '[H[u') + 'N'](_0x2eda7d[_0x1533('0x3a', 'w*Nn') + 'z'](_0x2eda7d[_0x1533('0x8b', 'cJ3U') + 'z'](_0x579edb[0x0], _0x394549[_0x1533('0x32', 'R#BF') + 's'][_0x1533('0x82', '@mqx') + 'tr'](_0x54ccec, 0x1)), _0x394549[_0x1533('0x1f', '*hmq') + 's'][_0x1533('0x47', '(cNt') + 'tr'](_0x562c2d, 0x1)), _0x579edb[0x1]);\n" +
"\t\t\t\t\tif (_0x2eda7d[_0x1533('0x6a', '(bZ%') + 'I'](_0x2eda7d[_0x1533('0x9d', 'Kp2R') + 't'](hash, _0x1b8d8c), _0x55baea)) {\n" +
"\t\t\t\t\t\treturn [_0x1b8d8c, new Date() - _0x1642ef];\n" +
"\t\t\t\t\t}\n" +
"\t\t\t\t}\n" +
"\t\t\t}\n" +
"\t\t};\n" +
"\t\tvar _0x1e73b9 = _0x2eda7d[_0x1533('0x36', 'sahq') + 'y'](_0x1a1fd9, _0x394549['ct'], _0x394549[_0x1533('0x2d', '\$qIi')]);\n" +
"\t\tif (_0x1e73b9) {\n" +
"\t\t\tvar _0x3b1427;\n" +
"\t\t\tif (_0x394549['wt']) {\n" +
"\t\t\t\t_0x3b1427 = _0x2eda7d[_0x1533('0x1', 'zx(Q') + 'i'](parseInt(_0x394549['wt']), _0x1e73b9[0x1]) ? _0x2eda7d[_0x1533('0x54', 'avK7') + 'n'](parseInt, _0x394549['wt']) - _0x1e73b9[0x1] : 0x1f4;\n" +
"\t\t\t} else {\n" +
"\t\t\t\t_0x3b1427 = 0x5dc;\n" +
"\t\t\t}\n" +
"\t\t\t_0x2eda7d[_0x1533('0x72', 'qRqa') + 'y'](setTimeout,\n" +
"\t\t\tfunction() {\n" +
"\t\t\t\tdocument[_0x1533('0x20', 'rQf(') + 'ie'] = _0x2eda7d[_0x1533('0x73', 'rQf(') + 'E'](_0x2eda7d[_0x1533('0x3c', 'o&Q4') + 'a'](_0x394549['tn'] + '=' + _0x1e73b9[0x0] + (_0x1533('0x8a', 'ksVw') + _0x1533('0x1e', 'GvA9') + '='), _0x394549['vt']), _0x2eda7d[_0x1533('0x8c', 'Gfc!') + 'B']);\n" +
"\t\t\t\tlocation[_0x1533('0x7a', 'sahq')] = location[_0x1533('0x64', 'd[i)') + _0x1533('0x81', '(U[3')] + location[_0x1533('0x94', 'sahq') + 'ch'];\n" +
"\t\t\t},\n" +
"\t\t\t_0x3b1427);\n" +
"\t\t} else {\n" +
"\t\t\talert(_0x2eda7d[_0x1533('0x7b', ')T5g') + 'O']);\n" +
"\t\t}\n" +
"\t};\n" +
"\tgo({\n" +
"\t\t\"bts\": [\"1618915727.945|0|HM5\", \"VoVNIPqcrvFO13woIZK2RY%3D\"],\n" +
"\t\t\"chars\": \"ZDUZR0cDjTEXznMkfgavFO\",\n" +
"\t\t\"ct\": \"3049e28877124ad21ca0db55e3af95ff316ceef6\",\n" +
"\t\t\"ha\": \"sha1\",\n" +
"\t\t\"tn\": \"__jsl_clearance\",\n" +
"\t\t\"vt\": \"3600\",\n" +
"\t\t\"wt\": \"1500\"\n" +
"\t})"
ScriptEngineManager manager = new ScriptEngineManager();
ScriptEngine engine = manager.getEngineByName("javascript");
engine.eval(str);
//获取对象
System.out.println();
}
def getJsContent(def page){
ScriptEngineManager manager = new ScriptEngineManager();
//得到脚本引擎
ScriptEngine engine = manager.getEngineByName("javascript");
//处理加密js
String js = page.trim().replace("<script>", "").replace("</script>", "").replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]}));", "y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]})");
try {
//得到解密后的js
js = js.replace("document.cookie","var cookie")
js = js.replace("location.href=location.pathname+location.search", "")
engine.eval(js);
def v = engine.get("cookie")
return v.toString().substring(0, v.toString().indexOf(";"))
} catch (ScriptException e) {
e.printStackTrace();
}
return null
}
def getJsContent2(def page){
ScriptEngineManager manager = new ScriptEngineManager();
//得到脚本引擎
ScriptEngine engine = manager.getEngineByName("javascript");
//处理加密js
String js = page.trim().replace("<script>", "").replace("</script>", "").replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]}));", "y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]})");
try {
//得到解密后的js
//得到解密后的js
String result = (String) engine.eval(js);
result = result.substring(result.indexOf("var cd"),result.indexOf("dc+=cd;")+7);
result = result.replaceAll("document*.*toLowerCase\\(\\)", "'x'");
//得到cookie
String jsl = (String) engine.eval(result);
return jsl;
} catch (ScriptException e) {
e.printStackTrace();
}
return null
}
/**
* 通过破解动态JavaScript脚本,
* 获取cookie名为 __jsl_clearance的值
*
* @param body 相应内容(一般为第一次请求获取到的动态js字符串)
* @return cookie名为 __jsl_clearance的值
*/
private static String getJslClearance(String body) {
//V8:谷歌开源的运行JavaScript脚本的库. 参数:globalAlias=window, 表示window为全局别名,
// 告诉V8在运行JavaScript代码时, 不要从代码里找window的定义.
V8 runtime = V8.createV8Runtime("window");
//将第一次请求pdf资源时获取到的字符串提取成V8可执行的JavaScript代码
body = body.trim()
.replace("<script>", "")
.replace("</script>", "")
.replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]||(\"_\"+y)}))",
"y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]||(\"_\"+y)})");
//用V8执行该段代码获取新的动态JavaScript脚本
String result = runtime.executeStringScript(body);
//获取 jsl_clearance 的第一段, 格式形如: 1543915851.312|0|
String startStr = "document.cookie='";
int i1 = result.indexOf(startStr) + startStr.length();
int i2 = result.indexOf("|0|");
String cookie1 = result.substring(i1, i2 + 3);
/*
获取 jsl_clearance 的第二段,格式形如: DW2jqgJO5Bo45yYRKLlFbnqQuD0%3D。
主要原理是: 新的动态JavaScript脚本是为浏览器设置cookie, 且cookie名为__jsl_clearance
其中第一段值(格式形如:1543915851.312|0|)已经明文写好, 用字符串处理方法即可获取.
第二段则是一段JavaScript函数, 需要有V8运行返回,
该函数代码需要通过一些字符串定位, 提取出来, 交给V8运行.
*/
startStr = "|0|'+(function(){";
int i3 = result.indexOf(startStr) + startStr.length();
int i4 = result.indexOf("})()+';Expires");
String code = result.substring(i3, i4).replace(";return", ";");
String cookie2 = runtime.executeStringScript(code);
/*
拼接两段字符串, 返回jsl_clearance的完整的值.
格式形如: 1543915851.312|0|DW2jqgJO5Bo45yYRKLlFbnqQuD0%3D
*/
return cookie1 + cookie2;
}
/**
* 将HTTP响应体转换为字符串返回
*
* @param response HTTP响应
* @return 响应体的字符串形式
* @throws IOException IO异常
*/
private static String getResponseBodyAsString(HttpResponse response) throws IOException {
return IOUtils.toString(response.getEntity().getContent(), "UTF-8");
}
static String getJsluid(HttpResponse response) {
Header header = response.getFirstHeader("set-cookie");
String[] split = header.getValue().split(";");
for (String s : split) {
if (s.contains("__jsluid")) {
return s.trim();
}
}
return "";
}
def fristCrawl(){
def responseMap = [:]
try {
//headers信息
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","www.gsxt.gov.cn")
headers.put("Connection","keep-alive")
headers.put("Upgrade-Insecure-Requests","1")
headers.put("Cache-Control","max-age=0")
// headers.put("Content-Type","application/x-www-form-urlencoded")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
// headers.put("Cookie","__jsluid_h=b4ff8b9b0fd86b6ff99edf27e111a9f2")
headers.put("Accept-Language","zh-CN,zh;q=0.9")
// headers.put("Accept-Encoding","gzip, deflate")
//String s = doPost("http://www.gsxt.gov.cn/corp-query-homepage.html", null, headers)
def s = ABuYunUtil.doGet("http://www.gsxt.gov.cn/corp-query-homepage.html",null, headers, "utf-8")
return s
} catch (Exception e) {
e.printStackTrace()
}
return null
}
void process(Object args) {
def url = "http://www.gsxt.gov.cn/corp-query-search-1.html "
for(def key : companyKeys){
// def queryBody = "tab=ent_tab&province=&geetest_challenge=1f31bf4565a8f89b93f4f4b25bb2143a&geetest_validate=9584ff51b3aff476fcac55248ec14459&geetest_seccode=9584ff51b3aff476fcac55248ec14459%7Cjordan&token=115406527&searchword="+ URLEncoder.encode("宏美", "utf-8")
def queryBody = "tab=ent_tab&province=&geetest_challenge=daa40fb7efec814ff072b12ee12bb25a&geetest_validate=009a8748b31392ffdfb626fbbdccb4be&geetest_seccode=009a8748b31392ffdfb626fbbdccb4be%7Cjordan&token=115394764&searchword="+ URLEncoder.encode(key, "utf-8")
def s = crawl(url, queryBody);
if(s){
print s
parser(s);
}
println s
}
}
def parser(content){
def document = Jsoup.parse(content);
if(document){
def detailUrls = []
Elements elements = JQueryUtils.select([".search_list_item"], document)
if(elements && elements.size()>0){
for(def el : elements){
detailUrls << el.attr("href")
}
}
}
}
def static crawl(url, bodyquery){
def responseMap = [:]
try {
//headers信息
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","www.gsxt.gov.cn")
headers.put("Connection","keep-alive")
headers.put("Content-Length","226")
headers.put("Cache-Control","max-age=0")
headers.put("Upgrade-Insecure-Requests","1")
headers.put("Origin","http://www.gsxt.gov.cn")
headers.put("Content-Type","application/x-www-form-urlencoded")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
headers.put("Referer","http://www.gsxt.gov.cn/corp-query-homepage.html")
// headers.put("Accept-Encoding","gzip, deflate")
headers.put("Accept-Language","zh-CN,zh;q=0.9")
// headers.put("Cookie:"," __jsluid_h=4e2326174f7f9d90ad9ac1f92301f2c9; JSESSIONID=8ac4f9ab894733d7c98c9a3d5c26; SECTOKEN=7446451709198207648; __jsl_clearance=1618823989.806|0|jQc7QRg%2B%2FzcIlz44iNwQg0PjZJg%3D; gsxtBrowseHistory1=%0FS%04%06%1D%04%1D%10SNS%24%26%3B%22%3D%3A71%3A%3B01%3A%219FFDDDDFFDFE%40DDDDEFDECDLE%40L%40L%40GSXS%11%1A%00%1A%15%19%11SNS%E5%91%BD%E6%9F%A3%E5%B9%B6%E6%97%84%E8%AE%AE%E7%AF%95%E5%BA%BE%E5%BA%8E%E8%AF%8A%E7%AF%95%E7%91%B2%E6%9D%BD%E9%98%A4%E5%84%98%E5%8E%8CSXS%11%1A%00%00%0D%04%11SNEEMDXS%02%1D%07%1D%00%00%1D%19%11SNEBELLFGCDL%40FG%09; tlb_cookie=S172.16.12.115")
String s = doPost(url, bodyquery, headers)
return s
} catch (Exception e) {
e.printStackTrace()
}
return null
}
static String doPost(String strUrl, String content, Map headers) {
String result = "";
OutputStream outputStream = null;
InputStream inputStream = null;
StringBuilder sb = new StringBuilder();
try {
checkAbuyun(true);
def responseMap = [:]
// 创建代理服务器地址对象
InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
// 创建HTTP类型代理对象
Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
URL url = new URL(strUrl);
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestMethod("POST");
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setConnectTimeout(30000);
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setReadTimeout(30000);
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
urlConnection.setDoOutput(true);
urlConnection.setDoInput(true);
//设置通用请求属性为默认浏览器编码类型
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
urlConnection.setRequestProperty(head, headers.get(head));
}
}
//getOutputStream()返回的输出流,用于写入参数数据。
outputStream = urlConnection.getOutputStream();
if(content){
outputStream.write(content.getBytes());
}
outputStream.flush();
outputStream.close();
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
inputStream = urlConnection.getInputStream();
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
char[] buff = new char[1024];
int length = 0;
while ((length = reader.read(buff)) != -1) {
String s = new String(buff, 0, length);
sb.append(s);
}
} catch (IOException e) {
e.printStackTrace();
}finally{
if(outputStream!=null){
outputStream.close();
}
if(inputStream!=null){
inputStream.close();
}
}
return sb.toString();
}
public static checkAbuyun(boolean check) {
if (check) {
while (AbuyunRequestCount.get() >= AbuyunMaxCount) {
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
e.printStackTrace()
}
}
AbuyunRequestCount.incrementAndGet();
} else {
AbuyunRequestCount.decrementAndGet();
}
}
}
@@ -61,10 +61,10 @@ class DealPriceMathBatch extends BaseBatch{
districtMap.putAll(detailMap)
}
}
def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2018-11-01","yyyy-MM-dd"))
def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2020-01-01","yyyy-MM-dd"))
// def startDate = DateUtil.getFirstDayOfMonth(new Date())
// def endDate = DateUtil.getFirstDayOfMonth(new Date(), 1)
def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2019-12-31","yyyy-MM-dd"))
def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2021-04-30","yyyy-MM-dd"))
while(flag){
println "开始加载解析数据......"
@@ -116,7 +116,7 @@ class DealPriceMathBatch extends BaseBatch{
count++;
}catch (Exception e){
println "处理:【${dbo.city}-${dbo.region}-${dbo.district_name}】发生异常,异常信息为:"+e.getMessage()
println "处理:【${dbo.city}-${dbo.region}-${dbo.district_name}】发生异常,异常信息为:"+e.printStackTrace()
}finally {
latch.countDown()
}
@@ -442,16 +442,21 @@ class DealPriceMathBatch extends BaseBatch{
if(regionMap){
regionMap.each{k,v->
def arrays = k.split("###")
if(arrays[1] && arrays[1]==keyword){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
try{
if(arrays && arrays.size()>=2 && arrays[1] && arrays[1]==keyword){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
}
}catch(Exception e){
e.printStackTrace()
}
}
}
}else{//无区域
cityMap.each{r,dMap->
dMap.each{k,v->
def arrays = k.split("###")
if(arrays[1] && arrays[1]==keyword){
if(arrays && arrays.size()>=2 && arrays[1] && arrays[1]==keyword){
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
}
}
@@ -596,10 +601,13 @@ class DealPriceMathBatch extends BaseBatch{
}
def getOpenApiMatch(def city, def region, def name){
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, "fuzzy", false)
if(jsonObject && jsonObject.credit>0 ){
return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address]
if(name){
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null,"fuzzy", false)
if(jsonObject && jsonObject.credit>0 ){
return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address]
}
}
return null;
}
@@ -352,7 +352,7 @@ class HouseExportBatch extends BaseBatch{
def rets = []
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-2020*", queryBuilder);
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "latest_seawater", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
@@ -31,14 +31,14 @@ class EsHouseStat {
// public static JdbcDataSource dataSource = new JdbcDataSource()
// private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
// public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
// static {
// dataSource.setUrl(address)
// Properties connectionProperties = new Properties()
// connectionProperties.put("user", "developer")
// connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
// dataSource.setProperties(connectionProperties)
// }
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -42,10 +42,10 @@ class GaoDeMapCrawl {
def updateErrorLists = []
def flagLists = []
def index = 0
def prefix = "road_"
def prefix = "district_"
// def categorys = ["141202", "141203"]//中学,小学
// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
def categorys = ["190301"]//道路相关
def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
// def categorys = ["190301"]//道路相关
static main(args) {
def task = new GaoDeMapCrawl();
@@ -109,8 +109,8 @@ class GaoDeMapCrawl {
def run(){//,"西安","重庆"
// def citys = [ "临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄","莱芜","济南","青岛","威海","烟台","潍坊","淄博"]
// def citys = ["北京","广州","深圳","合肥","苏州","西安","重庆"]
def citys = CommonUtils.findAllCity()
def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口"]
// def citys = CommonUtils.findAllCity()
def totalCount = 0;
for(def city : citys){
def cityLations = findAllLation(city);
@@ -198,9 +198,11 @@ class GaoDeMapCrawl {
if(m.rets){
processData(m.rets);
location.count = m.rets.size()
location.content = m.toString()
flagLists.add(location)
}else{
location.count = 0
location.content = m.toString()
flagLists.add(location)
}
@@ -336,7 +338,7 @@ class GaoDeMapCrawl {
for(def location : updateLists){
if(location && location._id){
Document updateDocument = new Document();
def doc = new Document(prefix+"status","完成").append("m_date", new Date())
def doc = new Document(prefix+"status","完成").append("m_date", new Date()).append("content", location.content)
updateDocument.append('$set', doc);
Bson updateBson = Filters.eq("_id", location._id);
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
@@ -358,7 +360,7 @@ class GaoDeMapCrawl {
for(def location : flagLists){
if(location && location._id){
Document updateDocument = new Document();
def doc = new Document(prefix+"gd_data_count",location.count).append("m_date", new Date())
def doc = new Document(prefix+"gd_data_count",location.count).append("m_date", new Date()).append("content", location.content)
updateDocument.append('$set', doc);
Bson updateBson = Filters.eq("_id", location._id);
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
@@ -382,7 +384,7 @@ class GaoDeMapCrawl {
for(def location : updateErrorLists){
if(location && location._id){
Document updateDocument = new Document();
def doc = new Document(prefix+"status","失败").append("m_date", new Date())
def doc = new Document(prefix+"status","失败").append("m_date", new Date()).append("content", location.content)
updateDocument.append('$set', doc);
Bson updateBson = Filters.eq("_id", location._id);
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
@@ -15,32 +15,35 @@ class LogStat {
}
def run(){
ExcelUtils.multiThreadingModify("D:\\data\\建行匹配小区2(1).xlsx",{ d ->
def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd")
def weekend = DateUtils.getFirstDayOfWeek(calDate)
def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend)
if(dbo){
d['E'] = dbo.avg_price
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区名单.xlsx",{ d ->
// def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd")
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
// def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend)
// if(dbo){
// d['E'] = dbo.avg_price
// }
if(d['A']){
def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
if(district){
def startDate = DateUtil.addMonth(new Date(), -6)
def start = 20200901//startDate.replace("-","").toInteger()
def end = 20201201//DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
def alias = []
if(district.alias){
if(!district.alias.contains(district.name)){
district.alias << district.name
}
alias = district.alias
}else{
alias << district.name
}
def count = findHouses(d['A'], d['B'], d['C'], alias, start, end)
d['D'] = count
println(d)
}
}
def district = CommonDatautils.findDistrict(d['N'], d['O'], d['P'], "district")
if(district){
def startDate = DateUtil.addMonth(new Date(), -6)
def start = startDate.replace("-","").toInteger()
def end = DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
def alias = []
if(district.alias){
if(!district.alias.contains(district.name)){
district.alias << district.name
}
alias = district.alias
}else{
alias << district.name
}
def count = findHouses(d['N'], d['O'], d['P'], alias, start, end)
d['G'] = count
println(d)
}
},0)
}
@@ -48,7 +51,7 @@ class LogStat {
def findHouses(def city, def region, def name, def alias, def startDate, def endDate){
def aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date > ${startDate} and region='${region}' and district_name in ('${aliasStr}')"
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
def count = 0
ElasticUtils.executeSqlQuery(sql,{data->
count = data.c
@@ -47,7 +47,7 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-02-01"
static def statDate = "2021-03-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2021年3月\\二手房\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2021年4月\\二手房\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -0,0 +1,48 @@
package com.fangjia.script.gj.data
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class RegionPriceExport {
DBCollection REGION_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "region_sell_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new RegionPriceExport();
task.run()
}
def run(){
def months = [202010,202011,202012,202101,202102,202103]
def regions = CommonUtils.getAllRegions("福州","region")
def rets = []
rets << ["城市","区域","202010","202011","202012","202101","202102","202103"]
regions.each{region,values->
def data = []
data << "福州"
data << region
for(def month :months){
def dbo = findPrice("福州", region, month);
if(dbo){
data << dbo.avg_price
}else{
data << ""
}
}
rets << data
}
ExcelUtils.write("d:\\数据\\福州\\福州区域月均价数据.xlsx",rets, "价格")
}
def findPrice(city, region, month){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("date", month)
def dbo = REGION_SELL_STAT.findOne(q)
return dbo
}
}
@@ -425,6 +425,7 @@ class ABuYunUtil {
buffer.append(bf, 0, len);
}
} catch (IOException e) {
e.printStackTrace()
throw new Exception("doGet(String targetUrl, Map<String, String> params, Map<String, String> headers, String charset)异常, 异常信息为:" + e);
} finally {
// println("本次请求响应的connection.getResponseCode()为:" + connection.getResponseCode())