Merge branch 'script_gj' into 'master'
Script gj See merge request !285
This commit is contained in:
@@ -494,5 +494,32 @@
|
||||
<artifactId>msgpack</artifactId>
|
||||
<version>0.6.12</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>com.asprise.ocr</groupId>
|
||||
<artifactId>java-ocr-api</artifactId>
|
||||
<version>[15,)</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>net.java.dev.jna</groupId>
|
||||
<artifactId>jna</artifactId>
|
||||
<version>4.1.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>net.sourceforge.tess4j</groupId>
|
||||
<artifactId>tess4j</artifactId>
|
||||
<version>3.4.0</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>com.sun.jna</groupId>
|
||||
<artifactId>jna</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
|
||||
</dependencies>
|
||||
</project>
|
||||
|
||||
+1
@@ -508,6 +508,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def q = new BasicDBObject()
|
||||
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate));
|
||||
q.put("area", new BasicDBObject('$gte', 20))
|
||||
q.put("status", 0)
|
||||
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
|
||||
|
||||
def rets = []
|
||||
|
||||
+1
-1
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220218, 20220319);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220318, 20220419);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||
//测试通过匹配接口配房源效果
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||
|
||||
+1
-1
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-03-1", "2022-04-1")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-04-1", "2022-05-1")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202202.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202203.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
@@ -13,7 +13,7 @@ import org.apache.commons.lang.StringUtils
|
||||
|
||||
class OpenApiUtils {
|
||||
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){
|
||||
address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","")
|
||||
address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","").replace("%","").replace("\"","")
|
||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
|
||||
if(preciseLevelflag){
|
||||
url = url +"&preciseLevel=strict"
|
||||
|
||||
+50
-13
@@ -88,9 +88,10 @@ def getDetailLinks(driver, cityList){
|
||||
driver.findElement(By.className("next")).click()
|
||||
}
|
||||
}
|
||||
println city +" "+urls.size()
|
||||
|
||||
def rets = batchParse(urls);
|
||||
saveToDB(rets);
|
||||
println city +" "+urls.size()
|
||||
}
|
||||
}
|
||||
println cityMap.provice +" 完成"
|
||||
@@ -123,7 +124,7 @@ def getCity(driver, proviceMap){
|
||||
for(def c : cityEles){
|
||||
def cityid = c.attr("data-cityid")
|
||||
def city = c.text()
|
||||
if(!["石家庄市","唐山市","秦皇岛市","邯郸市","邢台市","保定市"].contains(city)){
|
||||
if(!["石家庄市","太原市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){
|
||||
cityMap.put(city, cityid)
|
||||
}
|
||||
|
||||
@@ -170,8 +171,7 @@ def batchParse(def urls){
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def ret = []
|
||||
for(def map :urls){
|
||||
driver.get(map.url);
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def doc = getDocument(map.url, driver)
|
||||
def res = parse(map.province, map.city, map.url, doc);
|
||||
if(res){
|
||||
ret << res
|
||||
@@ -180,6 +180,30 @@ def batchParse(def urls){
|
||||
return ret;
|
||||
}
|
||||
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
if(doc){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=10){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
def parse(province, city, url,doc){
|
||||
if(doc){
|
||||
def result = [:]
|
||||
@@ -205,7 +229,12 @@ def parse(province, city, url,doc){
|
||||
|
||||
def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc)
|
||||
if (finishDateStr) {
|
||||
result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm");
|
||||
try{
|
||||
result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm");
|
||||
}catch(Exception e){
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def curPriceStr = JQueryUtils.selectText([".colorred"], doc)
|
||||
@@ -255,7 +284,12 @@ def parse(province, city, url,doc){
|
||||
startDateStr = startDateStr.toString().substring(startDateStr.indexOf("定于")+2, startDateStr.indexOf("日")+1).trim()
|
||||
}
|
||||
|
||||
result.startDate = com.fangjia.script.util.DateUtils.parseString2Date(startDateStr)
|
||||
try{
|
||||
result.startDate = com.fangjia.script.util.DateUtils.parseString2Date(startDateStr.trim())
|
||||
}catch(Exception e){
|
||||
|
||||
}
|
||||
|
||||
def imgs = JQueryUtils.select(".sf-pic-slide img", doc)
|
||||
if (imgs) {
|
||||
def imgLinks = []
|
||||
@@ -315,7 +349,7 @@ def saveToDB(rets){
|
||||
ret.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
|
||||
obj.source= "中拍网"
|
||||
if(dbo){
|
||||
obj.m_date = new Date()
|
||||
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
|
||||
@@ -346,15 +380,18 @@ def getAreaNumber(input, maxArea) {
|
||||
val = val.toString().substring(val.toString().indexOf("室")+1, val.toString().size())
|
||||
}
|
||||
def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", "");
|
||||
if (maxArea) {
|
||||
if (v.toDouble() < maxArea) {
|
||||
area = v;
|
||||
if(v && v.toString().isNumber()){
|
||||
if (maxArea) {
|
||||
if (v.toDouble() < maxArea) {
|
||||
area = v;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
area = v
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
area = v
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (area) {
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import org.apache.commons.codec.binary.Base64
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import sun.font.Font2D
|
||||
import sun.font.Font2DHandle
|
||||
import sun.font.TrueTypeFont
|
||||
import sun.font.TrueTypeGlyphMapper
|
||||
|
||||
import java.awt.Font
|
||||
import java.awt.font.FontRenderContext
|
||||
import java.awt.font.GlyphVector
|
||||
import java.awt.geom.AffineTransform
|
||||
import java.lang.reflect.Field
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DataUtil {
|
||||
static def getAreaNumber(def input, def maxArea) {
|
||||
Pattern pattern = Pattern.compile('\\d+(.\\d+)?(平方米|㎡|m2|方米|平)')
|
||||
Matcher matcher = pattern.matcher(input)
|
||||
|
||||
List<String> list = new ArrayList()
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group())
|
||||
}
|
||||
if (list.size() > 0) {
|
||||
def area = null
|
||||
|
||||
for (def val : list) {
|
||||
def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", "")
|
||||
if (maxArea) {
|
||||
if (v.isDouble()) {
|
||||
if (v.toDouble() < maxArea) {
|
||||
area = v
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
area = v
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (area) {
|
||||
return area
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def getPaiMaiType(def s) {
|
||||
if (s) {
|
||||
if (s.toString().contains("结束")) {
|
||||
return "已完成"
|
||||
} else if (s.toString().contains("进行") || s.toString().contains("正在拍卖")) {
|
||||
return "进行中"
|
||||
} else if (s.toString().contains("即将开始")) {
|
||||
return "即将开始"
|
||||
} else if (s.toString().contains("流拍")) {
|
||||
return "流拍"
|
||||
} else if (s.toString().contains("撤回")) {
|
||||
return "撤回"
|
||||
} else if (s.toString().contains("暂缓")) {
|
||||
return "暂缓"
|
||||
} else if (s.toString().contains("中止")) {
|
||||
return "中止"
|
||||
} else if (s.toString().contains("失败")) {
|
||||
return "失败"
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
|
||||
static def getDateRange(def s) {
|
||||
s = s.replace("起", "")
|
||||
Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}[月|日]\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)')
|
||||
Matcher matcher = pattern.matcher(s)
|
||||
|
||||
List<String> list = new ArrayList()
|
||||
while (matcher.find()) {
|
||||
def str = matcher.group()
|
||||
list.add(str)
|
||||
// if(str) {
|
||||
// def indx1 = str.indexOf("年")
|
||||
// def indx2 = str.indexOf("月")
|
||||
// def month = str.substring(indx1+1, indx2);
|
||||
// if(month.toInteger()<12) {
|
||||
// list.add(str);
|
||||
// }
|
||||
// }
|
||||
}
|
||||
if (list.size() >= 1) {
|
||||
String s1 = list.get(0)
|
||||
s1 = s1.replace("法院将于", "").replace(" ", "")
|
||||
def array = s1.split("至")
|
||||
return array
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
static def getStartPrice(def s) {
|
||||
Pattern pattern = Pattern.compile('起拍价:\\d+(.\\d+)?万?元')
|
||||
Matcher matcher = pattern.matcher(s)
|
||||
|
||||
List<String> list = new ArrayList()
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group())
|
||||
}
|
||||
if (list.size() >= 1) {
|
||||
String s1 = list.get(0)
|
||||
if (s1.contains("万元")) {
|
||||
s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "")
|
||||
return (int) (s1.toDouble() * 10000)
|
||||
} else {
|
||||
s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "")
|
||||
if (s1.isInteger()) {
|
||||
return s1.toInteger()
|
||||
} else {
|
||||
return (int) (s1.toDouble())
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def getEstateType(def s) {
|
||||
Pattern pattern = Pattern.compile('房屋类型:.*;')
|
||||
Matcher matcher = pattern.matcher(s)
|
||||
|
||||
List<String> list = new ArrayList()
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group())
|
||||
}
|
||||
if (list.size() >= 1) {
|
||||
String s1 = list.get(0)
|
||||
if (s1) {
|
||||
s1 = s1.substring(s1.indexOf(":") + 1, s1.indexOf(";"))
|
||||
return s1.replace(")", "").replace("】", "")
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def findEstateType(def s) {
|
||||
if (s) {
|
||||
def villTypes = ["独栋", "联排", "联列", "叠加", "双拼", "花园面积", "泳池", "别墅", "花园住宅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]
|
||||
for (def type : villTypes) {
|
||||
if (s.contains(type)) {
|
||||
return "别墅"
|
||||
}
|
||||
}
|
||||
|
||||
def districtTypes = ["住宅", "公寓"]
|
||||
for (def type : districtTypes) {
|
||||
if (s.contains(type)) {
|
||||
return "普通住宅"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
static def filterChar(def detailMap) {
|
||||
def newDetailMap = [:]
|
||||
detailMap.each { def k, v ->
|
||||
if (k.toString().contains(".")) {
|
||||
newDetailMap.put(k.toString().replace(".", "\\u002e"), v)
|
||||
} else {
|
||||
newDetailMap.put(k, v)
|
||||
}
|
||||
}
|
||||
return newDetailMap
|
||||
}
|
||||
|
||||
/**
|
||||
* 匹配符合条件的字符串
|
||||
*
|
||||
* @param str
|
||||
* @param pattern
|
||||
* @return
|
||||
*/
|
||||
static String match(String str, Pattern pattern) {
|
||||
if (str == null) {
|
||||
return null
|
||||
}
|
||||
Matcher matcher = pattern.matcher(str)
|
||||
if (matcher.find()) {
|
||||
return matcher.group()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* font-secret字符串专用解密工具
|
||||
*
|
||||
* @param key 密匙
|
||||
* @param encodeString 加密后的字符串
|
||||
* @return 解密后的字符串
|
||||
*/
|
||||
static String decodeString(String key, String encodeString) {
|
||||
if (!key) {
|
||||
return ""
|
||||
}
|
||||
try {
|
||||
//base64解码,初始化字体
|
||||
Base64 base64 = new Base64()
|
||||
byte[] ss = base64.decodeBase64(key)
|
||||
InputStream inputStream = new ByteArrayInputStream(ss)
|
||||
Font dynamicFont = Font.createFont(Font.TRUETYPE_FONT, inputStream)
|
||||
FontRenderContext fontRenderContext = new FontRenderContext(new AffineTransform(), false, false)
|
||||
GlyphVector glyphVector = dynamicFont.createGlyphVector(fontRenderContext, "")
|
||||
|
||||
//获取font中字形的映射关系,字段为private,使用反射
|
||||
Class<?> clazz = Font.class
|
||||
Field[] fs = clazz.getDeclaredFields()
|
||||
Font2DHandle font2DHandle = null
|
||||
for (int i = 0; i < fs.length; i++) {
|
||||
fs[i].setAccessible(true)// 将目标属性设置为可以访问
|
||||
if (fs[i].getName().equals("font2DHandle")) {
|
||||
font2DHandle = (Font2DHandle) fs[i].get(dynamicFont)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//得到映射关系
|
||||
Font2D font2D = font2DHandle.font2D
|
||||
TrueTypeFont trueTypeFont = (TrueTypeFont) font2D
|
||||
TrueTypeGlyphMapper charToGlyphMapper = (TrueTypeGlyphMapper) trueTypeFont.getMapper()
|
||||
|
||||
//开始解密,encodeString为加密后的字符串
|
||||
StringBuffer buffer = new StringBuffer()
|
||||
char[] chars = encodeString.toCharArray()
|
||||
for (int i = 0; i < chars.length; i++) {
|
||||
buffer.append(charToGlyphMapper.charToGlyph(chars[i]) - 1)
|
||||
}
|
||||
return buffer.toString()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
static List SITE_SUFFIXS = [".gov.cn",".com.cn", ".com", ".cn", ".net", ".org", ".cc", ".tm", ".biz", ".info", ".tv"]
|
||||
|
||||
static String urlToSiteName(String url) {
|
||||
if (StringUtils.isEmpty(url)) {
|
||||
return null
|
||||
}
|
||||
|
||||
for (String suffix : SITE_SUFFIXS) {
|
||||
if (url.indexOf(suffix) > -1) {
|
||||
String s = StringUtils.substringBefore(url, suffix)
|
||||
if (s.indexOf(".") > -1) {
|
||||
s = StringUtils.substringAfterLast(s, ".")
|
||||
if (StringUtils.isNotBlank(s)) {
|
||||
return s + suffix
|
||||
}
|
||||
} else if (s.indexOf("://") > -1) {
|
||||
s = StringUtils.substringAfterLast(s, "://")
|
||||
if (StringUtils.isNotBlank(s)) {
|
||||
return s + suffix
|
||||
}
|
||||
} else if (s.indexOf("://") == -1 && s.indexOf(".") == -1) {
|
||||
return s + suffix
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static double getRound(int scale, double dSource) {
|
||||
BigDecimal deSource = new BigDecimal(dSource)
|
||||
return deSource.setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue()
|
||||
}
|
||||
|
||||
static def filterEmpty(def s){
|
||||
if(s && s.contains("暂无")){
|
||||
return null
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
}
|
||||
+528
@@ -0,0 +1,528 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.nodes.Document
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
import javax.sound.midi.MidiSystem
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class JdPaimaiCrawl {
|
||||
|
||||
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
|
||||
|
||||
static main(args) {
|
||||
def task = new JdPaimaiCrawl();
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def existsIds = findAllIds()
|
||||
println "已存在:"+existsIds.size()
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def links = getList(driver)
|
||||
def ids = getDetailLinks(links, driver)
|
||||
println "共:"+ids.size()
|
||||
parserList( existsIds,ids, driver)
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
def getList(driver){
|
||||
def list = []
|
||||
for(def i=1;i<=32;i++){
|
||||
if(i==8){
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268"
|
||||
}else if(i==5){
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268"
|
||||
}else if(i==17){
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268"
|
||||
}else if(i==19){
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268"
|
||||
}else{
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268"
|
||||
}
|
||||
}
|
||||
def links = []
|
||||
for(def url : list){
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
//def document = ABuYunUtil.doPost(url, null, null, "utf-8", true)
|
||||
if(doc){
|
||||
def json = doc.toString().substring(doc.toString().indexOf("(")+1, doc.toString().lastIndexOf(")"))
|
||||
JSONObject obj = JSONObject.fromObject(json);
|
||||
if(obj && obj.total){
|
||||
def total = obj.total
|
||||
def page = total/40
|
||||
def p = page.toInteger()
|
||||
if(page > p){
|
||||
p = p +1
|
||||
}
|
||||
|
||||
for(def i=1;i<=p;i++){
|
||||
links << url+"&page="+i
|
||||
}
|
||||
// if(links){
|
||||
// break;
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return links;
|
||||
}
|
||||
|
||||
def getDetailLinks(links, driver){
|
||||
def rets = []
|
||||
def citys = ["北京","上海","天津","重庆"]
|
||||
for(def link : links){
|
||||
driver.get(link)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
if(doc) {
|
||||
def json = doc.toString().substring(doc.toString().indexOf("(") + 1, doc.toString().lastIndexOf(")"))
|
||||
JSONObject obj = JSONObject.fromObject(json);
|
||||
if (obj && obj.ls) {
|
||||
for(def o : obj.ls){
|
||||
def map = [:]
|
||||
if(citys.contains(o.province)){
|
||||
map.province = o.province
|
||||
map.city = o.province
|
||||
map.region = o.city
|
||||
}else{
|
||||
map.province = o.province
|
||||
map.city = o.city
|
||||
}
|
||||
map.id = o.id.toString()
|
||||
|
||||
rets << map
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def parserList(existsIds,list, driver){
|
||||
def rets = []
|
||||
for(def map : list) {
|
||||
if(!existsIds.contains(map.id)){
|
||||
def url = "https://paimai.jd.com/" + map.id
|
||||
def doc = getDocument(url, driver)
|
||||
if (doc) {
|
||||
def result = parsering(doc, url, map.province, map.city, map.region)
|
||||
if (result) {
|
||||
rets << result
|
||||
}
|
||||
if (rets.size() % 100 == 0) {
|
||||
def add= saveData(rets)
|
||||
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+" add:"+add
|
||||
rets = []
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(rets.size()>0){
|
||||
saveData(rets)
|
||||
print "完"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def saveData(def results){
|
||||
def add = 0
|
||||
for(def ret : results){
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", ret.url)
|
||||
def dbo = paimaiResultColl.findOne(q);
|
||||
|
||||
DBObject obj = new BasicDBObject();
|
||||
ret.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
|
||||
if(dbo){
|
||||
obj.m_date = new Date()
|
||||
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
|
||||
}else{
|
||||
add++
|
||||
obj.c_date = new Date()
|
||||
obj.m_date = new Date()
|
||||
try{
|
||||
paimaiResultColl.save(obj)
|
||||
}catch(Exception e){
|
||||
println obj.url
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return add
|
||||
}
|
||||
|
||||
def parsering(doc, def url, def province, def city, def region){
|
||||
def result = [:]
|
||||
if(doc) {
|
||||
result.proince = province
|
||||
result.city = city
|
||||
if(region){
|
||||
result.region = region
|
||||
}
|
||||
def title = JQueryUtils.selectText([".pm-name"], doc)
|
||||
result.title = title
|
||||
result.address = JQueryUtils.selectText([".pm-location em"], doc)
|
||||
if(result.address) {
|
||||
//调用获取城市
|
||||
// if(!city) {
|
||||
// city = RegionData.formatCity(cityMap.name)
|
||||
// }
|
||||
|
||||
if(city) {
|
||||
def rMap = OpenApiUtils.matchDistrict(city, result.title, null, true)
|
||||
if (rMap) {
|
||||
result.matchCity = rMap.city
|
||||
result.matchRegion = rMap.region
|
||||
result.matchName = rMap.name
|
||||
result.matchAddress = rMap.address
|
||||
result.houseNum = rMap.house_num
|
||||
result.roomNum = rMap.room_num
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
result.status = JQueryUtils.selectText([".pm-summary .state"], doc)
|
||||
|
||||
def enrollCount = JQueryUtils.selectText([".pm-summary .times span:eq(0) em"], doc)
|
||||
if(enrollCount) {
|
||||
result.enrollCount = enrollCount.toInteger()
|
||||
}
|
||||
|
||||
def lookCount = JQueryUtils.selectText([".pm-summary .times span:eq(1) em"], doc)
|
||||
if(lookCount) {
|
||||
result.lookCount = lookCount.toInteger()
|
||||
}
|
||||
|
||||
def callCount = JQueryUtils.selectText([".pm-summary .times span:eq(2) em"], doc)
|
||||
if(callCount) {
|
||||
result.callCount = callCount.toInteger()
|
||||
}
|
||||
|
||||
|
||||
def curPriceStr = JQueryUtils.selectText([".mc .current em"], doc)
|
||||
if(curPriceStr) {
|
||||
curPriceStr = curPriceStr.replace(",", "")
|
||||
if(curPriceStr.isInteger()) {
|
||||
result.curPrice = curPriceStr.toInteger()
|
||||
}else {
|
||||
result.curPrice = (int)(curPriceStr.toDouble())
|
||||
}
|
||||
}
|
||||
|
||||
def startPriceStr = JQueryUtils.selectText([".mc .base em"], doc)
|
||||
if(startPriceStr) {
|
||||
startPriceStr = startPriceStr.replace(",", "")
|
||||
|
||||
if(startPriceStr.isInteger()) {
|
||||
result.startPrice = startPriceStr.toInteger()
|
||||
}else {
|
||||
result.startPrice = (int)(startPriceStr.toDouble())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def tradePriceStr = JQueryUtils.selectText([".mc .deal em"], doc)
|
||||
if(tradePriceStr) {
|
||||
tradePriceStr = tradePriceStr.replace(",", "")
|
||||
if(tradePriceStr.isInteger()) {
|
||||
result.tradePrice = tradePriceStr.toInteger()
|
||||
}else {
|
||||
result.tradePrice = (int)(tradePriceStr.toDouble())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def finishDateStr = JQueryUtils.selectText([".endtime"], doc)
|
||||
if(finishDateStr && finishDateStr.contains("结束时间")) {
|
||||
finishDateStr = finishDateStr.substring(5, finishDateStr.indexOf(" "))
|
||||
result.finishDate = DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm:ss")
|
||||
}
|
||||
|
||||
def lis = JQueryUtils.select(".description ul li", doc)
|
||||
if(lis && lis.size()) {
|
||||
for(def li : lis) {
|
||||
def str = li.select("span").text()
|
||||
if(str.contains("起拍")) {
|
||||
def v= li.select("em").text().replace("¥", "").replace(",", "")
|
||||
if(v) {
|
||||
if(v.isInteger()) {
|
||||
result.startPrice = v.toDouble().toInteger()
|
||||
}else {
|
||||
result.startPrice = (int)(v.toDouble())
|
||||
}
|
||||
|
||||
}
|
||||
}else if(str.contains("评估价")) {
|
||||
def v = li.select("em").text().replace("¥", "").replace(",", "")
|
||||
if(v.isInteger()) {
|
||||
result.evaluationPrice = v.toDouble().toInteger()
|
||||
}else {
|
||||
result.evaluationPrice = (int)(v.toDouble())
|
||||
}
|
||||
}else if(str.contains("加价幅度")) {
|
||||
def v = li.select("em").text().replace("¥", "").replace(",", "")
|
||||
if(v) {
|
||||
result.addPrice = v.toDouble().toInteger()
|
||||
}
|
||||
}else if(str.contains("保证金")) {
|
||||
def v = li.select("em").text().replace("¥", "").replace(",", "")
|
||||
if(v) {
|
||||
if(v.isInteger()) {
|
||||
result.depositPrice = v.toDouble().toInteger()
|
||||
}else {
|
||||
result.depositPrice = (int)(v.toDouble())
|
||||
}
|
||||
|
||||
}
|
||||
}else if(str.contains("延时周期")) {
|
||||
|
||||
result.delayWeekly = li.select("em").text().replace("¥", "").replace(",", "")
|
||||
|
||||
}else if(str.contains("优先竞买权人")) {
|
||||
result.preemptivePerson = li.select("em").text().replace("¥", "").replace(",", "")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def trs = JQueryUtils.select([".floor-content div:eq(2) table tr"], doc)
|
||||
if(trs) {
|
||||
def detailMap = [:]
|
||||
for(def tr : trs) {
|
||||
def d_tds = tr.select("td")
|
||||
if(d_tds && d_tds.size()>=2) {
|
||||
if(d_tds.size()==3) {
|
||||
detailMap.put(d_tds[1].text(), d_tds[2].text())
|
||||
}else {
|
||||
detailMap.put(d_tds[0].text(), d_tds[1].text())
|
||||
}
|
||||
}else {
|
||||
if(d_tds[0] && d_tds[0].text().contains("拍品介绍")) {
|
||||
detailMap.put("拍品介绍", d_tds[0].text().replace("拍品介绍 ", ""))
|
||||
}
|
||||
}
|
||||
}
|
||||
if(detailMap) {
|
||||
result.detailMap = DataUtil.filterChar(detailMap)
|
||||
}
|
||||
if(!result.address) {
|
||||
result.address = detailMap.get("拍品名称")
|
||||
if(!result.address) {
|
||||
result.address = detailMap.get("标的名称")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def imgs =JQueryUtils.select(".floor-content div:eq(2) a", doc)
|
||||
if(imgs) {
|
||||
def imgLinks = []
|
||||
for(def img : imgs) {
|
||||
def href = img.attr("href")
|
||||
if(href.contains("http")) {
|
||||
imgLinks << href
|
||||
}else {
|
||||
imgLinks << url.substring(0, url.indexOf(":")+1)+href
|
||||
}
|
||||
}
|
||||
result.imgUrls = imgLinks
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//解析面积
|
||||
def article = JQueryUtils.selectText([".floor-content"], doc)
|
||||
if(article) {
|
||||
def area = DataUtil.getAreaNumber(article, 5000)
|
||||
if(!area) {
|
||||
article = JQueryUtils.selectText(["body"], doc)
|
||||
if(article) {
|
||||
area = DataUtil.getAreaNumber(article, 5000)
|
||||
}
|
||||
}
|
||||
|
||||
if(area) {
|
||||
result.area = area.toDouble()
|
||||
if(result.tradePrice) {
|
||||
result.tradeAvgPrice = (int)(result.tradePrice/area.toDouble())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def s1 = JQueryUtils.selectText(["li:contains(竞买公告) .floor-content"], doc)
|
||||
if(s1) {
|
||||
def start = s1.indexOf("法院将于")
|
||||
def end = s1.indexOf("止")
|
||||
if(start>-1 && end>-1 && end>start){
|
||||
s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止"))
|
||||
}
|
||||
def dateRangs = DataUtil.getDateRange(s1)
|
||||
if(dateRangs && dateRangs.size()==2) {
|
||||
|
||||
for(def j=0;j<dateRangs.size();j++) {
|
||||
def date = dateRangs[j]
|
||||
def pattern = null
|
||||
if(date.endsWith("日时")||date.endsWith("日")) {
|
||||
pattern = "yyyy年MM月dd日"
|
||||
}else {
|
||||
|
||||
def ind = date.indexOf("日")
|
||||
if(date.indexOf("时")==-1 && date.toString().length()>ind+1) {
|
||||
date = date +"时"
|
||||
}
|
||||
|
||||
pattern = "yyyy年MM月dd日HH时"
|
||||
}
|
||||
if(j==0) {
|
||||
try{
|
||||
result.startDate = DateUtils.parseFromTightByPattern(date, pattern)
|
||||
}catch(Exception e){
|
||||
|
||||
}
|
||||
|
||||
}else {
|
||||
if(!result.finishDate) {
|
||||
result.finishDate = DateUtils.parseFromTightByPattern(date, pattern)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!result.startPrice) {
|
||||
result.startPrice = DataUtil.getStartPrice(s1)
|
||||
}
|
||||
|
||||
def estateType = DataUtil.findEstateType(s1)
|
||||
if(estateType) {
|
||||
result.matchEstateType = estateType
|
||||
}
|
||||
|
||||
}
|
||||
result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
|
||||
result.introduce = JQueryUtils.select([".floor-content div:eq(2) table"], doc).toString()
|
||||
result.url = url
|
||||
result.city = city
|
||||
result.mDate = new Date()
|
||||
result.source = "京东"
|
||||
return result;
|
||||
}
|
||||
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
if(doc){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=10){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
def findAllIds(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", Pattern.compile(".*jd.com.*"))
|
||||
def dbcursor = paimaiResultColl.find(q)
|
||||
def ids = []
|
||||
for(def d : dbcursor){
|
||||
def url = d.url
|
||||
if(url){
|
||||
def id = url.substring(url.toString().lastIndexOf("/")+1, url.toString().length())
|
||||
ids << id
|
||||
}
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
}
|
||||
+53
-26
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
import java.util.zip.GZIPInputStream
|
||||
@@ -94,32 +95,32 @@ class PaimaiDataUtil {
|
||||
}
|
||||
|
||||
|
||||
public static def getDateRange(def s) {
|
||||
s = s.replace("起","")
|
||||
Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)');
|
||||
Matcher matcher = pattern.matcher(s);
|
||||
|
||||
List<String> list = new ArrayList();
|
||||
while (matcher.find()) {
|
||||
def str = matcher.group()
|
||||
list.add(str);
|
||||
// if(str) {
|
||||
// def indx1 = str.indexOf("年")
|
||||
// def indx2 = str.indexOf("月")
|
||||
// def month = str.substring(indx1+1, indx2);
|
||||
// if(month.toInteger()<12) {
|
||||
// list.add(str);
|
||||
// }
|
||||
// }
|
||||
}
|
||||
if(list.size()>=1) {
|
||||
String s1 = list.get(0)
|
||||
s1 = s1.replace("法院将于", "").replace(" ","")
|
||||
def array = s1.split("至");
|
||||
return array
|
||||
}
|
||||
return null;
|
||||
}
|
||||
// public static def getDateRange(def s) {
|
||||
// s = s.replace("起","")
|
||||
// Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)');
|
||||
// Matcher matcher = pattern.matcher(s);
|
||||
//
|
||||
// List<String> list = new ArrayList();
|
||||
// while (matcher.find()) {
|
||||
// def str = matcher.group()
|
||||
// list.add(str);
|
||||
//// if(str) {
|
||||
//// def indx1 = str.indexOf("年")
|
||||
//// def indx2 = str.indexOf("月")
|
||||
//// def month = str.substring(indx1+1, indx2);
|
||||
//// if(month.toInteger()<12) {
|
||||
//// list.add(str);
|
||||
//// }
|
||||
//// }
|
||||
// }
|
||||
// if(list.size()>=1) {
|
||||
// String s1 = list.get(0)
|
||||
// s1 = s1.replace("法院将于", "").replace(" ","")
|
||||
// def array = s1.split("至");
|
||||
// return array
|
||||
// }
|
||||
// return null;
|
||||
// }
|
||||
|
||||
|
||||
public static def getStartPrice(def s) {
|
||||
@@ -159,4 +160,30 @@ class PaimaiDataUtil {
|
||||
return newDetailMap;
|
||||
}
|
||||
|
||||
public static def getDateRange(def s) {
|
||||
s = s.replace("起","")
|
||||
Pattern pattern = Pattern.compile('法院将于(\\d+年\\d+月\\d+日\\d+时)至(\\d+年\\d+月\\d+日\\d+时)');
|
||||
Matcher matcher = pattern.matcher(s);
|
||||
|
||||
List<String> list = new ArrayList();
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group());
|
||||
}
|
||||
if(list.size()>=1) {
|
||||
String s1 = list.get(0)
|
||||
s1 = s1.replace("法院将于", "")
|
||||
def array = s1.split("至");
|
||||
return array
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static Date parseFromTightByPattern(String s, String pattern){
|
||||
if (s == null || s.length() == 0){
|
||||
return null
|
||||
}
|
||||
SimpleDateFormat format = new SimpleDateFormat(pattern);
|
||||
return format.parse(s)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+40
-13
@@ -83,15 +83,19 @@ class PaimaiTest {
|
||||
driver.get("https:"+data.href)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
if(citys.contains(data.proince)){
|
||||
def cityMap = [:]
|
||||
cityMap.proince = data.proince
|
||||
cityMap.city = data.proince
|
||||
cityMap.url = data.href
|
||||
cityList << cityMap
|
||||
def links = JQueryUtils.select(".condition .s-subtab a", doc)
|
||||
for(def ele : links) {
|
||||
def cityMap = [:]
|
||||
cityMap.proince = data.proince
|
||||
cityMap.city = data.proince
|
||||
cityMap.region =ele.text()
|
||||
cityMap.url = data.href
|
||||
cityList << cityMap
|
||||
}
|
||||
}else{
|
||||
def links = JQueryUtils.select(".condition .s-subtab a", doc)
|
||||
for(def ele : links){
|
||||
if(!["合肥市","芜湖市","蚌埠市","淮南市","马鞍山市","淮北市","铜陵市"].contains(ele.text())){
|
||||
if(!["乌鲁木齐市","克拉玛依市","合肥市","芜湖市","蚌埠市","淮南市","马鞍山市","淮北市","铜陵市"].contains(ele.text())){
|
||||
def cityMap = [:]
|
||||
cityMap.proince = data.proince
|
||||
cityMap.city = ele.text()
|
||||
@@ -109,7 +113,7 @@ class PaimaiTest {
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def datas = []
|
||||
def keys = ["上海","天津","重庆","北京","浙江","河北","山西","辽宁","吉林","江苏","福建","江西","山东","河南","湖北","湖南","广东"]
|
||||
def keys = [ "天津","上海","重庆","北京","浙江","河北","陕西","山西","新疆","辽宁","吉林","江苏","福建","江西","山东","河南","湖北","湖南","西藏","宁夏","甘肃","广东","安徽","广西","海南","四川","贵州","云南"]
|
||||
if(doc){
|
||||
def eles = JQueryUtils.select(".condition a", doc)
|
||||
for(def e : eles){
|
||||
@@ -123,11 +127,34 @@ class PaimaiTest {
|
||||
}
|
||||
return datas
|
||||
}
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
if(doc){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=10){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
def parser(detailDatas, driver, proince, city){
|
||||
def results = []
|
||||
for(def m : detailDatas){
|
||||
try{
|
||||
// try{
|
||||
driver.get("https:"+m.url)
|
||||
Thread.sleep(3000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
@@ -137,9 +164,9 @@ class PaimaiTest {
|
||||
obj.city = city
|
||||
results << obj
|
||||
}
|
||||
}catch(Exception e){
|
||||
println m.url+" 解析有问题!"
|
||||
}
|
||||
// }catch(Exception e){
|
||||
// println m.url+" 解析有问题!"
|
||||
// }
|
||||
|
||||
}
|
||||
return results
|
||||
@@ -410,7 +437,7 @@ class PaimaiTest {
|
||||
}
|
||||
}
|
||||
|
||||
// result.content = DataUtil.zip(s)
|
||||
result.content = DataUtil.zip(doc.toString())
|
||||
|
||||
def el = JQueryUtils.select([".d-block table:eq(1)"], doc)
|
||||
if(!el) {
|
||||
@@ -549,7 +576,7 @@ class PaimaiTest {
|
||||
ret.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
|
||||
obj.source = "公拍网"
|
||||
if(dbo){
|
||||
obj.m_date = new Date()
|
||||
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
|
||||
|
||||
+348
-71
@@ -1,12 +1,4 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.gj.paimai.PaimaiDataUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -14,66 +6,266 @@ import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.JavascriptExecutor
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.WebElement
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
import org.openqa.selenium.support.ui.WebDriverWait
|
||||
import org.openqa.selenium.support.ui.ExpectedCondition
|
||||
import org.openqa.selenium.support.ui.ExpectedConditions
|
||||
import org.openqa.selenium.support.ui.WebDriverWait
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
|
||||
|
||||
|
||||
def url = "https://www1.rmfysszc.gov.cn/projects.shtml?dh=3&gpstate=1&wsbm_slt=1"
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver.get(url);
|
||||
driver.findElementById("1").click()
|
||||
run(url, driver)
|
||||
//driver.get(url);
|
||||
//driver.findElementById("1").click()
|
||||
//driver.findElementById("search_sub").click()
|
||||
//
|
||||
//def provinces = getAllProvinceValue(driver.getPageSource())
|
||||
//for(def m : provinces){
|
||||
// def city_setting = driver.findElementById("city")
|
||||
// //${m.province_id}
|
||||
// city_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click()
|
||||
//
|
||||
////driver.manage().timeouts().implicitlyWait(30, TimeUnit.SECONDS);
|
||||
////WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
////wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.cssSelector("pagecur1"))))
|
||||
// Thread.sleep(10000)
|
||||
// def doc = Jsoup.parse(driver.getPageSource());
|
||||
// def citys = getAllCity(doc)
|
||||
// getDetailByCity(m.province, citys, driver)
|
||||
//}
|
||||
|
||||
def city_setting = driver.findElementById("city")
|
||||
city_setting.findElement(By.xpath("//option[@value='31']")).click()
|
||||
|
||||
driver.findElementById("search_sub").click()
|
||||
//driver.manage().timeouts().implicitlyWait(30, TimeUnit.SECONDS);
|
||||
//WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
//wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.cssSelector("pagecur1"))))
|
||||
Thread.sleep(10000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def links = getLinks(doc)
|
||||
def run(url, driver){
|
||||
def values = getAllCityRegionValue(url, driver)
|
||||
def finishCity = ["北京市","天津市","张家口市","邯郸市","保定市","邢台市","石家庄市","唐山市","秦皇岛市"]
|
||||
for(def m : values){
|
||||
if(!finishCity.contains(m.city)){
|
||||
driver.get(url);
|
||||
driver.findElementById("1").click()
|
||||
Thread.sleep(3000)
|
||||
|
||||
def pro_setting = driver.findElementById("city")
|
||||
pro_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click()
|
||||
Thread.sleep(3000)
|
||||
|
||||
def city_setting = driver.findElementById("city1")
|
||||
city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click()
|
||||
Thread.sleep(4000)
|
||||
|
||||
def region_setting = driver.findElementById("city2")
|
||||
region_setting.findElement(By.xpath("//option[@value='${m.region_id}']")).click()
|
||||
|
||||
driver.findElementById("search_sub").click()
|
||||
Thread.sleep(10000)
|
||||
getDetailByPage(m.province, m.city, m.region, driver)
|
||||
}
|
||||
|
||||
def eles = driver.findElementsByClassName("pagecur1")
|
||||
for(def ele : eles){
|
||||
ele.click();
|
||||
Thread.sleep(10000)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
links.addAll(getLinks(doc))
|
||||
}
|
||||
def rets = []
|
||||
for(def durl : links){
|
||||
driver.get(durl);
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
def res = parse(durl, doc);
|
||||
if(res){
|
||||
rets << res
|
||||
}
|
||||
}
|
||||
saveToDB(rets);
|
||||
|
||||
def getAllCityRegionValue(def url, def driver){
|
||||
driver.get(url);
|
||||
driver.findElementById("1").click()
|
||||
driver.findElementById("search_sub").click()
|
||||
|
||||
def provinces = getAllProvinceValue(driver.getPageSource())
|
||||
def values = []
|
||||
for(def m : provinces){
|
||||
def city_setting = driver.findElementById("city")
|
||||
|
||||
city_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click()
|
||||
|
||||
Thread.sleep(10000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def citys = getAllCity(doc)
|
||||
def cityRegions = getAllCityByRegion(m, citys, driver)
|
||||
values.addAll(cityRegions)
|
||||
//getDetailByCity(m.province, citys, driver)
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
def getDetailByCity(province, citys, driver){
|
||||
for(def m : citys){
|
||||
def city_setting = driver.findElementById("city1")
|
||||
city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click()
|
||||
Thread.sleep(10000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def regions = getAllRegion(doc)
|
||||
getDetailByRegion(province, m.city, regions, driver)
|
||||
}
|
||||
}
|
||||
|
||||
def getAllCityByRegion(provinceMap, citys, driver){
|
||||
def datas = []
|
||||
for(def m : citys){
|
||||
def city_setting = driver.findElementById("city1")
|
||||
city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click()
|
||||
Thread.sleep(10000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def regions = getAllRegion(provinceMap, m, doc)
|
||||
datas.addAll(regions)
|
||||
}
|
||||
return datas
|
||||
}
|
||||
|
||||
def getDetailByRegion(province, city, regions, driver){
|
||||
def links = []
|
||||
for(def r : regions){
|
||||
def city_setting = driver.findElementById("city2")
|
||||
city_setting.findElement(By.xpath("//option[@value='${r.region_id}']")).click()
|
||||
driver.findElementById("search_sub").click()
|
||||
Thread.sleep(10000)
|
||||
|
||||
def stop = false;
|
||||
while(!stop){
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
links.addAll(getLinks(r.region, doc))
|
||||
def eles = JQueryUtils.select("a:contains(下一页)", doc)
|
||||
if(eles){
|
||||
driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click()
|
||||
Thread.sleep(3000)
|
||||
}else{
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
def datas = parseDatas(province, city, links, driver)
|
||||
saveToDB(datas);
|
||||
println r.region +" "+ links.size()
|
||||
}
|
||||
|
||||
def getDetailByPage(province, city, region, driver){
|
||||
def links = []
|
||||
def stop = false;
|
||||
while(!stop){
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
links.addAll(getLinks(doc))
|
||||
def eles = JQueryUtils.select("a:contains(下一页)", doc)
|
||||
if(eles){
|
||||
driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click()
|
||||
Thread.sleep(3000)
|
||||
}else{
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
|
||||
def datas = parseDatas(province, city, region, links, driver)
|
||||
println province +" "+city+" "+region +" "+ links.size()
|
||||
saveToDB(datas);
|
||||
}
|
||||
|
||||
|
||||
def getAllProvinceValue(doc){
|
||||
def optionsEls = JQueryUtils.select("#city option", doc)
|
||||
def rets = []
|
||||
for(def eles : optionsEls){
|
||||
def val = eles.attr("value")
|
||||
if(val!="" && val!="0"){
|
||||
def province = eles.text()
|
||||
rets << ["province":province, "province_id":val]
|
||||
|
||||
def parse(url, doc){
|
||||
}
|
||||
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def getAllCity(doc){
|
||||
def optionsEls = JQueryUtils.select("#city1 option", doc)
|
||||
def rets = []
|
||||
for(def eles : optionsEls){
|
||||
def val = eles.attr("value")
|
||||
if(val!=""){
|
||||
def province = eles.text()
|
||||
rets << ["city":province, "city_id":val]
|
||||
}
|
||||
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def getAllRegion(provinceMap, cityMap,doc){
|
||||
def optionsEls = JQueryUtils.select("#city2 option", doc)
|
||||
def rets = []
|
||||
for(def eles : optionsEls){
|
||||
def val = eles.attr("value")
|
||||
if(val!=""){
|
||||
def region = eles.text()
|
||||
def regionMap = [:]
|
||||
regionMap.province = provinceMap.province
|
||||
regionMap.province_id = provinceMap.province_id
|
||||
regionMap.city = cityMap.city
|
||||
regionMap.city_id = cityMap.city_id
|
||||
regionMap.region = region
|
||||
regionMap.region_id = val
|
||||
rets << regionMap
|
||||
}
|
||||
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def parseDatas(province, city, region,links, driver){
|
||||
def datas = []
|
||||
for(def url: links){
|
||||
def doc = getDocument(url, driver)
|
||||
def result = parse(city, region, url, doc)
|
||||
if(result){
|
||||
result.province = province
|
||||
result.city = city
|
||||
result.region = region
|
||||
datas << result
|
||||
}
|
||||
}
|
||||
return datas
|
||||
}
|
||||
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
if(doc){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=10){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
def parse(city, region, url, doc){
|
||||
def result=[:]
|
||||
if(doc) {
|
||||
|
||||
def title = JQueryUtils.selectText(["#Title"], doc)
|
||||
result.title = title;
|
||||
result.address = title
|
||||
if(result.address){
|
||||
def rMap = OpenApiUtils.matchDistrict("上海",result.address)
|
||||
def rMap = OpenApiUtils.matchDistrict(city, result.address, null, true)
|
||||
if(rMap) {
|
||||
result.matchCity = rMap.city
|
||||
result.matchRegion = rMap.region
|
||||
@@ -99,6 +291,7 @@ def parse(url, doc){
|
||||
|
||||
def enrollCount = JQueryUtils.selectText([".bmnumber"], doc)
|
||||
if(enrollCount) {
|
||||
enrollCount = enrollCount.replace("人报名","")
|
||||
if(enrollCount.isInteger()){
|
||||
result.enrollCount = enrollCount.toInteger()
|
||||
}
|
||||
@@ -107,6 +300,7 @@ def parse(url, doc){
|
||||
|
||||
def callCount = JQueryUtils.selectText([".sc"], doc)
|
||||
if(callCount) {
|
||||
callCount =callCount.replace("人收藏","")
|
||||
if(callCount.isInteger()){
|
||||
result.callCount = callCount.toInteger()
|
||||
}
|
||||
@@ -114,6 +308,7 @@ def parse(url, doc){
|
||||
|
||||
def lookCount = JQueryUtils.selectText([".hits"], doc)
|
||||
if(lookCount) {
|
||||
lookCount = lookCount.replace("次围观","")
|
||||
if(lookCount.isInteger()){
|
||||
result.lookCount = lookCount.toInteger()
|
||||
}
|
||||
@@ -123,8 +318,8 @@ def parse(url, doc){
|
||||
def curPriceStr = JQueryUtils.selectText(["#price div:eq(0)"], doc)
|
||||
if(curPriceStr && curPriceStr.contains("起拍价")) {
|
||||
curPriceStr = curPriceStr.replace("起拍价", "").replace("¥","")
|
||||
if(curPriceStr.contains("万元")){
|
||||
curPriceStr = curPriceStr.replace("万元","")
|
||||
if(curPriceStr.contains("万")){
|
||||
curPriceStr = curPriceStr.replace("万元","").replace("万","")
|
||||
if(curPriceStr.isInteger()){
|
||||
result.startPrice = curPriceStr.toInteger()*10000
|
||||
}else{
|
||||
@@ -144,6 +339,44 @@ def parse(url, doc){
|
||||
}
|
||||
}
|
||||
|
||||
def tableEles = JQueryUtils.select("#bg1", doc)
|
||||
if(tableEles){
|
||||
def evaluatePriceStr = tableEles.select("td:contains(值:)").text()
|
||||
if(evaluatePriceStr && evaluatePriceStr.contains(":")){
|
||||
def arrs = evaluatePriceStr.split(":")
|
||||
if(arrs.size()==2){
|
||||
evaluatePriceStr = arrs[1]
|
||||
}
|
||||
}
|
||||
|
||||
def depositPriceStr = tableEles.select("td:contains(金:)").text()
|
||||
if(depositPriceStr && depositPriceStr.contains(":")){
|
||||
def arrs = depositPriceStr.split(":")
|
||||
if(arrs.size()==2){
|
||||
depositPriceStr = arrs[1]
|
||||
}
|
||||
}
|
||||
if(depositPriceStr){
|
||||
depositPriceStr = depositPriceStr.replace("万元","")
|
||||
if(depositPriceStr.toString().isNumber()){
|
||||
result.depositPrice = depositPriceStr.toDouble()*10000
|
||||
}
|
||||
}
|
||||
|
||||
def courtStr = tableEles.select("td:contains(院:)").text()
|
||||
if(courtStr && courtStr.contains(":")){
|
||||
result.court = courtStr.substring(courtStr.indexOf(":")+1, courtStr.size())
|
||||
}
|
||||
def personStr = tableEles.select("td:contains(人:)").text()
|
||||
if(personStr && personStr.contains(":")){
|
||||
result.person = personStr.substring(personStr.indexOf(":")+1, personStr.size())
|
||||
}
|
||||
|
||||
def countStr = tableEles.select("td:contains(拍卖阶段:)").text().replace("拍卖阶段:","")
|
||||
if(countStr && countStr.contains("第") && countStr.contains("次")){
|
||||
result.count = countStr.substring(countStr.indexOf("第")+1, countStr.indexOf("次"))
|
||||
}
|
||||
}
|
||||
|
||||
def lis = JQueryUtils.select("#bg1 div:eq(0) table tr", doc)
|
||||
if(lis && lis.size()) {
|
||||
@@ -151,18 +384,23 @@ def parse(url, doc){
|
||||
def str = li.select("span").text()
|
||||
if(str.contains("评估值")) {
|
||||
def v = li.select("span:eq(1)").text().replace("万元", "")
|
||||
if(v.isInteger()) {
|
||||
result.evaluationPrice = v.toInteger()*10000
|
||||
}else{
|
||||
result.evaluationPrice = (int)(v.toDouble()*10000)
|
||||
if(v.isNumber()){
|
||||
if(v.isInteger()) {
|
||||
result.evaluationPrice = v.toInteger()*10000
|
||||
}else{
|
||||
result.evaluationPrice = (int)(v.toDouble()*10000)
|
||||
}
|
||||
}
|
||||
|
||||
}else if(str.contains("保证金")) {
|
||||
def v = li.select("span:eq(1)").text().replace("万元", "")
|
||||
if(v) {
|
||||
if(v.isInteger()) {
|
||||
result.depositPrice = v.toInteger()*10000
|
||||
}else {
|
||||
result.depositPrice = (int)(v.toDouble()*10000)
|
||||
if(v.isNumber()) {
|
||||
if (v) {
|
||||
if (v.isInteger()) {
|
||||
result.depositPrice = v.toInteger() * 10000
|
||||
} else {
|
||||
result.depositPrice = (int) (v.toDouble() * 10000)
|
||||
}
|
||||
}
|
||||
}
|
||||
}else if (str.contains("公告日期:")){
|
||||
@@ -177,13 +415,32 @@ def parse(url, doc){
|
||||
def trs = tables[1].select("tr")
|
||||
if(trs) {
|
||||
def detailMap = [:]
|
||||
|
||||
for(def tr : trs) {
|
||||
def d_tds = tr.select("td")
|
||||
if(tr.text().contains("调查情况表")){
|
||||
continue
|
||||
}
|
||||
if(d_tds.size()>1){
|
||||
def lastEleStr = d_tds[d_tds.size()-1].toString()
|
||||
if(lastEleStr.contains('height="42"')){
|
||||
d_tds.remove(d_tds[d_tds.size()-1])
|
||||
}
|
||||
}
|
||||
|
||||
if(d_tds.size()>3){
|
||||
continue
|
||||
}
|
||||
if(d_tds && d_tds.size()>=2) {
|
||||
if(d_tds.size()==3) {
|
||||
detailMap.put(d_tds[1].text(), d_tds[2].text())
|
||||
if(d_tds[1].text() && d_tds[2].text()){
|
||||
detailMap.put(d_tds[1].text(), d_tds[2].text())
|
||||
}
|
||||
|
||||
}else {
|
||||
detailMap.put(d_tds[0].text(), d_tds[1].text())
|
||||
if(d_tds[0].text() && d_tds[1].text()){
|
||||
detailMap.put(d_tds[0].text(), d_tds[1].text())
|
||||
}
|
||||
}
|
||||
}else {
|
||||
if(d_tds[0] && d_tds[0].text().contains("拍品介绍")) {
|
||||
@@ -202,8 +459,6 @@ def parse(url, doc){
|
||||
result.introduce = tables[1].toString()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def imgs =JQueryUtils.select("#bdjs11 img", doc)
|
||||
if(imgs) {
|
||||
def imgLinks = []
|
||||
@@ -232,6 +487,20 @@ def parse(url, doc){
|
||||
result.tradeAvgPrice = (int)(result.tradePrice/result.area)
|
||||
}
|
||||
|
||||
def s1 = JQueryUtils.selectText(["#pmgg"], doc)
|
||||
if(s1) {
|
||||
def start = s1.indexOf("法院将于")
|
||||
def end = s1.indexOf("止")
|
||||
if(start>-1 && end>-1 && end>start){
|
||||
s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止"))
|
||||
}
|
||||
def dateRangs = PaimaiDataUtil.getDateRange(s1)
|
||||
if(dateRangs && dateRangs.size()==2) {
|
||||
result.startDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[0], "yyyy年MM月dd日HH时")
|
||||
result.finishDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[1], "yyyy年MM月dd日HH时")
|
||||
}
|
||||
}
|
||||
|
||||
result.content = DataUtil.zip(doc.toString())
|
||||
|
||||
result.url = url
|
||||
@@ -246,7 +515,7 @@ def getLinks(doc){
|
||||
if(eles){
|
||||
for(def el :eles){
|
||||
def href = el.attr("href")
|
||||
urls << "https://www.rmfysszc.gov.cn/statichtml/rm_obj/"+href.substring(href.indexOf("/")+1,href.length())
|
||||
urls <<"https://www.rmfysszc.gov.cn/statichtml/rm_obj/"+href.substring(href.indexOf("/")+1,href.length())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -264,13 +533,19 @@ def saveToDB(rets){
|
||||
ret.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
|
||||
if(dbo){
|
||||
obj.remove("url")
|
||||
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
|
||||
}else{
|
||||
paimaiResultColl.save(obj)
|
||||
}
|
||||
obj.source= "人民法院诉讼资产网"
|
||||
try{
|
||||
if(dbo){
|
||||
obj.m_date = new Date()
|
||||
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
|
||||
}else{
|
||||
obj.c_date = new Date()
|
||||
obj.m_date = new Date()
|
||||
paimaiResultColl.save(obj)
|
||||
}
|
||||
}catch(Exception e){
|
||||
println "error:"+obj.url
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -287,14 +562,16 @@ def getAreaNumber(input, maxArea) {
|
||||
|
||||
for (def val : list) {
|
||||
def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", "");
|
||||
if (maxArea) {
|
||||
if (v.toDouble() < maxArea) {
|
||||
area = v;
|
||||
if(v && v.isNumber()){
|
||||
if (maxArea) {
|
||||
if (v.toDouble() < maxArea) {
|
||||
area = v;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
area = v
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
area = v
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
package com.fangjia.script.loudong
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class CqfdcMatch {
|
||||
|
||||
static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018)
|
||||
static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018)
|
||||
static main(args) {
|
||||
def task = new CqfdcMatch();
|
||||
// task.run()
|
||||
// task.stat()
|
||||
task.export2()
|
||||
}
|
||||
def export(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_city", new BasicDBObject('$exists', false))
|
||||
|
||||
def field = new BasicDBObject()
|
||||
field.put("city",1)
|
||||
field.put("region",1)
|
||||
field.put("district_name",1)
|
||||
field.put("address",1)
|
||||
field.put("type",1)
|
||||
def dbcursor = loudongConn.find(q, field).batchSize(5000)
|
||||
def keys = []
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","地址","类型"]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city +"##"+d.district_name
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
rets << [d.city, d.region, d.district_name, d.address, d.type]
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\重庆楼栋未匹配1.xlsx",rets, "数据")
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas= findAddress2()
|
||||
def results = match(datas);
|
||||
update(results)
|
||||
}
|
||||
|
||||
|
||||
def export2(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_city", new BasicDBObject('$exists', false))
|
||||
|
||||
def field = new BasicDBObject()
|
||||
field.put("city",1)
|
||||
field.put("name",1)
|
||||
field.put("use_type",1)
|
||||
def dbcursor = loudongFdcConn.find(q, field).batchSize(5000)
|
||||
def keys = []
|
||||
def rets = []
|
||||
rets << ["城市","地址","类型"]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city +"##"+d.name
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
rets << [d.city, d.name,d.use_type]
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\重庆楼栋未匹配2.xlsx",rets, "数据")
|
||||
}
|
||||
|
||||
def update(results){
|
||||
for(def m : results){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", m.old_city)
|
||||
q.put("district_name", m.old_name)
|
||||
q.put("region",m.old_region)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("match_city", m.match_city)
|
||||
udata.put("match_region", m.match_region)
|
||||
udata.put("match_name", m.match_name)
|
||||
loudongConn.update(q, new BasicDBObject('$set',udata),false, true)
|
||||
}
|
||||
}
|
||||
|
||||
def match(def datas){
|
||||
def rets = []
|
||||
for(def d : datas){
|
||||
def jsonObject = OpenApiClient.addressMatch(d.city, d.name, "property", false, d.region, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
def map = [:]
|
||||
map.match_city = jsonObject.get("city")
|
||||
map.match_region = jsonObject.get("district")
|
||||
map.match_name = jsonObject.get("name")
|
||||
map.old_name = d.name
|
||||
map.old_city = d.city
|
||||
map.old_region = d.region
|
||||
rets << map
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findAddress(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = loudongConn.find(q)
|
||||
def rets = []
|
||||
def keys = []
|
||||
for(def d : dbcursor){
|
||||
if(!keys.contains(d.name)){
|
||||
|
||||
keys << d.name
|
||||
def map = [:]
|
||||
map.city = d.city
|
||||
map.name = d.name
|
||||
rets << map
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findAddress2(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = loudongConn.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
def keys = []
|
||||
for(def d : dbcursor){
|
||||
def key = d.city +"##"+d.region+"##"+d.district_name
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
def map = [:]
|
||||
map.city = d.city
|
||||
map.region = d.region
|
||||
map.name = d.district_name
|
||||
rets << map
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def stat(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_city", new BasicDBObject('$exists', true))
|
||||
|
||||
def field = new BasicDBObject()
|
||||
field.put("match_city",1)
|
||||
field.put("match_region",1)
|
||||
field.put("match_name",1)
|
||||
def dbcursor = loudongConn.find(q, field).batchSize(2000)
|
||||
def keys = []
|
||||
for(def d : dbcursor){
|
||||
def key = d.match_city+"##"+d.match_region+"##"+d.match_name
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
}
|
||||
}
|
||||
|
||||
def dbcursor2 = loudongFdcConn.find(q, field).batchSize(5000)
|
||||
for(def d : dbcursor2){
|
||||
def key = d.match_city+"##"+d.match_region+"##"+d.match_name
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
}
|
||||
}
|
||||
println "小区:"+keys.size()
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
+1
-1
@@ -76,7 +76,7 @@ class SeaweedExport {
|
||||
ret[1] = obj.developer
|
||||
ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent)
|
||||
ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent)
|
||||
ret[4] = SeaweedUtils.formatAlias(obj.alias)
|
||||
ret[4] = SeaweedUtils.formatAlias(obj.mapping_alias)
|
||||
ret[5] = SeaweedUtils.formatCarSiteCount(obj.car_site_count)
|
||||
ret[6] = SeaweedUtils.calcCarAndHouseholdRadio(obj.car_site_count, obj.household_count)
|
||||
ret[7] = ""
|
||||
|
||||
+4
-4
@@ -40,11 +40,11 @@ class DataStat {
|
||||
|
||||
|
||||
def run(){
|
||||
def source = "搜房"//
|
||||
def citys = findCrawlCity(source, 202112);
|
||||
def source = "58同城"//
|
||||
def citys = findCrawlCity(source, 202202);
|
||||
for(def city : citys){
|
||||
def count1 = findCount(source, city, 202112);
|
||||
def count2 = findCount(source, city, 202202);
|
||||
def count1 = findCount(source, city, 202202);
|
||||
def count2 = findCount(source, city, 202203);
|
||||
println city +" "+count1+" "+count2
|
||||
// println count1
|
||||
// println "--- ----"
|
||||
|
||||
+44
-39
@@ -40,19 +40,17 @@ class Esf58Crawl extends BaseBatch{
|
||||
def static useDistKeys = []
|
||||
static def totalCount = new AtomicLong()
|
||||
|
||||
def cityList = [
|
||||
def cityList = ["成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413",
|
||||
"保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872",
|
||||
"沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556",
|
||||
"石家庄":"sjz|241","太原":"ty|740","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606",
|
||||
"绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572",
|
||||
"淮北":"huaibei|9357","滁州":"chuzhou|10266",
|
||||
"福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158",
|
||||
"兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362",
|
||||
"兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","潍坊":"wf|362",
|
||||
"芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222",
|
||||
"南通":"nt|394","东莞":"dg|413","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325",
|
||||
"南通":"nt|394","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325",
|
||||
"呼和浩特":"hu|811",
|
||||
"济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096",
|
||||
"临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","北京":"bj|1","上海":"sh|2",
|
||||
"三亚":"sanya|2422","烟台":"yt|228","宁波":"nb|135","金华":"jh|531", "绍兴":"sx|355","湖州":"huzhou|831","张家口":"zjk|3328","珠海":"zh|910","海口":"haikou|2053","保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872",
|
||||
"沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556",
|
||||
"深圳":"sz|4","石家庄":"sjz|241","太原":"ty|740","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606",
|
||||
"绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572",
|
||||
"淮北":"huaibei|9357","滁州":"chuzhou|10266",
|
||||
"马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323",
|
||||
"池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296",
|
||||
"宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232",
|
||||
@@ -76,7 +74,7 @@ class Esf58Crawl extends BaseBatch{
|
||||
"五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136",
|
||||
"文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184",
|
||||
"东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","白沙":"baish|10380",
|
||||
"儋州":"danzhou|10394", "郑州":"zz|342","新乡":"xx|1016","许昌":"xc|977",
|
||||
"儋州":"danzhou|10394", "新乡":"xx|1016","许昌":"xc|977",
|
||||
"平顶山":"pds|1005","焦作":"jiaozuo|3266","开封":"kaifeng|2342",
|
||||
"濮阳":"puyang|2346","周口":"zk|933","驻马店":"zmd|1067","漯河":"luohe|2347",
|
||||
"三门峡":"smx|9317","鹤壁":"hb|2344","济源":"jiyuan|9918","明港":"mg|8541","鄢陵":"yanling|9123",
|
||||
@@ -98,7 +96,7 @@ class Esf58Crawl extends BaseBatch{
|
||||
"广水":"guangshuishi|9657","赤壁":"chibishi|9623","老河口":"laohekou|895","谷城":"gucheng|899","宜城":"yichengshi|897",
|
||||
"南漳":"nanzhang|898","云梦":"yunmeng|3438","安陆":"anlu|3442","大悟":"dawu|3437","孝昌":"xiaochang|3436",
|
||||
"当阳":"dangyang|865","枝江":"zhijiang|866","嘉鱼":"jiayuxian|9624","随县":"suixia|9660",
|
||||
"长沙":"cs|414","益阳":"yiyang|10198","衡阳":"hy|914",
|
||||
"益阳":"yiyang|10198","衡阳":"hy|914",
|
||||
"湘潭":"xiangtan|2047","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756",
|
||||
"永州":"yongzhou|2307","娄底":"ld|9481","湘西":"xiangxi|10219","张家界":"zjj|6788","醴陵":"liling|1091",
|
||||
"澧县":"lixian|876","桂阳":"czguiyang|5699","资兴":"zixing|5698","永兴":"yongxing|5701","常宁":"changningshi|921",
|
||||
@@ -181,7 +179,10 @@ class Esf58Crawl extends BaseBatch{
|
||||
"长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536",
|
||||
"安吉":"anji|836","苍南":"cangnanxian|7579","临海":"linhai|407","永康":"yongkang|537","玉环":"yuhuan|409",
|
||||
"平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361",
|
||||
"江山":"jiangshanshi|6796","平阳":"pingyangxian|7575"
|
||||
"威海":"weihai|518","安阳":"ay|1096",
|
||||
"临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","北京":"bj|1","上海":"sh|2",
|
||||
"三亚":"sanya|2422","烟台":"yt|228","宁波":"nb|135","金华":"jh|531", "绍兴":"sx|355","湖州":"huzhou|831","张家口":"zjk|3328","珠海":"zh|910","海口":"aihkou|2053",
|
||||
"江山":"jiangshanshi|6796","平阳":"pingyangxian|7575","济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|27 6","商丘":"sq|1029","大同":"dt|6964",
|
||||
]
|
||||
|
||||
static main(args) {
|
||||
@@ -191,37 +192,37 @@ class Esf58Crawl extends BaseBatch{
|
||||
void process(Object args) {
|
||||
//addContent("福州","https://fz.58.com/xiaoqu/", 70);
|
||||
|
||||
// def citys = ["新余","松原","楚雄","吕梁","雅安","本溪","鹤壁","拉萨","黔东南","忻州","兴安盟","乌海","贵港","三门峡","仙桃","安康","白城","平凉","庆阳","潮州","白银","吴忠","防城港","延安","丽江","黔西南","喀什","钦州","鸡西","萍乡","石嘴山","鹰潭","保山","哈密","铜仁","武威","普洱","河池","酒泉","朔州","百色","鹤岗","张家界","定西","湘西","济源","昭通","中卫","张掖","克拉玛依","黑河","铜川","辽源","云浮","博尔塔拉","固原","白山","天门","潜江","贺州","临沧","伊春","临夏","石河子","来宾","双鸭山","嘉峪关","海东"]
|
||||
// for(def city : citys){
|
||||
// def str = cityList.get(city)
|
||||
def citys = ["佛山","潍坊","长春","太原","南通","厦门","呼和浩特","常州","中山","德阳","镇江","泰安","银川","襄阳","江门","荆州","上海","衡阳","滨州","遂宁","鞍山","赣州","阜阳","枣庄","聊城","芜湖","吉林","德州","信阳","新乡","宿迁","沧州","桂林","包头","常德","乐山","遵义","眉山","株洲","驻马店","日照","许昌","成都","衡水","宜宾","泸州","营口","达州","宝鸡","广安","吉安","孝感","鄂尔多斯","十堰","清远","湘潭","上饶","漯河","邢台","平顶山","菏泽","宜春","承德","赤峰","延边","茂名","焦作","内江","韶关","梅州","郴州","黄冈","红河","绥化","娄底","丹东","邵阳","开封","阳江","黄石","周口","牡丹江","长治","咸宁","汉中","自贡","衢州","晋中","永州","北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏","重庆"]
|
||||
for(def city : citys){
|
||||
def str = cityList.get(city)
|
||||
for(def i=0;i<1;i++){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||
if(districts){
|
||||
def arrs = str.split("\\|")
|
||||
def url = "https://"+arrs[0]+".58.com/xiaoqu/"
|
||||
distMap.put(city, districts)
|
||||
addCrawlKeys(city);
|
||||
addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cityList.each{city, str->
|
||||
// for(def i=0;i<1;i++){
|
||||
// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||
// if(districts){
|
||||
// def arrs = str.split("\\|")
|
||||
// def url = "https://"+arrs[0]+".58.com/xiaoqu/"
|
||||
// distMap.put(city, districts)
|
||||
// addCrawlKeys(city);
|
||||
// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
|
||||
// if(citys.contains(city)){
|
||||
// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||
// if(districts){
|
||||
// def arrs = str.split("\\|")
|
||||
// def url = "https://"+arrs[0]+".58.com/xiaoqu/"
|
||||
// distMap.put(city, districts)
|
||||
//
|
||||
// addCrawlKeys(city);
|
||||
// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
cityList.each{city, str->
|
||||
for(def i=0;i<1;i++){
|
||||
// if(citys.contains(city)){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||
if(districts){
|
||||
def arrs = str.split("\\|")
|
||||
def url = "https://"+arrs[0]+".58.com/xiaoqu/"
|
||||
distMap.put(city, districts)
|
||||
|
||||
addCrawlKeys(city);
|
||||
addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -230,6 +231,10 @@ class Esf58Crawl extends BaseBatch{
|
||||
for(def i=0;i<districts.size();i++){
|
||||
def districtName = districts[i].name
|
||||
println "${city}【${i}/${districts.size()}】 "+districtName
|
||||
def total = 0
|
||||
if(districtName.length()>5){
|
||||
districtName = districtName.substring(0,3)
|
||||
}
|
||||
for(def j=1;j<=districtName.length()-1;j++) {
|
||||
// if(j<=3){
|
||||
def keyword = districtName.toString().substring(0, j);
|
||||
|
||||
+3
-3
@@ -15,10 +15,10 @@ class SynchDistrictPrice {
|
||||
def static final SHORT_DATE = Utils.getLastMonth()
|
||||
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
"anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
|
||||
// def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
|
||||
//"fang.com":"ËÑ·¿",
|
||||
// def sourceMap = ["fang.com":"搜房"]
|
||||
def sourceMap = ["ke.com":"贝壳找房","fang.com":"搜房","cityhouse.cn":"城市房产"]
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
task.run()
|
||||
|
||||
Reference in New Issue
Block a user