Merge branch 'script_gj' into 'master'

Script gj

See merge request !285
This commit is contained in:
2022-04-18 14:10:38 +08:00
17 changed files with 1566 additions and 174 deletions
+27
View File
@@ -494,5 +494,32 @@
<artifactId>msgpack</artifactId>
<version>0.6.12</version>
</dependency>
<dependency>
<groupId>com.asprise.ocr</groupId>
<artifactId>java-ocr-api</artifactId>
<version>[15,)</version>
</dependency>
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
<version>4.1.0</version>
</dependency>
<dependency>
<groupId>net.sourceforge.tess4j</groupId>
<artifactId>tess4j</artifactId>
<version>3.4.0</version>
<exclusions>
<exclusion>
<groupId>com.sun.jna</groupId>
<artifactId>jna</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</project>
@@ -508,6 +508,7 @@ class BigdataDealBatch extends BaseBatch{
def q = new BasicDBObject()
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate));
q.put("area", new BasicDBObject('$gte', 20))
q.put("status", 0)
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
def rets = []
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220218, 20220319);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220318, 20220419);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2022-03-1", "2022-04-1")
def prices = CcbCommonUtils.findPersonPrice(city, "2022-04-1", "2022-05-1")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202202.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202203.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -13,7 +13,7 @@ import org.apache.commons.lang.StringUtils
class OpenApiUtils {
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){
address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","")
address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","").replace("%","").replace("\"","")
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
if(preciseLevelflag){
url = url +"&preciseLevel=strict"
@@ -88,9 +88,10 @@ def getDetailLinks(driver, cityList){
driver.findElement(By.className("next")).click()
}
}
println city +" "+urls.size()
def rets = batchParse(urls);
saveToDB(rets);
println city +" "+urls.size()
}
}
println cityMap.provice +" 完成"
@@ -123,7 +124,7 @@ def getCity(driver, proviceMap){
for(def c : cityEles){
def cityid = c.attr("data-cityid")
def city = c.text()
if(!["石家庄市","唐山市","秦皇岛市","邯郸市","邢台市","保定市"].contains(city)){
if(!["石家庄市","太原市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){
cityMap.put(city, cityid)
}
@@ -170,8 +171,7 @@ def batchParse(def urls){
WebDriver driver = new ChromeDriver()
def ret = []
for(def map :urls){
driver.get(map.url);
def doc = Jsoup.parse(driver.getPageSource());
def doc = getDocument(map.url, driver)
def res = parse(map.province, map.city, map.url, doc);
if(res){
ret << res
@@ -180,6 +180,30 @@ def batchParse(def urls){
return ret;
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc){
stop = true;
}else{
error++
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=10){
stop = true;
}
}
return doc
}
def parse(province, city, url,doc){
if(doc){
def result = [:]
@@ -205,7 +229,12 @@ def parse(province, city, url,doc){
def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc)
if (finishDateStr) {
result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm");
try{
result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm");
}catch(Exception e){
}
}
def curPriceStr = JQueryUtils.selectText([".colorred"], doc)
@@ -255,7 +284,12 @@ def parse(province, city, url,doc){
startDateStr = startDateStr.toString().substring(startDateStr.indexOf("定于")+2, startDateStr.indexOf("日")+1).trim()
}
result.startDate = com.fangjia.script.util.DateUtils.parseString2Date(startDateStr)
try{
result.startDate = com.fangjia.script.util.DateUtils.parseString2Date(startDateStr.trim())
}catch(Exception e){
}
def imgs = JQueryUtils.select(".sf-pic-slide img", doc)
if (imgs) {
def imgLinks = []
@@ -315,7 +349,7 @@ def saveToDB(rets){
ret.each{k,v->
obj.put(k,v)
}
obj.source= "中拍网"
if(dbo){
obj.m_date = new Date()
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
@@ -346,15 +380,18 @@ def getAreaNumber(input, maxArea) {
val = val.toString().substring(val.toString().indexOf("室")+1, val.toString().size())
}
def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", "");
if (maxArea) {
if (v.toDouble() < maxArea) {
area = v;
if(v && v.toString().isNumber()){
if (maxArea) {
if (v.toDouble() < maxArea) {
area = v;
break;
}
} else {
area = v
break;
}
} else {
area = v
break;
}
}
if (area) {
@@ -0,0 +1,290 @@
package com.fangjia.script.gj.paimai
import org.apache.commons.codec.binary.Base64
import org.apache.commons.lang.StringUtils
import sun.font.Font2D
import sun.font.Font2DHandle
import sun.font.TrueTypeFont
import sun.font.TrueTypeGlyphMapper
import java.awt.Font
import java.awt.font.FontRenderContext
import java.awt.font.GlyphVector
import java.awt.geom.AffineTransform
import java.lang.reflect.Field
import java.util.regex.Matcher
import java.util.regex.Pattern
class DataUtil {
static def getAreaNumber(def input, def maxArea) {
Pattern pattern = Pattern.compile('\\d+(.\\d+)?(平方米|㎡|m2|方米|平)')
Matcher matcher = pattern.matcher(input)
List<String> list = new ArrayList()
while (matcher.find()) {
list.add(matcher.group())
}
if (list.size() > 0) {
def area = null
for (def val : list) {
def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", "")
if (maxArea) {
if (v.isDouble()) {
if (v.toDouble() < maxArea) {
area = v
break
}
}
} else {
area = v
break
}
}
if (area) {
return area
}
}
return null
}
static def getPaiMaiType(def s) {
if (s) {
if (s.toString().contains("结束")) {
return "已完成"
} else if (s.toString().contains("进行") || s.toString().contains("正在拍卖")) {
return "进行中"
} else if (s.toString().contains("即将开始")) {
return "即将开始"
} else if (s.toString().contains("流拍")) {
return "流拍"
} else if (s.toString().contains("撤回")) {
return "撤回"
} else if (s.toString().contains("暂缓")) {
return "暂缓"
} else if (s.toString().contains("中止")) {
return "中止"
} else if (s.toString().contains("失败")) {
return "失败"
}
}
return s
}
static def getDateRange(def s) {
s = s.replace("起", "")
Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}[月|日]\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)')
Matcher matcher = pattern.matcher(s)
List<String> list = new ArrayList()
while (matcher.find()) {
def str = matcher.group()
list.add(str)
// if(str) {
// def indx1 = str.indexOf("年")
// def indx2 = str.indexOf("月")
// def month = str.substring(indx1+1, indx2);
// if(month.toInteger()<12) {
// list.add(str);
// }
// }
}
if (list.size() >= 1) {
String s1 = list.get(0)
s1 = s1.replace("法院将于", "").replace(" ", "")
def array = s1.split("至")
return array
}
return null
}
static def getStartPrice(def s) {
Pattern pattern = Pattern.compile('起拍价:\\d+(.\\d+)?万?元')
Matcher matcher = pattern.matcher(s)
List<String> list = new ArrayList()
while (matcher.find()) {
list.add(matcher.group())
}
if (list.size() >= 1) {
String s1 = list.get(0)
if (s1.contains("万元")) {
s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "")
return (int) (s1.toDouble() * 10000)
} else {
s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "")
if (s1.isInteger()) {
return s1.toInteger()
} else {
return (int) (s1.toDouble())
}
}
}
return null
}
static def getEstateType(def s) {
Pattern pattern = Pattern.compile('房屋类型:.*;')
Matcher matcher = pattern.matcher(s)
List<String> list = new ArrayList()
while (matcher.find()) {
list.add(matcher.group())
}
if (list.size() >= 1) {
String s1 = list.get(0)
if (s1) {
s1 = s1.substring(s1.indexOf(":") + 1, s1.indexOf(";"))
return s1.replace(")", "").replace("】", "")
}
}
return null
}
static def findEstateType(def s) {
if (s) {
def villTypes = ["独栋", "联排", "联列", "叠加", "双拼", "花园面积", "泳池", "别墅", "花园住宅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]
for (def type : villTypes) {
if (s.contains(type)) {
return "别墅"
}
}
def districtTypes = ["住宅", "公寓"]
for (def type : districtTypes) {
if (s.contains(type)) {
return "普通住宅"
}
}
}
return null
}
static def filterChar(def detailMap) {
def newDetailMap = [:]
detailMap.each { def k, v ->
if (k.toString().contains(".")) {
newDetailMap.put(k.toString().replace(".", "\\u002e"), v)
} else {
newDetailMap.put(k, v)
}
}
return newDetailMap
}
/**
* 匹配符合条件的字符串
*
* @param str
* @param pattern
* @return
*/
static String match(String str, Pattern pattern) {
if (str == null) {
return null
}
Matcher matcher = pattern.matcher(str)
if (matcher.find()) {
return matcher.group()
}
return null
}
/**
* font-secret字符串专用解密工具
*
* @param key 密匙
* @param encodeString 加密后的字符串
* @return 解密后的字符串
*/
static String decodeString(String key, String encodeString) {
if (!key) {
return ""
}
try {
//base64解码,初始化字体
Base64 base64 = new Base64()
byte[] ss = base64.decodeBase64(key)
InputStream inputStream = new ByteArrayInputStream(ss)
Font dynamicFont = Font.createFont(Font.TRUETYPE_FONT, inputStream)
FontRenderContext fontRenderContext = new FontRenderContext(new AffineTransform(), false, false)
GlyphVector glyphVector = dynamicFont.createGlyphVector(fontRenderContext, "")
//获取font中字形的映射关系,字段为private,使用反射
Class<?> clazz = Font.class
Field[] fs = clazz.getDeclaredFields()
Font2DHandle font2DHandle = null
for (int i = 0; i < fs.length; i++) {
fs[i].setAccessible(true)// 将目标属性设置为可以访问
if (fs[i].getName().equals("font2DHandle")) {
font2DHandle = (Font2DHandle) fs[i].get(dynamicFont)
}
}
//得到映射关系
Font2D font2D = font2DHandle.font2D
TrueTypeFont trueTypeFont = (TrueTypeFont) font2D
TrueTypeGlyphMapper charToGlyphMapper = (TrueTypeGlyphMapper) trueTypeFont.getMapper()
//开始解密,encodeString为加密后的字符串
StringBuffer buffer = new StringBuffer()
char[] chars = encodeString.toCharArray()
for (int i = 0; i < chars.length; i++) {
buffer.append(charToGlyphMapper.charToGlyph(chars[i]) - 1)
}
return buffer.toString()
} catch (Exception e) {
e.printStackTrace()
}
return ""
}
static List SITE_SUFFIXS = [".gov.cn",".com.cn", ".com", ".cn", ".net", ".org", ".cc", ".tm", ".biz", ".info", ".tv"]
static String urlToSiteName(String url) {
if (StringUtils.isEmpty(url)) {
return null
}
for (String suffix : SITE_SUFFIXS) {
if (url.indexOf(suffix) > -1) {
String s = StringUtils.substringBefore(url, suffix)
if (s.indexOf(".") > -1) {
s = StringUtils.substringAfterLast(s, ".")
if (StringUtils.isNotBlank(s)) {
return s + suffix
}
} else if (s.indexOf("://") > -1) {
s = StringUtils.substringAfterLast(s, "://")
if (StringUtils.isNotBlank(s)) {
return s + suffix
}
} else if (s.indexOf("://") == -1 && s.indexOf(".") == -1) {
return s + suffix
}
return null
}
}
return null
}
static double getRound(int scale, double dSource) {
BigDecimal deSource = new BigDecimal(dSource)
return deSource.setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue()
}
static def filterEmpty(def s){
if(s && s.contains("暂无")){
return null
}
return s
}
}
@@ -0,0 +1,528 @@
package com.fangjia.script.gj.paimai
import com.fangjia.core.RegionData
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import net.sf.json.JSONObject
import org.apache.poi.ss.util.RegionUtil
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
import javax.sound.midi.MidiSystem
import java.util.regex.Pattern
class JdPaimaiCrawl {
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
static main(args) {
def task = new JdPaimaiCrawl();
task.run();
}
def run(){
def existsIds = findAllIds()
println "已存在:"+existsIds.size()
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def links = getList(driver)
def ids = getDetailLinks(links, driver)
println "共:"+ids.size()
parserList( existsIds,ids, driver)
}
def getList(driver){
def list = []
for(def i=1;i<=32;i++){
if(i==8){
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268"
}else if(i==5){
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268"
}else if(i==17){
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268"
}else if(i==19){
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268"
}else{
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268"
}
}
def links = []
for(def url : list){
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
//def document = ABuYunUtil.doPost(url, null, null, "utf-8", true)
if(doc){
def json = doc.toString().substring(doc.toString().indexOf("(")+1, doc.toString().lastIndexOf(")"))
JSONObject obj = JSONObject.fromObject(json);
if(obj && obj.total){
def total = obj.total
def page = total/40
def p = page.toInteger()
if(page > p){
p = p +1
}
for(def i=1;i<=p;i++){
links << url+"&page="+i
}
// if(links){
// break;
// }
}
}
}
return links;
}
def getDetailLinks(links, driver){
def rets = []
def citys = ["北京","上海","天津","重庆"]
for(def link : links){
driver.get(link)
def doc = Jsoup.parse(driver.getPageSource());
if(doc) {
def json = doc.toString().substring(doc.toString().indexOf("(") + 1, doc.toString().lastIndexOf(")"))
JSONObject obj = JSONObject.fromObject(json);
if (obj && obj.ls) {
for(def o : obj.ls){
def map = [:]
if(citys.contains(o.province)){
map.province = o.province
map.city = o.province
map.region = o.city
}else{
map.province = o.province
map.city = o.city
}
map.id = o.id.toString()
rets << map
}
}
}
}
return rets
}
def parserList(existsIds,list, driver){
def rets = []
for(def map : list) {
if(!existsIds.contains(map.id)){
def url = "https://paimai.jd.com/" + map.id
def doc = getDocument(url, driver)
if (doc) {
def result = parsering(doc, url, map.province, map.city, map.region)
if (result) {
rets << result
}
if (rets.size() % 100 == 0) {
def add= saveData(rets)
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+" add:"+add
rets = []
}
}
}
}
if(rets.size()>0){
saveData(rets)
print "完"
}
}
def saveData(def results){
def add = 0
for(def ret : results){
def q = new BasicDBObject()
q.put("url", ret.url)
def dbo = paimaiResultColl.findOne(q);
DBObject obj = new BasicDBObject();
ret.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.m_date = new Date()
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
}else{
add++
obj.c_date = new Date()
obj.m_date = new Date()
try{
paimaiResultColl.save(obj)
}catch(Exception e){
println obj.url
}
}
}
return add
}
def parsering(doc, def url, def province, def city, def region){
def result = [:]
if(doc) {
result.proince = province
result.city = city
if(region){
result.region = region
}
def title = JQueryUtils.selectText([".pm-name"], doc)
result.title = title
result.address = JQueryUtils.selectText([".pm-location em"], doc)
if(result.address) {
//调用获取城市
// if(!city) {
// city = RegionData.formatCity(cityMap.name)
// }
if(city) {
def rMap = OpenApiUtils.matchDistrict(city, result.title, null, true)
if (rMap) {
result.matchCity = rMap.city
result.matchRegion = rMap.region
result.matchName = rMap.name
result.matchAddress = rMap.address
result.houseNum = rMap.house_num
result.roomNum = rMap.room_num
}
}
}
result.status = JQueryUtils.selectText([".pm-summary .state"], doc)
def enrollCount = JQueryUtils.selectText([".pm-summary .times span:eq(0) em"], doc)
if(enrollCount) {
result.enrollCount = enrollCount.toInteger()
}
def lookCount = JQueryUtils.selectText([".pm-summary .times span:eq(1) em"], doc)
if(lookCount) {
result.lookCount = lookCount.toInteger()
}
def callCount = JQueryUtils.selectText([".pm-summary .times span:eq(2) em"], doc)
if(callCount) {
result.callCount = callCount.toInteger()
}
def curPriceStr = JQueryUtils.selectText([".mc .current em"], doc)
if(curPriceStr) {
curPriceStr = curPriceStr.replace(",", "")
if(curPriceStr.isInteger()) {
result.curPrice = curPriceStr.toInteger()
}else {
result.curPrice = (int)(curPriceStr.toDouble())
}
}
def startPriceStr = JQueryUtils.selectText([".mc .base em"], doc)
if(startPriceStr) {
startPriceStr = startPriceStr.replace(",", "")
if(startPriceStr.isInteger()) {
result.startPrice = startPriceStr.toInteger()
}else {
result.startPrice = (int)(startPriceStr.toDouble())
}
}
def tradePriceStr = JQueryUtils.selectText([".mc .deal em"], doc)
if(tradePriceStr) {
tradePriceStr = tradePriceStr.replace(",", "")
if(tradePriceStr.isInteger()) {
result.tradePrice = tradePriceStr.toInteger()
}else {
result.tradePrice = (int)(tradePriceStr.toDouble())
}
}
def finishDateStr = JQueryUtils.selectText([".endtime"], doc)
if(finishDateStr && finishDateStr.contains("结束时间")) {
finishDateStr = finishDateStr.substring(5, finishDateStr.indexOf(" "))
result.finishDate = DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm:ss")
}
def lis = JQueryUtils.select(".description ul li", doc)
if(lis && lis.size()) {
for(def li : lis) {
def str = li.select("span").text()
if(str.contains("起拍")) {
def v= li.select("em").text().replace("¥", "").replace(",", "")
if(v) {
if(v.isInteger()) {
result.startPrice = v.toDouble().toInteger()
}else {
result.startPrice = (int)(v.toDouble())
}
}
}else if(str.contains("评估价")) {
def v = li.select("em").text().replace("¥", "").replace(",", "")
if(v.isInteger()) {
result.evaluationPrice = v.toDouble().toInteger()
}else {
result.evaluationPrice = (int)(v.toDouble())
}
}else if(str.contains("加价幅度")) {
def v = li.select("em").text().replace("¥", "").replace(",", "")
if(v) {
result.addPrice = v.toDouble().toInteger()
}
}else if(str.contains("保证金")) {
def v = li.select("em").text().replace("¥", "").replace(",", "")
if(v) {
if(v.isInteger()) {
result.depositPrice = v.toDouble().toInteger()
}else {
result.depositPrice = (int)(v.toDouble())
}
}
}else if(str.contains("延时周期")) {
result.delayWeekly = li.select("em").text().replace("¥", "").replace(",", "")
}else if(str.contains("优先竞买权人")) {
result.preemptivePerson = li.select("em").text().replace("¥", "").replace(",", "")
}
}
}
def trs = JQueryUtils.select([".floor-content div:eq(2) table tr"], doc)
if(trs) {
def detailMap = [:]
for(def tr : trs) {
def d_tds = tr.select("td")
if(d_tds && d_tds.size()>=2) {
if(d_tds.size()==3) {
detailMap.put(d_tds[1].text(), d_tds[2].text())
}else {
detailMap.put(d_tds[0].text(), d_tds[1].text())
}
}else {
if(d_tds[0] && d_tds[0].text().contains("拍品介绍")) {
detailMap.put("拍品介绍", d_tds[0].text().replace("拍品介绍 ", ""))
}
}
}
if(detailMap) {
result.detailMap = DataUtil.filterChar(detailMap)
}
if(!result.address) {
result.address = detailMap.get("拍品名称")
if(!result.address) {
result.address = detailMap.get("标的名称")
}
}
}
def imgs =JQueryUtils.select(".floor-content div:eq(2) a", doc)
if(imgs) {
def imgLinks = []
for(def img : imgs) {
def href = img.attr("href")
if(href.contains("http")) {
imgLinks << href
}else {
imgLinks << url.substring(0, url.indexOf(":")+1)+href
}
}
result.imgUrls = imgLinks
}
}
//解析面积
def article = JQueryUtils.selectText([".floor-content"], doc)
if(article) {
def area = DataUtil.getAreaNumber(article, 5000)
if(!area) {
article = JQueryUtils.selectText(["body"], doc)
if(article) {
area = DataUtil.getAreaNumber(article, 5000)
}
}
if(area) {
result.area = area.toDouble()
if(result.tradePrice) {
result.tradeAvgPrice = (int)(result.tradePrice/area.toDouble())
}
}
}
def s1 = JQueryUtils.selectText(["li:contains(竞买公告) .floor-content"], doc)
if(s1) {
def start = s1.indexOf("法院将于")
def end = s1.indexOf("止")
if(start>-1 && end>-1 && end>start){
s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止"))
}
def dateRangs = DataUtil.getDateRange(s1)
if(dateRangs && dateRangs.size()==2) {
for(def j=0;j<dateRangs.size();j++) {
def date = dateRangs[j]
def pattern = null
if(date.endsWith("日时")||date.endsWith("日")) {
pattern = "yyyy年MM月dd日"
}else {
def ind = date.indexOf("日")
if(date.indexOf("时")==-1 && date.toString().length()>ind+1) {
date = date +"时"
}
pattern = "yyyy年MM月dd日HH时"
}
if(j==0) {
try{
result.startDate = DateUtils.parseFromTightByPattern(date, pattern)
}catch(Exception e){
}
}else {
if(!result.finishDate) {
result.finishDate = DateUtils.parseFromTightByPattern(date, pattern)
}
}
}
}
if(!result.startPrice) {
result.startPrice = DataUtil.getStartPrice(s1)
}
def estateType = DataUtil.findEstateType(s1)
if(estateType) {
result.matchEstateType = estateType
}
}
result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
result.introduce = JQueryUtils.select([".floor-content div:eq(2) table"], doc).toString()
result.url = url
result.city = city
result.mDate = new Date()
result.source = "京东"
return result;
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc){
stop = true;
}else{
error++
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=10){
stop = true;
}
}
return doc
}
def findAllIds(){
def q = new BasicDBObject()
q.put("url", Pattern.compile(".*jd.com.*"))
def dbcursor = paimaiResultColl.find(q)
def ids = []
for(def d : dbcursor){
def url = d.url
if(url){
def id = url.substring(url.toString().lastIndexOf("/")+1, url.toString().length())
ids << id
}
}
return ids;
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.paimai
import java.text.SimpleDateFormat
import java.util.regex.Matcher
import java.util.regex.Pattern
import java.util.zip.GZIPInputStream
@@ -94,32 +95,32 @@ class PaimaiDataUtil {
}
public static def getDateRange(def s) {
s = s.replace("起","")
Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)');
Matcher matcher = pattern.matcher(s);
List<String> list = new ArrayList();
while (matcher.find()) {
def str = matcher.group()
list.add(str);
// if(str) {
// def indx1 = str.indexOf("年")
// def indx2 = str.indexOf("月")
// def month = str.substring(indx1+1, indx2);
// if(month.toInteger()<12) {
// list.add(str);
// }
// }
}
if(list.size()>=1) {
String s1 = list.get(0)
s1 = s1.replace("法院将于", "").replace(" ","")
def array = s1.split("至");
return array
}
return null;
}
// public static def getDateRange(def s) {
// s = s.replace("起","")
// Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)');
// Matcher matcher = pattern.matcher(s);
//
// List<String> list = new ArrayList();
// while (matcher.find()) {
// def str = matcher.group()
// list.add(str);
//// if(str) {
//// def indx1 = str.indexOf("年")
//// def indx2 = str.indexOf("月")
//// def month = str.substring(indx1+1, indx2);
//// if(month.toInteger()<12) {
//// list.add(str);
//// }
//// }
// }
// if(list.size()>=1) {
// String s1 = list.get(0)
// s1 = s1.replace("法院将于", "").replace(" ","")
// def array = s1.split("至");
// return array
// }
// return null;
// }
public static def getStartPrice(def s) {
@@ -159,4 +160,30 @@ class PaimaiDataUtil {
return newDetailMap;
}
public static def getDateRange(def s) {
s = s.replace("起","")
Pattern pattern = Pattern.compile('法院将于(\\d+年\\d+月\\d+日\\d+时)至(\\d+年\\d+月\\d+日\\d+时)');
Matcher matcher = pattern.matcher(s);
List<String> list = new ArrayList();
while (matcher.find()) {
list.add(matcher.group());
}
if(list.size()>=1) {
String s1 = list.get(0)
s1 = s1.replace("法院将于", "")
def array = s1.split("至");
return array
}
return null;
}
public static Date parseFromTightByPattern(String s, String pattern){
if (s == null || s.length() == 0){
return null
}
SimpleDateFormat format = new SimpleDateFormat(pattern);
return format.parse(s)
}
}
@@ -83,15 +83,19 @@ class PaimaiTest {
driver.get("https:"+data.href)
def doc = Jsoup.parse(driver.getPageSource());
if(citys.contains(data.proince)){
def cityMap = [:]
cityMap.proince = data.proince
cityMap.city = data.proince
cityMap.url = data.href
cityList << cityMap
def links = JQueryUtils.select(".condition .s-subtab a", doc)
for(def ele : links) {
def cityMap = [:]
cityMap.proince = data.proince
cityMap.city = data.proince
cityMap.region =ele.text()
cityMap.url = data.href
cityList << cityMap
}
}else{
def links = JQueryUtils.select(".condition .s-subtab a", doc)
for(def ele : links){
if(!["合肥市","芜湖市","蚌埠市","淮南市","马鞍山市","淮北市","铜陵市"].contains(ele.text())){
if(!["乌鲁木齐市","克拉玛依市","合肥市","芜湖市","蚌埠市","淮南市","马鞍山市","淮北市","铜陵市"].contains(ele.text())){
def cityMap = [:]
cityMap.proince = data.proince
cityMap.city = ele.text()
@@ -109,7 +113,7 @@ class PaimaiTest {
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def datas = []
def keys = ["上海","天津","重庆","北京","浙江","河北","山西","辽宁","吉林","江苏","福建","江西","山东","河南","湖北","湖南","广东"]
def keys = [ "天津","上海","重庆","北京","浙江","河北","陕西","山西","新疆","辽宁","吉林","江苏","福建","江西","山东","河南","湖北","湖南","西藏","宁夏","甘肃","广东","安徽","广西","海南","四川","贵州","云南"]
if(doc){
def eles = JQueryUtils.select(".condition a", doc)
for(def e : eles){
@@ -123,11 +127,34 @@ class PaimaiTest {
}
return datas
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc){
stop = true;
}else{
error++
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=10){
stop = true;
}
}
return doc
}
def parser(detailDatas, driver, proince, city){
def results = []
for(def m : detailDatas){
try{
// try{
driver.get("https:"+m.url)
Thread.sleep(3000)
def doc = Jsoup.parse(driver.getPageSource());
@@ -137,9 +164,9 @@ class PaimaiTest {
obj.city = city
results << obj
}
}catch(Exception e){
println m.url+" 解析有问题!"
}
// }catch(Exception e){
// println m.url+" 解析有问题!"
// }
}
return results
@@ -410,7 +437,7 @@ class PaimaiTest {
}
}
// result.content = DataUtil.zip(s)
result.content = DataUtil.zip(doc.toString())
def el = JQueryUtils.select([".d-block table:eq(1)"], doc)
if(!el) {
@@ -549,7 +576,7 @@ class PaimaiTest {
ret.each{k,v->
obj.put(k,v)
}
obj.source = "公拍网"
if(dbo){
obj.m_date = new Date()
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
@@ -1,12 +1,4 @@
package com.fangjia.script.gj.paimai
import com.fangjia.core.util.DataUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.gj.paimai.PaimaiDataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
@@ -14,66 +6,266 @@ import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.JavascriptExecutor
import org.openqa.selenium.WebDriver
import org.openqa.selenium.WebElement
import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.support.ui.WebDriverWait
import org.openqa.selenium.support.ui.ExpectedCondition
import org.openqa.selenium.support.ui.ExpectedConditions
import org.openqa.selenium.support.ui.WebDriverWait
import com.fangjia.script.gj.OpenApiUtils
import java.util.concurrent.TimeUnit
import java.util.regex.Matcher
import java.util.regex.Pattern
import com.fangjia.core.util.DataUtil
import com.fangjia.script.crawl.JQueryUtils
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
def url = "https://www1.rmfysszc.gov.cn/projects.shtml?dh=3&gpstate=1&wsbm_slt=1"
WebDriver driver = new ChromeDriver()
driver.get(url);
driver.findElementById("1").click()
run(url, driver)
//driver.get(url);
//driver.findElementById("1").click()
//driver.findElementById("search_sub").click()
//
//def provinces = getAllProvinceValue(driver.getPageSource())
//for(def m : provinces){
// def city_setting = driver.findElementById("city")
// //${m.province_id}
// city_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click()
//
////driver.manage().timeouts().implicitlyWait(30, TimeUnit.SECONDS);
////WebDriverWait wait = new WebDriverWait(driver,20)
////wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.cssSelector("pagecur1"))))
// Thread.sleep(10000)
// def doc = Jsoup.parse(driver.getPageSource());
// def citys = getAllCity(doc)
// getDetailByCity(m.province, citys, driver)
//}
def city_setting = driver.findElementById("city")
city_setting.findElement(By.xpath("//option[@value='31']")).click()
driver.findElementById("search_sub").click()
//driver.manage().timeouts().implicitlyWait(30, TimeUnit.SECONDS);
//WebDriverWait wait = new WebDriverWait(driver,20)
//wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.cssSelector("pagecur1"))))
Thread.sleep(10000)
def doc = Jsoup.parse(driver.getPageSource());
def links = getLinks(doc)
def run(url, driver){
def values = getAllCityRegionValue(url, driver)
def finishCity = ["北京市","天津市","张家口市","邯郸市","保定市","邢台市","石家庄市","唐山市","秦皇岛市"]
for(def m : values){
if(!finishCity.contains(m.city)){
driver.get(url);
driver.findElementById("1").click()
Thread.sleep(3000)
def pro_setting = driver.findElementById("city")
pro_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click()
Thread.sleep(3000)
def city_setting = driver.findElementById("city1")
city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click()
Thread.sleep(4000)
def region_setting = driver.findElementById("city2")
region_setting.findElement(By.xpath("//option[@value='${m.region_id}']")).click()
driver.findElementById("search_sub").click()
Thread.sleep(10000)
getDetailByPage(m.province, m.city, m.region, driver)
}
def eles = driver.findElementsByClassName("pagecur1")
for(def ele : eles){
ele.click();
Thread.sleep(10000)
doc = Jsoup.parse(driver.getPageSource());
links.addAll(getLinks(doc))
}
def rets = []
for(def durl : links){
driver.get(durl);
doc = Jsoup.parse(driver.getPageSource());
def res = parse(durl, doc);
if(res){
rets << res
}
}
saveToDB(rets);
def getAllCityRegionValue(def url, def driver){
driver.get(url);
driver.findElementById("1").click()
driver.findElementById("search_sub").click()
def provinces = getAllProvinceValue(driver.getPageSource())
def values = []
for(def m : provinces){
def city_setting = driver.findElementById("city")
city_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click()
Thread.sleep(10000)
def doc = Jsoup.parse(driver.getPageSource());
def citys = getAllCity(doc)
def cityRegions = getAllCityByRegion(m, citys, driver)
values.addAll(cityRegions)
//getDetailByCity(m.province, citys, driver)
}
return values
}
def getDetailByCity(province, citys, driver){
for(def m : citys){
def city_setting = driver.findElementById("city1")
city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click()
Thread.sleep(10000)
def doc = Jsoup.parse(driver.getPageSource());
def regions = getAllRegion(doc)
getDetailByRegion(province, m.city, regions, driver)
}
}
def getAllCityByRegion(provinceMap, citys, driver){
def datas = []
for(def m : citys){
def city_setting = driver.findElementById("city1")
city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click()
Thread.sleep(10000)
def doc = Jsoup.parse(driver.getPageSource());
def regions = getAllRegion(provinceMap, m, doc)
datas.addAll(regions)
}
return datas
}
def getDetailByRegion(province, city, regions, driver){
def links = []
for(def r : regions){
def city_setting = driver.findElementById("city2")
city_setting.findElement(By.xpath("//option[@value='${r.region_id}']")).click()
driver.findElementById("search_sub").click()
Thread.sleep(10000)
def stop = false;
while(!stop){
def doc = Jsoup.parse(driver.getPageSource());
links.addAll(getLinks(r.region, doc))
def eles = JQueryUtils.select("a:contains(下一页)", doc)
if(eles){
driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click()
Thread.sleep(3000)
}else{
stop = true;
}
}
}
def datas = parseDatas(province, city, links, driver)
saveToDB(datas);
println r.region +" "+ links.size()
}
def getDetailByPage(province, city, region, driver){
def links = []
def stop = false;
while(!stop){
def doc = Jsoup.parse(driver.getPageSource());
links.addAll(getLinks(doc))
def eles = JQueryUtils.select("a:contains(下一页)", doc)
if(eles){
driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click()
Thread.sleep(3000)
}else{
stop = true;
}
}
def datas = parseDatas(province, city, region, links, driver)
println province +" "+city+" "+region +" "+ links.size()
saveToDB(datas);
}
def getAllProvinceValue(doc){
def optionsEls = JQueryUtils.select("#city option", doc)
def rets = []
for(def eles : optionsEls){
def val = eles.attr("value")
if(val!="" && val!="0"){
def province = eles.text()
rets << ["province":province, "province_id":val]
def parse(url, doc){
}
}
return rets
}
def getAllCity(doc){
def optionsEls = JQueryUtils.select("#city1 option", doc)
def rets = []
for(def eles : optionsEls){
def val = eles.attr("value")
if(val!=""){
def province = eles.text()
rets << ["city":province, "city_id":val]
}
}
return rets
}
def getAllRegion(provinceMap, cityMap,doc){
def optionsEls = JQueryUtils.select("#city2 option", doc)
def rets = []
for(def eles : optionsEls){
def val = eles.attr("value")
if(val!=""){
def region = eles.text()
def regionMap = [:]
regionMap.province = provinceMap.province
regionMap.province_id = provinceMap.province_id
regionMap.city = cityMap.city
regionMap.city_id = cityMap.city_id
regionMap.region = region
regionMap.region_id = val
rets << regionMap
}
}
return rets
}
def parseDatas(province, city, region,links, driver){
def datas = []
for(def url: links){
def doc = getDocument(url, driver)
def result = parse(city, region, url, doc)
if(result){
result.province = province
result.city = city
result.region = region
datas << result
}
}
return datas
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc){
stop = true;
}else{
error++
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=10){
stop = true;
}
}
return doc
}
def parse(city, region, url, doc){
def result=[:]
if(doc) {
def title = JQueryUtils.selectText(["#Title"], doc)
result.title = title;
result.address = title
if(result.address){
def rMap = OpenApiUtils.matchDistrict("上海",result.address)
def rMap = OpenApiUtils.matchDistrict(city, result.address, null, true)
if(rMap) {
result.matchCity = rMap.city
result.matchRegion = rMap.region
@@ -99,6 +291,7 @@ def parse(url, doc){
def enrollCount = JQueryUtils.selectText([".bmnumber"], doc)
if(enrollCount) {
enrollCount = enrollCount.replace("人报名","")
if(enrollCount.isInteger()){
result.enrollCount = enrollCount.toInteger()
}
@@ -107,6 +300,7 @@ def parse(url, doc){
def callCount = JQueryUtils.selectText([".sc"], doc)
if(callCount) {
callCount =callCount.replace("人收藏","")
if(callCount.isInteger()){
result.callCount = callCount.toInteger()
}
@@ -114,6 +308,7 @@ def parse(url, doc){
def lookCount = JQueryUtils.selectText([".hits"], doc)
if(lookCount) {
lookCount = lookCount.replace("次围观","")
if(lookCount.isInteger()){
result.lookCount = lookCount.toInteger()
}
@@ -123,8 +318,8 @@ def parse(url, doc){
def curPriceStr = JQueryUtils.selectText(["#price div:eq(0)"], doc)
if(curPriceStr && curPriceStr.contains("起拍价")) {
curPriceStr = curPriceStr.replace("起拍价", "").replace("¥","")
if(curPriceStr.contains("万元")){
curPriceStr = curPriceStr.replace("万元","")
if(curPriceStr.contains("万")){
curPriceStr = curPriceStr.replace("万元","").replace("万","")
if(curPriceStr.isInteger()){
result.startPrice = curPriceStr.toInteger()*10000
}else{
@@ -144,6 +339,44 @@ def parse(url, doc){
}
}
def tableEles = JQueryUtils.select("#bg1", doc)
if(tableEles){
def evaluatePriceStr = tableEles.select("td:contains(值:)").text()
if(evaluatePriceStr && evaluatePriceStr.contains(":")){
def arrs = evaluatePriceStr.split(":")
if(arrs.size()==2){
evaluatePriceStr = arrs[1]
}
}
def depositPriceStr = tableEles.select("td:contains(金:)").text()
if(depositPriceStr && depositPriceStr.contains(":")){
def arrs = depositPriceStr.split(":")
if(arrs.size()==2){
depositPriceStr = arrs[1]
}
}
if(depositPriceStr){
depositPriceStr = depositPriceStr.replace("万元","")
if(depositPriceStr.toString().isNumber()){
result.depositPrice = depositPriceStr.toDouble()*10000
}
}
def courtStr = tableEles.select("td:contains(院:)").text()
if(courtStr && courtStr.contains(":")){
result.court = courtStr.substring(courtStr.indexOf(":")+1, courtStr.size())
}
def personStr = tableEles.select("td:contains(人:)").text()
if(personStr && personStr.contains(":")){
result.person = personStr.substring(personStr.indexOf(":")+1, personStr.size())
}
def countStr = tableEles.select("td:contains(拍卖阶段:)").text().replace("拍卖阶段:","")
if(countStr && countStr.contains("第") && countStr.contains("次")){
result.count = countStr.substring(countStr.indexOf("第")+1, countStr.indexOf("次"))
}
}
def lis = JQueryUtils.select("#bg1 div:eq(0) table tr", doc)
if(lis && lis.size()) {
@@ -151,18 +384,23 @@ def parse(url, doc){
def str = li.select("span").text()
if(str.contains("评估值")) {
def v = li.select("span:eq(1)").text().replace("万元", "")
if(v.isInteger()) {
result.evaluationPrice = v.toInteger()*10000
}else{
result.evaluationPrice = (int)(v.toDouble()*10000)
if(v.isNumber()){
if(v.isInteger()) {
result.evaluationPrice = v.toInteger()*10000
}else{
result.evaluationPrice = (int)(v.toDouble()*10000)
}
}
}else if(str.contains("保证金")) {
def v = li.select("span:eq(1)").text().replace("万元", "")
if(v) {
if(v.isInteger()) {
result.depositPrice = v.toInteger()*10000
}else {
result.depositPrice = (int)(v.toDouble()*10000)
if(v.isNumber()) {
if (v) {
if (v.isInteger()) {
result.depositPrice = v.toInteger() * 10000
} else {
result.depositPrice = (int) (v.toDouble() * 10000)
}
}
}
}else if (str.contains("公告日期:")){
@@ -177,13 +415,32 @@ def parse(url, doc){
def trs = tables[1].select("tr")
if(trs) {
def detailMap = [:]
for(def tr : trs) {
def d_tds = tr.select("td")
if(tr.text().contains("调查情况表")){
continue
}
if(d_tds.size()>1){
def lastEleStr = d_tds[d_tds.size()-1].toString()
if(lastEleStr.contains('height="42"')){
d_tds.remove(d_tds[d_tds.size()-1])
}
}
if(d_tds.size()>3){
continue
}
if(d_tds && d_tds.size()>=2) {
if(d_tds.size()==3) {
detailMap.put(d_tds[1].text(), d_tds[2].text())
if(d_tds[1].text() && d_tds[2].text()){
detailMap.put(d_tds[1].text(), d_tds[2].text())
}
}else {
detailMap.put(d_tds[0].text(), d_tds[1].text())
if(d_tds[0].text() && d_tds[1].text()){
detailMap.put(d_tds[0].text(), d_tds[1].text())
}
}
}else {
if(d_tds[0] && d_tds[0].text().contains("拍品介绍")) {
@@ -202,8 +459,6 @@ def parse(url, doc){
result.introduce = tables[1].toString()
}
def imgs =JQueryUtils.select("#bdjs11 img", doc)
if(imgs) {
def imgLinks = []
@@ -232,6 +487,20 @@ def parse(url, doc){
result.tradeAvgPrice = (int)(result.tradePrice/result.area)
}
def s1 = JQueryUtils.selectText(["#pmgg"], doc)
if(s1) {
def start = s1.indexOf("法院将于")
def end = s1.indexOf("止")
if(start>-1 && end>-1 && end>start){
s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止"))
}
def dateRangs = PaimaiDataUtil.getDateRange(s1)
if(dateRangs && dateRangs.size()==2) {
result.startDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[0], "yyyy年MM月dd日HH时")
result.finishDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[1], "yyyy年MM月dd日HH时")
}
}
result.content = DataUtil.zip(doc.toString())
result.url = url
@@ -246,7 +515,7 @@ def getLinks(doc){
if(eles){
for(def el :eles){
def href = el.attr("href")
urls << "https://www.rmfysszc.gov.cn/statichtml/rm_obj/"+href.substring(href.indexOf("/")+1,href.length())
urls <<"https://www.rmfysszc.gov.cn/statichtml/rm_obj/"+href.substring(href.indexOf("/")+1,href.length())
}
}
}
@@ -264,13 +533,19 @@ def saveToDB(rets){
ret.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.remove("url")
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
}else{
paimaiResultColl.save(obj)
}
obj.source= "人民法院诉讼资产网"
try{
if(dbo){
obj.m_date = new Date()
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
}else{
obj.c_date = new Date()
obj.m_date = new Date()
paimaiResultColl.save(obj)
}
}catch(Exception e){
println "error:"+obj.url
}
}
}
@@ -287,14 +562,16 @@ def getAreaNumber(input, maxArea) {
for (def val : list) {
def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", "");
if (maxArea) {
if (v.toDouble() < maxArea) {
area = v;
if(v && v.isNumber()){
if (maxArea) {
if (v.toDouble() < maxArea) {
area = v;
break;
}
} else {
area = v
break;
}
} else {
area = v
break;
}
}
@@ -0,0 +1,173 @@
package com.fangjia.script.loudong
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class CqfdcMatch {
static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018)
static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018)
static main(args) {
def task = new CqfdcMatch();
// task.run()
// task.stat()
task.export2()
}
def export(){
def q = new BasicDBObject()
q.put("match_city", new BasicDBObject('$exists', false))
def field = new BasicDBObject()
field.put("city",1)
field.put("region",1)
field.put("district_name",1)
field.put("address",1)
field.put("type",1)
def dbcursor = loudongConn.find(q, field).batchSize(5000)
def keys = []
def rets = []
rets << ["城市","区域","名称","地址","类型"]
for(def d : dbcursor){
def key = d.city +"##"+d.district_name
if(!keys.contains(key)){
keys << key
rets << [d.city, d.region, d.district_name, d.address, d.type]
}
}
ExcelUtils.write("D:\\数据\\重庆农商行\\重庆楼栋未匹配1.xlsx",rets, "数据")
}
def run(){
def datas= findAddress2()
def results = match(datas);
update(results)
}
def export2(){
def q = new BasicDBObject()
q.put("match_city", new BasicDBObject('$exists', false))
def field = new BasicDBObject()
field.put("city",1)
field.put("name",1)
field.put("use_type",1)
def dbcursor = loudongFdcConn.find(q, field).batchSize(5000)
def keys = []
def rets = []
rets << ["城市","地址","类型"]
for(def d : dbcursor){
def key = d.city +"##"+d.name
if(!keys.contains(key)){
keys << key
rets << [d.city, d.name,d.use_type]
}
}
ExcelUtils.write("D:\\数据\\重庆农商行\\重庆楼栋未匹配2.xlsx",rets, "数据")
}
def update(results){
for(def m : results){
def q = new BasicDBObject()
q.put("city", m.old_city)
q.put("district_name", m.old_name)
q.put("region",m.old_region)
def udata = new BasicDBObject()
udata.put("match_city", m.match_city)
udata.put("match_region", m.match_region)
udata.put("match_name", m.match_name)
loudongConn.update(q, new BasicDBObject('$set',udata),false, true)
}
}
def match(def datas){
def rets = []
for(def d : datas){
def jsonObject = OpenApiClient.addressMatch(d.city, d.name, "property", false, d.region, null)
if(jsonObject != null && jsonObject.precise!=-1) {
def map = [:]
map.match_city = jsonObject.get("city")
map.match_region = jsonObject.get("district")
map.match_name = jsonObject.get("name")
map.old_name = d.name
map.old_city = d.city
map.old_region = d.region
rets << map
}
}
return rets;
}
def findAddress(){
def q = new BasicDBObject()
def dbcursor = loudongConn.find(q)
def rets = []
def keys = []
for(def d : dbcursor){
if(!keys.contains(d.name)){
keys << d.name
def map = [:]
map.city = d.city
map.name = d.name
rets << map
}
}
return rets;
}
def findAddress2(){
def q = new BasicDBObject()
def dbcursor = loudongConn.find(q).batchSize(5000)
def rets = []
def keys = []
for(def d : dbcursor){
def key = d.city +"##"+d.region+"##"+d.district_name
if(!keys.contains(key)){
keys << key
def map = [:]
map.city = d.city
map.region = d.region
map.name = d.district_name
rets << map
}
}
return rets;
}
def stat(){
def q = new BasicDBObject()
q.put("match_city", new BasicDBObject('$exists', true))
def field = new BasicDBObject()
field.put("match_city",1)
field.put("match_region",1)
field.put("match_name",1)
def dbcursor = loudongConn.find(q, field).batchSize(2000)
def keys = []
for(def d : dbcursor){
def key = d.match_city+"##"+d.match_region+"##"+d.match_name
if(!keys.contains(key)){
keys << key
}
}
def dbcursor2 = loudongFdcConn.find(q, field).batchSize(5000)
for(def d : dbcursor2){
def key = d.match_city+"##"+d.match_region+"##"+d.match_name
if(!keys.contains(key)){
keys << key
}
}
println "小区:"+keys.size()
}
}
@@ -76,7 +76,7 @@ class SeaweedExport {
ret[1] = obj.developer
ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent)
ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent)
ret[4] = SeaweedUtils.formatAlias(obj.alias)
ret[4] = SeaweedUtils.formatAlias(obj.mapping_alias)
ret[5] = SeaweedUtils.formatCarSiteCount(obj.car_site_count)
ret[6] = SeaweedUtils.calcCarAndHouseholdRadio(obj.car_site_count, obj.household_count)
ret[7] = ""
@@ -40,11 +40,11 @@ class DataStat {
def run(){
def source = "搜房"//
def citys = findCrawlCity(source, 202112);
def source = "58同城"//
def citys = findCrawlCity(source, 202202);
for(def city : citys){
def count1 = findCount(source, city, 202112);
def count2 = findCount(source, city, 202202);
def count1 = findCount(source, city, 202202);
def count2 = findCount(source, city, 202203);
println city +" "+count1+" "+count2
// println count1
// println "--- ----"
@@ -40,19 +40,17 @@ class Esf58Crawl extends BaseBatch{
def static useDistKeys = []
static def totalCount = new AtomicLong()
def cityList = [
def cityList = ["成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413",
"保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872",
"沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556",
"石家庄":"sjz|241","太原":"ty|740","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606",
"绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572",
"淮北":"huaibei|9357","滁州":"chuzhou|10266",
"福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158",
"兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362",
"兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","潍坊":"wf|362",
"芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222",
"南通":"nt|394","东莞":"dg|413","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325",
"南通":"nt|394","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325",
"呼和浩特":"hu|811",
"济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096",
"临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","北京":"bj|1","上海":"sh|2",
"三亚":"sanya|2422","烟台":"yt|228","宁波":"nb|135","金华":"jh|531", "绍兴":"sx|355","湖州":"huzhou|831","张家口":"zjk|3328","珠海":"zh|910","海口":"haikou|2053","保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872",
"沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556",
"深圳":"sz|4","石家庄":"sjz|241","太原":"ty|740","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606",
"绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572",
"淮北":"huaibei|9357","滁州":"chuzhou|10266",
"马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323",
"池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296",
"宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232",
@@ -76,7 +74,7 @@ class Esf58Crawl extends BaseBatch{
"五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136",
"文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184",
"东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","白沙":"baish|10380",
"儋州":"danzhou|10394", "郑州":"zz|342","新乡":"xx|1016","许昌":"xc|977",
"儋州":"danzhou|10394", "新乡":"xx|1016","许昌":"xc|977",
"平顶山":"pds|1005","焦作":"jiaozuo|3266","开封":"kaifeng|2342",
"濮阳":"puyang|2346","周口":"zk|933","驻马店":"zmd|1067","漯河":"luohe|2347",
"三门峡":"smx|9317","鹤壁":"hb|2344","济源":"jiyuan|9918","明港":"mg|8541","鄢陵":"yanling|9123",
@@ -98,7 +96,7 @@ class Esf58Crawl extends BaseBatch{
"广水":"guangshuishi|9657","赤壁":"chibishi|9623","老河口":"laohekou|895","谷城":"gucheng|899","宜城":"yichengshi|897",
"南漳":"nanzhang|898","云梦":"yunmeng|3438","安陆":"anlu|3442","大悟":"dawu|3437","孝昌":"xiaochang|3436",
"当阳":"dangyang|865","枝江":"zhijiang|866","嘉鱼":"jiayuxian|9624","随县":"suixia|9660",
"长沙":"cs|414","益阳":"yiyang|10198","衡阳":"hy|914",
"益阳":"yiyang|10198","衡阳":"hy|914",
"湘潭":"xiangtan|2047","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756",
"永州":"yongzhou|2307","娄底":"ld|9481","湘西":"xiangxi|10219","张家界":"zjj|6788","醴陵":"liling|1091",
"澧县":"lixian|876","桂阳":"czguiyang|5699","资兴":"zixing|5698","永兴":"yongxing|5701","常宁":"changningshi|921",
@@ -181,7 +179,10 @@ class Esf58Crawl extends BaseBatch{
"长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536",
"安吉":"anji|836","苍南":"cangnanxian|7579","临海":"linhai|407","永康":"yongkang|537","玉环":"yuhuan|409",
"平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361",
"江山":"jiangshanshi|6796","平阳":"pingyangxian|7575"
"威海":"weihai|518","安阳":"ay|1096",
"临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","北京":"bj|1","上海":"sh|2",
"三亚":"sanya|2422","烟台":"yt|228","宁波":"nb|135","金华":"jh|531", "绍兴":"sx|355","湖州":"huzhou|831","张家口":"zjk|3328","珠海":"zh|910","海口":"aihkou|2053",
"江山":"jiangshanshi|6796","平阳":"pingyangxian|7575","济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|27 6","商丘":"sq|1029","大同":"dt|6964",
]
static main(args) {
@@ -191,37 +192,37 @@ class Esf58Crawl extends BaseBatch{
void process(Object args) {
//addContent("福州","https://fz.58.com/xiaoqu/", 70);
// def citys = ["新余","松原","楚雄","吕梁","雅安","本溪","鹤壁","拉萨","黔东南","忻州","兴安盟","乌海","贵港","三门峡","仙桃","安康","白城","平凉","庆阳","潮州","白银","吴忠","防城港","延安","丽江","黔西南","喀什","钦州","鸡西","萍乡","石嘴山","鹰潭","保山","哈密","铜仁","武威","普洱","河池","酒泉","朔州","百色","鹤岗","张家界","定西","湘西","济源","昭通","中卫","张掖","克拉玛依","黑河","铜川","辽源","云浮","博尔塔拉","固原","白山","天门","潜江","贺州","临沧","伊春","临夏","石河子","来宾","双鸭山","嘉峪关","海东"]
// for(def city : citys){
// def str = cityList.get(city)
def citys = ["佛山","潍坊","长春","太原","南通","厦门","呼和浩特","常州","中山","德阳","镇江","泰安","银川","襄阳","江门","荆州","上海","衡阳","滨州","遂宁","鞍山","赣州","阜阳","枣庄","聊城","芜湖","吉林","德州","信阳","新乡","宿迁","沧州","桂林","包头","常德","乐山","遵义","眉山","株洲","驻马店","日照","许昌","成都","衡水","宜宾","泸州","营口","达州","宝鸡","广安","吉安","孝感","鄂尔多斯","十堰","清远","湘潭","上饶","漯河","邢台","平顶山","菏泽","宜春","承德","赤峰","延边","茂名","焦作","内江","韶关","梅州","郴州","黄冈","红河","绥化","娄底","丹东","邵阳","开封","阳江","黄石","周口","牡丹江","长治","咸宁","汉中","自贡","衢州","晋中","永州","北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏","重庆"]
for(def city : citys){
def str = cityList.get(city)
for(def i=0;i<1;i++){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
if(districts){
def arrs = str.split("\\|")
def url = "https://"+arrs[0]+".58.com/xiaoqu/"
distMap.put(city, districts)
addCrawlKeys(city);
addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
}
}
}
// cityList.each{city, str->
// for(def i=0;i<1;i++){
// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
// if(districts){
// def arrs = str.split("\\|")
// def url = "https://"+arrs[0]+".58.com/xiaoqu/"
// distMap.put(city, districts)
// addCrawlKeys(city);
// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
// if(citys.contains(city)){
// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
// if(districts){
// def arrs = str.split("\\|")
// def url = "https://"+arrs[0]+".58.com/xiaoqu/"
// distMap.put(city, districts)
//
// addCrawlKeys(city);
// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
// }
// }
// }
// }
cityList.each{city, str->
for(def i=0;i<1;i++){
// if(citys.contains(city)){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
if(districts){
def arrs = str.split("\\|")
def url = "https://"+arrs[0]+".58.com/xiaoqu/"
distMap.put(city, districts)
addCrawlKeys(city);
addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
}
// }
}
}
}
@@ -230,6 +231,10 @@ class Esf58Crawl extends BaseBatch{
for(def i=0;i<districts.size();i++){
def districtName = districts[i].name
println "${city}【${i}/${districts.size()}】 "+districtName
def total = 0
if(districtName.length()>5){
districtName = districtName.substring(0,3)
}
for(def j=1;j<=districtName.length()-1;j++) {
// if(j<=3){
def keyword = districtName.toString().substring(0, j);
@@ -15,10 +15,10 @@ class SynchDistrictPrice {
def static final SHORT_DATE = Utils.getLastMonth()
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
"anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
// def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
//"fang.com":"ËÑ·¿",
// def sourceMap = ["fang.com":"搜房"]
def sourceMap = ["ke.com":"贝壳找房","fang.com":"搜房","cityhouse.cn":"城市房产"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()