diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml
index c71006bb8..9469b6d02 100644
--- a/projects/fangjia-script/pom.xml
+++ b/projects/fangjia-script/pom.xml
@@ -494,5 +494,32 @@
msgpack
0.6.12
+
+
+
+ com.asprise.ocr
+ java-ocr-api
+ [15,)
+
+
+
+ net.java.dev.jna
+ jna
+ 4.1.0
+
+
+
+ net.sourceforge.tess4j
+ tess4j
+ 3.4.0
+
+
+ com.sun.jna
+ jna
+
+
+
+
+
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy
index aa12539b9..d34dedcc2 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy
@@ -508,6 +508,7 @@ class BigdataDealBatch extends BaseBatch{
def q = new BasicDBObject()
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate));
q.put("area", new BasicDBObject('$gte', 20))
+ q.put("status", 0)
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
def rets = []
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy
index 2749ed7f4..4978bd962 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
- exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220218, 20220319);
+ exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220318, 20220419);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy
index 412677ae5..2476de4c3 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
- def prices = CcbCommonUtils.findPersonPrice(city, "2022-03-1", "2022-04-1")
+ def prices = CcbCommonUtils.findPersonPrice(city, "2022-04-1", "2022-05-1")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy
index a8a616b7c..5a6b21aac 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
-def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202202.csv")
+def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202203.csv")
def count0 = 0
def count1 = 0
def count2 = 0
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy
index 244ae41c0..77239e3ea 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy
@@ -13,7 +13,7 @@ import org.apache.commons.lang.StringUtils
class OpenApiUtils {
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){
- address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","")
+ address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","").replace("%","").replace("\"","")
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
if(preciseLevelflag){
url = url +"&preciseLevel=strict"
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy
index 4229926a1..afaae338f 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy
@@ -88,9 +88,10 @@ def getDetailLinks(driver, cityList){
driver.findElement(By.className("next")).click()
}
}
- println city +" "+urls.size()
+
def rets = batchParse(urls);
saveToDB(rets);
+ println city +" "+urls.size()
}
}
println cityMap.provice +" 完成"
@@ -123,7 +124,7 @@ def getCity(driver, proviceMap){
for(def c : cityEles){
def cityid = c.attr("data-cityid")
def city = c.text()
- if(!["石家庄市","唐山市","秦皇岛市","邯郸市","邢台市","保定市"].contains(city)){
+ if(!["石家庄市","太原市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){
cityMap.put(city, cityid)
}
@@ -170,8 +171,7 @@ def batchParse(def urls){
WebDriver driver = new ChromeDriver()
def ret = []
for(def map :urls){
- driver.get(map.url);
- def doc = Jsoup.parse(driver.getPageSource());
+ def doc = getDocument(map.url, driver)
def res = parse(map.province, map.city, map.url, doc);
if(res){
ret << res
@@ -180,6 +180,30 @@ def batchParse(def urls){
return ret;
}
+def getDocument(url, driver){
+ def stop = false
+ def error = 0
+ def doc = null
+ while(!stop){
+ try{
+ driver.get(url)
+ doc = Jsoup.parse(driver.getPageSource());
+ if(doc){
+ stop = true;
+ }else{
+ error++
+ Thread.sleep(5000)
+ }
+ }catch(Exception e){
+ error++
+ }
+ if(error>=10){
+ stop = true;
+ }
+ }
+ return doc
+}
+
def parse(province, city, url,doc){
if(doc){
def result = [:]
@@ -205,7 +229,12 @@ def parse(province, city, url,doc){
def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc)
if (finishDateStr) {
- result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm");
+ try{
+ result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm");
+ }catch(Exception e){
+
+ }
+
}
def curPriceStr = JQueryUtils.selectText([".colorred"], doc)
@@ -255,7 +284,12 @@ def parse(province, city, url,doc){
startDateStr = startDateStr.toString().substring(startDateStr.indexOf("定于")+2, startDateStr.indexOf("日")+1).trim()
}
- result.startDate = com.fangjia.script.util.DateUtils.parseString2Date(startDateStr)
+ try{
+ result.startDate = com.fangjia.script.util.DateUtils.parseString2Date(startDateStr.trim())
+ }catch(Exception e){
+
+ }
+
def imgs = JQueryUtils.select(".sf-pic-slide img", doc)
if (imgs) {
def imgLinks = []
@@ -315,7 +349,7 @@ def saveToDB(rets){
ret.each{k,v->
obj.put(k,v)
}
-
+ obj.source= "中拍网"
if(dbo){
obj.m_date = new Date()
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
@@ -346,15 +380,18 @@ def getAreaNumber(input, maxArea) {
val = val.toString().substring(val.toString().indexOf("室")+1, val.toString().size())
}
def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", "");
- if (maxArea) {
- if (v.toDouble() < maxArea) {
- area = v;
+ if(v && v.toString().isNumber()){
+ if (maxArea) {
+ if (v.toDouble() < maxArea) {
+ area = v;
+ break;
+ }
+ } else {
+ area = v
break;
}
- } else {
- area = v
- break;
}
+
}
if (area) {
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy
new file mode 100644
index 000000000..8cce171c6
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy
@@ -0,0 +1,290 @@
+package com.fangjia.script.gj.paimai
+
+import org.apache.commons.codec.binary.Base64
+import org.apache.commons.lang.StringUtils
+import sun.font.Font2D
+import sun.font.Font2DHandle
+import sun.font.TrueTypeFont
+import sun.font.TrueTypeGlyphMapper
+
+import java.awt.Font
+import java.awt.font.FontRenderContext
+import java.awt.font.GlyphVector
+import java.awt.geom.AffineTransform
+import java.lang.reflect.Field
+import java.util.regex.Matcher
+import java.util.regex.Pattern
+
+class DataUtil {
+ static def getAreaNumber(def input, def maxArea) {
+ Pattern pattern = Pattern.compile('\\d+(.\\d+)?(平方米|㎡|m2|方米|平)')
+ Matcher matcher = pattern.matcher(input)
+
+ List list = new ArrayList()
+ while (matcher.find()) {
+ list.add(matcher.group())
+ }
+ if (list.size() > 0) {
+ def area = null
+
+ for (def val : list) {
+ def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", "")
+ if (maxArea) {
+ if (v.isDouble()) {
+ if (v.toDouble() < maxArea) {
+ area = v
+ break
+ }
+ }
+ } else {
+ area = v
+ break
+ }
+ }
+
+ if (area) {
+ return area
+ }
+ }
+ return null
+ }
+
+ static def getPaiMaiType(def s) {
+ if (s) {
+ if (s.toString().contains("结束")) {
+ return "已完成"
+ } else if (s.toString().contains("进行") || s.toString().contains("正在拍卖")) {
+ return "进行中"
+ } else if (s.toString().contains("即将开始")) {
+ return "即将开始"
+ } else if (s.toString().contains("流拍")) {
+ return "流拍"
+ } else if (s.toString().contains("撤回")) {
+ return "撤回"
+ } else if (s.toString().contains("暂缓")) {
+ return "暂缓"
+ } else if (s.toString().contains("中止")) {
+ return "中止"
+ } else if (s.toString().contains("失败")) {
+ return "失败"
+ }
+ }
+ return s
+ }
+
+
+ static def getDateRange(def s) {
+ s = s.replace("起", "")
+ Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}[月|日]\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)')
+ Matcher matcher = pattern.matcher(s)
+
+ List list = new ArrayList()
+ while (matcher.find()) {
+ def str = matcher.group()
+ list.add(str)
+// if(str) {
+// def indx1 = str.indexOf("年")
+// def indx2 = str.indexOf("月")
+// def month = str.substring(indx1+1, indx2);
+// if(month.toInteger()<12) {
+// list.add(str);
+// }
+// }
+ }
+ if (list.size() >= 1) {
+ String s1 = list.get(0)
+ s1 = s1.replace("法院将于", "").replace(" ", "")
+ def array = s1.split("至")
+ return array
+ }
+ return null
+ }
+
+
+ static def getStartPrice(def s) {
+ Pattern pattern = Pattern.compile('起拍价:\\d+(.\\d+)?万?元')
+ Matcher matcher = pattern.matcher(s)
+
+ List list = new ArrayList()
+ while (matcher.find()) {
+ list.add(matcher.group())
+ }
+ if (list.size() >= 1) {
+ String s1 = list.get(0)
+ if (s1.contains("万元")) {
+ s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "")
+ return (int) (s1.toDouble() * 10000)
+ } else {
+ s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "")
+ if (s1.isInteger()) {
+ return s1.toInteger()
+ } else {
+ return (int) (s1.toDouble())
+ }
+ }
+ }
+ return null
+ }
+
+ static def getEstateType(def s) {
+ Pattern pattern = Pattern.compile('房屋类型:.*;')
+ Matcher matcher = pattern.matcher(s)
+
+ List list = new ArrayList()
+ while (matcher.find()) {
+ list.add(matcher.group())
+ }
+ if (list.size() >= 1) {
+ String s1 = list.get(0)
+ if (s1) {
+ s1 = s1.substring(s1.indexOf(":") + 1, s1.indexOf(";"))
+ return s1.replace(")", "").replace("】", "")
+ }
+ }
+ return null
+ }
+
+ static def findEstateType(def s) {
+ if (s) {
+ def villTypes = ["独栋", "联排", "联列", "叠加", "双拼", "花园面积", "泳池", "别墅", "花园住宅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]
+ for (def type : villTypes) {
+ if (s.contains(type)) {
+ return "别墅"
+ }
+ }
+
+ def districtTypes = ["住宅", "公寓"]
+ for (def type : districtTypes) {
+ if (s.contains(type)) {
+ return "普通住宅"
+ }
+ }
+ }
+
+ return null
+
+ }
+
+ static def filterChar(def detailMap) {
+ def newDetailMap = [:]
+ detailMap.each { def k, v ->
+ if (k.toString().contains(".")) {
+ newDetailMap.put(k.toString().replace(".", "\\u002e"), v)
+ } else {
+ newDetailMap.put(k, v)
+ }
+ }
+ return newDetailMap
+ }
+
+ /**
+ * 匹配符合条件的字符串
+ *
+ * @param str
+ * @param pattern
+ * @return
+ */
+ static String match(String str, Pattern pattern) {
+ if (str == null) {
+ return null
+ }
+ Matcher matcher = pattern.matcher(str)
+ if (matcher.find()) {
+ return matcher.group()
+ }
+ return null
+ }
+
+ /**
+ * font-secret字符串专用解密工具
+ *
+ * @param key 密匙
+ * @param encodeString 加密后的字符串
+ * @return 解密后的字符串
+ */
+ static String decodeString(String key, String encodeString) {
+ if (!key) {
+ return ""
+ }
+ try {
+ //base64解码,初始化字体
+ Base64 base64 = new Base64()
+ byte[] ss = base64.decodeBase64(key)
+ InputStream inputStream = new ByteArrayInputStream(ss)
+ Font dynamicFont = Font.createFont(Font.TRUETYPE_FONT, inputStream)
+ FontRenderContext fontRenderContext = new FontRenderContext(new AffineTransform(), false, false)
+ GlyphVector glyphVector = dynamicFont.createGlyphVector(fontRenderContext, "")
+
+ //获取font中字形的映射关系,字段为private,使用反射
+ Class> clazz = Font.class
+ Field[] fs = clazz.getDeclaredFields()
+ Font2DHandle font2DHandle = null
+ for (int i = 0; i < fs.length; i++) {
+ fs[i].setAccessible(true)// 将目标属性设置为可以访问
+ if (fs[i].getName().equals("font2DHandle")) {
+ font2DHandle = (Font2DHandle) fs[i].get(dynamicFont)
+ }
+
+ }
+
+ //得到映射关系
+ Font2D font2D = font2DHandle.font2D
+ TrueTypeFont trueTypeFont = (TrueTypeFont) font2D
+ TrueTypeGlyphMapper charToGlyphMapper = (TrueTypeGlyphMapper) trueTypeFont.getMapper()
+
+ //开始解密,encodeString为加密后的字符串
+ StringBuffer buffer = new StringBuffer()
+ char[] chars = encodeString.toCharArray()
+ for (int i = 0; i < chars.length; i++) {
+ buffer.append(charToGlyphMapper.charToGlyph(chars[i]) - 1)
+ }
+ return buffer.toString()
+ } catch (Exception e) {
+ e.printStackTrace()
+ }
+ return ""
+ }
+
+ static List SITE_SUFFIXS = [".gov.cn",".com.cn", ".com", ".cn", ".net", ".org", ".cc", ".tm", ".biz", ".info", ".tv"]
+
+ static String urlToSiteName(String url) {
+ if (StringUtils.isEmpty(url)) {
+ return null
+ }
+
+ for (String suffix : SITE_SUFFIXS) {
+ if (url.indexOf(suffix) > -1) {
+ String s = StringUtils.substringBefore(url, suffix)
+ if (s.indexOf(".") > -1) {
+ s = StringUtils.substringAfterLast(s, ".")
+ if (StringUtils.isNotBlank(s)) {
+ return s + suffix
+ }
+ } else if (s.indexOf("://") > -1) {
+ s = StringUtils.substringAfterLast(s, "://")
+ if (StringUtils.isNotBlank(s)) {
+ return s + suffix
+ }
+ } else if (s.indexOf("://") == -1 && s.indexOf(".") == -1) {
+ return s + suffix
+ }
+
+ return null
+ }
+ }
+ return null
+ }
+
+ static double getRound(int scale, double dSource) {
+ BigDecimal deSource = new BigDecimal(dSource)
+ return deSource.setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue()
+ }
+
+ static def filterEmpty(def s){
+ if(s && s.contains("暂无")){
+ return null
+ }
+ return s
+ }
+
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy
new file mode 100644
index 000000000..aac9c7315
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy
@@ -0,0 +1,528 @@
+package com.fangjia.script.gj.paimai
+
+import com.fangjia.core.RegionData
+import com.fangjia.script.crawl.JQueryUtils
+import com.fangjia.script.gj.OpenApiUtils
+import com.fangjia.script.thirdPriceCrawl.CrawalUtil
+import com.fangjia.script.util.ABuYunUtil
+import com.fangjia.script.util.DateUtils
+import com.fangjia.script.util.MongoUtils
+import com.mongodb.BasicDBObject
+import com.mongodb.DBCollection
+import com.mongodb.DBObject
+import net.sf.json.JSONObject
+import org.apache.poi.ss.util.RegionUtil
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
+import org.openqa.selenium.WebDriver
+import org.openqa.selenium.chrome.ChromeDriver
+
+import javax.sound.midi.MidiSystem
+import java.util.regex.Pattern
+
+class JdPaimaiCrawl {
+
+ DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
+
+ static main(args) {
+ def task = new JdPaimaiCrawl();
+ task.run();
+ }
+ def run(){
+ def existsIds = findAllIds()
+ println "已存在:"+existsIds.size()
+ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
+ System.setProperty("http.nonProxyHosts", "localhost");
+ WebDriver driver = new ChromeDriver()
+ def links = getList(driver)
+ def ids = getDetailLinks(links, driver)
+ println "共:"+ids.size()
+ parserList( existsIds,ids, driver)
+
+
+
+ }
+
+ def getList(driver){
+ def list = []
+ for(def i=1;i<=32;i++){
+ if(i==8){
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268"
+ }else if(i==5){
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268"
+ }else if(i==17){
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268"
+ }else if(i==19){
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268"
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268"
+ }else{
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268"
+ }
+ }
+ def links = []
+ for(def url : list){
+ driver.get(url)
+ def doc = Jsoup.parse(driver.getPageSource());
+ //def document = ABuYunUtil.doPost(url, null, null, "utf-8", true)
+ if(doc){
+ def json = doc.toString().substring(doc.toString().indexOf("(")+1, doc.toString().lastIndexOf(")"))
+ JSONObject obj = JSONObject.fromObject(json);
+ if(obj && obj.total){
+ def total = obj.total
+ def page = total/40
+ def p = page.toInteger()
+ if(page > p){
+ p = p +1
+ }
+
+ for(def i=1;i<=p;i++){
+ links << url+"&page="+i
+ }
+// if(links){
+// break;
+// }
+ }
+ }
+ }
+ return links;
+ }
+
+ def getDetailLinks(links, driver){
+ def rets = []
+ def citys = ["北京","上海","天津","重庆"]
+ for(def link : links){
+ driver.get(link)
+ def doc = Jsoup.parse(driver.getPageSource());
+ if(doc) {
+ def json = doc.toString().substring(doc.toString().indexOf("(") + 1, doc.toString().lastIndexOf(")"))
+ JSONObject obj = JSONObject.fromObject(json);
+ if (obj && obj.ls) {
+ for(def o : obj.ls){
+ def map = [:]
+ if(citys.contains(o.province)){
+ map.province = o.province
+ map.city = o.province
+ map.region = o.city
+ }else{
+ map.province = o.province
+ map.city = o.city
+ }
+ map.id = o.id.toString()
+
+ rets << map
+ }
+ }
+ }
+ }
+ return rets
+ }
+
+ def parserList(existsIds,list, driver){
+ def rets = []
+ for(def map : list) {
+ if(!existsIds.contains(map.id)){
+ def url = "https://paimai.jd.com/" + map.id
+ def doc = getDocument(url, driver)
+ if (doc) {
+ def result = parsering(doc, url, map.province, map.city, map.region)
+ if (result) {
+ rets << result
+ }
+ if (rets.size() % 100 == 0) {
+ def add= saveData(rets)
+ println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+" add:"+add
+ rets = []
+ }
+ }
+ }
+ }
+ if(rets.size()>0){
+ saveData(rets)
+ print "完"
+ }
+
+ }
+
+ def saveData(def results){
+ def add = 0
+ for(def ret : results){
+ def q = new BasicDBObject()
+ q.put("url", ret.url)
+ def dbo = paimaiResultColl.findOne(q);
+
+ DBObject obj = new BasicDBObject();
+ ret.each{k,v->
+ obj.put(k,v)
+ }
+
+ if(dbo){
+ obj.m_date = new Date()
+ paimaiResultColl.update(q, new BasicDBObject('$set', obj))
+ }else{
+ add++
+ obj.c_date = new Date()
+ obj.m_date = new Date()
+ try{
+ paimaiResultColl.save(obj)
+ }catch(Exception e){
+ println obj.url
+ }
+
+ }
+ }
+ return add
+ }
+
+ def parsering(doc, def url, def province, def city, def region){
+ def result = [:]
+ if(doc) {
+ result.proince = province
+ result.city = city
+ if(region){
+ result.region = region
+ }
+ def title = JQueryUtils.selectText([".pm-name"], doc)
+ result.title = title
+ result.address = JQueryUtils.selectText([".pm-location em"], doc)
+ if(result.address) {
+ //调用获取城市
+// if(!city) {
+// city = RegionData.formatCity(cityMap.name)
+// }
+
+ if(city) {
+ def rMap = OpenApiUtils.matchDistrict(city, result.title, null, true)
+ if (rMap) {
+ result.matchCity = rMap.city
+ result.matchRegion = rMap.region
+ result.matchName = rMap.name
+ result.matchAddress = rMap.address
+ result.houseNum = rMap.house_num
+ result.roomNum = rMap.room_num
+ }
+ }
+
+ }
+
+ result.status = JQueryUtils.selectText([".pm-summary .state"], doc)
+
+ def enrollCount = JQueryUtils.selectText([".pm-summary .times span:eq(0) em"], doc)
+ if(enrollCount) {
+ result.enrollCount = enrollCount.toInteger()
+ }
+
+ def lookCount = JQueryUtils.selectText([".pm-summary .times span:eq(1) em"], doc)
+ if(lookCount) {
+ result.lookCount = lookCount.toInteger()
+ }
+
+ def callCount = JQueryUtils.selectText([".pm-summary .times span:eq(2) em"], doc)
+ if(callCount) {
+ result.callCount = callCount.toInteger()
+ }
+
+
+ def curPriceStr = JQueryUtils.selectText([".mc .current em"], doc)
+ if(curPriceStr) {
+ curPriceStr = curPriceStr.replace(",", "")
+ if(curPriceStr.isInteger()) {
+ result.curPrice = curPriceStr.toInteger()
+ }else {
+ result.curPrice = (int)(curPriceStr.toDouble())
+ }
+ }
+
+ def startPriceStr = JQueryUtils.selectText([".mc .base em"], doc)
+ if(startPriceStr) {
+ startPriceStr = startPriceStr.replace(",", "")
+
+ if(startPriceStr.isInteger()) {
+ result.startPrice = startPriceStr.toInteger()
+ }else {
+ result.startPrice = (int)(startPriceStr.toDouble())
+ }
+
+ }
+
+ def tradePriceStr = JQueryUtils.selectText([".mc .deal em"], doc)
+ if(tradePriceStr) {
+ tradePriceStr = tradePriceStr.replace(",", "")
+ if(tradePriceStr.isInteger()) {
+ result.tradePrice = tradePriceStr.toInteger()
+ }else {
+ result.tradePrice = (int)(tradePriceStr.toDouble())
+ }
+
+ }
+
+ def finishDateStr = JQueryUtils.selectText([".endtime"], doc)
+ if(finishDateStr && finishDateStr.contains("结束时间")) {
+ finishDateStr = finishDateStr.substring(5, finishDateStr.indexOf(" "))
+ result.finishDate = DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm:ss")
+ }
+
+ def lis = JQueryUtils.select(".description ul li", doc)
+ if(lis && lis.size()) {
+ for(def li : lis) {
+ def str = li.select("span").text()
+ if(str.contains("起拍")) {
+ def v= li.select("em").text().replace("¥", "").replace(",", "")
+ if(v) {
+ if(v.isInteger()) {
+ result.startPrice = v.toDouble().toInteger()
+ }else {
+ result.startPrice = (int)(v.toDouble())
+ }
+
+ }
+ }else if(str.contains("评估价")) {
+ def v = li.select("em").text().replace("¥", "").replace(",", "")
+ if(v.isInteger()) {
+ result.evaluationPrice = v.toDouble().toInteger()
+ }else {
+ result.evaluationPrice = (int)(v.toDouble())
+ }
+ }else if(str.contains("加价幅度")) {
+ def v = li.select("em").text().replace("¥", "").replace(",", "")
+ if(v) {
+ result.addPrice = v.toDouble().toInteger()
+ }
+ }else if(str.contains("保证金")) {
+ def v = li.select("em").text().replace("¥", "").replace(",", "")
+ if(v) {
+ if(v.isInteger()) {
+ result.depositPrice = v.toDouble().toInteger()
+ }else {
+ result.depositPrice = (int)(v.toDouble())
+ }
+
+ }
+ }else if(str.contains("延时周期")) {
+
+ result.delayWeekly = li.select("em").text().replace("¥", "").replace(",", "")
+
+ }else if(str.contains("优先竞买权人")) {
+ result.preemptivePerson = li.select("em").text().replace("¥", "").replace(",", "")
+ }
+ }
+ }
+
+ def trs = JQueryUtils.select([".floor-content div:eq(2) table tr"], doc)
+ if(trs) {
+ def detailMap = [:]
+ for(def tr : trs) {
+ def d_tds = tr.select("td")
+ if(d_tds && d_tds.size()>=2) {
+ if(d_tds.size()==3) {
+ detailMap.put(d_tds[1].text(), d_tds[2].text())
+ }else {
+ detailMap.put(d_tds[0].text(), d_tds[1].text())
+ }
+ }else {
+ if(d_tds[0] && d_tds[0].text().contains("拍品介绍")) {
+ detailMap.put("拍品介绍", d_tds[0].text().replace("拍品介绍 ", ""))
+ }
+ }
+ }
+ if(detailMap) {
+ result.detailMap = DataUtil.filterChar(detailMap)
+ }
+ if(!result.address) {
+ result.address = detailMap.get("拍品名称")
+ if(!result.address) {
+ result.address = detailMap.get("标的名称")
+ }
+ }
+
+ }
+
+
+ def imgs =JQueryUtils.select(".floor-content div:eq(2) a", doc)
+ if(imgs) {
+ def imgLinks = []
+ for(def img : imgs) {
+ def href = img.attr("href")
+ if(href.contains("http")) {
+ imgLinks << href
+ }else {
+ imgLinks << url.substring(0, url.indexOf(":")+1)+href
+ }
+ }
+ result.imgUrls = imgLinks
+ }
+
+ }
+
+ //解析面积
+ def article = JQueryUtils.selectText([".floor-content"], doc)
+ if(article) {
+ def area = DataUtil.getAreaNumber(article, 5000)
+ if(!area) {
+ article = JQueryUtils.selectText(["body"], doc)
+ if(article) {
+ area = DataUtil.getAreaNumber(article, 5000)
+ }
+ }
+
+ if(area) {
+ result.area = area.toDouble()
+ if(result.tradePrice) {
+ result.tradeAvgPrice = (int)(result.tradePrice/area.toDouble())
+ }
+ }
+ }
+
+ def s1 = JQueryUtils.selectText(["li:contains(竞买公告) .floor-content"], doc)
+ if(s1) {
+ def start = s1.indexOf("法院将于")
+ def end = s1.indexOf("止")
+ if(start>-1 && end>-1 && end>start){
+ s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止"))
+ }
+ def dateRangs = DataUtil.getDateRange(s1)
+ if(dateRangs && dateRangs.size()==2) {
+
+ for(def j=0;jind+1) {
+ date = date +"时"
+ }
+
+ pattern = "yyyy年MM月dd日HH时"
+ }
+ if(j==0) {
+ try{
+ result.startDate = DateUtils.parseFromTightByPattern(date, pattern)
+ }catch(Exception e){
+
+ }
+
+ }else {
+ if(!result.finishDate) {
+ result.finishDate = DateUtils.parseFromTightByPattern(date, pattern)
+ }
+ }
+ }
+ }
+ if(!result.startPrice) {
+ result.startPrice = DataUtil.getStartPrice(s1)
+ }
+
+ def estateType = DataUtil.findEstateType(s1)
+ if(estateType) {
+ result.matchEstateType = estateType
+ }
+
+ }
+ result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
+ result.introduce = JQueryUtils.select([".floor-content div:eq(2) table"], doc).toString()
+ result.url = url
+ result.city = city
+ result.mDate = new Date()
+ result.source = "京东"
+ return result;
+ }
+
+ def getDocument(url, driver){
+ def stop = false
+ def error = 0
+ def doc = null
+ while(!stop){
+ try{
+ driver.get(url)
+ doc = Jsoup.parse(driver.getPageSource());
+ if(doc){
+ stop = true;
+ }else{
+ error++
+ Thread.sleep(5000)
+ }
+ }catch(Exception e){
+ error++
+ }
+ if(error>=10){
+ stop = true;
+ }
+ }
+ return doc
+ }
+ def findAllIds(){
+ def q = new BasicDBObject()
+ q.put("url", Pattern.compile(".*jd.com.*"))
+ def dbcursor = paimaiResultColl.find(q)
+ def ids = []
+ for(def d : dbcursor){
+ def url = d.url
+ if(url){
+ def id = url.substring(url.toString().lastIndexOf("/")+1, url.toString().length())
+ ids << id
+ }
+ }
+ return ids;
+ }
+
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataUtil.groovy
index f98faa3ee..90a450a66 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataUtil.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataUtil.groovy
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.paimai
+import java.text.SimpleDateFormat
import java.util.regex.Matcher
import java.util.regex.Pattern
import java.util.zip.GZIPInputStream
@@ -94,32 +95,32 @@ class PaimaiDataUtil {
}
- public static def getDateRange(def s) {
- s = s.replace("起","")
- Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)');
- Matcher matcher = pattern.matcher(s);
-
- List list = new ArrayList();
- while (matcher.find()) {
- def str = matcher.group()
- list.add(str);
-// if(str) {
-// def indx1 = str.indexOf("年")
-// def indx2 = str.indexOf("月")
-// def month = str.substring(indx1+1, indx2);
-// if(month.toInteger()<12) {
-// list.add(str);
-// }
-// }
- }
- if(list.size()>=1) {
- String s1 = list.get(0)
- s1 = s1.replace("法院将于", "").replace(" ","")
- def array = s1.split("至");
- return array
- }
- return null;
- }
+// public static def getDateRange(def s) {
+// s = s.replace("起","")
+// Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)');
+// Matcher matcher = pattern.matcher(s);
+//
+// List list = new ArrayList();
+// while (matcher.find()) {
+// def str = matcher.group()
+// list.add(str);
+//// if(str) {
+//// def indx1 = str.indexOf("年")
+//// def indx2 = str.indexOf("月")
+//// def month = str.substring(indx1+1, indx2);
+//// if(month.toInteger()<12) {
+//// list.add(str);
+//// }
+//// }
+// }
+// if(list.size()>=1) {
+// String s1 = list.get(0)
+// s1 = s1.replace("法院将于", "").replace(" ","")
+// def array = s1.split("至");
+// return array
+// }
+// return null;
+// }
public static def getStartPrice(def s) {
@@ -159,4 +160,30 @@ class PaimaiDataUtil {
return newDetailMap;
}
+ public static def getDateRange(def s) {
+ s = s.replace("起","")
+ Pattern pattern = Pattern.compile('法院将于(\\d+年\\d+月\\d+日\\d+时)至(\\d+年\\d+月\\d+日\\d+时)');
+ Matcher matcher = pattern.matcher(s);
+
+ List list = new ArrayList();
+ while (matcher.find()) {
+ list.add(matcher.group());
+ }
+ if(list.size()>=1) {
+ String s1 = list.get(0)
+ s1 = s1.replace("法院将于", "")
+ def array = s1.split("至");
+ return array
+ }
+ return null;
+ }
+
+ public static Date parseFromTightByPattern(String s, String pattern){
+ if (s == null || s.length() == 0){
+ return null
+ }
+ SimpleDateFormat format = new SimpleDateFormat(pattern);
+ return format.parse(s)
+ }
+
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy
index e35e9f251..acced0959 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy
@@ -83,15 +83,19 @@ class PaimaiTest {
driver.get("https:"+data.href)
def doc = Jsoup.parse(driver.getPageSource());
if(citys.contains(data.proince)){
- def cityMap = [:]
- cityMap.proince = data.proince
- cityMap.city = data.proince
- cityMap.url = data.href
- cityList << cityMap
+ def links = JQueryUtils.select(".condition .s-subtab a", doc)
+ for(def ele : links) {
+ def cityMap = [:]
+ cityMap.proince = data.proince
+ cityMap.city = data.proince
+ cityMap.region =ele.text()
+ cityMap.url = data.href
+ cityList << cityMap
+ }
}else{
def links = JQueryUtils.select(".condition .s-subtab a", doc)
for(def ele : links){
- if(!["合肥市","芜湖市","蚌埠市","淮南市","马鞍山市","淮北市","铜陵市"].contains(ele.text())){
+ if(!["乌鲁木齐市","克拉玛依市","合肥市","芜湖市","蚌埠市","淮南市","马鞍山市","淮北市","铜陵市"].contains(ele.text())){
def cityMap = [:]
cityMap.proince = data.proince
cityMap.city = ele.text()
@@ -109,7 +113,7 @@ class PaimaiTest {
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def datas = []
- def keys = ["上海","天津","重庆","北京","浙江","河北","山西","辽宁","吉林","江苏","福建","江西","山东","河南","湖北","湖南","广东"]
+ def keys = [ "天津","上海","重庆","北京","浙江","河北","陕西","山西","新疆","辽宁","吉林","江苏","福建","江西","山东","河南","湖北","湖南","西藏","宁夏","甘肃","广东","安徽","广西","海南","四川","贵州","云南"]
if(doc){
def eles = JQueryUtils.select(".condition a", doc)
for(def e : eles){
@@ -123,11 +127,34 @@ class PaimaiTest {
}
return datas
}
+ def getDocument(url, driver){
+ def stop = false
+ def error = 0
+ def doc = null
+ while(!stop){
+ try{
+ driver.get(url)
+ doc = Jsoup.parse(driver.getPageSource());
+ if(doc){
+ stop = true;
+ }else{
+ error++
+ Thread.sleep(5000)
+ }
+ }catch(Exception e){
+ error++
+ }
+ if(error>=10){
+ stop = true;
+ }
+ }
+ return doc
+ }
def parser(detailDatas, driver, proince, city){
def results = []
for(def m : detailDatas){
- try{
+ // try{
driver.get("https:"+m.url)
Thread.sleep(3000)
def doc = Jsoup.parse(driver.getPageSource());
@@ -137,9 +164,9 @@ class PaimaiTest {
obj.city = city
results << obj
}
- }catch(Exception e){
- println m.url+" 解析有问题!"
- }
+// }catch(Exception e){
+// println m.url+" 解析有问题!"
+// }
}
return results
@@ -410,7 +437,7 @@ class PaimaiTest {
}
}
-// result.content = DataUtil.zip(s)
+ result.content = DataUtil.zip(doc.toString())
def el = JQueryUtils.select([".d-block table:eq(1)"], doc)
if(!el) {
@@ -549,7 +576,7 @@ class PaimaiTest {
ret.each{k,v->
obj.put(k,v)
}
-
+ obj.source = "公拍网"
if(dbo){
obj.m_date = new Date()
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/RmfysszcCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/RmfysszcCrawl.groovy
index 244cc0d60..a7c7dccb2 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/RmfysszcCrawl.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/RmfysszcCrawl.groovy
@@ -1,12 +1,4 @@
-package com.fangjia.script.gj.paimai
-
-import com.fangjia.core.util.DataUtil
-import com.fangjia.script.BaseBatch
-import com.fangjia.script.BatchOwner
-import com.fangjia.script.Watcher
-import com.fangjia.script.crawl.JQueryUtils
-import com.fangjia.script.gj.OpenApiUtils
-import com.fangjia.script.util.ABuYunUtil
+import com.fangjia.script.gj.paimai.PaimaiDataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
@@ -14,66 +6,266 @@ import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.jsoup.Jsoup
import org.openqa.selenium.By
-import org.openqa.selenium.JavascriptExecutor
import org.openqa.selenium.WebDriver
-import org.openqa.selenium.WebElement
import org.openqa.selenium.chrome.ChromeDriver
+import org.openqa.selenium.support.ui.WebDriverWait
import org.openqa.selenium.support.ui.ExpectedCondition
import org.openqa.selenium.support.ui.ExpectedConditions
-import org.openqa.selenium.support.ui.WebDriverWait
+import com.fangjia.script.gj.OpenApiUtils
-import java.util.concurrent.TimeUnit
import java.util.regex.Matcher
import java.util.regex.Pattern
-
+import com.fangjia.core.util.DataUtil
+import com.fangjia.script.crawl.JQueryUtils
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
+
+
def url = "https://www1.rmfysszc.gov.cn/projects.shtml?dh=3&gpstate=1&wsbm_slt=1"
WebDriver driver = new ChromeDriver()
-driver.get(url);
-driver.findElementById("1").click()
+run(url, driver)
+//driver.get(url);
+//driver.findElementById("1").click()
+//driver.findElementById("search_sub").click()
+//
+//def provinces = getAllProvinceValue(driver.getPageSource())
+//for(def m : provinces){
+// def city_setting = driver.findElementById("city")
+// //${m.province_id}
+// city_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click()
+//
+////driver.manage().timeouts().implicitlyWait(30, TimeUnit.SECONDS);
+////WebDriverWait wait = new WebDriverWait(driver,20)
+////wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.cssSelector("pagecur1"))))
+// Thread.sleep(10000)
+// def doc = Jsoup.parse(driver.getPageSource());
+// def citys = getAllCity(doc)
+// getDetailByCity(m.province, citys, driver)
+//}
-def city_setting = driver.findElementById("city")
-city_setting.findElement(By.xpath("//option[@value='31']")).click()
-driver.findElementById("search_sub").click()
-//driver.manage().timeouts().implicitlyWait(30, TimeUnit.SECONDS);
-//WebDriverWait wait = new WebDriverWait(driver,20)
-//wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.cssSelector("pagecur1"))))
-Thread.sleep(10000)
-def doc = Jsoup.parse(driver.getPageSource());
-def links = getLinks(doc)
+def run(url, driver){
+ def values = getAllCityRegionValue(url, driver)
+ def finishCity = ["北京市","天津市","张家口市","邯郸市","保定市","邢台市","石家庄市","唐山市","秦皇岛市"]
+ for(def m : values){
+ if(!finishCity.contains(m.city)){
+ driver.get(url);
+ driver.findElementById("1").click()
+ Thread.sleep(3000)
+
+ def pro_setting = driver.findElementById("city")
+ pro_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click()
+ Thread.sleep(3000)
+
+ def city_setting = driver.findElementById("city1")
+ city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click()
+ Thread.sleep(4000)
+
+ def region_setting = driver.findElementById("city2")
+ region_setting.findElement(By.xpath("//option[@value='${m.region_id}']")).click()
+
+ driver.findElementById("search_sub").click()
+ Thread.sleep(10000)
+ getDetailByPage(m.province, m.city, m.region, driver)
+ }
-def eles = driver.findElementsByClassName("pagecur1")
-for(def ele : eles){
- ele.click();
- Thread.sleep(10000)
- doc = Jsoup.parse(driver.getPageSource());
- links.addAll(getLinks(doc))
-}
-def rets = []
-for(def durl : links){
- driver.get(durl);
- doc = Jsoup.parse(driver.getPageSource());
- def res = parse(durl, doc);
- if(res){
- rets << res
}
}
-saveToDB(rets);
+
+def getAllCityRegionValue(def url, def driver){
+ driver.get(url);
+ driver.findElementById("1").click()
+ driver.findElementById("search_sub").click()
+
+ def provinces = getAllProvinceValue(driver.getPageSource())
+ def values = []
+ for(def m : provinces){
+ def city_setting = driver.findElementById("city")
+
+ city_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click()
+
+ Thread.sleep(10000)
+ def doc = Jsoup.parse(driver.getPageSource());
+ def citys = getAllCity(doc)
+ def cityRegions = getAllCityByRegion(m, citys, driver)
+ values.addAll(cityRegions)
+ //getDetailByCity(m.province, citys, driver)
+ }
+ return values
+}
+
+def getDetailByCity(province, citys, driver){
+ for(def m : citys){
+ def city_setting = driver.findElementById("city1")
+ city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click()
+ Thread.sleep(10000)
+ def doc = Jsoup.parse(driver.getPageSource());
+ def regions = getAllRegion(doc)
+ getDetailByRegion(province, m.city, regions, driver)
+ }
+}
+
+def getAllCityByRegion(provinceMap, citys, driver){
+ def datas = []
+ for(def m : citys){
+ def city_setting = driver.findElementById("city1")
+ city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click()
+ Thread.sleep(10000)
+ def doc = Jsoup.parse(driver.getPageSource());
+ def regions = getAllRegion(provinceMap, m, doc)
+ datas.addAll(regions)
+ }
+ return datas
+}
+
+def getDetailByRegion(province, city, regions, driver){
+ def links = []
+ for(def r : regions){
+ def city_setting = driver.findElementById("city2")
+ city_setting.findElement(By.xpath("//option[@value='${r.region_id}']")).click()
+ driver.findElementById("search_sub").click()
+ Thread.sleep(10000)
+
+ def stop = false;
+ while(!stop){
+ def doc = Jsoup.parse(driver.getPageSource());
+ links.addAll(getLinks(r.region, doc))
+ def eles = JQueryUtils.select("a:contains(下一页)", doc)
+ if(eles){
+ driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click()
+ Thread.sleep(3000)
+ }else{
+ stop = true;
+ }
+ }
+
+ }
+ def datas = parseDatas(province, city, links, driver)
+ saveToDB(datas);
+ println r.region +" "+ links.size()
+}
+
+def getDetailByPage(province, city, region, driver){
+ def links = []
+ def stop = false;
+ while(!stop){
+ def doc = Jsoup.parse(driver.getPageSource());
+ links.addAll(getLinks(doc))
+ def eles = JQueryUtils.select("a:contains(下一页)", doc)
+ if(eles){
+ driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click()
+ Thread.sleep(3000)
+ }else{
+ stop = true;
+ }
+ }
+
+ def datas = parseDatas(province, city, region, links, driver)
+ println province +" "+city+" "+region +" "+ links.size()
+ saveToDB(datas);
+}
+def getAllProvinceValue(doc){
+ def optionsEls = JQueryUtils.select("#city option", doc)
+ def rets = []
+ for(def eles : optionsEls){
+ def val = eles.attr("value")
+ if(val!="" && val!="0"){
+ def province = eles.text()
+ rets << ["province":province, "province_id":val]
-def parse(url, doc){
+ }
+
+ }
+ return rets
+}
+
+def getAllCity(doc){
+ def optionsEls = JQueryUtils.select("#city1 option", doc)
+ def rets = []
+ for(def eles : optionsEls){
+ def val = eles.attr("value")
+ if(val!=""){
+ def province = eles.text()
+ rets << ["city":province, "city_id":val]
+ }
+
+ }
+ return rets
+}
+
+def getAllRegion(provinceMap, cityMap,doc){
+ def optionsEls = JQueryUtils.select("#city2 option", doc)
+ def rets = []
+ for(def eles : optionsEls){
+ def val = eles.attr("value")
+ if(val!=""){
+ def region = eles.text()
+ def regionMap = [:]
+ regionMap.province = provinceMap.province
+ regionMap.province_id = provinceMap.province_id
+ regionMap.city = cityMap.city
+ regionMap.city_id = cityMap.city_id
+ regionMap.region = region
+ regionMap.region_id = val
+ rets << regionMap
+ }
+
+ }
+ return rets
+}
+
+def parseDatas(province, city, region,links, driver){
+ def datas = []
+ for(def url: links){
+ def doc = getDocument(url, driver)
+ def result = parse(city, region, url, doc)
+ if(result){
+ result.province = province
+ result.city = city
+ result.region = region
+ datas << result
+ }
+ }
+ return datas
+}
+
+def getDocument(url, driver){
+ def stop = false
+ def error = 0
+ def doc = null
+ while(!stop){
+ try{
+ driver.get(url)
+ doc = Jsoup.parse(driver.getPageSource());
+ if(doc){
+ stop = true;
+ }else{
+ error++
+ Thread.sleep(5000)
+ }
+ }catch(Exception e){
+ error++
+ }
+ if(error>=10){
+ stop = true;
+ }
+ }
+ return doc
+}
+
+def parse(city, region, url, doc){
def result=[:]
if(doc) {
+
def title = JQueryUtils.selectText(["#Title"], doc)
result.title = title;
result.address = title
if(result.address){
- def rMap = OpenApiUtils.matchDistrict("上海",result.address)
+ def rMap = OpenApiUtils.matchDistrict(city, result.address, null, true)
if(rMap) {
result.matchCity = rMap.city
result.matchRegion = rMap.region
@@ -99,6 +291,7 @@ def parse(url, doc){
def enrollCount = JQueryUtils.selectText([".bmnumber"], doc)
if(enrollCount) {
+ enrollCount = enrollCount.replace("人报名","")
if(enrollCount.isInteger()){
result.enrollCount = enrollCount.toInteger()
}
@@ -107,6 +300,7 @@ def parse(url, doc){
def callCount = JQueryUtils.selectText([".sc"], doc)
if(callCount) {
+ callCount =callCount.replace("人收藏","")
if(callCount.isInteger()){
result.callCount = callCount.toInteger()
}
@@ -114,6 +308,7 @@ def parse(url, doc){
def lookCount = JQueryUtils.selectText([".hits"], doc)
if(lookCount) {
+ lookCount = lookCount.replace("次围观","")
if(lookCount.isInteger()){
result.lookCount = lookCount.toInteger()
}
@@ -123,8 +318,8 @@ def parse(url, doc){
def curPriceStr = JQueryUtils.selectText(["#price div:eq(0)"], doc)
if(curPriceStr && curPriceStr.contains("起拍价")) {
curPriceStr = curPriceStr.replace("起拍价", "").replace("¥","")
- if(curPriceStr.contains("万元")){
- curPriceStr = curPriceStr.replace("万元","")
+ if(curPriceStr.contains("万")){
+ curPriceStr = curPriceStr.replace("万元","").replace("万","")
if(curPriceStr.isInteger()){
result.startPrice = curPriceStr.toInteger()*10000
}else{
@@ -144,6 +339,44 @@ def parse(url, doc){
}
}
+ def tableEles = JQueryUtils.select("#bg1", doc)
+ if(tableEles){
+ def evaluatePriceStr = tableEles.select("td:contains(值:)").text()
+ if(evaluatePriceStr && evaluatePriceStr.contains(":")){
+ def arrs = evaluatePriceStr.split(":")
+ if(arrs.size()==2){
+ evaluatePriceStr = arrs[1]
+ }
+ }
+
+ def depositPriceStr = tableEles.select("td:contains(金:)").text()
+ if(depositPriceStr && depositPriceStr.contains(":")){
+ def arrs = depositPriceStr.split(":")
+ if(arrs.size()==2){
+ depositPriceStr = arrs[1]
+ }
+ }
+ if(depositPriceStr){
+ depositPriceStr = depositPriceStr.replace("万元","")
+ if(depositPriceStr.toString().isNumber()){
+ result.depositPrice = depositPriceStr.toDouble()*10000
+ }
+ }
+
+ def courtStr = tableEles.select("td:contains(院:)").text()
+ if(courtStr && courtStr.contains(":")){
+ result.court = courtStr.substring(courtStr.indexOf(":")+1, courtStr.size())
+ }
+ def personStr = tableEles.select("td:contains(人:)").text()
+ if(personStr && personStr.contains(":")){
+ result.person = personStr.substring(personStr.indexOf(":")+1, personStr.size())
+ }
+
+ def countStr = tableEles.select("td:contains(拍卖阶段:)").text().replace("拍卖阶段:","")
+ if(countStr && countStr.contains("第") && countStr.contains("次")){
+ result.count = countStr.substring(countStr.indexOf("第")+1, countStr.indexOf("次"))
+ }
+ }
def lis = JQueryUtils.select("#bg1 div:eq(0) table tr", doc)
if(lis && lis.size()) {
@@ -151,18 +384,23 @@ def parse(url, doc){
def str = li.select("span").text()
if(str.contains("评估值")) {
def v = li.select("span:eq(1)").text().replace("万元", "")
- if(v.isInteger()) {
- result.evaluationPrice = v.toInteger()*10000
- }else{
- result.evaluationPrice = (int)(v.toDouble()*10000)
+ if(v.isNumber()){
+ if(v.isInteger()) {
+ result.evaluationPrice = v.toInteger()*10000
+ }else{
+ result.evaluationPrice = (int)(v.toDouble()*10000)
+ }
}
+
}else if(str.contains("保证金")) {
def v = li.select("span:eq(1)").text().replace("万元", "")
- if(v) {
- if(v.isInteger()) {
- result.depositPrice = v.toInteger()*10000
- }else {
- result.depositPrice = (int)(v.toDouble()*10000)
+ if(v.isNumber()) {
+ if (v) {
+ if (v.isInteger()) {
+ result.depositPrice = v.toInteger() * 10000
+ } else {
+ result.depositPrice = (int) (v.toDouble() * 10000)
+ }
}
}
}else if (str.contains("公告日期:")){
@@ -177,13 +415,32 @@ def parse(url, doc){
def trs = tables[1].select("tr")
if(trs) {
def detailMap = [:]
+
for(def tr : trs) {
def d_tds = tr.select("td")
+ if(tr.text().contains("调查情况表")){
+ continue
+ }
+ if(d_tds.size()>1){
+ def lastEleStr = d_tds[d_tds.size()-1].toString()
+ if(lastEleStr.contains('height="42"')){
+ d_tds.remove(d_tds[d_tds.size()-1])
+ }
+ }
+
+ if(d_tds.size()>3){
+ continue
+ }
if(d_tds && d_tds.size()>=2) {
if(d_tds.size()==3) {
- detailMap.put(d_tds[1].text(), d_tds[2].text())
+ if(d_tds[1].text() && d_tds[2].text()){
+ detailMap.put(d_tds[1].text(), d_tds[2].text())
+ }
+
}else {
- detailMap.put(d_tds[0].text(), d_tds[1].text())
+ if(d_tds[0].text() && d_tds[1].text()){
+ detailMap.put(d_tds[0].text(), d_tds[1].text())
+ }
}
}else {
if(d_tds[0] && d_tds[0].text().contains("拍品介绍")) {
@@ -202,8 +459,6 @@ def parse(url, doc){
result.introduce = tables[1].toString()
}
-
-
def imgs =JQueryUtils.select("#bdjs11 img", doc)
if(imgs) {
def imgLinks = []
@@ -232,6 +487,20 @@ def parse(url, doc){
result.tradeAvgPrice = (int)(result.tradePrice/result.area)
}
+ def s1 = JQueryUtils.selectText(["#pmgg"], doc)
+ if(s1) {
+ def start = s1.indexOf("法院将于")
+ def end = s1.indexOf("止")
+ if(start>-1 && end>-1 && end>start){
+ s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止"))
+ }
+ def dateRangs = PaimaiDataUtil.getDateRange(s1)
+ if(dateRangs && dateRangs.size()==2) {
+ result.startDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[0], "yyyy年MM月dd日HH时")
+ result.finishDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[1], "yyyy年MM月dd日HH时")
+ }
+ }
+
result.content = DataUtil.zip(doc.toString())
result.url = url
@@ -246,7 +515,7 @@ def getLinks(doc){
if(eles){
for(def el :eles){
def href = el.attr("href")
- urls << "https://www.rmfysszc.gov.cn/statichtml/rm_obj/"+href.substring(href.indexOf("/")+1,href.length())
+ urls <<"https://www.rmfysszc.gov.cn/statichtml/rm_obj/"+href.substring(href.indexOf("/")+1,href.length())
}
}
}
@@ -264,13 +533,19 @@ def saveToDB(rets){
ret.each{k,v->
obj.put(k,v)
}
-
- if(dbo){
- obj.remove("url")
- paimaiResultColl.update(q, new BasicDBObject('$set', obj))
- }else{
- paimaiResultColl.save(obj)
- }
+ obj.source= "人民法院诉讼资产网"
+ try{
+ if(dbo){
+ obj.m_date = new Date()
+ paimaiResultColl.update(q, new BasicDBObject('$set', obj))
+ }else{
+ obj.c_date = new Date()
+ obj.m_date = new Date()
+ paimaiResultColl.save(obj)
+ }
+ }catch(Exception e){
+ println "error:"+obj.url
+ }
}
}
@@ -287,14 +562,16 @@ def getAreaNumber(input, maxArea) {
for (def val : list) {
def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", "");
- if (maxArea) {
- if (v.toDouble() < maxArea) {
- area = v;
+ if(v && v.isNumber()){
+ if (maxArea) {
+ if (v.toDouble() < maxArea) {
+ area = v;
+ break;
+ }
+ } else {
+ area = v
break;
}
- } else {
- area = v
- break;
}
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy
new file mode 100644
index 000000000..1ce1c48c7
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy
@@ -0,0 +1,173 @@
+package com.fangjia.script.loudong
+
+import com.fangjia.script.util.ExcelUtils
+import com.fangjia.script.util.MongoUtils
+import com.fangjia.script.util.OpenApiClient
+import com.mongodb.BasicDBObject
+import com.mongodb.DBCollection
+
+class CqfdcMatch {
+
+ static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018)
+ static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018)
+ static main(args) {
+ def task = new CqfdcMatch();
+// task.run()
+// task.stat()
+ task.export2()
+ }
+ def export(){
+ def q = new BasicDBObject()
+ q.put("match_city", new BasicDBObject('$exists', false))
+
+ def field = new BasicDBObject()
+ field.put("city",1)
+ field.put("region",1)
+ field.put("district_name",1)
+ field.put("address",1)
+ field.put("type",1)
+ def dbcursor = loudongConn.find(q, field).batchSize(5000)
+ def keys = []
+ def rets = []
+ rets << ["城市","区域","名称","地址","类型"]
+ for(def d : dbcursor){
+ def key = d.city +"##"+d.district_name
+ if(!keys.contains(key)){
+ keys << key
+ rets << [d.city, d.region, d.district_name, d.address, d.type]
+ }
+ }
+ ExcelUtils.write("D:\\数据\\重庆农商行\\重庆楼栋未匹配1.xlsx",rets, "数据")
+ }
+
+ def run(){
+ def datas= findAddress2()
+ def results = match(datas);
+ update(results)
+ }
+
+
+ def export2(){
+ def q = new BasicDBObject()
+ q.put("match_city", new BasicDBObject('$exists', false))
+
+ def field = new BasicDBObject()
+ field.put("city",1)
+ field.put("name",1)
+ field.put("use_type",1)
+ def dbcursor = loudongFdcConn.find(q, field).batchSize(5000)
+ def keys = []
+ def rets = []
+ rets << ["城市","地址","类型"]
+ for(def d : dbcursor){
+ def key = d.city +"##"+d.name
+ if(!keys.contains(key)){
+ keys << key
+ rets << [d.city, d.name,d.use_type]
+ }
+ }
+ ExcelUtils.write("D:\\数据\\重庆农商行\\重庆楼栋未匹配2.xlsx",rets, "数据")
+ }
+
+ def update(results){
+ for(def m : results){
+ def q = new BasicDBObject()
+ q.put("city", m.old_city)
+ q.put("district_name", m.old_name)
+ q.put("region",m.old_region)
+
+ def udata = new BasicDBObject()
+ udata.put("match_city", m.match_city)
+ udata.put("match_region", m.match_region)
+ udata.put("match_name", m.match_name)
+ loudongConn.update(q, new BasicDBObject('$set',udata),false, true)
+ }
+ }
+
+ def match(def datas){
+ def rets = []
+ for(def d : datas){
+ def jsonObject = OpenApiClient.addressMatch(d.city, d.name, "property", false, d.region, null)
+ if(jsonObject != null && jsonObject.precise!=-1) {
+ def map = [:]
+ map.match_city = jsonObject.get("city")
+ map.match_region = jsonObject.get("district")
+ map.match_name = jsonObject.get("name")
+ map.old_name = d.name
+ map.old_city = d.city
+ map.old_region = d.region
+ rets << map
+ }
+ }
+ return rets;
+ }
+
+ def findAddress(){
+ def q = new BasicDBObject()
+ def dbcursor = loudongConn.find(q)
+ def rets = []
+ def keys = []
+ for(def d : dbcursor){
+ if(!keys.contains(d.name)){
+
+ keys << d.name
+ def map = [:]
+ map.city = d.city
+ map.name = d.name
+ rets << map
+ }
+ }
+ return rets;
+ }
+
+ def findAddress2(){
+ def q = new BasicDBObject()
+ def dbcursor = loudongConn.find(q).batchSize(5000)
+ def rets = []
+ def keys = []
+ for(def d : dbcursor){
+ def key = d.city +"##"+d.region+"##"+d.district_name
+ if(!keys.contains(key)){
+ keys << key
+ def map = [:]
+ map.city = d.city
+ map.region = d.region
+ map.name = d.district_name
+ rets << map
+ }
+ }
+ return rets;
+ }
+
+
+ def stat(){
+ def q = new BasicDBObject()
+ q.put("match_city", new BasicDBObject('$exists', true))
+
+ def field = new BasicDBObject()
+ field.put("match_city",1)
+ field.put("match_region",1)
+ field.put("match_name",1)
+ def dbcursor = loudongConn.find(q, field).batchSize(2000)
+ def keys = []
+ for(def d : dbcursor){
+ def key = d.match_city+"##"+d.match_region+"##"+d.match_name
+ if(!keys.contains(key)){
+ keys << key
+ }
+ }
+
+ def dbcursor2 = loudongFdcConn.find(q, field).batchSize(5000)
+ for(def d : dbcursor2){
+ def key = d.match_city+"##"+d.match_region+"##"+d.match_name
+ if(!keys.contains(key)){
+ keys << key
+ }
+ }
+ println "小区:"+keys.size()
+
+ }
+
+
+
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy
index a437ea0c3..9817c00cb 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy
@@ -76,7 +76,7 @@ class SeaweedExport {
ret[1] = obj.developer
ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent)
ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent)
- ret[4] = SeaweedUtils.formatAlias(obj.alias)
+ ret[4] = SeaweedUtils.formatAlias(obj.mapping_alias)
ret[5] = SeaweedUtils.formatCarSiteCount(obj.car_site_count)
ret[6] = SeaweedUtils.calcCarAndHouseholdRadio(obj.car_site_count, obj.household_count)
ret[7] = ""
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy
index 97dc2e689..97af5b6dc 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy
@@ -40,11 +40,11 @@ class DataStat {
def run(){
- def source = "搜房"//
- def citys = findCrawlCity(source, 202112);
+ def source = "58同城"//
+ def citys = findCrawlCity(source, 202202);
for(def city : citys){
- def count1 = findCount(source, city, 202112);
- def count2 = findCount(source, city, 202202);
+ def count1 = findCount(source, city, 202202);
+ def count2 = findCount(source, city, 202203);
println city +" "+count1+" "+count2
// println count1
// println "--- ----"
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy
index 9362389ca..ee0edcfe2 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy
@@ -40,19 +40,17 @@ class Esf58Crawl extends BaseBatch{
def static useDistKeys = []
static def totalCount = new AtomicLong()
- def cityList = [
+ def cityList = ["成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413",
+ "保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872",
+ "沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556",
+ "石家庄":"sjz|241","太原":"ty|740","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606",
+ "绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572",
+ "淮北":"huaibei|9357","滁州":"chuzhou|10266",
"福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158",
- "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362",
+ "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","潍坊":"wf|362",
"芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222",
- "南通":"nt|394","东莞":"dg|413","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325",
+ "南通":"nt|394","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325",
"呼和浩特":"hu|811",
- "济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096",
- "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","北京":"bj|1","上海":"sh|2",
- "三亚":"sanya|2422","烟台":"yt|228","宁波":"nb|135","金华":"jh|531", "绍兴":"sx|355","湖州":"huzhou|831","张家口":"zjk|3328","珠海":"zh|910","海口":"haikou|2053","保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872",
- "沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556",
- "深圳":"sz|4","石家庄":"sjz|241","太原":"ty|740","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606",
- "绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572",
- "淮北":"huaibei|9357","滁州":"chuzhou|10266",
"马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323",
"池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296",
"宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232",
@@ -76,7 +74,7 @@ class Esf58Crawl extends BaseBatch{
"五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136",
"文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184",
"东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","白沙":"baish|10380",
- "儋州":"danzhou|10394", "郑州":"zz|342","新乡":"xx|1016","许昌":"xc|977",
+ "儋州":"danzhou|10394", "新乡":"xx|1016","许昌":"xc|977",
"平顶山":"pds|1005","焦作":"jiaozuo|3266","开封":"kaifeng|2342",
"濮阳":"puyang|2346","周口":"zk|933","驻马店":"zmd|1067","漯河":"luohe|2347",
"三门峡":"smx|9317","鹤壁":"hb|2344","济源":"jiyuan|9918","明港":"mg|8541","鄢陵":"yanling|9123",
@@ -98,7 +96,7 @@ class Esf58Crawl extends BaseBatch{
"广水":"guangshuishi|9657","赤壁":"chibishi|9623","老河口":"laohekou|895","谷城":"gucheng|899","宜城":"yichengshi|897",
"南漳":"nanzhang|898","云梦":"yunmeng|3438","安陆":"anlu|3442","大悟":"dawu|3437","孝昌":"xiaochang|3436",
"当阳":"dangyang|865","枝江":"zhijiang|866","嘉鱼":"jiayuxian|9624","随县":"suixia|9660",
- "长沙":"cs|414","益阳":"yiyang|10198","衡阳":"hy|914",
+ "益阳":"yiyang|10198","衡阳":"hy|914",
"湘潭":"xiangtan|2047","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756",
"永州":"yongzhou|2307","娄底":"ld|9481","湘西":"xiangxi|10219","张家界":"zjj|6788","醴陵":"liling|1091",
"澧县":"lixian|876","桂阳":"czguiyang|5699","资兴":"zixing|5698","永兴":"yongxing|5701","常宁":"changningshi|921",
@@ -181,7 +179,10 @@ class Esf58Crawl extends BaseBatch{
"长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536",
"安吉":"anji|836","苍南":"cangnanxian|7579","临海":"linhai|407","永康":"yongkang|537","玉环":"yuhuan|409",
"平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361",
- "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575"
+ "威海":"weihai|518","安阳":"ay|1096",
+ "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","北京":"bj|1","上海":"sh|2",
+ "三亚":"sanya|2422","烟台":"yt|228","宁波":"nb|135","金华":"jh|531", "绍兴":"sx|355","湖州":"huzhou|831","张家口":"zjk|3328","珠海":"zh|910","海口":"aihkou|2053",
+ "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575","济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|27 6","商丘":"sq|1029","大同":"dt|6964",
]
static main(args) {
@@ -191,37 +192,37 @@ class Esf58Crawl extends BaseBatch{
void process(Object args) {
//addContent("福州","https://fz.58.com/xiaoqu/", 70);
-// def citys = ["新余","松原","楚雄","吕梁","雅安","本溪","鹤壁","拉萨","黔东南","忻州","兴安盟","乌海","贵港","三门峡","仙桃","安康","白城","平凉","庆阳","潮州","白银","吴忠","防城港","延安","丽江","黔西南","喀什","钦州","鸡西","萍乡","石嘴山","鹰潭","保山","哈密","铜仁","武威","普洱","河池","酒泉","朔州","百色","鹤岗","张家界","定西","湘西","济源","昭通","中卫","张掖","克拉玛依","黑河","铜川","辽源","云浮","博尔塔拉","固原","白山","天门","潜江","贺州","临沧","伊春","临夏","石河子","来宾","双鸭山","嘉峪关","海东"]
-// for(def city : citys){
-// def str = cityList.get(city)
+ def citys = ["佛山","潍坊","长春","太原","南通","厦门","呼和浩特","常州","中山","德阳","镇江","泰安","银川","襄阳","江门","荆州","上海","衡阳","滨州","遂宁","鞍山","赣州","阜阳","枣庄","聊城","芜湖","吉林","德州","信阳","新乡","宿迁","沧州","桂林","包头","常德","乐山","遵义","眉山","株洲","驻马店","日照","许昌","成都","衡水","宜宾","泸州","营口","达州","宝鸡","广安","吉安","孝感","鄂尔多斯","十堰","清远","湘潭","上饶","漯河","邢台","平顶山","菏泽","宜春","承德","赤峰","延边","茂名","焦作","内江","韶关","梅州","郴州","黄冈","红河","绥化","娄底","丹东","邵阳","开封","阳江","黄石","周口","牡丹江","长治","咸宁","汉中","自贡","衢州","晋中","永州","北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏","重庆"]
+ for(def city : citys){
+ def str = cityList.get(city)
+ for(def i=0;i<1;i++){
+ def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
+ if(districts){
+ def arrs = str.split("\\|")
+ def url = "https://"+arrs[0]+".58.com/xiaoqu/"
+ distMap.put(city, districts)
+ addCrawlKeys(city);
+ addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
+ }
+ }
+ }
+
+// cityList.each{city, str->
// for(def i=0;i<1;i++){
-// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
-// if(districts){
-// def arrs = str.split("\\|")
-// def url = "https://"+arrs[0]+".58.com/xiaoqu/"
-// distMap.put(city, districts)
-// addCrawlKeys(city);
-// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
+// if(citys.contains(city)){
+// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
+// if(districts){
+// def arrs = str.split("\\|")
+// def url = "https://"+arrs[0]+".58.com/xiaoqu/"
+// distMap.put(city, districts)
+//
+// addCrawlKeys(city);
+// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
+// }
// }
// }
// }
- cityList.each{city, str->
- for(def i=0;i<1;i++){
-// if(citys.contains(city)){
- def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
- if(districts){
- def arrs = str.split("\\|")
- def url = "https://"+arrs[0]+".58.com/xiaoqu/"
- distMap.put(city, districts)
-
- addCrawlKeys(city);
- addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
- }
-// }
- }
- }
-
}
@@ -230,6 +231,10 @@ class Esf58Crawl extends BaseBatch{
for(def i=0;i5){
+ districtName = districtName.substring(0,3)
+ }
for(def j=1;j<=districtName.length()-1;j++) {
// if(j<=3){
def keyword = districtName.toString().substring(0, j);
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy
index 0ca7c1afb..279f5f3c1 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy
@@ -15,10 +15,10 @@ class SynchDistrictPrice {
def static final SHORT_DATE = Utils.getLastMonth()
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
- def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
- "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
+// def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
+// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
//"fang.com":"搜房",
-// def sourceMap = ["fang.com":"搜房"]
+ def sourceMap = ["ke.com":"贝壳找房","fang.com":"搜房","cityhouse.cn":"城市房产"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()