This commit is contained in:
2023-05-27 11:04:47 +08:00
parent cd4029bbb2
commit 7f38659cec
9 changed files with 110 additions and 105 deletions
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳三月份成交数据.xls", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\3月坪山龙华盐田罗湖龙岗成交数据.xls", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -29,7 +29,7 @@ for(def d : datas){
map.area = d['F'].toString().toDouble()
map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
map.avg_price = (int)(d['H'].toString().toDouble()*10000)
def trade_date = DateUtils.getRandomDate("2023-03-01", "2023-03-31")
def trade_date = DateUtils.getRandomDate("2023-04-01", "2023-04-30")
map.trade_date = DateUtils.parseFromTightByPattern(trade_date, "yyyy-MM-dd")
map.source = "深房家族"
map.create_date = new Date()
@@ -56,8 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets)
@@ -78,10 +78,10 @@ class BigdataCommunityBatch extends BaseBatch{
println "txt 生成->导Oracle"
// delPgData()
// saveToPgSql(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
delPgData()
saveToPgSql(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
// delPgData()
// saveToPgSql2(datas)
@@ -259,7 +259,7 @@ class BigdataCommunityBatch extends BaseBatch{
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
sqlInstance.withBatch(1, sql){ ps->
sqlInstance.withBatch(50000, sql){ ps->
for(def r : rows) {
try{
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
@@ -430,7 +430,7 @@ class CcbCommonUtils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
// q.put("name","锦南浦三小区")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -1,5 +1,8 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore
import com.mongodb.BasicDBObject
import groovy.sql.Sql
import java.util.concurrent.ConcurrentHashMap
@@ -12,15 +15,24 @@ class TestRepeat {
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
// def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
private DSCollection seabird = new DataStore("192.168.0.100", 37017).getCollection("fangjia.seabird")
static main(args) {
def task = new TestRepeat()
task.run()
}
def run(){
def q = new BasicDBObject()
q.put("cat", "sell")
q.put("city", "ÏåÑô")
def count = seabird.count(q)
println count
// for(def d : dbcursor){
// println d.name
// }
}
@@ -39,48 +39,49 @@ class ExportDistrictRongHe extends BaseBatch{
citys = CcbCommonUtils.getRunningCitys(args)
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def new_dists = []//["350604","350605","410307","410308","450181"]
//按天导(每天一个文件夹)
// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
//// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
// exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
//
// if(curDate.toString().endsWith("10")){
// exportBigdataCommunityGJRH()//
// exportBigdataBuildingFJW()//
// exportBigdataCommfeeGj()//
//
// exportBigdataCommbuildstruGJ()//
// exportBigdataCommhousetypeGJ()//
// exportBigdataCommbuildyearGj()//
// exportBigdataCommhouseuseGJ()//
// exportBigdataPositionGJ()//
// exportBigdataDistrictFJW()//
// exportBigdataAliasFJW()//
//
// exportBigdataBuildprojFjw()
// exportBigdataCommlayoutpicFjw()
// exportEvaluateRate()
//
// exportBigdataCityPrice(queryDate)
// exportBigdataDistPrice(queryDate)
// exportBigdataCommPrice(queryDate)
//
//
//
// //1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据
//// exportCommunityBasePrice(lastMonth)
//// exportCommunityPdescribeGj(lastMonth)
//// exportGuideStatistics(lastMonth)
//// exportBigdataSupplierStatistics(lastMonth)
//// exportBigdataModelingBase(lastMonth)
//
// //2月8号 临时导出苏州一年的数据
// exportPgBigdataForsale(houseStartDate,DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出
// }
if(curDate.toString().endsWith("10")){
exportBigdataCommunityGJRH()//
exportBigdataBuildingFJW()//
exportBigdataCommfeeGj()//
exportBigdataCommbuildstruGJ()//
exportBigdataCommhousetypeGJ()//
exportBigdataCommbuildyearGj()//
exportBigdataCommhouseuseGJ()//
exportBigdataPositionGJ()//
exportBigdataDistrictFJW()//
exportBigdataAliasFJW()//
exportBigdataBuildprojFjw()
exportBigdataCommlayoutpicFjw()
exportEvaluateRate()
exportBigdataCityPrice(queryDate)
exportBigdataDistPrice(queryDate)
exportBigdataCommPrice(queryDate)
//1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据
// exportCommunityBasePrice(lastMonth)
// exportCommunityPdescribeGj(lastMonth)
// exportGuideStatistics(lastMonth)
// exportBigdataSupplierStatistics(lastMonth)
// exportBigdataModelingBase(lastMonth)
//2月8号 临时导出苏州一年的数据
exportPgBigdataForsale(houseStartDate,DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出
}
if(curDate.toString().endsWith("28")){
if(curDate.toString().endsWith("20")){
exportPgBigdataSupplierStatistics(lastMonth)
//因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供
def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
@@ -92,10 +93,10 @@ class ExportDistrictRongHe extends BaseBatch{
exportPgGuideStatistics(lastMonth)
}
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
exportOk(curDate)
// SSHHelper jschCommand = new SSHHelper();
// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
//
// exportOk(curDate)
}
//
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2023-03-01"
static def statDate = "2023-04-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年3月\\查询1.DBF");//友达二手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年5月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -83,7 +83,7 @@ class FileUtils {
def f = new File(file)
def ret = [];
if(f.exists()){
def lines = org.apache.commons.io.FileUtils.readLines(f)
def lines = org.apache.commons.io.FileUtils.readLines(f,"utf-8")
for(line in lines) {
ret << line;
}
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.paimai
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
@@ -49,27 +50,8 @@ public class BeiKeCrawlByChrome {
WebDriver driver = new ChromeDriver()
def citys = getAllCityUrls(driver)
// citys = citys.reverse()
// def index = 0
// for(def c : citys){
// index++
// if(c.city=="合肥"){
// break
// }
// }
// citys = citys[index..citys.size()-1]
def start = 0
// for(def i=0;i<citys.size();i++){
// def m = citys[i]
// println i+" "+ m.city
// if(m.city=="天津"){
// start = i
// break;
// }
//
// }
// citys = citys[(start+1)..(citys.size()-1)]
driver = getLogin(driver)
driver = getLogin(driver, "https://sh.ke.com/")
crawlData(driver, citys)
@@ -113,7 +95,7 @@ public class BeiKeCrawlByChrome {
def flag = true
while(flag){
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
driver = getLogin(driver)
driver = getLogin(driver, url)
driver.get(url)
Thread.sleep(5000)
s = driver.getPageSource()
@@ -127,8 +109,14 @@ public class BeiKeCrawlByChrome {
def crawlData(driver, citys){
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def runedCitys = FileUtils.readTxt("d:\\数据\\成交抓取\\贝壳已抓城市.txt")
for(def m : citys){
def city = m[0]
if(runedCitys.contains(city)){
println city +" 已运行"
continue
}
def url = m[1]
def newUrl = url+"/chengjiao/"
driver.get(newUrl)
@@ -145,10 +133,11 @@ public class BeiKeCrawlByChrome {
println url +" 详情链接:"+detailUrls.size()
}
}
FileUtils.batchCreateFile([[city]], "d:\\数据\\成交抓取\\贝壳已抓城市.txt", [], "utf-8", true);
// def detailUrls = getPageDeail(driver, m.city, m.url)
// println m.city +" 详情链接:"+detailUrls.size()
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
}
}
@@ -434,27 +423,32 @@ public class BeiKeCrawlByChrome {
return list
}
def getLogin(driver){
def getLogin(driver, url){
// WebDriver driver = new ChromeDriver()
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F");
def root = url.toString().substring(url.toString().indexOf("//")+2, url.toString().indexOf(".com")+4)
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252F"+root+"%252Fchengjiao%252F");
WebDriverWait wait = new WebDriverWait(driver,20)
Thread.sleep(10000)
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
try{
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
elemUsername.sendKeys("15021384995");
elemUsername.sendKeys("15021384995");
elemPassword.sendKeys("12qwaszx");
elemPassword.sendKeys("12qwaszx");
btn.submit(); // 点击行为;
Thread.sleep(10000)
btn.submit(); // 点击行为;
Thread.sleep(10000)
}catch(Exception e){
Thread.sleep(60000)
}
return driver
}
}
@@ -118,6 +118,7 @@ public class LianjiaCrawlByChrome {
}
def run(){
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
@@ -523,24 +524,21 @@ public class LianjiaCrawlByChrome {
Thread.sleep(10000)
try{
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
elemUsername.sendKeys("15021384995");
elemPassword.sendKeys("12qwaszx");
btn.submit(); // 点击行为;
Thread.sleep(10000)
}catch(Exception e){
Thread.sleep(60000)
getLogin(driver, url)
Thread.sleep(10000)
}
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
elemUsername.sendKeys("15021384995");
elemPassword.sendKeys("12qwaszx");
btn.submit(); // 点击行为;
Thread.sleep(20000)
return driver
}
}