Merge branch 'script_gj' into 'master'
Script gj See merge request !632
This commit is contained in:
+2
-2
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳三月份成交数据.xls", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\3月坪山龙华盐田罗湖龙岗成交数据.xls", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
@@ -29,7 +29,7 @@ for(def d : datas){
|
||||
map.area = d['F'].toString().toDouble()
|
||||
map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
|
||||
map.avg_price = (int)(d['H'].toString().toDouble()*10000)
|
||||
def trade_date = DateUtils.getRandomDate("2023-03-01", "2023-03-31")
|
||||
def trade_date = DateUtils.getRandomDate("2023-04-01", "2023-04-30")
|
||||
map.trade_date = DateUtils.parseFromTightByPattern(trade_date, "yyyy-MM-dd")
|
||||
map.source = "深房家族"
|
||||
map.create_date = new Date()
|
||||
|
||||
+6
-6
@@ -56,8 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
|
||||
|
||||
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
|
||||
// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
|
||||
def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets)
|
||||
rows.addAll(rets)
|
||||
@@ -78,10 +78,10 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
println "txt 生成->导Oracle"
|
||||
|
||||
|
||||
// delPgData()
|
||||
// saveToPgSql(rows)
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
|
||||
delPgData()
|
||||
saveToPgSql(rows)
|
||||
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
|
||||
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
|
||||
// delPgData()
|
||||
// saveToPgSql2(datas)
|
||||
@@ -259,7 +259,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
sqlInstance.withBatch(1, sql){ ps->
|
||||
sqlInstance.withBatch(50000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
try{
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
|
||||
|
||||
+1
-1
@@ -430,7 +430,7 @@ class CcbCommonUtils {
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉
|
||||
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
|
||||
// q.put("name","锦南浦三小区")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
||||
|
||||
+15
-3
@@ -1,5 +1,8 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.mongodb.BasicDBObject
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
@@ -12,15 +15,24 @@ class TestRepeat {
|
||||
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
// def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
private DSCollection seabird = new DataStore("192.168.0.100", 37017).getCollection("fangjia.seabird")
|
||||
|
||||
static main(args) {
|
||||
def task = new TestRepeat()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("cat", "sell")
|
||||
q.put("city", "ÏåÑô")
|
||||
def count = seabird.count(q)
|
||||
println count
|
||||
// for(def d : dbcursor){
|
||||
// println d.name
|
||||
// }
|
||||
}
|
||||
|
||||
|
||||
|
||||
+40
-39
@@ -39,48 +39,49 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
|
||||
def new_dists = []//["350604","350605","410307","410308","450181"]
|
||||
|
||||
//按天导(每天一个文件夹)
|
||||
// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
|
||||
exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
|
||||
//
|
||||
// if(curDate.toString().endsWith("10")){
|
||||
// exportBigdataCommunityGJRH()//
|
||||
// exportBigdataBuildingFJW()//
|
||||
// exportBigdataCommfeeGj()//
|
||||
//
|
||||
// exportBigdataCommbuildstruGJ()//
|
||||
// exportBigdataCommhousetypeGJ()//
|
||||
// exportBigdataCommbuildyearGj()//
|
||||
// exportBigdataCommhouseuseGJ()//
|
||||
// exportBigdataPositionGJ()//
|
||||
// exportBigdataDistrictFJW()//
|
||||
// exportBigdataAliasFJW()//
|
||||
//
|
||||
// exportBigdataBuildprojFjw()
|
||||
// exportBigdataCommlayoutpicFjw()
|
||||
// exportEvaluateRate()
|
||||
//
|
||||
// exportBigdataCityPrice(queryDate)
|
||||
// exportBigdataDistPrice(queryDate)
|
||||
// exportBigdataCommPrice(queryDate)
|
||||
//
|
||||
//
|
||||
//
|
||||
// //1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据
|
||||
//// exportCommunityBasePrice(lastMonth)
|
||||
//// exportCommunityPdescribeGj(lastMonth)
|
||||
//// exportGuideStatistics(lastMonth)
|
||||
//// exportBigdataSupplierStatistics(lastMonth)
|
||||
//// exportBigdataModelingBase(lastMonth)
|
||||
//
|
||||
// //2月8号 临时导出苏州一年的数据
|
||||
// exportPgBigdataForsale(houseStartDate,DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出
|
||||
// }
|
||||
|
||||
if(curDate.toString().endsWith("10")){
|
||||
exportBigdataCommunityGJRH()//
|
||||
exportBigdataBuildingFJW()//
|
||||
exportBigdataCommfeeGj()//
|
||||
|
||||
exportBigdataCommbuildstruGJ()//
|
||||
exportBigdataCommhousetypeGJ()//
|
||||
exportBigdataCommbuildyearGj()//
|
||||
exportBigdataCommhouseuseGJ()//
|
||||
exportBigdataPositionGJ()//
|
||||
exportBigdataDistrictFJW()//
|
||||
exportBigdataAliasFJW()//
|
||||
|
||||
exportBigdataBuildprojFjw()
|
||||
exportBigdataCommlayoutpicFjw()
|
||||
exportEvaluateRate()
|
||||
|
||||
exportBigdataCityPrice(queryDate)
|
||||
exportBigdataDistPrice(queryDate)
|
||||
exportBigdataCommPrice(queryDate)
|
||||
|
||||
|
||||
|
||||
//1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据
|
||||
// exportCommunityBasePrice(lastMonth)
|
||||
// exportCommunityPdescribeGj(lastMonth)
|
||||
// exportGuideStatistics(lastMonth)
|
||||
// exportBigdataSupplierStatistics(lastMonth)
|
||||
// exportBigdataModelingBase(lastMonth)
|
||||
|
||||
//2月8号 临时导出苏州一年的数据
|
||||
exportPgBigdataForsale(houseStartDate,DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出
|
||||
}
|
||||
|
||||
if(curDate.toString().endsWith("28")){
|
||||
if(curDate.toString().endsWith("20")){
|
||||
exportPgBigdataSupplierStatistics(lastMonth)
|
||||
//因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供
|
||||
def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
@@ -92,10 +93,10 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
exportPgGuideStatistics(lastMonth)
|
||||
}
|
||||
|
||||
SSHHelper jschCommand = new SSHHelper();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
|
||||
|
||||
exportOk(curDate)
|
||||
// SSHHelper jschCommand = new SSHHelper();
|
||||
// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
|
||||
//
|
||||
// exportOk(curDate)
|
||||
}
|
||||
|
||||
//
|
||||
|
||||
+2
-2
@@ -48,11 +48,11 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2023-03-01"
|
||||
static def statDate = "2023-04-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2023年3月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2023年5月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
|
||||
@@ -83,7 +83,7 @@ class FileUtils {
|
||||
def f = new File(file)
|
||||
def ret = [];
|
||||
if(f.exists()){
|
||||
def lines = org.apache.commons.io.FileUtils.readLines(f)
|
||||
def lines = org.apache.commons.io.FileUtils.readLines(f,"utf-8")
|
||||
for(line in lines) {
|
||||
ret << line;
|
||||
}
|
||||
|
||||
+27
-33
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.paimai
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
@@ -49,27 +50,8 @@ public class BeiKeCrawlByChrome {
|
||||
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def citys = getAllCityUrls(driver)
|
||||
// citys = citys.reverse()
|
||||
// def index = 0
|
||||
// for(def c : citys){
|
||||
// index++
|
||||
// if(c.city=="合肥"){
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// citys = citys[index..citys.size()-1]
|
||||
def start = 0
|
||||
// for(def i=0;i<citys.size();i++){
|
||||
// def m = citys[i]
|
||||
// println i+" "+ m.city
|
||||
// if(m.city=="天津"){
|
||||
// start = i
|
||||
// break;
|
||||
// }
|
||||
//
|
||||
// }
|
||||
// citys = citys[(start+1)..(citys.size()-1)]
|
||||
driver = getLogin(driver)
|
||||
|
||||
driver = getLogin(driver, "https://sh.ke.com/")
|
||||
crawlData(driver, citys)
|
||||
|
||||
|
||||
@@ -113,7 +95,7 @@ public class BeiKeCrawlByChrome {
|
||||
def flag = true
|
||||
while(flag){
|
||||
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
|
||||
driver = getLogin(driver)
|
||||
driver = getLogin(driver, url)
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
@@ -127,8 +109,14 @@ public class BeiKeCrawlByChrome {
|
||||
def crawlData(driver, citys){
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def runedCitys = FileUtils.readTxt("d:\\数据\\成交抓取\\贝壳已抓城市.txt")
|
||||
|
||||
for(def m : citys){
|
||||
def city = m[0]
|
||||
if(runedCitys.contains(city)){
|
||||
println city +" 已运行"
|
||||
continue
|
||||
}
|
||||
def url = m[1]
|
||||
def newUrl = url+"/chengjiao/"
|
||||
driver.get(newUrl)
|
||||
@@ -145,10 +133,11 @@ public class BeiKeCrawlByChrome {
|
||||
println url +" 详情链接:"+detailUrls.size()
|
||||
}
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile([[city]], "d:\\数据\\成交抓取\\贝壳已抓城市.txt", [], "utf-8", true);
|
||||
// def detailUrls = getPageDeail(driver, m.city, m.url)
|
||||
// println m.city +" 详情链接:"+detailUrls.size()
|
||||
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -434,27 +423,32 @@ public class BeiKeCrawlByChrome {
|
||||
return list
|
||||
}
|
||||
|
||||
def getLogin(driver){
|
||||
def getLogin(driver, url){
|
||||
// WebDriver driver = new ChromeDriver()
|
||||
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F");
|
||||
def root = url.toString().substring(url.toString().indexOf("//")+2, url.toString().indexOf(".com")+4)
|
||||
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252F"+root+"%252Fchengjiao%252F");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
Thread.sleep(10000)
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
|
||||
try{
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
|
||||
|
||||
|
||||
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
|
||||
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
|
||||
|
||||
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
|
||||
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
|
||||
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
|
||||
elemUsername.sendKeys("15021384995");
|
||||
elemUsername.sendKeys("15021384995");
|
||||
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(10000)
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(10000)
|
||||
}catch(Exception e){
|
||||
Thread.sleep(60000)
|
||||
}
|
||||
return driver
|
||||
}
|
||||
}
|
||||
|
||||
+14
-16
@@ -118,6 +118,7 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
@@ -523,24 +524,21 @@ public class LianjiaCrawlByChrome {
|
||||
Thread.sleep(10000)
|
||||
try{
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
|
||||
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
|
||||
|
||||
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
|
||||
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
elemUsername.sendKeys("15021384995");
|
||||
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(10000)
|
||||
}catch(Exception e){
|
||||
Thread.sleep(60000)
|
||||
getLogin(driver, url)
|
||||
Thread.sleep(10000)
|
||||
}
|
||||
|
||||
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
|
||||
|
||||
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
|
||||
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
|
||||
elemUsername.sendKeys("15021384995");
|
||||
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(20000)
|
||||
return driver
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user