Merge branch 'script_gj' into 'master'

Script gj

See merge request !235
This commit is contained in:
2021-12-28 16:50:21 +08:00
8 changed files with 327 additions and 107 deletions
@@ -36,9 +36,9 @@ class BigdataBuildingBatch extends BaseBatch{
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap)
println city + " " + rets.size()
def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat"
def fileName = "/home/goojia/ccb/bigdata_building_fjw.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat"
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_building_fjw.dat"
}
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true);
}
@@ -24,30 +24,29 @@ class ExportDistrictRongHe extends BaseBatch{
@Override
void process(Object args) {
// def citys = CcbCommonUtils.citys;
def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"]
def citys = CcbCommonUtils.citys;
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// exportAllBigdataDealFJW()
for(def city : citys) {
def cityCode = cityCodeDataMap.get(city)
// exportBigdataCommunityGJRH(cityCode)
// exportBigdataBuildingFJW(cityCode)
// exportBigdataCommfeeGj(cityCode)
//
exportBigdataDealFJW(cityCode)
// exportBigdataCommbuildstruGJ(cityCode)
// exportBigdataCommhousetypeGJ(cityCode)
// exportBigdataCommbuildyearGj(cityCode)
// exportBigdataCommhouseuseGJ(cityCode)
// exportBigdataPositionGJ(cityCode)
// exportBigdataDistrictFJW(cityCode)
exportBigdataCommunityGJRH(cityCode)
exportBigdataBuildingFJW(cityCode)
exportBigdataCommfeeGj(cityCode)
exportBigdataDealFJW(cityCode, '2021-11-01')
exportBigdataCommbuildstruGJ(cityCode)
exportBigdataCommhousetypeGJ(cityCode)
exportBigdataCommbuildyearGj(cityCode)
exportBigdataCommhouseuseGJ(cityCode)
exportBigdataPositionGJ(cityCode)
exportBigdataDistrictFJW(cityCode)
exportBigdataAliasFJW(cityCode)
//
// exportBigdataBuildprojFjw(cityCode)
// exportBigdataCommlayoutpicFjw(cityCode)
exportBigdataBuildprojFjw(cityCode)
exportBigdataCommlayoutpicFjw(cityCode)
}
// exportEvaluateRate()
exportEvaluateRate()
}
@@ -311,94 +310,110 @@ class ExportDistrictRongHe extends BaseBatch{
println "ADDRESS_ALIAS_FJW 上传Ftp完成......"
}
def exportBigdataDealFJW(cityCode){
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}' and dealdate>=to_date('2021-06-01','yyyy-MM-dd')";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DEALID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.STREET
data << r.STREETNO
data << r.COMMID
data << r.COMMNAME
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.UNITID
data << r.UNITNO
data << r.UNITNAME
data << r.HOUSEID
data << r.ADDRESS
data << r.BUILDYEAR
data << r.PROPTYPE
data << r.PROPYEARS
data << r.PERIODNUM
data << r.FLOORNUM
data << r.FLOORTYPE
data << r.FLOORCNT
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.BUILDSTRU
data << r.BUILDAREA
data << r.LIVINGAREA
data << r.HOUSESTYLE
data << r.ROOMCNT
data << r.LIVINGROOMCNT
data << r.KITCHENCNT
data << r.BATHROOMCNT
data << r.HOUSEUSE
data << r.HOUSESTRU
data << r.HOUSEASPECT
data << r.FITMENT
data << r.EVALUATENO
data << r.DEALUNITPRICE
data << r.DEALTOTALPRICE
data << r.DECLAREPRICE
data << r.EVALUATEPRICE
data << r.CONTRACTNO
data << r.DEALDATE
data << r.DEALTYPE
data << r.PAYTYPE
data << r.DEALPERIOD
data << r.IFELEVATOR
data << r.COMPAREAGARDEN
data << r.COMPAREATERRACE
data << r.COMPAREABALCONY
data << r.COMPAREABAYWINDOW
data << r.COMPAREAATTIC
data << r.COMPAREAGARAGE
data << r.COMPAREABASEMENT
data << r.COMPAREAOTHER
data << r.DATASOURCE
data << r.AGENCYID
data << r.AGENCYNAME
data << r.REMARK
data << r.CREATETIME1
data << r.UPDATETIME1
data << r.SOURCECODE
data << r.WORKDATE
allRows << data
});
def exportBigdataDealFJW(cityCode, dealdate){
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def stop = false
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
def index = 0
while(!stop){
index++
def dealStartStr = startDateStr
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${dealStartStr.replace("-","")}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${dealStartStr.replace("-","")}_0001.dat"
}
def dealEndStr = DateUtils.add(startDateStr, 1)
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}' and dealdate>=to_date('${dealStartStr}','yyyy-MM-dd') and dealdate<to_date('${dealEndStr}','yyyy-MM-dd')";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DEALID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.STREET
data << r.STREETNO
data << r.COMMID
data << r.COMMNAME
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.UNITID
data << r.UNITNO
data << r.UNITNAME
data << r.HOUSEID
data << r.ADDRESS
data << r.BUILDYEAR
data << r.PROPTYPE
data << r.PROPYEARS
data << r.PERIODNUM
data << r.FLOORNUM
data << r.FLOORTYPE
data << r.FLOORCNT
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.BUILDSTRU
data << r.BUILDAREA
data << r.LIVINGAREA
data << r.HOUSESTYLE
data << r.ROOMCNT
data << r.LIVINGROOMCNT
data << r.KITCHENCNT
data << r.BATHROOMCNT
data << r.HOUSEUSE
data << r.HOUSESTRU
data << r.HOUSEASPECT
data << r.FITMENT
data << r.EVALUATENO
data << r.DEALUNITPRICE
data << r.DEALTOTALPRICE
data << r.DECLAREPRICE
data << r.EVALUATEPRICE
data << r.CONTRACTNO
data << r.DEALDATE
data << r.DEALTYPE
data << r.PAYTYPE
data << r.DEALPERIOD
data << r.IFELEVATOR
data << r.COMPAREAGARDEN
data << r.COMPAREATERRACE
data << r.COMPAREABALCONY
data << r.COMPAREABAYWINDOW
data << r.COMPAREAATTIC
data << r.COMPAREAGARAGE
data << r.COMPAREABASEMENT
data << r.COMPAREAOTHER
data << r.DATASOURCE
data << r.AGENCYID
data << r.AGENCYNAME
data << r.REMARK
data << r.CREATETIME1
data << r.UPDATETIME1
data << r.SOURCECODE
data << r.WORKDATE
allRows << data
});
if(dealStartStr == endDateStr){
stop = true
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
println "BIGDATA_DEAL_FJW 上传Ftp完成......${dealStartStr}"
startDateStr = dealEndStr
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
println "BIGDATA_DEAL_FJW 上传Ftp完成......"
}
def exportAllBigdataDealFJW(){
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ";
def allRows = []
@@ -0,0 +1,114 @@
package com.fangjia.script.gj
import com.fangjia.core.util.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.text.ParseException
import java.text.SimpleDateFormat
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class UpdateStatWeekend {
public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", mongoDbUrl)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
static main(args) {
def task = new UpdateStatWeekend()
task.run()
}
def run(){
//"合肥","上海","北京","广州","杭州","深圳","石家庄","天津","武汉","西安","重庆",
def citys = ["苏州"]
for(def city : citys){
def rets = findAddData(city)
process(city, rets)
}
}
def process(def city, def rets){
def index = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def r = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
index.addAndGet(1)
updateProcess(r)
println index+"/"+rets.size() +" "+r.id
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def updateProcess(def r){
def curDate = DateUtils.parseFromTightYYYYMMDDhhmmss("2021-12-28 11:00:00")
if(r.m_date.getTime() < curDate.getTime()){
def weekendStr = DateUtils.formatDateByPattern(r.weekend, "yyyy-MM-dd")
weekendStr = DateUtils.add(weekendStr, -1)
def weekend = DateUtils.parseFromTightYYYYMMDD(weekendStr);
weekend = mongoDateTran(weekend)
DBObject data = new BasicDBObject()
data.put("weekend", weekend)
data.put("m_date", new Date())
try{
DIST_STAT_COLL.update(new BasicDBObject('_id', r.id), new BasicDBObject('$set', data))
}catch(Exception e){
DIST_STAT_COLL.remove(new BasicDBObject('_id', r.id))
println "[重复] "+ r.id
}
}
}
public static Date mongoDateTran(Date date) {
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String dateStr = format.format(date);
format.setCalendar(new GregorianCalendar(new SimpleTimeZone(-57600000, "GMT")));
try {
return format.parse(dateStr);
} catch (ParseException e) {
return date;
}
}
def findAddData(city){
def q = new BasicDBObject()
q.put("city",city)
q.put("c_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-12-27")))
def dbcursor = DIST_STAT_COLL.find(q)
dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def rets = []
for(def d : dbcursor){
rets << ["id":d._id, "weekend":d.weekend,"c_date":d.c_date,"m_date":d.m_date]
if(rets.size()/10000==0){
print "."
}
}
return rets;
}
}
@@ -0,0 +1,91 @@
package com.fangjia.script.temp.shenjian.batch
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import org.apache.http.client.utils.DateUtils
import java.util.regex.Matcher
import java.util.regex.Pattern
/**
* 只保留86机器最近一个月索引
*/
@BatchOwner(Watcher.ShenJian)
class RemoveOldIndexOfEs86 extends BaseBatch {
static void main(String[] args) {
runBatch(new RemoveOldIndexOfEs86(), args)
}
@Override
void process(Object args) {
String url = "http://192.168.0.86:9200/_cat/indices"
// 必须分开写,不能有空格
String[] cmds = ["curl", "-XGET", url, "--user", "elastic:goojia123456"]
List<List<String>> indices = execCurl(cmds)
Pattern yearMonthPattern = Pattern.compile("\\d{6}")
Pattern yearMonthPattern1 = Pattern.compile("\\d{4}\\.\\d{2}")
List<String> indexNeedDel = new ArrayList<>()
for (List<String> index : indices) {
String name = index.get(2)
// 忽略系统索引
if (name.startsWith(".")) {
continue
}
// 匹配含有年月的索引
Matcher matcher = yearMonthPattern.matcher(name)
if (matcher.find()) {
String date = matcher.group()
if (dateBeforeCurrMonth(date)) {
indexNeedDel.add(name)
}
continue
}
matcher = yearMonthPattern1.matcher(name)
if (matcher.find()) {
String date = matcher.group()
if (dateBeforeCurrMonth(date)) {
indexNeedDel.add(name)
}
}
}
for (String index : indexNeedDel) {
println "删除索引" + index
String urlPrefix = "http://192.168.0.86:9200/"
cmds = ["curl", "-XDELETE", urlPrefix + index, "--user", "elastic:goojia123456"]
execCurl(cmds)
}
}
static List<List<String>> execCurl(String[] cmds) {
ProcessBuilder process = new ProcessBuilder(cmds)
Process p
try {
List<String> result = new ArrayList<>()
p = process.start()
BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream()))
String line = null
while ((line = reader.readLine()) != null) {
result.add(line.split("{1} +"))
}
return result
} catch (IOException e) {
e.printStackTrace()
}
return null
}
def boolean dateBeforeCurrMonth(String date) {
String dateCurr = DateUtils.formatDate(new Date(), "yyyyMM");
if (date.contains(dateCurr)) {
return false
}
dateCurr = DateUtils.formatDate(new Date(), "yyyy.MM");
if (date.contains(dateCurr)) {
return false
}
return true
}
}
@@ -22,7 +22,7 @@ import java.util.concurrent.Executors
class SeaweedAroundCommittee {
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi"
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi"
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) {
@@ -28,7 +28,7 @@ import java.util.concurrent.Executors
class SeaweedAroundPoi extends BaseBatch {
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi"
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi"
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) {
@@ -28,7 +28,7 @@ import java.util.concurrent.Executors
class SeaweedAroundPoiSingal {
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi"
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi"
static void main(String[] args) {
def types = listAllType()
@@ -14,7 +14,7 @@ import org.apache.commons.collections.CollectionUtils
*/
class YinlianLngLatAndAround {
static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi"
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi"
static void main(String[] args) {
this.getAround()