43改为域名方式

This commit is contained in:
2025-04-14 14:02:02 +08:00
parent b958ae87a3
commit 78e0f3c202
129 changed files with 480 additions and 240 deletions
+12
View File
@@ -543,6 +543,18 @@
<version>1.18.2</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-text</artifactId>
<version>1.9</version>
</dependency>
<dependency>
<groupId>com.hankcs.hanlp.restful</groupId>
<artifactId>hanlp-restful</artifactId>
<version>0.0.12</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.fangjia</groupId>-->
<!-- <artifactId>fangjia-open</artifactId>-->
@@ -21,7 +21,7 @@ import java.sql.DriverManager
@BatchOwner(Watcher.WangWenYan)
class ClickHouseDistrictStatMarkupCalcBatch extends BaseBatch {
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 27017)
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo", 27017)
static void main(String[] args) {
runBatch(new ClickHouseDistrictStatMarkupCalcBatch(), args)
}
@@ -18,7 +18,7 @@ import com.mongodb.DBCollection
@BatchOwner(Watcher.WangWenYan)
class DistrictStatMarkupCalcBatch extends BaseBatch {
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 27017)
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com", 27017)
Date oneYearAgo = DateUtil.add(new Date(), Calendar.MONTH, -1)
@@ -8,8 +8,10 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2025年01月深圳二手房成交的.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
//没有成交时间,随机生成
@@ -18,7 +18,7 @@ import java.util.concurrent.atomic.AtomicLong
*/
class AddThirdPriceSample {
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
public static final DBCollection NO_PRICE_DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "deal_price_all", "mongo.db.fangjia.com")
@@ -43,7 +43,7 @@ class BigdataDealBatch extends BaseBatch{
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -45,7 +45,7 @@ class DistrictDelete {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static main(args) {
def task = new DistrictDelete()
@@ -27,7 +27,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07",
@@ -36,7 +36,7 @@ class TradeStatLog {
def pssqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
@@ -18,10 +18,10 @@ import com.mongodb.QueryOperators
import groovy.sql.Sql
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
DBCollection CITY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "city_stat", "192.168.0.43")
DBCollection CITY_INFO = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "192.168.0.43")
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
DBCollection CITY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "city_stat", "mongo.db.fangjia.com")
DBCollection CITY_INFO = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "mongo.db.fangjia.com")
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -12,7 +12,7 @@ class MatchDealPriceRegion {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
@@ -21,7 +21,7 @@ import org.apache.lucene.queryparser.classic.QueryParser
*/
class EvaluationBatch {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -18,7 +18,7 @@ class FumingTest {
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
@@ -38,7 +38,7 @@ class HailanTest {
}
static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
@@ -19,7 +19,7 @@ import java.util.concurrent.atomic.AtomicLong
class PocStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl)
public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", lineUrl)
@@ -15,7 +15,7 @@ class RuiFenPocTest {
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
@@ -99,7 +99,7 @@ class CommonDatautils {
}
public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009")
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com", 28017)
public static final DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com")
@@ -23,7 +23,7 @@ import org.apache.commons.lang.StringUtils
class FewerHouseDistrictCrawlCommandBatch extends BaseBatch {
private static
final DBCollection ARTEMIS_COLL = MongoUtils.getCollectionByName("fangjia_log", "artemis_message", "mongo.db.fangjia.com")
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43")
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static void main(String[] args) {
runBatch(new FewerHouseDistrictCrawlCommandBatch(), args)
@@ -15,7 +15,7 @@ import org.bson.types.ObjectId
//new FewerHouseDistrictCrawlCommandBatch().process(null);
//return;
//
//DBCollection col_43_artemis_message = MongoUtils.getCollectionByName("fangjia_log", "artemis_message", "192.168.0.43", 27017);
//DBCollection col_43_artemis_message = MongoUtils.getCollectionByName("fangjia_log", "artemis_message", "mongo.db.fangjia.com", 27017);
//DBCursor cursor_artemis_message = col_43_artemis_message.find(new BasicDBObject("status", 0));
//int count = 0;
//while(cursor_artemis_message.hasNext()){
@@ -21,8 +21,8 @@ import java.util.concurrent.TimeUnit
@BatchOwner(Watcher.GaoJuan)
class DistrictTradeHouseStatBatch extends BaseBatch{
public static DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
public static DBCollection DISTRICT_TRADE_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_trade_stat", "192.168.0.43")
public static DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static DBCollection DISTRICT_TRADE_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_trade_stat", "mongo.db.fangjia.com")
static main(args) {
runBatch(new DistrictTradeHouseStatBatch(), args)
@@ -23,7 +23,7 @@ import org.elasticsearch.common.xcontent.XContentType
*/
@BatchOwner(Watcher.GaoJuan)
class SynchDealPriceToEs extends BaseBatch{
static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com",28017)
static main(args) {
runBatch(new SynchDealPriceToEs(), args)
@@ -20,7 +20,7 @@ class AbcEvaluationRowsBatch extends BaseBatch{
static final def fjtSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", dbUrl)
static main(args) {
@@ -29,7 +29,7 @@ class DistrictInfoExportBatch extends BaseBatch{
def charset = "GBK"
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection FZ_ADDR_COLL = MongoUtils.getCollectionByName2("business", "fz_district_address", lineUrl)
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName2("business", "fz_district", lineUrl)
@@ -23,7 +23,7 @@ class FzDataResultStat {
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName2("business", "fz_district", lineUrl)
@@ -16,7 +16,7 @@ import net.sf.json.JSONObject
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
import com.fangjia.core.util.HttpRequest
@@ -169,7 +169,7 @@ DingDingMessageUtil.sendTextMessage(access_token, s)
//
//
//def count(name, sDate){
// DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
// DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
// def q = new BasicDBObject()
// q.put("city","¸£ÖÝ")
// q.put("s_date", sDate)
@@ -21,7 +21,7 @@ class ThirdPriceExportBatch extends BaseBatch{
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
@@ -16,7 +16,7 @@ import java.util.concurrent.CountDownLatch
class ThirdTest {
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
static main(args) {
@@ -25,7 +25,7 @@ class TradeExportBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
static def districtMap = [:]
@@ -30,7 +30,7 @@ import java.util.concurrent.atomic.AtomicInteger
class CzBankStat {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("business", "average_statistics", "192.168.0.43")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("business", "average_statistics", "mongo.db.fangjia.com")
def static final DBCollection CZLOG = MongoUtils.getCollectionByName("fangjia_log", "cz_log2", "192.168.0.136")
static main(args) {
def task = new CzBankStat();
@@ -16,7 +16,7 @@ import org.apache.commons.lang.StringUtils
import java.util.regex.Pattern
class DealPriceProcess {
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
def total = 0
static main(args) {
@@ -17,7 +17,7 @@ import java.util.concurrent.Executors
class DistrictDetailBatch {
public static DBCollection DISTRICT_DETAIL_COLL = MongoUtils.getCollectionByName("fangjia_youfang", "district_detail", "192.168.0.43")
public static DBCollection DISTRICT_DETAIL_COLL = MongoUtils.getCollectionByName("fangjia_youfang", "district_detail", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
@@ -40,7 +40,7 @@ class EsHouseStat {
dataSource.setProperties(connectionProperties)
}
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
public static void main(String[] args){
@@ -16,8 +16,8 @@ import java.sql.ResultSet
class FZDataExport {
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
@@ -8,7 +8,7 @@ import com.mongodb.DBCollection
class FZHouseEvaluation {
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", dbUrl)
def regionMap = [:]
@@ -8,7 +8,7 @@ import com.mongodb.DBCollection
class FjwUserStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection USER_COLL = MongoUtils.getCollectionByName2("fangjia_coresite", "user", lineUrl)
@@ -18,7 +18,7 @@ import org.bson.Document;
import org.bson.conversions.Bson;
class HouseLouDongMatch {
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", dbUrl)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
@@ -22,7 +22,7 @@ class LandDetailCrawl {
DBCollection LAND_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "land_deal_price1", "192.168.0.235", 27018)
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection FJT_LAND_COLL = MongoUtils.getCollectionByName2("business", "land_distinct", lineUrl)
public static final DBCollection OFF_FJT_LAND_COLL = MongoUtils.getCollectionByName2("business", "land_distinct", offlineUrl)
@@ -14,10 +14,10 @@ import org.apache.lucene.queryparser.classic.QueryParser
class LogStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static main(args) {
def task = new LogStat()
@@ -21,8 +21,8 @@ import java.util.concurrent.Executors
@BatchOwner(Watcher.GaoJuan)
class NoHouseDistrictStatBatch extends BaseBatch{
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
// public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
// public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", dbUrl)
public static final DBCollection AVERAGER_CONFIG_COLL = MongoUtils.getCollectionByName2("fangjia", "averager_config", dbUrl)
@@ -13,7 +13,7 @@ class PingAnAddressLevalTest {
String token = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB";
DBCollection DIST_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_grade", "192.168.0.43")
DBCollection DIST_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_grade", "mongo.db.fangjia.com")
static main(args) {
def task = new PingAnAddressLevalTest()
task.run();
@@ -329,7 +329,7 @@ def findDistrictPrice(def city, def region){
q.put("region", region);
}
//districtStatConn.addOption(Bytes.QUERYOPTION_SLAVEOK);
final def districtStatConn = Mongo3Utils.getDbCollection("mongodb://developer:G00IDCDEV12385@192.168.0.43:28017","fangjia_stat", "district_stat");
final def districtStatConn = Mongo3Utils.getDbCollection("mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017","fangjia_stat", "district_stat");
def count = districtStatConn.count(q)
return count;
}
@@ -343,7 +343,7 @@ def findDistrictPrice(def city, def region){
q.put("region", region);
}
//districtStatConn.addOption(Bytes.QUERYOPTION_SLAVEOK);
final def districtStatConn = Mongo3Utils.getDbCollection("mongodb://developer:G00IDCDEV12385@192.168.0.43:28017","fangjia_stat", "district_stat");
final def districtStatConn = Mongo3Utils.getDbCollection("mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017","fangjia_stat", "district_stat");
def count = districtStatConn.count(q)
return count;
}
@@ -6,7 +6,7 @@ import com.mongodb.DBCollection
class RegionBlockSetting {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection REGION_BLOCK_COLL = MongoUtils.getCollectionByName2("fangjia", "region_block", lineUrl)
static main(args) {
@@ -17,8 +17,8 @@ import com.mongodb.DBObject
class SynchDistrictStatBatch extends BaseBatch{
//public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
//public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com", 28017)
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", lineUrl)
@@ -15,7 +15,7 @@ import java.util.concurrent.atomic.AtomicLong
class UpdateThirdPrice {
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
def static final DBCollection AVERAGE_STATISTICS_OFFLINE = CrawalDBUtil.getAverage_statistics_col();
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
@@ -14,7 +14,7 @@ import groovy.sql.Sql
class ApiUserCountAnalyBatch extends BaseBatch{
static final dbFangjia = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia?autoReconnect=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver");
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection AUTHORITY_COLL = MongoUtils.getCollectionByName2("fangjia", "authority.api_user_authority", lineUrl)
DBCollection VISIT_COLL = MongoUtils.getCollectionByName2("business", "api_visit_count", lineUrl)
@@ -10,7 +10,7 @@ import com.mongodb.DBCollection
class CcbCityPriceStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
@@ -20,7 +20,7 @@ import com.mongodb.DBCollection
*/
class PocTest2023 {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
@@ -15,7 +15,7 @@ import org.apache.commons.lang.math.RandomUtils
class DistrictSelectHouse {
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static main(args) {
def task = new DistrictSelectHouse();
@@ -9,8 +9,8 @@ import com.mongodb.QueryOperators
class TestPrice {
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
@@ -17,7 +17,7 @@ import org.elasticsearch.index.query.QueryBuilders
class CityStatExport {
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", dbUrl)
@@ -16,7 +16,7 @@ import java.util.concurrent.atomic.AtomicLong
class CityStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", lineUrl)
public static final DBCollection BAD_CASE_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "bad_case", lineUrl)
@@ -21,13 +21,13 @@ import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43")
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
//def citys = ["ÎÚ³ľÆë","²ý¼ª","°¢¿ËËÕ","ÒÁÀç","¿¦Ê²"]
@@ -11,7 +11,7 @@ import com.mongodb.QueryOperators
class BestDistrictPriceExport {
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static main(args) {
def task = new BestDistrictPriceExport();
@@ -5,10 +5,8 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import java.util.regex.Pattern
/**
* Ãû¼¼¯
*/
@@ -34,6 +32,7 @@ class MinglujiData {
def prov = "½­ËÕÊ¡"
def city = "ÎÞÎý"
exportDatas(city)
// exportDatas(city)
// matchDev(prov, city)
// matchCommunity(city)
// fiveLevelSplit(city)
@@ -4,9 +4,12 @@ import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
@@ -33,10 +36,14 @@ class RegionStandardBatch extends BaseBatch{
void process(Object args) {
seaweedRegionStandard()
// poiRegionStandard()
// seaweedRegionStandard()
// poiRegionStandard()
//通过乡镇街道修改区域 如:东莞市
// updateRegionByTown()
}
def seaweedRegionStandard(){
seaweedRegionStandardByCat("district")
seaweedRegionStandardByCat("office")
}
def poiRegionStandard(){
@@ -93,7 +100,7 @@ class RegionStandardBatch extends BaseBatch{
def seaweedRegionStandardByCat(cat){
def citys = CommonDatautils.findAllCity()
def citys = ["伊春"]//CommonDatautils.findAllCity()
for(def city : citys){
def rMap = regionMap.get(city)
@@ -148,8 +155,12 @@ class RegionStandardBatch extends BaseBatch{
def streetMap = CommonDatautils.queryAllStreetNameByCity(fullCityName)
if(streetMap){
set = streetMap.keySet()
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"])
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","lng","lat"])
println city +" "+ districts.size()
def index = 0
for(def d : districts){
index ++
println index +" "+ d.region
if(set.contains(d.region)){
continue
}
@@ -157,8 +168,12 @@ class RegionStandardBatch extends BaseBatch{
def updateRegion = null
def flag = null
for(def key : set){
def alias = streetMap.get(key)
if((alias && alias.contains(d.region)) || (!alias && key.contains(d.region))){
def obj = streetMap.get(key)
def alias = null
if(obj && obj.alias){
alias = obj.alias
}
if((alias && alias.contains(d.region)) || (!alias && key==d.region)){
def newRegion = RegionData.formatRegion(d.city, key)
if(newRegion!=d.region){
updateRegion = newRegion
@@ -166,13 +181,26 @@ class RegionStandardBatch extends BaseBatch{
flag = "pass"
break;
}
if(obj.shape){
def inAreaFlag = InsidePolygon.isInArea(d.lng/10000000000.0, d.lat/10000000000.0, InsidePolygon.createPoint4(obj.shape.coordinates))
if(inAreaFlag){
def newRegion = obj.street
newRegion = RegionData.formatRegion(d.city, newRegion)
if(newRegion!=d.region){
updateRegion = newRegion
}
flag = "pass"
break;
}
}
}
if(updateRegion){
def udata = new BasicDBObject()
udata.put("region", updateRegion)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name +" ÇøÓòÐÞ¸ÄΪ£º"+updateRegion
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}else{
if(flag!="pass"){
def udata = new BasicDBObject()
@@ -402,4 +430,27 @@ class RegionStandardBatch extends BaseBatch{
return rets;
}
def updateRegionByTown(){
ExcelUtils.modify("d:\\数据\\城市与区域相同写字楼.xlsx",{d->
def town = RegionData.formatRegion(d['B'], d['F'])
if(town){
def obj = CommonDatautils.findDistrict(d['B'], town, d['E'], "office")
if(!obj){
def udata = new BasicDBObject()
udata.put("region", town)
udata.put("m_date", new Date())
println d['F']+" "+" 区域修改为:" + town
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
}else{
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("status", 1)
println d['F']+" "+" 删除:" + town
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
}
}
})
}
}
@@ -16,7 +16,7 @@ class RegionUpdateBatch {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static main(args) {
def task = new RegionUpdateBatch()
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
@@ -219,34 +220,41 @@ class TempExportData {
}
def exportOfficeHouseCount(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\写字楼\\待核实写字楼.xlsx",{d->
ExcelUtils.multiThreadingModify("D:\\数据\\高德写字楼-房源第三方验证.xlsx",{d->
// def obj = findYuanQu(d['B'], d['D'])
// if(obj){
// d['T'] = "ǰհ´æÔÚ"
// }
// if(!d['T']){
// def obj = findYuanQu(d['B'], d['D'])
// def city = RegionData.formatCity(d['C'])
// def obj = findYuanQu(city, d['E'])
// if(obj){
// d['T'] = "前瞻存在"
// d['U'] = obj.name
// }
// }
// if(!d['T']) {
// def count = findHouseCountByCat(d['B'], d['C'], d['D'], "office_sell")
// def city = RegionData.formatCity(d['C'])
// def region = RegionData.formatRegion(city, d['D'])
//
// def count = findHouseCountByCat(city, region, d['E'], "office_sell")
// if (!count) {
// count = findHouseCountByCat(d['B'], d['C'], d['D'], "office_rent")
// count = findHouseCountByCat(city, region, d['E'], "office_rent")
// }
// d['S'] = count
// d['T'] = count
// }
if(!d['J']){
def datas = CommonDatautils.findThirdNetWorkOffices(d['B'], d['C'], d['D'])
if(!d['T']){
def city = RegionData.formatCity(d['C'])
def region = RegionData.formatRegion(city, d['D'])
def datas = CommonDatautils.findThirdNetWorkOffices(city, region, d['E'])
if(datas){
def sources = []
for(def o : datas){
sources << o.source
}
if(sources){
d['K'] = sources.join(",")
d['V'] = sources.join(",")
}
}
}
@@ -1,58 +1,57 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.math.RandomUtils
def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl)
def city = "昌江"
ExcelUtils.modify("D:\\数据\\海南\\海南省城市均价.xlsx",{d->
def date = d['B']
def start = DateUtils.getFirstDayOfMonth(date)
def end = DateUtils.getLastDayOfMonth(date)
def weekends = DateUtils.getWeekendBetweenDates(start, end)
for(def weekend : weekends){
def avg_price = d['I']
def val = random(-200,200)
def new_price = (int)(avg_price*val/10000 + avg_price)
def last_weekend = DateUtils.add(DateUtils.formatDateByPattern(weekend, "yyyy-MM-dd"), -7)
def last_price = CommonDatautils.queryCityPriceByWeekend(city, last_weekend)
def o = new BasicDBObject()
o.put("avg_price", new_price)
o.put("c_date" , new Date())
o.put("city", city)
o.put("m_date" , new Date())
o.put("weekend", weekend)
o.put("cat","城市")
o.put("moving_average" , (int)avg_price)
if(!last_price){
o.put("weekend_markup" , 0)
}else{
def weekend_markup = (int)((new_price-last_price)*100/last_price)
o.put("weekend_markup" , weekend_markup)
}
def q = new BasicDBObject("cat", "城市").append("city", city).append("weekend", weekend)
REGION_STAT_COLL.update(q, new BasicDBObject('$set', o),true, false)
}
},1)
public static final int random(int i1, int i2) {
int max = Math.max(i1, i2);
int min = Math.min(i1, i2);
return min + RandomUtils.nextInt(max - min + 1);
}
import org.bson.types.ObjectId
ObjectId objectId = new ObjectId("659c9ae92ae7732fd7d54e40");
Date creationDate = objectId.getDate();
System.out.println(creationDate);
//def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
//DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl)
//
//
//def city = "昌江"
//ExcelUtils.modify("D:\\数据\\海南\\海南省城市均价.xlsx",{d->
// def date = d['B']
// def start = DateUtils.getFirstDayOfMonth(date)
// def end = DateUtils.getLastDayOfMonth(date)
// def weekends = DateUtils.getWeekendBetweenDates(start, end)
// for(def weekend : weekends){
// def avg_price = d['I']
// def val = random(-200,200)
// def new_price = (int)(avg_price*val/10000 + avg_price)
// def last_weekend = DateUtils.add(DateUtils.formatDateByPattern(weekend, "yyyy-MM-dd"), -7)
// def last_price = CommonDatautils.queryCityPriceByWeekend(city, last_weekend)
//
// def o = new BasicDBObject()
// o.put("avg_price", new_price)
// o.put("c_date" , new Date())
// o.put("city", city)
// o.put("m_date" , new Date())
// o.put("weekend", weekend)
// o.put("cat","城市")
// o.put("moving_average" , (int)avg_price)
// if(!last_price){
// o.put("weekend_markup" , 0)
// }else{
// def weekend_markup = (int)((new_price-last_price)*100/last_price)
// o.put("weekend_markup" , weekend_markup)
// }
//
// def q = new BasicDBObject("cat", "城市").append("city", city).append("weekend", weekend)
// REGION_STAT_COLL.update(q, new BasicDBObject('$set', o),true, false)
// }
//},1)
//
//
//public static final int random(int i1, int i2) {
// int max = Math.max(i1, i2);
// int min = Math.min(i1, i2);
// return min + RandomUtils.nextInt(max - min + 1);
//}
//
//
//
//
//
@@ -3,23 +3,22 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.BaiduMapUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.client.AggregateIterable
import org.bson.Document
import net.sf.json.JSONObject
import org.bson.types.ObjectId
import java.util.regex.Pattern
/**
* 乡镇街道中间表
* 包含市,区,乡镇街道别名,来源 统计局community表
@@ -45,6 +44,8 @@ class TownMiddleProcess {
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
static DBCollection POLYLINECOLL2025 = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline_2025")
def fullCityMap = CommonDatautils.queryFullNameCity()
static main(args) {
@@ -96,6 +97,12 @@ class TownMiddleProcess {
// checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中
// addSpecialCityTownAlias()
// updateNewRegionPoyline()//更新新的区域围栏
updateMappingPolyline()
}
@@ -681,7 +688,7 @@ class TownMiddleProcess {
}
def regionPolyline(){
def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district"))
def dbcursor = POLYLINECOLL2025.find(new BasicDBObject("level","district"))
def map = [:]
for(def d : dbcursor){
def resultMap = [:]
@@ -1356,4 +1363,90 @@ class TownMiddleProcess {
keys.put("street", 1)
return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).toList()
}
def updateNewRegionPoyline(){
def regionPolylineMap = regionPolyline()
def q = new BasicDBObject()
q.put("type","区域")
// q.put("city", "苏州市")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
for(def d : dbcursor){
def shapeData = regionPolylineMap.get(d.city+"|@|"+d.district)
println d.city+" "+d.district
if(shapeData){
println "新围栏数:"+shapeData.shape.coordinates.size()
}
if(d.shape){
println "旧围栏数:"+d.shape.coordinates.size()
}
if(d.shape && shapeData && shapeData.shape!=d.shape){
def udata = new BasicDBObject()
udata.shape = shapeData.shape
udata.location = shapeData.location
udata.m_date = new Date()
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
// if(shapeData && d.shape && shapeData.shape.type=="polygon" && d.shape.type=="polygon"){
// double area = com.fangjia.script.util.GeoHelper.calculateArea(shapeData.shape.coordinates[0])
// double area1 = com.fangjia.script.util.GeoHelper.calculateArea(d.shape.coordinates[0])
// if(Math.abs(area1-area)>=1){
// println "新:"+area
// println "旧:"+area1
// def udata = new BasicDBObject()
// udata.shape = shapeData.shape
// udata.location = shapeData.location
// //DMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
// }
// }
}
}
}
def updateMappingPolyline(){
def q = new BasicDBObject()
q.put("type","区域")
q.put("shape.type","polygon")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
for(def d : dbcursor){
updateAdministrativeMapping(d._id.toString(), d.shape)
}
}
def updateAdministrativeMapping(def id, def shape){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.shape = params.newshape",
"params":{
"newshape":{
"type":"${shape.type}",
"coordinates":${shape.coordinates}
}
}
},
"query": {
"term": {
"_id": "${id}"
}
}
}
"""
HttpRequest request = new HttpRequest()
try{
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
println(response)
}catch(Exception e){
println "error : "+e.getMessage()
}
}
}
@@ -31,7 +31,6 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 区域标准化
* 1.小区库乡镇街道标准化,包含小区,商铺,写字楼
@@ -629,7 +628,7 @@ class TownStandardBatch extends BaseBatch{
// //通过调取地图api接口获取乡镇街道
// if(!saveObj.township){
// def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon)
def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon)
// if(gaodeMap && gaodeMap.addressComponent) {
// def addressComponent = gaodeMap.addressComponent
// if(addressComponent){
@@ -1,16 +1,9 @@
package com.fangjia.script.gj.district
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.addrepo.InvokeHttpRequest
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.util.HttpRequestUtil
import com.mongodb.DBObject
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang.StringUtils
class FangjiaClientUtils {
@@ -9,7 +9,7 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def city = "³£ÖÝ"
@@ -14,7 +14,7 @@ import org.apache.lucene.queryparser.classic.QueryParser
class EsMonthHouseStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
@@ -38,7 +38,7 @@ class FjwBigdataDealBatch extends BaseBatch{
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
@@ -29,7 +29,7 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07",
@@ -27,7 +27,7 @@ class GuangfaBlankSelectDistrict {
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static main(args) {
def task = new GuangfaBlankSelectDistrict();
@@ -31,7 +31,7 @@ import java.util.concurrent.atomic.AtomicLong
public class BeiKeCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
WebDriver driver = null;
@@ -118,7 +118,7 @@ public class BeiKeCrawlByChrome {
}
def crawlData(driver, citys){
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def runedCitys = FileUtils.readTxt("d:\\数据\\成交抓取\\贝壳已抓城市.txt")
@@ -15,7 +15,7 @@ import java.util.regex.Pattern
class ExportPaimai {
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
static main(args) {
@@ -31,7 +31,7 @@ import java.util.concurrent.atomic.AtomicLong
public class LianjiaCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
WebDriver driver = null;
@@ -189,7 +189,7 @@ public class LianjiaCrawlByChrome {
}
def crawlData(driver, cityList){
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def runedCitys = FileUtils.readTxt("d:\\数据\\成交抓取\\链家已抓城市.txt")
@@ -28,7 +28,7 @@ class PaimaiDataSend {
DBCollection jd_coll = MongoUtils.getCollectionByName("fangjia_crawl", "fapai_jd_detail", "192.168.0.235", 27018)
DBCollection rmfysszc_coll = MongoUtils.getCollectionByName("fangjia_crawl", "fapai_rmfysszc_detail", "192.168.0.235", 27018)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
@@ -26,7 +26,7 @@ class PaimaiTest {
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
@@ -30,7 +30,7 @@ class AddPriceTest {
def pgsqlInstance235 = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com")
static main(args) {
@@ -230,6 +230,8 @@ class AddPriceTest {
if(newPriceMap){
d['G'] = newPriceMap.avg_price
d['H'] = newPriceMap.month
d['H'] = map.calc_type
// d['AM'] = map.credit_value
}
}
})
@@ -13,8 +13,8 @@ import java.util.concurrent.atomic.AtomicInteger
class ExportToDealPrice {
DBCollection DEAL_PRICE_MATCH_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price_match", "192.168.0.43")
DBCollection DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43")
DBCollection DEAL_PRICE_MATCH_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price_match", "mongo.db.fangjia.com")
DBCollection DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
DBCollection TRADE_MATCH_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "trade_match_price", "192.168.0.136")
@@ -14,7 +14,7 @@ import java.util.regex.Pattern
class FzBlankTest {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection FZ_ADDR_COLL = MongoUtils.getCollectionByName2("business", "fz_district_address", lineUrl)
static main(args) {
@@ -20,7 +20,7 @@ import java.util.regex.Pattern
class JiangShuBlankDataStat {
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static main(args) {
def task = new JiangShuBlankDataStat()
@@ -12,7 +12,7 @@ import com.mongodb.DBCollection
class TestArroundPrice {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DIST_DATA_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_analy_data", lineUrl)
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
@@ -15,7 +15,7 @@ import java.util.concurrent.atomic.AtomicLong
class StreetProcess {
final DBCollection TOWNSHIP = MongoUtils.getCollectionByName("fangjia", "township", "192.168.0.43")
final DBCollection TOWNSHIP = MongoUtils.getCollectionByName("fangjia", "township", "mongo.db.fangjia.com")
public static final def MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
public static final DBCollection TOWNSHIP100 = MongoUtils.getCollectionByName2("fangjia", "township", MONGODB_URL)
@@ -27,9 +27,9 @@ class NetworkDistanceBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
final DBCollection NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network", "192.168.0.43")
final DBCollection NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network", "mongo.db.fangjia.com")
final DBCollection MAP_NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network_map", "192.168.0.43")
final DBCollection MAP_NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network_map", "mongo.db.fangjia.com")
static main(args) {
runBatch(new NetworkDistanceBatch(), args)
@@ -20,10 +20,10 @@ import java.util.regex.Pattern
class ConsumptionRatingBatch extends BaseBatch {
def weekend = DateUtil.truncateDay(DateUtil.getFirstDayOfWeek(new Date()))
public static
final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43")
final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
public static
final DBCollection CONSUMPTION_GRADE_COLL = MongoUtils.getCollectionByName("fangjia_stat", "consumption_grade", "192.168.0.85")
final DBCollection CONSUMPTION_GRADE_COLL = MongoUtils.getCollectionByName("fangjia_stat", "consumption_grade", "mongo.db.fangjia.com")
final DBCollection AMAP_POI_New_COLL = MongoUtils.getCollectionByName("m_dianping", "dianping_9", "114.80.150.196", 27777)
@@ -52,7 +52,7 @@ class AliyunSmsClient {
}
public static void main(String[] args) {
sendSms("13916076378", "SMS_135220034", '{"code":12345}')
sendSms("15021384995", "SMS_482915107", '{"code":12345}')
}
@@ -30,7 +30,7 @@ import java.sql.SQLException
class HousingTransactionExportBatch extends BaseBatch {
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
static DBCollection _db_esf_transaction = MongoUtils.getCollectionByName("vip_site", "esf_transaction", "192.168.0.235", 27018);
static MongoCollection<Document> PAI_MAI = Mongo3Utils.getDbCollection(Constants.FANGJIA_MONGO_URI, "fangjia_crawl", "pai_mai")
@@ -23,7 +23,7 @@ import java.util.regex.Pattern
class CqfdcMatch {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl)
static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018)
@@ -53,8 +53,8 @@ def pcmap = [:]
provinceList.remove("广西省")
provinceList.add("广西壮族自治区")
DSCollection seabird = new DataStore("192.168.0.43", 28017).getCollection("fangjia_base.baidu_place_poi2")
DBCollection m43 = MongoUtils.getCollectionByName("fangjia_base", "baidu_place_poi2", "192.168.0.43", 28017)
DSCollection seabird = new DataStore("mongo.db.fangjia.com", 28017).getCollection("fangjia_base.baidu_place_poi2")
DBCollection m43 = MongoUtils.getCollectionByName("fangjia_base", "baidu_place_poi2", "mongo.db.fangjia.com", 28017)
//def type3Not = ["学校内部设施", "进港", "出港", "候船室", "住宿服务相关"]
//
//def type2Not = ["学校", "专科医院", "综合医院", "诊所", "银行", "自动提款机", "便民商店/便利店", "家电电子卖场", "超级市场", "商场", "美容美发店", "地铁站", "港口码头", "住宿服务"]
@@ -16,9 +16,9 @@ import com.mongodb.DBCollection
//@BatchOwner(Watcher.MaJinTao)
class CommunityHouseSampleBatch extends BaseBatch {
DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.43", 28017)
DBCollection STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
DBCollection HOUSE_CASE = MongoUtils.getCollectionByName("business", "house_case", "192.168.0.43", 28017)
DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com", 28017)
DBCollection STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com", 28017)
DBCollection HOUSE_CASE = MongoUtils.getCollectionByName("business", "house_case", "mongo.db.fangjia.com", 28017)
DSCollection SEABIRD = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird")
static void main(String[] args) {
@@ -160,13 +160,13 @@ import com.mongodb.DBCollection
@BatchOwner(Watcher.MaJinTao)
class CommunityHouseSampleBatch extends BaseBatch {
DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.43", 28017)
DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com", 28017)
DBCollection STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
DBCollection STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com", 28017)
DBCollection HOUSE_CASE = MongoUtils.getCollectionByName("business", "house_case", "192.168.0.43", 28017)
DBCollection HOUSE_CASE = MongoUtils.getCollectionByName("business", "house_case", "mongo.db.fangjia.com", 28017)
// DBCollection SEABIRD = MongoUtils.getCollectionByName("fangjia", "seabird", "192.168.0.43", 28017)
// DBCollection SEABIRD = MongoUtils.getCollectionByName("fangjia", "seabird", "1mongo.db.fangjia.com", 28017)
DSCollection SEABIRD = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird")
@@ -16,7 +16,7 @@ import java.sql.Statement
@BatchOwner(Watcher.WangWenYan)
class DistrictStatClickhouseSyncBatch extends BaseBatch{
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 27017)
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com", 27017)
static void main(String[] args) {
runBatch(new DistrictStatClickhouseSyncBatch(), args)
@@ -19,9 +19,9 @@ import java.util.concurrent.*
class DealPriceDetailExport {
// 小区均价集合
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com", 28017)
// 小区成交集合
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43", 28017)
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com", 28017)
static final Map retMap = new ConcurrentHashMap()
static ExecutorService executorService = Executors.newFixedThreadPool(10)
static ExecutorService districtExecutorService = Executors.newFixedThreadPool(30)
@@ -23,9 +23,9 @@ import java.util.concurrent.atomic.AtomicInteger
class PPE15Export {
// 小区均价集合
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017)
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com",28017)
// 小区成交集合
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com",28017)
static Map retMap = new ConcurrentHashMap()
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static ExecutorService districtExecutorService = Executors.newFixedThreadPool(30)
@@ -23,9 +23,9 @@ import java.util.concurrent.atomic.AtomicInteger
class PPE15ExportTest {
// 小区均价集合
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017)
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com",28017)
// 小区成交集合
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com",28017)
static Map retMap = new ConcurrentHashMap()
static main(args) {
@@ -21,7 +21,7 @@ import java.util.concurrent.atomic.AtomicInteger
*/
class RentExport {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia", "district_stat_rent", "192.168.0.43",28017)
static DBCollection collection = MongoUtils.getCollectionByName("fangjia", "district_stat_rent", "mongo.db.fangjia.com",28017)
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static Map retMap = new ConcurrentHashMap()
@@ -20,9 +20,9 @@ import java.util.concurrent.CountDownLatch
class TotalTestExport {
// 小区均价集合
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017)
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com",28017)
// 小区成交集合
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com",28017)
static Map retMap = new ConcurrentHashMap()
// 在此更改时间
static final def TRADE_DATE_START = "2020-03-28"
@@ -20,9 +20,9 @@ import java.util.concurrent.CountDownLatch
class ValueRateCityInfoExport {
// 小区均价集合
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017)
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com",28017)
// 小区成交集合
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com",28017)
static Map retMap = new ConcurrentHashMap()
// 在此更改时间
static final def TRADE_DATE_START = "2020-03-28"
@@ -16,7 +16,7 @@ import org.apache.commons.lang3.StringUtils
class BusinessDianpingExport {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018)
static DBCollection cityCollection = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
static DBCollection cityCollection = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com", 28017)
static void main(String[] args) {
def cityCodes = ["320500","610100","340100"];
@@ -28,7 +28,7 @@ import java.util.concurrent.atomic.AtomicInteger
*/
class AroundExportByNewWay {
static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017)
static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "mongo.db.fangjia.com", 28017)
static def resultMap = new ConcurrentHashMap()
static def unMatchedMap = new ConcurrentHashMap()
static ExecutorService executorService = Executors.newFixedThreadPool(30)
@@ -28,7 +28,7 @@ import java.util.concurrent.atomic.AtomicInteger
*/
class AroundFormatExport {
static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017)
static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "mongo.db.fangjia.com", 28017)
static def resultMap = new ConcurrentHashMap()
static def unMatchedMap = new ConcurrentHashMap()
static ExecutorService executorService = Executors.newFixedThreadPool(30)
@@ -14,7 +14,7 @@ import com.mongodb.DBObject
*/
class ImportSchool {
static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017)
static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "mongo.db.fangjia.com", 28017)
static void main(String[] args) {
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\重庆学校_0902.xlsx", 1)
@@ -16,7 +16,7 @@ import sun.swing.BakedArrayList
*/
class MarkUnNeedExport {
static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017)
static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "mongo.db.fangjia.com", 28017)
static void main(String[] args) {
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边补充.xlsx", 1)
@@ -22,7 +22,7 @@ import java.util.concurrent.atomic.AtomicInteger
class MoveArroundTemp {
static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018)
static DBCollection conn43 = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017)
static DBCollection conn43 = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "mongo.db.fangjia.com", 28017)
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) {
@@ -16,7 +16,7 @@ import java.util.regex.Pattern
*/
class POIRule {
static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017)
static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "mongo.db.fangjia.com", 28017)
static void main(String[] args) {

Some files were not shown because too many files have changed in this diff Show More