Compare commits
695 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2d266a30c2 | |||
| 0d65af4be7 | |||
| fb8f6400be | |||
| 0b7caf6a9f | |||
| 92802d2d50 | |||
| e79e154547 | |||
| 8f502b42b7 | |||
| 3e927f2c6d | |||
| 21be81690f | |||
| 3a301eea2e | |||
| cccafbe651 | |||
| 6264d35a54 | |||
| 9134b28713 | |||
| 5711fb84a8 | |||
| a5e0ae9a0d | |||
| df9dd81487 | |||
| 2b72b6597a | |||
| dba583f1c4 | |||
| b2f92c12f2 | |||
| c4556a9271 | |||
| 3bdbc5ca8c | |||
| 9800abcbc2 | |||
| 5603858a93 | |||
| bf2923bdc4 | |||
| fff45fbd37 | |||
| 726ac630f5 | |||
| 5110fee974 | |||
| 77186ee35c | |||
| ac01d98e08 | |||
| d04bdbd464 | |||
| 6dae2f895c | |||
| 3f09013747 | |||
| 4721f23827 | |||
| e36d16cbdd | |||
| b7534064aa | |||
| a11383312e | |||
| eddb99892a | |||
| aee4c00628 | |||
| bfcdb591d3 | |||
| 9467a62152 | |||
| 327b36e5a0 | |||
| aed37e0759 | |||
| ea0a812f09 | |||
| afa57f7261 | |||
| 8f5b35679b | |||
| 009e3559c9 | |||
| 9faf6baa72 | |||
| 6d88b5e31d | |||
| 740353e0a7 | |||
| 8fd76993fc | |||
| 4b9c8f6d19 | |||
| ab7d061946 | |||
| 6289e30cfe | |||
| 21f6354658 | |||
| 70224430b9 | |||
| 54012fc635 | |||
| 94f509725a | |||
| 2e49c448d4 | |||
| 2de5f549ed | |||
| 37ecd3095d | |||
| 06551bff10 | |||
| 80d960dd4c | |||
| eec268365b | |||
| d4fdc75146 | |||
| 7ea3f83374 | |||
| 1b030c9522 | |||
| 2f6a40d64a | |||
| e6922e4c6b | |||
| 617db91a11 | |||
| f42e32065e | |||
| 641356bd07 | |||
| 02a1c27c0c | |||
| c1168b02a8 | |||
| ef5601e0e0 | |||
| 203920ad6c | |||
| ece4633ab1 | |||
| e5d39afc20 | |||
| fde96a1886 | |||
| 27a8f78f62 | |||
| 50597d9a5a | |||
| 7f286359e1 | |||
| e31aaf6794 | |||
| 93c5d9501b | |||
| 125937a96c | |||
| 5ef8c21a3b | |||
| 5b8fe6947c | |||
| 047b826691 | |||
| f892dcee0a | |||
| 7880ccf802 | |||
| a97cf5035b | |||
| 9d1a9a769e | |||
| 399151ff58 | |||
| 9ce31e9b56 | |||
| 3975d029f8 | |||
| d0e6714ecd | |||
| d0ebcb451a | |||
| 663157757b | |||
| 405f40f0b8 | |||
| 1605ef8165 | |||
| 5cd3470091 | |||
| 7363524890 | |||
| 241de34a92 | |||
| 51fda5fdf2 | |||
| ffafaa919c | |||
| 6c69efaed4 | |||
| e150527534 | |||
| 4868577bef | |||
| b5583f4e66 | |||
| d3b666d3e7 | |||
| 4572a84568 | |||
| a16b7e4404 | |||
| 3a00ac99c1 | |||
| 5562054ea3 | |||
| b70761874a | |||
| b3fbf74e5c | |||
| 8b9ff84b64 | |||
| e01efaca40 | |||
| f08fadb7c9 | |||
| e079a20472 | |||
| 27319ed28d | |||
| cea2711e5f | |||
| f36d7bd51f | |||
| 7b0542a36a | |||
| b1a7892268 | |||
| 235fd10e3f | |||
| 1f04a0a198 | |||
| ddaccd7600 | |||
| 36b4cc59a9 | |||
| 22c2711485 | |||
| 3a7c6b683c | |||
| 019d59ef51 | |||
| 9750eb7697 | |||
| 48ad7629a3 | |||
| b6502d0051 | |||
| e60ec4dfaa | |||
| bc7f275d06 | |||
| a6e60819ad | |||
| 92ce23a11d | |||
| 6ce61b6336 | |||
| 96c795be4a | |||
| df71f89b07 | |||
| cb938f81f0 | |||
| 77cc4fb096 | |||
| 24cbd81454 | |||
| 43e9b5af14 | |||
| a09c8b2feb | |||
| ac357b1d9e | |||
| 0d8c221dd3 | |||
| 6375437b7f | |||
| 0fee4586b2 | |||
| 351bad8b57 | |||
| 04ab71fc1c | |||
| 6ba24850fc | |||
| b4b1a39118 | |||
| 2f3a64a018 | |||
| dc67eeba57 | |||
| 2428254918 | |||
| 7409e0edcf | |||
| 2ddec57ba3 | |||
| 982f42fe14 | |||
| 338870334c | |||
| 7072183435 | |||
| 43f35fa63e | |||
| b064138c46 | |||
| 02e1aab542 | |||
| 5e6ae745a1 | |||
| ca1cf0f15c | |||
| 20b606a0dd | |||
| 27de5a7ec9 | |||
| e3e3ebc981 | |||
| f502d7f05e | |||
| 49a387f9ed | |||
| 4bb93c977f | |||
| 7d9d16d33a | |||
| 237e88d79f | |||
| 165e50ae16 | |||
| c4368c1421 | |||
| ba1d065337 | |||
| f30f940e0b | |||
| b9d9e728ad | |||
| c8acedbd14 | |||
| 82119764c6 | |||
| 82f7359410 | |||
| 16aea01faf | |||
| 72106ba0ae | |||
| 2728bbbdad | |||
| bc7a02d319 | |||
| 47bb142987 | |||
| d6df49c9a6 | |||
| 4b01ed8fd6 | |||
| c4566e8426 | |||
| cde3fd5e40 | |||
| 9962cf5ef3 | |||
| e5ceaddfc4 | |||
| f8c27a33a2 | |||
| 66bb733a0b | |||
| d49b907f38 | |||
| 7c8a704922 | |||
| 46a7501cca | |||
| 9881b93d7f | |||
| 5b67698197 | |||
| db93c0fef2 | |||
| f1a8e3e001 | |||
| 7f91150550 | |||
| f4b49bd870 | |||
| 744ef7c9f5 | |||
| ef8161abe8 | |||
| fb33ddc969 | |||
| 80b4867cf3 | |||
| 3e72a9a6af | |||
| 6fe9688eac | |||
| bc43c1457f | |||
| 916804115b | |||
| 0e2f9d0cae | |||
| fb8f307596 | |||
| 3c88276173 | |||
| 713e6ce9ba | |||
| 9968c6ed0c | |||
| 390b869d3e | |||
| 88b1e939ff | |||
| 80d391b358 | |||
| acc10c6c3d | |||
| c45f480a42 | |||
| 3c77ac061c | |||
| 027ba5ef1f | |||
| 6b4e6ee71e | |||
| 146fe2e680 | |||
| ddfb1cde70 | |||
| 608f492b29 | |||
| 53e7ba8c99 | |||
| 4a2e34c25f | |||
| 6c6841988e | |||
| 82c3191983 | |||
| 88e6099e0b | |||
| 83dfdb7ab0 | |||
| 9d2c12653d | |||
| 7d25815e7e | |||
| 747c79eef2 | |||
| 51e4abdcfc | |||
| b61199d0be | |||
| 153a1aa418 | |||
| f943abb51e | |||
| 12f1dcf7cf | |||
| f06263e434 | |||
| c1e55c08a4 | |||
| d183ca5c6c | |||
| c936b2c2c5 | |||
| 8723806fab | |||
| 56519e159c | |||
| c640d914cc | |||
| b0708d3a3e | |||
| 319ef5847c | |||
| 43118a7267 | |||
| 9f8a34ac8d | |||
| db3c29c3c9 | |||
| 7b17928545 | |||
| ca4b7782ef | |||
| dcce114957 | |||
| 515351d56b | |||
| 8eb6cb3876 | |||
| b36485473c | |||
| 812888eb79 | |||
| 0c751f413a | |||
| 46515fe238 | |||
| a91fe73747 | |||
| 61f39b36b6 | |||
| 7f38681f70 | |||
| 13dabf5682 | |||
| 81438e9d39 | |||
| 8a3b2fdcf4 | |||
| 0067fc6775 | |||
| 4f9fe0dce9 | |||
| 7568cefde6 | |||
| 15c3ec9857 | |||
| 401cc4fbe9 | |||
| bb217cc4c3 | |||
| 0fa69ae9c0 | |||
| 5f9c42623c | |||
| 079652a557 | |||
| 7df202a638 | |||
| 6fe4240883 | |||
| fd586603d9 | |||
| 2e806a81cb | |||
| 50e30d9dc0 | |||
| 0c6a7ad583 | |||
| e3579708e4 | |||
| d89ecf29f0 | |||
| 5b5252c185 | |||
| 3a30ed8880 | |||
| ad778373e4 | |||
| c0e2f6aaad | |||
| 92f7ca4362 | |||
| 95dea214e7 | |||
| a091466906 | |||
| 241597d1ae | |||
| c2cf59acb8 | |||
| 2d6ff5e11c | |||
| 2d68181688 | |||
| a78c8010fc | |||
| 0202976699 | |||
| bcca0c0311 | |||
| 8bf59f7b50 | |||
| ecd912f8e6 | |||
| 12bcea225a | |||
| b6f79e808f | |||
| 72fcacd73f | |||
| a226629f9c | |||
| cb469f2f41 | |||
| 528a257113 | |||
| 075577afd7 | |||
| b8b2680b00 | |||
| 5913f8fff0 | |||
| db8cbf0edb | |||
| 1cd5079ac4 | |||
| 67eed5a9df | |||
| 3ed3bf7641 | |||
| b67bd8bd1e | |||
| 7ab06bd585 | |||
| 9172e1a4f9 | |||
| b07f22cf33 | |||
| 00e258e4a8 | |||
| da778f9a02 | |||
| a82c958915 | |||
| 850ba31f49 | |||
| a97f0262b1 | |||
| d6beb9df39 | |||
| 937d370b0b | |||
| 1bbf0c3fb6 | |||
| 8f6744966d | |||
| 8d07f4daff | |||
| aa0f3719a5 | |||
| ac8efee40b | |||
| 2d7aaf647f | |||
| 9b962728e6 | |||
| 3408b712cf | |||
| 9655f2cbe5 | |||
| a9abba222d | |||
| aebb2bd467 | |||
| 9f82d9e0f4 | |||
| 4c55387790 | |||
| 6fe480ee66 | |||
| f113be7ba0 | |||
| 1194467e5f | |||
| 6e56ac4c02 | |||
| 1f60c344bd | |||
| 7e60e06c3f | |||
| eb0e37884b | |||
| cda9cd919a | |||
| 7dc6a7eea6 | |||
| 3289317575 | |||
| cc2a198a83 | |||
| a4fc57a52a | |||
| 3bb8da4722 | |||
| 10f2e97cd2 | |||
| 1de9f959e4 | |||
| 3ffb594338 | |||
| 5c3d38f171 | |||
| 00c27ba1c1 | |||
| 446088c7d9 | |||
| 9f8aa6c7a7 | |||
| 0bdbed9454 | |||
| 0f44db29cb | |||
| 0fb7e9186f | |||
| 32378a2073 | |||
| ad255ba0db | |||
| 202a5b9198 | |||
| 89c472d131 | |||
| 3cda5cd211 | |||
| 5d94eb7bd0 | |||
| 5dd1991a18 | |||
| 9f9c3b6555 | |||
| 058ed8448d | |||
| 6c311a3e05 | |||
| aafafccb28 | |||
| 840eb8408a | |||
| 16999f22b2 | |||
| 8c5b16e519 | |||
| 5fdb0ac7ce | |||
| 7c8dbeed6c | |||
| 1c24b23e63 | |||
| 5ac62440b1 | |||
| 52716c3778 | |||
| 668eb29a8d | |||
| c360db45c9 | |||
| 728dfa0e94 | |||
| 4cc53dfbf5 | |||
| 9601ede115 | |||
| 24d7200b75 | |||
| 5d8397ec85 | |||
| 8c5779fd07 | |||
| 2fb76d1f55 | |||
| 726490f872 | |||
| fdedc4cbf2 | |||
| b761162d28 | |||
| cb2ab3c418 | |||
| 3146952bd1 | |||
| 6ac82bce70 | |||
| a6c7076e14 | |||
| 780f4e469d | |||
| f03bc87f6f | |||
| 339c84ea24 | |||
| c4e9f38b51 | |||
| 18ee0b05ac | |||
| 2723cb6ef0 | |||
| 8fd13e2dd8 | |||
| 278206fab5 | |||
| 35cb0fdbcb | |||
| 2d8465c219 | |||
| 83665e7e81 | |||
| 6fee8096dd | |||
| 7f8e044640 | |||
| 5192fbd93c | |||
| 89e5814468 | |||
| 6b4cb7f78a | |||
| 5877566c2e | |||
| 2f267ee038 | |||
| d82c2f1b2b | |||
| 036a6791a7 | |||
| fab0bb6d6d | |||
| 25f30657bf | |||
| 100d89e556 | |||
| 95462de314 | |||
| 0077622e63 | |||
| 58ec14eb1b | |||
| 33f65f6199 | |||
| 3801ddcc70 | |||
| 787380ad94 | |||
| 4e0deb2159 | |||
| 8760b4d513 | |||
| 6cfd477e30 | |||
| f6faf38ab0 | |||
| 27c5fcacc1 | |||
| 6d35355baf | |||
| 1a921c08fd | |||
| d4a0ecc971 | |||
| bc1af1c4db | |||
| 0fa6b8821f | |||
| db23742520 | |||
| ac063da374 | |||
| c9ee9a2381 | |||
| 7adb2a18a7 | |||
| 117affd8ce | |||
| 48de7df4b2 | |||
| 98c1ad5baa | |||
| 6c3ef4ebbe | |||
| 7313a03f76 | |||
| a4861702c6 | |||
| 4f16894602 | |||
| 19ff0909f5 | |||
| 3c2884bee9 | |||
| 89ef0a32e2 | |||
| 4f6853f710 | |||
| f8c67d5e9f | |||
| 2ed24c67b7 | |||
| 8174eaa78d | |||
| 35c374fc46 | |||
| 09444c04b7 | |||
| cf6508a2c5 | |||
| a9392e1337 | |||
| d2fc14b43a | |||
| f98a75cc2d | |||
| 43038a0478 | |||
| 4a44be7c29 | |||
| 279663f351 | |||
| 3da15c2b5c | |||
| 13bb857701 | |||
| 83cdba3406 | |||
| 40aad80afe | |||
| b3910631db | |||
| e1b9abbda3 | |||
| d4dfcb7146 | |||
| 0611777787 | |||
| 456fd66319 | |||
| 66dec65a06 | |||
| 8598b007ae | |||
| b061834407 | |||
| d3386f5b10 | |||
| 499ee47be0 | |||
| ee60711af1 | |||
| ee82ee0846 | |||
| 0792dccd92 | |||
| 13747baa12 | |||
| 6a0d4149b8 | |||
| 2d0c87b701 | |||
| 4818ce566a | |||
| 8ab375aa5d | |||
| f4f5c86e1e | |||
| bdc9a3c62a | |||
| db2b2005dd | |||
| 29f827e2cc | |||
| 2f7478f017 | |||
| 9c07c89801 | |||
| 767a0d040e | |||
| 0b5dff7111 | |||
| b915e1fe1b | |||
| 2a9fa3574f | |||
| 6ef0d4adfc | |||
| c3d937e326 | |||
| 74e3211fb7 | |||
| 0b912ab106 | |||
| d1e5b5579d | |||
| 2a75739658 | |||
| e66e8e75ce | |||
| c48580c31e | |||
| feecd858fa | |||
| 5107701487 | |||
| f082ff8739 | |||
| c8537800a8 | |||
| 0ee261a545 | |||
| 9f379f106c | |||
| 4008f7937f | |||
| e39c52849c | |||
| 8f170e86fa | |||
| 6488f4a40a | |||
| 2666137af9 | |||
| a5982ecfa0 | |||
| bf7b1174f3 | |||
| b7ab8b9192 | |||
| feab049058 | |||
| 3f9e8a7528 | |||
| 2eecb42410 | |||
| 3ab1d31852 | |||
| cf9c99a416 | |||
| 98017d34f0 | |||
| 9abc7a5676 | |||
| d43a950e83 | |||
| e0e18e5d36 | |||
| b41c78fef4 | |||
| b7c567b2fb | |||
| 16b812967c | |||
| e8c124a1e9 | |||
| 2373b2f879 | |||
| 7838d54e8c | |||
| 0fae0996ef | |||
| 27e7f3fae6 | |||
| bb9981a5d9 | |||
| b5da624d93 | |||
| 02bcba8a29 | |||
| 0727376478 | |||
| 29d42f33d6 | |||
| 3d2acbddaa | |||
| 65d128dde2 | |||
| 8c439e3201 | |||
| 0fe9d8f99a | |||
| ea6ae494cd | |||
| 85669b6bd7 | |||
| a2fa2be77a | |||
| 650e7e0795 | |||
| 940f7d85e9 | |||
| ac42eb2812 | |||
| 109db091e0 | |||
| ba51ae5ee3 | |||
| 713e358000 | |||
| db22cb2cb5 | |||
| 7c99f1b0ac | |||
| 11d76204a7 | |||
| caf18c2d8a | |||
| da08f4b06c | |||
| 0cfe28ae48 | |||
| 0876b47a47 | |||
| 15bc05797e | |||
| e04026491d | |||
| 891419b45a | |||
| d8543eae5b | |||
| 8406955a7e | |||
| 2654fcfe67 | |||
| 94cf7da75d | |||
| 94f2b602dc | |||
| cf60427ef0 | |||
| 034a2a80ba | |||
| e5172f0d6c | |||
| c31e32598f | |||
| a1c65a7633 | |||
| 50cbe55556 | |||
| b8e8d79821 | |||
| fe592e20ac | |||
| 3572add969 | |||
| f43f99bfaf | |||
| 70e5bbaf28 | |||
| e2573a5f1c | |||
| 0767704ed0 | |||
| 9df55a9883 | |||
| 49cb6a3023 | |||
| 60de058719 | |||
| e8bca1f2cb | |||
| 37997feb6e | |||
| 202111ef0f | |||
| 1c241ef841 | |||
| 15453e103c | |||
| 999f98e088 | |||
| c4e801d537 | |||
| 6c4385b4b4 | |||
| c2455331a6 | |||
| 3e23948677 | |||
| ea796a823b | |||
| 08950f621f | |||
| 78f210db5d | |||
| c0fcafc2fc | |||
| 6821a74975 | |||
| 25c2972c54 | |||
| 652c48b73c | |||
| db24839786 | |||
| 15ac2a3501 | |||
| 5d31462b43 | |||
| dda9c85c48 | |||
| 5b15017380 | |||
| 4db4e6b19d | |||
| 6b2728d693 | |||
| d327ad1324 | |||
| 071990bf85 | |||
| 7b6c28794e | |||
| 449076a561 | |||
| abf65edf14 | |||
| f8cfc224ae | |||
| a286e5e8af | |||
| 128eeb47e9 | |||
| f2dfc4da85 | |||
| d201872c66 | |||
| c8093fce87 | |||
| e581409541 | |||
| 34b25bdfd2 | |||
| 15acb1eb5c | |||
| b747587402 | |||
| 3b3ac26094 | |||
| dea4e5b23d | |||
| 3cff0c88d4 | |||
| b2abcf1200 | |||
| 63b6d49dac | |||
| f707184d93 | |||
| e065f8632b | |||
| 8345545b0e | |||
| 568a06fe0b | |||
| 5cac0cd831 | |||
| 1dc4d674f4 | |||
| 28096a64b8 | |||
| 0a98c28ba3 | |||
| b4056ef74a | |||
| 84607517db | |||
| e5e699c8da | |||
| 0b78d4c5ce | |||
| bcf64ccca1 | |||
| 71d00d28dd | |||
| c9e51dd426 | |||
| 98466f4b22 | |||
| 437bb336bd | |||
| ccc566f6d1 | |||
| 261270221b | |||
| 7aa9778413 | |||
| e90b282e73 | |||
| 44597fd073 | |||
| 0466cd3772 | |||
| b131fa673e | |||
| 2fa43377fe | |||
| f31502935d | |||
| 32798ccefc | |||
| 09a2733f8e | |||
| 48ceec2760 | |||
| 6688e93b99 | |||
| 57e5a5e649 | |||
| a76f9b14a2 | |||
| b43c7857c2 | |||
| 484c65ce15 | |||
| ed57487aef | |||
| 42d78f95e5 | |||
| 9245b3fe96 | |||
| 8d37a92cf6 | |||
| 4d6145fb89 | |||
| 9e7d8d1743 | |||
| 25d046cc6e | |||
| 44cee71ce3 | |||
| c9e802da90 | |||
| 68e154d390 | |||
| 9e07fdec9a | |||
| 66a623a060 | |||
| 00c76af03a | |||
| a423fe10c1 | |||
| 6ce1606b51 | |||
| 1ad3654416 | |||
| 0cc1670467 | |||
| 087ef47d4e | |||
| afefe99b0f | |||
| b7d4180cde | |||
| 0ded5b6901 | |||
| 05ee877888 | |||
| 98266e9c2b | |||
| f746d854c7 | |||
| b00f3a67a1 | |||
| 4d6c59d043 | |||
| 87e931f200 | |||
| acc58f437a | |||
| 37aa051bf9 | |||
| 010a9db93b | |||
| ffe2f5fcbd | |||
| c6ef25d6b4 | |||
| c0b9224679 |
@@ -1,6 +0,0 @@
|
|||||||
poi/business/.*
|
|
||||||
stat/evaluate/*
|
|
||||||
stat/community/*
|
|
||||||
mt/supply/*
|
|
||||||
business/business/*
|
|
||||||
plugin/**
|
|
||||||
@@ -7,4 +7,5 @@ pip install -r requirements.txt -i https://repo.fangjia.com/repository/pypi-publ
|
|||||||
|
|
||||||
- 具体业务逻辑放入xxx/business中,如poi/business
|
- 具体业务逻辑放入xxx/business中,如poi/business
|
||||||
- 添加.airflowignore xxx/business/*, 如poi/business/*
|
- 添加.airflowignore xxx/business/*, 如poi/business/*
|
||||||
- dag放在xxx目录下即可,命名为xx_dag.py
|
- dag放在xxx目录下即可,命名为dag_xx.py,之前命名为xx_dag导致xx与报名相同而加入airflowignore后dag不显示问题,
|
||||||
|
如merge_dag会不显示,而dag_merge即可,怀疑是jh/merge/*的bug
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
poi/business/*
|
||||||
|
stat/evaluate/*
|
||||||
|
stat/community/*
|
||||||
|
stats/evaluate/*
|
||||||
|
stats/community/*
|
||||||
|
stats/aiprice/*
|
||||||
|
stats/cityPrice/*
|
||||||
|
stats/test/*
|
||||||
|
mt/supply/*
|
||||||
|
business/business/*
|
||||||
|
jh/merge/*
|
||||||
|
jh/para/*
|
||||||
|
jh/price/*
|
||||||
|
jh/sample/*
|
||||||
|
db/*
|
||||||
|
office/rent/*
|
||||||
|
office/sell/*
|
||||||
|
shop/rent/*
|
||||||
|
shop/sell/*
|
||||||
|
company/company/*
|
||||||
|
ai/stock/*
|
||||||
|
price_exponents/region_price_month.py
|
||||||
|
price_exponents/seaweed_price_month.py
|
||||||
|
test/*
|
||||||
|
train/*
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
# from airflow.providers.standard.operators.bash import BashOperator
|
||||||
|
# import pendulum
|
||||||
|
# from airflow import DAG
|
||||||
|
#
|
||||||
|
# with DAG(
|
||||||
|
# dag_id="ai_stock_dag",
|
||||||
|
# # 每天运行
|
||||||
|
# schedule="30 8 * * 1-5",
|
||||||
|
# start_date=pendulum.datetime(2025, 11, 10, tz="Asia/Shanghai"),
|
||||||
|
# catchup=True,
|
||||||
|
# tags=["ai_stock"],
|
||||||
|
# ) as dag:
|
||||||
|
#
|
||||||
|
# t1 = BashOperator(
|
||||||
|
# task_id='stock_strategy',
|
||||||
|
# bash_command='python /opt/airflow/dags/repo/dags/ai/stock/stock_strategy.py',
|
||||||
|
# )
|
||||||
|
#
|
||||||
|
# t2 = BashOperator(
|
||||||
|
# task_id='ai_stock',
|
||||||
|
# bash_command='python /opt/airflow/dags/repo/dags/ai/stock/stock.py',
|
||||||
|
# )
|
||||||
|
# t1 >> t2
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
# from goojia_utils import MysqlClient, ArkAIClient, dingutils
|
||||||
|
# from goojia_utils import dateutils
|
||||||
|
# from datetime import datetime
|
||||||
|
# import requests
|
||||||
|
#
|
||||||
|
# db_util = MysqlClient(
|
||||||
|
# host="192.168.0.93",
|
||||||
|
# port=3306,
|
||||||
|
# database="stock",
|
||||||
|
# user="root",
|
||||||
|
# password="Goojia123456"
|
||||||
|
# )
|
||||||
|
# date = int(dateutils.format_date(dateutils.add_months(datetime.today(), months=-1), fm='%Y%m%d'))
|
||||||
|
# select_stocks = db_util.scan_by_sql_to_data_frame("select ts_code as 股票代码, stock_name as 股票名称,close as 股价, pe as PE,turnover_rate as 换手率 from mu_stocks_new where `trade_date`= (select max(`trade_date`) from mu_stocks_new)")
|
||||||
|
# stat = db_util.scan_by_sql_to_data_frame(f"select ts_code as 股票代码, count(ts_code) as 命中次数 from mu_stocks_new where `trade_date` > {date} group by ts_code")
|
||||||
|
# select_stocks = select_stocks.merge(stat, on='股票代码', how='left')
|
||||||
|
# select_stocks['命中次数'] = select_stocks['命中次数'].fillna(0)
|
||||||
|
# select_stocks = select_stocks.sort_values(by=['命中次数', 'PE'], ascending=[False, True])
|
||||||
|
# def df_to_nested_list(df):
|
||||||
|
# """转换为嵌套的 Markdown 列表"""
|
||||||
|
# lines = []
|
||||||
|
# i= 1
|
||||||
|
# for _, row in df.iterrows():
|
||||||
|
# lines.append(f"{i}. **{row['股票名称']}** ({row['股票代码']}) PE: {row['PE']} 股价: {row['股价']:.2f} 出现次数: {row['命中次数']} 换手率: {row['换手率']:.2f}")
|
||||||
|
# i = i + 1
|
||||||
|
# return '\n'.join(lines)
|
||||||
|
# data = {
|
||||||
|
# "msgtype": "markdown",
|
||||||
|
# "markdown": {
|
||||||
|
# "title": dateutils.format_date(datetime.today(), fm='%Y%m%d') + "选股策略播报",
|
||||||
|
# "text": df_to_nested_list(select_stocks)
|
||||||
|
# }
|
||||||
|
# }
|
||||||
|
# access_token = "1e5f9aaeb90b091bc79652b52800cad2f1dee1ac0990149252511965f6d1c12a"
|
||||||
|
# # 发送消息至群
|
||||||
|
# msg = requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=' + access_token, json=data).json()
|
||||||
|
# msg = requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=45ac47a34dc5491535d77d643512244a85d91d7e74676e9c42446dc06357326e', json=data).json()
|
||||||
|
# print(msg)
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
import datetime
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from dateutil.relativedelta import relativedelta
|
||||||
|
import pandas as pd
|
||||||
|
from sklearn.preprocessing import StandardScaler
|
||||||
|
from sklearn.mixture import GaussianMixture
|
||||||
|
import numpy as np
|
||||||
|
from collections import defaultdict
|
||||||
|
engine = create_engine('mysql+pymysql://root:Goojia123456@192.168.0.93:3306/stock')
|
||||||
|
mu_stocks_all = pd.read_sql(f"select * from mu_stocks_new", con = engine)
|
||||||
|
stock_basic_infos = pd.read_sql(f"select * from stock_basic", con=engine)
|
||||||
|
trade_dates = mu_stocks_all['trade_date'].tolist()
|
||||||
|
today = datetime.datetime.now()
|
||||||
|
def gmm_trade(get_data,df):
|
||||||
|
gmm = GaussianMixture(n_components=100, covariance_type='spherical', random_state=0)
|
||||||
|
data = df.values
|
||||||
|
data = np.array(data, dtype='float64')
|
||||||
|
x = data
|
||||||
|
gmm.fit(x)
|
||||||
|
y_hat = gmm.predict(x)
|
||||||
|
gmm_means_ = [i[0] for i in gmm.means_]
|
||||||
|
d = defaultdict(list)
|
||||||
|
for k, va in [(v, i) for i, v in enumerate(gmm_means_)]:
|
||||||
|
d[va] = k
|
||||||
|
gmm_means_list = []
|
||||||
|
for label in y_hat:
|
||||||
|
mean_value = d[label]
|
||||||
|
gmm_means_list.append(mean_value)
|
||||||
|
get_data["rating"] = y_hat
|
||||||
|
get_data["mean"] = gmm_means_list
|
||||||
|
return get_data
|
||||||
|
def stock_gmm(last_day_str,last_month_str):
|
||||||
|
start = datetime.datetime.now()
|
||||||
|
stock_all = pd.read_sql(f"select * from stock_index_daily where trade_date>'{last_month_str}' and trade_date<='{last_day_str}'", con = engine)
|
||||||
|
end = datetime.datetime.now()
|
||||||
|
print('MySQL Query Running time: %s Seconds' % (end - start))
|
||||||
|
stock_all = stock_all.sort_values(by="trade_date", ascending=False)
|
||||||
|
stock_select = stock_all[((stock_all['ts_code'].str.startswith('600')) | (stock_all['ts_code'].str.startswith('00')))]
|
||||||
|
stock_data = stock_select.dropna(subset=['turnover_rate_f', 'volume_ratio', 'pe', 'pe_ttm', 'pb', 'ps_ttm', 'ps'])
|
||||||
|
stock_group = stock_data.groupby(['ts_code'])
|
||||||
|
scaler = StandardScaler()
|
||||||
|
import_col = ['turnover_rate_f', 'volume_ratio', 'pe', 'pe_ttm', 'pb', 'ps_ttm', 'ps']
|
||||||
|
scaler_col = [str(i) for i in range(len(import_col))]
|
||||||
|
res = []
|
||||||
|
for ts_code, ts_detail in stock_group:
|
||||||
|
if len(ts_detail) > 3:
|
||||||
|
ts_detail[scaler_col] = ts_detail.apply(lambda x: (0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0), axis=1,
|
||||||
|
result_type="expand")
|
||||||
|
ts_detail[scaler_col] = scaler.fit_transform(ts_detail[import_col].values)
|
||||||
|
for i, row in ts_detail.iterrows():
|
||||||
|
res.append(row)
|
||||||
|
stock_res = pd.DataFrame(res)
|
||||||
|
stock_res.to_excel('E:\\工作文件\\avm_output\\airflow_scaled_df.xlsx', encoding='utf-8', index=False)
|
||||||
|
stock_all_newest = stock_all[stock_all.trade_date == stock_all.trade_date.max()]
|
||||||
|
stock_all_newest = stock_all_newest[['ts_code', 'turnover_rate', 'close', 'amount', 'pe', 'pe_ttm', 'total_mv', 'pct_chg']]
|
||||||
|
X = stock_res[['0', '1', '2', '3', '4', '5', '6']]
|
||||||
|
gmm_df = gmm_trade(stock_res, X)
|
||||||
|
gmm_df.to_excel('E:\\工作文件\\avm_output\\airflow_gmm_df.xlsx', encoding='utf-8', index=False)
|
||||||
|
gmm_group_df = gmm_df.groupby(['ts_code', 'rating'])['stock_name'].count().reset_index()
|
||||||
|
gmm_group_df.rename(columns={'stock_name': 'count'}, inplace=True)
|
||||||
|
gmm_df = gmm_df.merge(gmm_group_df, how='left', on=['ts_code', 'rating'])
|
||||||
|
min_mean = gmm_df['mean'].min()
|
||||||
|
slecct_df = gmm_df[gmm_df['mean'] == min_mean]
|
||||||
|
count_max = slecct_df['count'].max()
|
||||||
|
slecct_df = slecct_df[slecct_df['count'] == count_max]
|
||||||
|
slecct_df.drop_duplicates(subset=["ts_code"], keep="first", inplace=True)
|
||||||
|
block_list = list(slecct_df['stock_name'])
|
||||||
|
select_block = []
|
||||||
|
for name in block_list:
|
||||||
|
status = 1
|
||||||
|
for i in ['N', 'ST']:
|
||||||
|
if i in name:
|
||||||
|
status = 0
|
||||||
|
if status == 1:
|
||||||
|
select_block.append(name)
|
||||||
|
return stock_all_newest,select_block,slecct_df
|
||||||
|
|
||||||
|
def stock_select(i):
|
||||||
|
last_day = today+relativedelta(days=-i)
|
||||||
|
last_month = last_day+relativedelta(days=-15)
|
||||||
|
last_day_str = last_day.strftime('%Y%m%d')
|
||||||
|
last_day_str = '20260325'
|
||||||
|
# if last_day_str in trade_dates:
|
||||||
|
# return last_day_str
|
||||||
|
# if int(last_day_str)<20200101:
|
||||||
|
# return last_day_str
|
||||||
|
last_month_str = last_month.strftime('%Y%m%d')
|
||||||
|
last_month_str = '20260304'
|
||||||
|
stock_all_newest,select_block,slecct_df=stock_gmm(last_day_str,last_month_str)
|
||||||
|
db_stock = slecct_df[['ts_code', 'stock_name', 'open', 'close', 'high', 'low',
|
||||||
|
'vol', 'amount', 'pre_close', 'change', 'pct_chg', 'turnover_rate',
|
||||||
|
'turnover_rate_f', 'volume_ratio', 'pe', 'pe_ttm', 'pb', 'ps_ttm', 'ps',
|
||||||
|
'total_share', 'float_share', 'free_share', 'total_mv', 'circ_mv']].copy()
|
||||||
|
db_stock['trade_date'] = last_day_str
|
||||||
|
print(select_block)
|
||||||
|
# db_stock['ts_code'] = db_stock['ts_code'].apply(lambda x: x[0:6])
|
||||||
|
# db_stock.to_sql("mu_stocks_new",con=engine, index=False, if_exists="append")
|
||||||
|
return last_day_str
|
||||||
|
if __name__ == '__main__':
|
||||||
|
stock_select(0)
|
||||||
@@ -0,0 +1,208 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
import os
|
||||||
|
import datetime
|
||||||
|
import calendar
|
||||||
|
import json
|
||||||
|
import pandas as pd
|
||||||
|
from goojia_utils import LOG_MONGO_CLIENT
|
||||||
|
from airflow import DAG
|
||||||
|
from airflow.operators.python import PythonOperator
|
||||||
|
from airflow.utils.email import send_email
|
||||||
|
import pendulum
|
||||||
|
|
||||||
|
|
||||||
|
def get_last_month_range(ref_date: datetime.date = None):
|
||||||
|
"""
|
||||||
|
获取参考日期的上个月第一天 00:00:00 与最后一天 23:59:59.999999
|
||||||
|
以北京时间为准
|
||||||
|
"""
|
||||||
|
if ref_date is None:
|
||||||
|
ref_date = datetime.date.today()
|
||||||
|
|
||||||
|
first_day_current_month = ref_date.replace(day=1)
|
||||||
|
last_day_last_month = first_day_current_month - datetime.timedelta(days=1)
|
||||||
|
first_day_last_month = last_day_last_month.replace(day=1)
|
||||||
|
|
||||||
|
start_time = datetime.datetime.combine(first_day_last_month, datetime.time.min)
|
||||||
|
end_time = datetime.datetime.combine(last_day_last_month, datetime.time.max)
|
||||||
|
|
||||||
|
return start_time, end_time, first_day_last_month.year, first_day_last_month.month
|
||||||
|
|
||||||
|
|
||||||
|
def parse_dict(val):
|
||||||
|
"""将字符串类型的 JSON 或字典解析为 dict"""
|
||||||
|
if isinstance(val, dict):
|
||||||
|
return val
|
||||||
|
if isinstance(val, str) and val.strip():
|
||||||
|
try:
|
||||||
|
return json.loads(val)
|
||||||
|
except Exception:
|
||||||
|
return {}
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
# Excel 中文表头定义:区分【输入】和【输出】
|
||||||
|
EXCEL_COLUMNS = [
|
||||||
|
'请求时间',
|
||||||
|
# 输入参数
|
||||||
|
'输入-城市',
|
||||||
|
'输入-地址',
|
||||||
|
# 响应参数 (取自 res_map.result)
|
||||||
|
'输出-省份',
|
||||||
|
'输出-省份编码',
|
||||||
|
'输出-城市名称',
|
||||||
|
'输出-城市编码',
|
||||||
|
'输出-区域',
|
||||||
|
'输出-区域编码',
|
||||||
|
'输出-乡镇街道',
|
||||||
|
'输出-乡镇街道编码',
|
||||||
|
'输出-是否涉农',
|
||||||
|
'输出-所属区域',
|
||||||
|
'输出-地址验证',
|
||||||
|
'输出-是否为县级市',
|
||||||
|
'输出-是否县城城区',
|
||||||
|
'输出-是否为省国家级开发区',
|
||||||
|
'输出-省国家级开发区名称',
|
||||||
|
'输出-开发区等级'
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def export_and_send_njcb_statement(**context):
|
||||||
|
"""
|
||||||
|
查询上月对公涉农API日志,生成全中文表头Excel对账单并通过邮件发送给顾老师
|
||||||
|
"""
|
||||||
|
now = datetime.datetime.now()
|
||||||
|
start_time, end_time, last_year, last_month = get_last_month_range(now.date())
|
||||||
|
|
||||||
|
print(f"统计时间范围: {start_time} 至 {end_time} (归属年月: {last_year}年{last_month:02d}月)")
|
||||||
|
|
||||||
|
open_uri_log_conn = LOG_MONGO_CLIENT.get_collection('business', 'open_uri_log')
|
||||||
|
|
||||||
|
# 查询条件:
|
||||||
|
# user_id = 1007801, request_uri = /addrSvc/njcbVillageIdentify, c_date在时间范围内, 且只输出code是200的响应
|
||||||
|
query = {
|
||||||
|
'user_id': '1007801',
|
||||||
|
'request_uri': '/addrSvc/njcbVillageIdentify',
|
||||||
|
'c_date': {
|
||||||
|
'$gte': start_time,
|
||||||
|
'$lte': end_time
|
||||||
|
},
|
||||||
|
'$or': [
|
||||||
|
{'code': 200},
|
||||||
|
{'code': '200'}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
print(f"开始查询 MongoDB business.open_uri_log,条件: {query}")
|
||||||
|
cursor = open_uri_log_conn.find(query).batch_size(2000)
|
||||||
|
|
||||||
|
rows = []
|
||||||
|
scanned_count = 0
|
||||||
|
for doc in cursor:
|
||||||
|
scanned_count += 1
|
||||||
|
param_map = parse_dict(doc.get('parameter_map'))
|
||||||
|
res_map = parse_dict(doc.get('res_map'))
|
||||||
|
|
||||||
|
# res_map 结构中有 code, msg, result 嵌套
|
||||||
|
# 实际业务返回字段在 res_map['result'] 中,若无 result 则回退直接使用 res_map
|
||||||
|
result_map = parse_dict(res_map.get('result')) if res_map.get('result') else res_map
|
||||||
|
|
||||||
|
# 过滤只保留"对公涉农"的请求
|
||||||
|
# 涉农类型: 1 对私,2 对公 (默认为对私)
|
||||||
|
req_type = str(param_map.get('type', '1')).strip()
|
||||||
|
if req_type != '2':
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 仅输出"输出-是否涉农"为"农"的部分
|
||||||
|
verify_flag = str(result_map.get('verifyFlag', '')).strip()
|
||||||
|
if verify_flag != '农':
|
||||||
|
continue
|
||||||
|
|
||||||
|
c_date = doc.get('c_date')
|
||||||
|
if isinstance(c_date, datetime.datetime):
|
||||||
|
c_date_str = c_date.strftime('%Y-%m-%d %H:%M:%S')
|
||||||
|
else:
|
||||||
|
c_date_str = str(c_date) if c_date is not None else ''
|
||||||
|
|
||||||
|
row = {
|
||||||
|
'请求时间': c_date_str,
|
||||||
|
# 输入参数
|
||||||
|
'输入-城市': param_map.get('city', ''),
|
||||||
|
'输入-地址': param_map.get('address', ''),
|
||||||
|
# 响应参数 (取自 result_map)
|
||||||
|
'输出-省份': result_map.get('province', ''),
|
||||||
|
'输出-省份编码': result_map.get('provinceCode', ''),
|
||||||
|
'输出-城市名称': result_map.get('city', ''),
|
||||||
|
'输出-城市编码': result_map.get('cityCode', ''),
|
||||||
|
'输出-区域': result_map.get('district', ''),
|
||||||
|
'输出-区域编码': result_map.get('districtCode', ''),
|
||||||
|
'输出-乡镇街道': result_map.get('town', ''),
|
||||||
|
'输出-乡镇街道编码': result_map.get('townCode', ''),
|
||||||
|
'输出-是否涉农': verify_flag,
|
||||||
|
'输出-所属区域': result_map.get('addrArea', ''),
|
||||||
|
'输出-地址验证': result_map.get('addrVerify', ''),
|
||||||
|
'输出-是否为县级市': result_map.get('countyLevelCity', ''),
|
||||||
|
'输出-是否县城城区': result_map.get('countyGov', ''),
|
||||||
|
'输出-是否为省国家级开发区': result_map.get('devDistrict', ''),
|
||||||
|
'输出-省国家级开发区名称': result_map.get('devDistrictName', ''),
|
||||||
|
'输出-开发区等级': result_map.get('devLevel', '')
|
||||||
|
}
|
||||||
|
rows.append(row)
|
||||||
|
|
||||||
|
total_count = len(rows)
|
||||||
|
print(f"扫描命中记录: {scanned_count} 条,其中符合对公且是否涉农为'农'的记录: {total_count} 条")
|
||||||
|
|
||||||
|
# 生成 Excel 文件
|
||||||
|
tmp_dir = os.path.join(os.path.expanduser('~'), 'tmp', 'njcb_statement')
|
||||||
|
os.makedirs(tmp_dir, exist_ok=True)
|
||||||
|
filename = f"南京银行涉农API对公涉农调用流水_{last_year}{last_month:02d}.xlsx"
|
||||||
|
excel_path = os.path.join(tmp_dir, filename)
|
||||||
|
|
||||||
|
if rows:
|
||||||
|
df = pd.DataFrame(rows, columns=EXCEL_COLUMNS)
|
||||||
|
else:
|
||||||
|
df = pd.DataFrame(columns=EXCEL_COLUMNS)
|
||||||
|
|
||||||
|
df.to_excel(excel_path, index=False, engine='openpyxl')
|
||||||
|
print(f"Excel文件生成成功: {excel_path}")
|
||||||
|
|
||||||
|
# 发送邮件
|
||||||
|
receiver = 'guyzh@njcb.com.cn'
|
||||||
|
cc_receiver = ['wangwenyan@fangjia.com', 'daijinshui@fangjia.com']
|
||||||
|
subject = f"南京银行{last_year}年{last_month:02d}月涉农API对公涉农调用流水"
|
||||||
|
html_content = f"""
|
||||||
|
<h3>顾老师:</h3>
|
||||||
|
<p>您好!</p>
|
||||||
|
<p>附件为南京银行 <b>{last_year}年{last_month:02d}月</b> 涉农识别API(/addrSvc/njcbVillageIdentify)对公涉农调用流水,请查收。</p>
|
||||||
|
<ul>
|
||||||
|
<li><b>对账周期:</b> {start_time.strftime('%Y-%m-%d')} 至 {end_time.strftime('%Y-%m-%d')}</li>
|
||||||
|
<li><b>调用有效成功条数(Code=200 且 对公且判定为农):</b> {total_count} 条</li>
|
||||||
|
</ul>
|
||||||
|
<p>如有任何疑问,请随时与我们联系。</p>
|
||||||
|
"""
|
||||||
|
|
||||||
|
print(f"正在发送邮件至: {receiver}, 抄送: {cc_receiver}, 附件: {excel_path}")
|
||||||
|
send_email(
|
||||||
|
to=receiver,
|
||||||
|
subject=subject,
|
||||||
|
html_content=html_content,
|
||||||
|
files=[excel_path],
|
||||||
|
cc=cc_receiver
|
||||||
|
)
|
||||||
|
print("邮件发送完成。")
|
||||||
|
|
||||||
|
|
||||||
|
# Airflow DAG 定义:每月2号上午9:00执行(北京时间)
|
||||||
|
with DAG(
|
||||||
|
dag_id="njcb_village_identify_statement_dag",
|
||||||
|
schedule="0 9 2 * *",
|
||||||
|
start_date=pendulum.datetime(2024, 1, 1, tz="Asia/Shanghai"),
|
||||||
|
catchup=False,
|
||||||
|
description="每月2号给南京银行顾老师发送上个月涉农API对公涉农调用流水",
|
||||||
|
tags=["api", "njcb", "statement"]
|
||||||
|
) as dag:
|
||||||
|
|
||||||
|
send_statement_task = PythonOperator(
|
||||||
|
task_id="send_njcb_village_statement",
|
||||||
|
python_callable=export_and_send_njcb_statement
|
||||||
|
)
|
||||||
@@ -1,27 +1,32 @@
|
|||||||
from goojia_utils import REDIS_CLIENT
|
from goojia_utils import REDIS_CLIENT
|
||||||
import json
|
import json
|
||||||
|
import logging
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
"""
|
"""
|
||||||
高德key每日定时生成
|
高德key每日定时生成
|
||||||
"""
|
"""
|
||||||
#amap_key = {"4e9592208336433ee1b7aa4d1d434418", "10bcb421d2ee4e51ed9268d6dca58ad1"}
|
#amap_key = {"4e9592208336433ee1b7aa4d1d434418", "10bcb421d2ee4e51ed9268d6dca58ad1"}
|
||||||
|
|
||||||
|
log.info("测试生成key")
|
||||||
|
|
||||||
#老板新注册key,限制15万
|
#老板新注册key,限制15万
|
||||||
amap_key = {"6e7b0b0539ff51a714678cd69183fe32"}
|
# amap_key = {"6e7b0b0539ff51a714678cd69183fe32"}
|
||||||
|
#
|
||||||
service = {'v3/geocode/geo': 150000,
|
# service = {'v3/geocode/geo': 150000,
|
||||||
'v3/geocode/regeo': 150000,
|
# 'v3/geocode/regeo': 150000,
|
||||||
'v3/place/text': 150000,
|
# 'v3/place/text': 150000,
|
||||||
'v5/place/text': 150000,
|
# 'v5/place/text': 150000,
|
||||||
'v3/place/around': 150000,
|
# 'v3/place/around': 150000,
|
||||||
'v5/place/around': 150000,
|
# 'v5/place/around': 150000,
|
||||||
'v3/place/detail': 150000,
|
# 'v3/place/detail': 150000,
|
||||||
'v5/place/detail': 150000}
|
# 'v5/place/detail': 150000}
|
||||||
|
#
|
||||||
for service_key in service.keys():
|
# for service_key in service.keys():
|
||||||
val = dict()
|
# val = dict()
|
||||||
for key in amap_key:
|
# for key in amap_key:
|
||||||
val[key] = service[service_key]
|
# val[key] = service[service_key]
|
||||||
|
#
|
||||||
REDIS_CLIENT.hset("amapKey", service_key, json.dumps(val))
|
# REDIS_CLIENT.hset("amapKey", service_key, json.dumps(val))
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
from goojia_utils import LOG_MONGO_CLIENT,dingutils
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
import datetime
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
# mysql连接
|
||||||
|
engine = create_engine('mysql+pymysql://sa:G00jia@192.168.0.42:4306/fangjia?charset=utf8')
|
||||||
|
# mongodb连接
|
||||||
|
open_uri_log_conn = LOG_MONGO_CLIENT.get_collection('business', 'open_uri_log')
|
||||||
|
# 计算查询时间范围
|
||||||
|
now = datetime.datetime.now()
|
||||||
|
last = now - datetime.timedelta(hours=1)
|
||||||
|
|
||||||
|
now_year = now.year
|
||||||
|
now_month = now.month
|
||||||
|
now_day = now.day
|
||||||
|
now_hour = now.hour
|
||||||
|
|
||||||
|
last_year = last.year
|
||||||
|
last_month = last.month
|
||||||
|
last_day = last.day
|
||||||
|
last_hour = last.hour
|
||||||
|
|
||||||
|
def sync():
|
||||||
|
|
||||||
|
user_ids = []
|
||||||
|
|
||||||
|
df_mysql = pd.DataFrame(pd.read_sql("SELECT ID as user_id, IS_TEST FROM api_user WHERE IS_MONITOR = 1", con=engine))
|
||||||
|
|
||||||
|
for index in df_mysql.index:
|
||||||
|
user_id = df_mysql['user_id'].get(index)
|
||||||
|
user_ids.append(str(user_id))
|
||||||
|
|
||||||
|
df_mongo = pd.DataFrame(list(open_uri_log_conn.find(
|
||||||
|
{'user_id': {'$in': user_ids},
|
||||||
|
'c_date':
|
||||||
|
{'$gte': datetime.datetime(last_year, last_month, last_day, last_hour, 0),
|
||||||
|
'$lt': datetime.datetime(now_year, now_month, now_day, now_hour, 0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
).batch_size(10000)))
|
||||||
|
|
||||||
|
if len(df_mongo) > 0:
|
||||||
|
|
||||||
|
df_mongo = df_mongo.groupby(by=['user_id', 'user_name', 'request_uri', 'company_name']).agg(
|
||||||
|
{'_id': 'count', 'time': 'sum'}).reset_index()
|
||||||
|
|
||||||
|
df_mongo['user_id'] = df_mongo['user_id'].astype('int')
|
||||||
|
df = pd.merge(df_mongo, df_mysql, how='left', on=['user_id'])
|
||||||
|
df
|
||||||
|
text = ''
|
||||||
|
user_id_last = ''
|
||||||
|
for index in df.index:
|
||||||
|
user_id = df['user_id'].get(index)
|
||||||
|
request_uri = df['request_uri'].get(index)
|
||||||
|
count = df['_id'].get(index)
|
||||||
|
total_time = df['time'].get(index)
|
||||||
|
if user_id_last == user_id:
|
||||||
|
text = text + ' 使用的接口: ' + request_uri + ' 使用次数: ' + str(count) + ' 总计耗时: ' + str(total_time)
|
||||||
|
else:
|
||||||
|
user_name = df['user_name'].get(index)
|
||||||
|
company_name = df['company_name'].get(index)
|
||||||
|
if df['IS_TEST'].get(index) == 0:
|
||||||
|
user_type = '正式账户'
|
||||||
|
else:
|
||||||
|
user_type = '测试账户'
|
||||||
|
text = text + '\n' + company_name + ' 用户ID: ' + str(
|
||||||
|
user_id) + '(' + user_type + ')' + ' 用户名: ' + user_name + ' 使用的接口: ' + request_uri + ' 使用次数: ' + str(
|
||||||
|
count) + ' 总计耗时: ' + str(total_time)
|
||||||
|
user_id_last = user_id
|
||||||
|
sendDingDing(text)
|
||||||
|
|
||||||
|
def sendDingDing(text):
|
||||||
|
text = '【Api接口使用提醒】' + text
|
||||||
|
# robot_password = "SECf69d6dc5be4ca7e52b598d7eac75eb2e7d3bd6707c9f0a3ccb3b8a68ecf6f3fc"
|
||||||
|
# robot_webhook = "https://oapi.dingtalk.com/robot/send?access_token=f617e5b73808597d38101393b34c609cbe5353f9ed9a70cabda0b9e6e8ca371e"
|
||||||
|
token = '1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea'
|
||||||
|
# 发送钉钉群消息
|
||||||
|
dingutils.send_text(text,token)
|
||||||
|
|
||||||
|
sync()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@@ -29,18 +29,27 @@ def do():
|
|||||||
global count
|
global count
|
||||||
is_old = False
|
is_old = False
|
||||||
time_now = int(time.mktime(time.strptime(datetime.datetime.now().strftime('%Y-%m-%d'), '%Y-%m-%d')))
|
time_now = int(time.mktime(time.strptime(datetime.datetime.now().strftime('%Y-%m-%d'), '%Y-%m-%d')))
|
||||||
if 'open_date' in x and pd.notna(x['open_date']):
|
try:
|
||||||
|
if 'open_date' in x and pd.notna(x['open_date']) and str(x['open_date']) != '':
|
||||||
time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(x['open_date'], years=3), '%Y-%m-%d'), '%Y-%m-%d')))
|
time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(x['open_date'], years=3), '%Y-%m-%d'), '%Y-%m-%d')))
|
||||||
if time_old < time_now:
|
if time_old < time_now:
|
||||||
is_old = True
|
is_old = True
|
||||||
if is_old is False and 'live_date' in x and pd.notna(x['live_date']):
|
except Exception as e:
|
||||||
|
log.error(e)
|
||||||
|
try:
|
||||||
|
if is_old is False and 'live_date' in x and pd.notna(x['live_date']) and str(x['live_date']) != '':
|
||||||
time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(dateutils.parse_date(x['live_date']), years=3), '%Y-%m-%d'), '%Y-%m-%d')))
|
time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(dateutils.parse_date(x['live_date']), years=3), '%Y-%m-%d'), '%Y-%m-%d')))
|
||||||
if time_old < time_now:
|
if time_old < time_now:
|
||||||
is_old = True
|
is_old = True
|
||||||
if is_old is False and 'create_year' in x and pd.notna(x['create_year']):
|
except Exception as e:
|
||||||
time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(dateutils.parse_date(x['create_year'], '%Y-%m'), years=3), '%Y-%m-%d'), '%Y-%m-%d')))
|
log.error(e)
|
||||||
|
try:
|
||||||
|
if is_old is False and 'create_year' in x and pd.notna(x['create_year']) and str(x['create_year']) != '':
|
||||||
|
time_old = int(time.mktime(time.strptime(dateutils.format_date(dateutils.add_months(dateutils.parse_date(x['create_year'], '%Y'), years=3), '%Y-%m-%d'), '%Y-%m-%d')))
|
||||||
if time_old < time_now:
|
if time_old < time_now:
|
||||||
is_old = True
|
is_old = True
|
||||||
|
except Exception as e:
|
||||||
|
log.error(e)
|
||||||
# 查询样本库
|
# 查询样本库
|
||||||
names = [x['name']]
|
names = [x['name']]
|
||||||
if type(x['alias']) == list:
|
if type(x['alias']) == list:
|
||||||
|
|||||||
@@ -3,46 +3,71 @@
|
|||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
|
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, MAIN_MONGO_CLIENT
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
import numpy as np
|
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
import sys
|
import sys
|
||||||
|
from threading import Lock
|
||||||
|
|
||||||
|
mutex = Lock()
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||||
|
es89_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||||
|
|
||||||
index_name = "community_summary"
|
index_name = "community_summary"
|
||||||
m_date = dateutils.iso_format()
|
m_date = dateutils.iso_format()
|
||||||
path = "/home/airflow/data/community"
|
path = "/home/airflow/data/community"
|
||||||
|
file_path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||||
|
|
||||||
|
avg_cost_min_max_df = pd.read_excel(file_path + '/全国各区域人均消费中位值.xlsx')
|
||||||
|
avg_cost_city_region_dict = dict()
|
||||||
|
|
||||||
|
|
||||||
def find_around_price(lng, lat):
|
# 加载各区域人均消费最大最小值
|
||||||
|
def deal_avg_cost(x):
|
||||||
|
city_now = x['city']
|
||||||
|
region_now = x['region']
|
||||||
|
if pd.notna(city_now) and pd.notna(region_now):
|
||||||
|
avg_cost_city_region_dict[city_now + region_now] = {'min': x['3'], 'max': x['97']}
|
||||||
|
elif pd.notna(city_now):
|
||||||
|
avg_cost_city_region_dict[city_now] = {'min': x['3'], 'max': x['97']}
|
||||||
|
|
||||||
|
|
||||||
|
avg_cost_min_max_df.apply(lambda x: deal_avg_cost(x), axis=1)
|
||||||
|
|
||||||
|
|
||||||
|
def find_around_price(seaweed):
|
||||||
"""
|
"""
|
||||||
查找周边均价
|
查找周边均价
|
||||||
|
返回取值为10~300,为所有有价格的1-99百分位
|
||||||
"""
|
"""
|
||||||
if pd.isna(lng):
|
lng = commutils.format_lat_lng(seaweed['lng'])
|
||||||
return None
|
lat = commutils.format_lat_lng(seaweed['lat'])
|
||||||
|
city_short = seaweed['city']
|
||||||
|
region_short = seaweed['region']
|
||||||
|
city_display = areautils.get_city_display(city_short)
|
||||||
|
|
||||||
|
filter_query = [{"terms": {"tag": ["美食;中餐厅", "美食;外国餐厅"]}}]
|
||||||
|
if pd.notna(city_display):
|
||||||
|
filter_query.append({"term": {"city": {"value": city_display}}})
|
||||||
|
|
||||||
|
min_now = 10
|
||||||
|
max_now = 300
|
||||||
|
if (city_short+region_short) in avg_cost_city_region_dict:
|
||||||
|
min_now = avg_cost_city_region_dict[city_short + region_short]['min']
|
||||||
|
max_now = avg_cost_city_region_dict[city_short + region_short]['max']
|
||||||
|
elif city_short in avg_cost_city_region_dict:
|
||||||
|
min_now = avg_cost_city_region_dict[city_short]['min']
|
||||||
|
max_now = avg_cost_city_region_dict[city_short]['max']
|
||||||
|
|
||||||
|
filter_query.append({"range": {"price": {"gte": min_now, "lte": max_now}}})
|
||||||
|
filter_query.append({"geo_distance": {"distance": 1000, "location": [lng, lat]}})
|
||||||
|
|
||||||
ret_es = es206_client.elastic_client.search({
|
ret_es = es206_client.elastic_client.search({
|
||||||
"query": {
|
"query": {
|
||||||
"bool": {
|
"bool": {
|
||||||
"must": [
|
"filter": filter_query
|
||||||
{
|
|
||||||
"term": {
|
|
||||||
"status": {
|
|
||||||
"value": 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"filter": [
|
|
||||||
{
|
|
||||||
"geo_distance": {
|
|
||||||
"distance": 1000,
|
|
||||||
"location": [lng, lat]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"size": 0,
|
"size": 0,
|
||||||
@@ -59,72 +84,51 @@ def find_around_price(lng, lat):
|
|||||||
|
|
||||||
def build_doc(item):
|
def build_doc(item):
|
||||||
doc_dict = dict()
|
doc_dict = dict()
|
||||||
if 'avg_cost' in item and pd.notna(item['avg_cost']):
|
if 'avg_cost' in item:
|
||||||
doc_dict['avg_cost'] = item['avg_cost']
|
if pd.notna(item['avg_cost']) or item['avg_cost'] > 0:
|
||||||
|
doc_dict['avg_cost'] = int(item['avg_cost'])
|
||||||
|
else:
|
||||||
|
doc_dict['avg_cost'] = None
|
||||||
return doc_dict
|
return doc_dict
|
||||||
|
|
||||||
|
|
||||||
def update_status(new_df):
|
find_count = 0
|
||||||
actions = list()
|
|
||||||
|
|
||||||
def deal(item):
|
|
||||||
action_doc = {
|
|
||||||
"_op_type": "update",
|
|
||||||
"_index": "bd_poi",
|
|
||||||
"_id": item['_id'],
|
|
||||||
"doc": {'status': 0}
|
|
||||||
}
|
|
||||||
actions.append(action_doc)
|
|
||||||
if len(actions) == 5000:
|
|
||||||
helpers.bulk(es206_client.elastic_client, actions)
|
|
||||||
actions.clear()
|
|
||||||
return item
|
|
||||||
|
|
||||||
new_df.apply(lambda x: deal(x), axis=1)
|
def find_around_price_item(seaweed):
|
||||||
if len(actions) > 0:
|
global find_count
|
||||||
helpers.bulk(es206_client.elastic_client, actions)
|
find_count += 1
|
||||||
actions.clear()
|
if find_count % 1000 == 0:
|
||||||
|
log.info("已查找" + str(find_count))
|
||||||
|
if 'lng' not in seaweed:
|
||||||
|
return
|
||||||
|
price = find_around_price(seaweed)
|
||||||
|
mutex.acquire(timeout=60)
|
||||||
|
if pd.notna(price):
|
||||||
|
seaweed['avg_cost'] = int(price)
|
||||||
|
else:
|
||||||
|
seaweed['avg_cost'] = None
|
||||||
|
mutex.release()
|
||||||
|
|
||||||
|
|
||||||
def sync(city=None):
|
def sync(city=None):
|
||||||
log.info("开始同步消费数据...")
|
log.info("开始同步消费数据...")
|
||||||
query_sql = "select id, city, tag, type, price from bd_poi where price > 0 and tag in('美食;中餐厅', '美食;外国餐厅') "
|
query = {}
|
||||||
if pd.notna(city):
|
if pd.notna(city):
|
||||||
query_sql += " and city='" + areautils.get_city_display(city) + "'"
|
query['city'] = city
|
||||||
df = es206_client.scan_by_sql_to_data_frame(query_sql)
|
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
|
||||||
|
|
||||||
# 按百分位5-95筛选数据
|
seaweed_list = list()
|
||||||
cities = df['city'].unique()
|
seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1)
|
||||||
for city_now in cities:
|
|
||||||
df.loc[df['city'] == city_now, 'minPrice5'] = np.percentile(a=df[df['city'] == city_now]['price'], q=5, axis=0)
|
|
||||||
df.loc[df['city'] == city_now, 'minPrice95'] = np.percentile(a=df[df['city'] == city_now]['price'], q=95, axis=0)
|
|
||||||
|
|
||||||
new_df = df[(df['price'] >= df['minPrice5']) & (df['price'] <= df['minPrice95'])]
|
|
||||||
new_df['new_mean'] = new_df.groupby(by=['city'])['price'].transform(np.mean)
|
|
||||||
|
|
||||||
# 按最新的百分位更新库中价格状态
|
|
||||||
update_status(new_df)
|
|
||||||
|
|
||||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
|
||||||
if pd.notna(city):
|
|
||||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
|
||||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
|
||||||
|
|
||||||
def find_around_price_item(index, item):
|
|
||||||
if index % 1000 == 0:
|
|
||||||
log.info("已查找" + str(index))
|
|
||||||
if 'lng' not in item:
|
|
||||||
return
|
|
||||||
lng = commutils.format_lat_lng(item['lng'])
|
|
||||||
lat = commutils.format_lat_lng(item['lat'])
|
|
||||||
price = find_around_price(lng, lat)
|
|
||||||
seaweed_df.loc[index, 'avg_cost'] = price
|
|
||||||
|
|
||||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||||
for index, item in seaweed_df.iterrows():
|
for seaweed in seaweed_list:
|
||||||
pool.submit(find_around_price_item, index, item)
|
pool.submit(find_around_price_item, seaweed)
|
||||||
# for index, item in seaweed_df.iterrows():
|
|
||||||
# find_around_price_item(index, item)
|
# for seaweed in seaweed_list:
|
||||||
|
# find_around_price_item(seaweed)
|
||||||
|
|
||||||
|
seaweed_df = pd.DataFrame(seaweed_list)
|
||||||
actions = list()
|
actions = list()
|
||||||
count = 0
|
count = 0
|
||||||
for index, item in seaweed_df.iterrows():
|
for index, item in seaweed_df.iterrows():
|
||||||
@@ -140,16 +144,21 @@ def sync(city=None):
|
|||||||
|
|
||||||
actions.append(action)
|
actions.append(action)
|
||||||
if len(actions) == 5000:
|
if len(actions) == 5000:
|
||||||
helpers.bulk(es206_client.elastic_client, actions)
|
helpers.bulk(es89_client.elastic_client, actions)
|
||||||
count += len(actions)
|
count += len(actions)
|
||||||
log.info("更新:" + str(count))
|
log.info("更新:" + str(count))
|
||||||
actions.clear()
|
actions.clear()
|
||||||
if len(actions) > 0:
|
if len(actions) > 0:
|
||||||
helpers.bulk(es206_client.elastic_client, actions)
|
helpers.bulk(es89_client.elastic_client, actions)
|
||||||
actions.clear()
|
actions.clear()
|
||||||
|
|
||||||
log.info("完成同步消费数据...")
|
log.info("完成同步消费数据...")
|
||||||
|
|
||||||
|
|
||||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
sync(city)
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
|
|||||||
@@ -3,14 +3,15 @@
|
|||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from goojia_utils import ElasticSearchClient, commutils, dateutils
|
from goojia_utils import ElasticSearchClient, commutils, dateutils, MAIN_MONGO_CLIENT
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from threading import Lock
|
from threading import Lock
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||||
|
|
||||||
index_name = "community_summary"
|
index_name = "community_summary"
|
||||||
path = "/home/airflow/data/community"
|
path = "/home/airflow/data/community"
|
||||||
@@ -89,10 +90,10 @@ def deal(actions, item):
|
|||||||
|
|
||||||
def sync(city=None):
|
def sync(city=None):
|
||||||
log.info(f"开始同步商圈至206 {index_name}")
|
log.info(f"开始同步商圈至206 {index_name}")
|
||||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
query = {}
|
||||||
if pd.notna(city):
|
if pd.notna(city):
|
||||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
query['city'] = city
|
||||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
|
||||||
|
|
||||||
actions = list()
|
actions = list()
|
||||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||||
@@ -105,5 +106,10 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步商圈至206 {index_name}")
|
log.info(f"完成同步商圈至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
sync(city)
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
"""
|
||||||
|
城市等级及城市价格等级计算
|
||||||
|
"""
|
||||||
|
from goojia_utils import ElasticSearchClient
|
||||||
|
import pandas as pd
|
||||||
|
from sklearn.mixture import GaussianMixture
|
||||||
|
from sklearn.preprocessing import QuantileTransformer
|
||||||
|
import logging
|
||||||
|
from elasticsearch import helpers
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
def sync():
|
||||||
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||||
|
|
||||||
|
new_df = pd.read_excel(path + '/城市等级及城市价格等级信息.xlsx')
|
||||||
|
community_summary_df = es206_client.scan_by_sql_to_data_frame(
|
||||||
|
'select city, region, name, consumption_potential from community_summary')
|
||||||
|
|
||||||
|
merge_df = pd.merge(community_summary_df, new_df, how='left', on=['city'])
|
||||||
|
city_grade = ['一线', '新一线', '二线', '三四线', '五线']
|
||||||
|
rating = [1, 2, 3, 4, 5]
|
||||||
|
index_name = "community_summary"
|
||||||
|
|
||||||
|
def filter_min_max(community_summary_df, source):
|
||||||
|
community_summary_df['normal_consumption_potential'] = QuantileTransformer(
|
||||||
|
output_distribution="normal").fit_transform(community_summary_df[[source]])
|
||||||
|
return community_summary_df
|
||||||
|
|
||||||
|
def gmm_train(total_rate, consumption_df, source, target):
|
||||||
|
# 高斯混合模型
|
||||||
|
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(
|
||||||
|
consumption_df[[source]])
|
||||||
|
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
|
||||||
|
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
|
||||||
|
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense',
|
||||||
|
ascending=False).astype(
|
||||||
|
int)
|
||||||
|
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
|
||||||
|
return consumption_df
|
||||||
|
|
||||||
|
log.info("开始按城市等级评级...")
|
||||||
|
merge_df = merge_df[merge_df['consumption_potential'].notna()].copy()
|
||||||
|
merge_df['consumption_potential_city_grade'] = merge_df['consumption_potential'] * merge_df['city_grade_ratio']
|
||||||
|
merge_df['consumption_potential_city_grade'].fillna(merge_df['consumption_potential'], inplace=True)
|
||||||
|
|
||||||
|
city_grade_ret_df = pd.DataFrame()
|
||||||
|
for grade in city_grade:
|
||||||
|
temp_df = merge_df[merge_df['city_grade'] == grade].copy()
|
||||||
|
log.info(f"当前城市等级{grade}, 评级数量{len(temp_df)}")
|
||||||
|
temp_df = filter_min_max(temp_df, 'consumption_potential_city_grade')
|
||||||
|
source = "normal_consumption_potential"
|
||||||
|
target = "city_level_rating"
|
||||||
|
temp_df = gmm_train(5, temp_df, source, target)
|
||||||
|
city_grade_ret_df = pd.concat([city_grade_ret_df, temp_df])
|
||||||
|
log.info("完成按城市等级评级...")
|
||||||
|
|
||||||
|
log.info("开始按城市价格等级评级")
|
||||||
|
merge_df = merge_df[merge_df['consumption_potential'].notna()].copy()
|
||||||
|
merge_df['consumption_potential_rating'] = merge_df['consumption_potential'] * merge_df['rating_ratio']
|
||||||
|
merge_df['consumption_potential_rating'].fillna(merge_df['consumption_potential'], inplace=True)
|
||||||
|
|
||||||
|
rating_ret_df = pd.DataFrame()
|
||||||
|
for grade in rating:
|
||||||
|
temp_df = merge_df[merge_df['rating'] == grade].copy()
|
||||||
|
log.info(f"当前城市价格等级{grade}, 评级数量{len(temp_df)}")
|
||||||
|
temp_df = filter_min_max(temp_df, 'consumption_potential_rating')
|
||||||
|
source = "normal_consumption_potential"
|
||||||
|
target = "city_price_level_rating"
|
||||||
|
temp_df = gmm_train(5, temp_df, source, target)
|
||||||
|
rating_ret_df = pd.concat([rating_ret_df, temp_df])
|
||||||
|
log.info("完成按城市价格等级评级")
|
||||||
|
|
||||||
|
city_grade_ret_df = city_grade_ret_df[['_id', 'city', 'region', 'name', 'city_grade', 'city_level_rating']]
|
||||||
|
rating_ret_df = rating_ret_df[['_id', 'city', 'region', 'name', 'rating', 'city_price_level_rating']]
|
||||||
|
|
||||||
|
aim_df = pd.merge(city_grade_ret_df, rating_ret_df, how='outer', on=['_id', 'city', 'region', 'name'])
|
||||||
|
|
||||||
|
log.info(f"开始写入ES206索引{index_name}")
|
||||||
|
actions = list()
|
||||||
|
count = 0
|
||||||
|
|
||||||
|
city_grade_dict = {'一线': 1, '新一线': 2, '二线': 3, '三四线': 4, '五线': 5}
|
||||||
|
for index, item in aim_df.iterrows():
|
||||||
|
row = dict()
|
||||||
|
_id = str(item["_id"])
|
||||||
|
if pd.notna(item['city_grade']):
|
||||||
|
row['city_level'] = city_grade_dict[item['city_grade']]
|
||||||
|
if pd.notna(item['city_level_rating']):
|
||||||
|
row['city_level_rating'] = item['city_level_rating']
|
||||||
|
if pd.notna(item['rating']):
|
||||||
|
row['city_price_level'] = int(item['rating'])
|
||||||
|
if pd.notna(item['city_price_level_rating']):
|
||||||
|
row['city_price_level_rating'] = item['city_price_level_rating']
|
||||||
|
|
||||||
|
action = {'_op_type': 'update',
|
||||||
|
'_index': index_name,
|
||||||
|
'_id': _id, # 请勿忘记ID
|
||||||
|
'doc': row,
|
||||||
|
"_retry_on_conflict": 5
|
||||||
|
}
|
||||||
|
actions.append(action)
|
||||||
|
if len(actions) == 5000:
|
||||||
|
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||||
|
count += len(actions)
|
||||||
|
actions.clear()
|
||||||
|
log.info("评级中: 当前已评级{}".format(count))
|
||||||
|
if len(actions) > 0:
|
||||||
|
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||||
|
count += len(actions)
|
||||||
|
actions.clear()
|
||||||
|
log.info("评级完成,共更新: {}".format(count))
|
||||||
|
log.info(f"完成写入ES206索引{index_name}")
|
||||||
|
|
||||||
|
|
||||||
|
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
# 对于该评级等级来说,不能单个城市处理,只能全量
|
||||||
|
if pd.isna(city):
|
||||||
|
sync()
|
||||||
@@ -1,9 +1,39 @@
|
|||||||
"""
|
"""
|
||||||
同步小区基本信息至ES206
|
同步小区基本信息至ES206
|
||||||
"""
|
"""
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
# 强制设置 Python 进程内部本地时区为北京时间(CST / UTC+8),消除 Docker 容器内的 UTC 时区日志差异
|
||||||
|
os.environ['TZ'] = 'Asia/Shanghai'
|
||||||
|
if hasattr(time, 'tzset'):
|
||||||
|
time.tzset()
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
# 兼容性补丁:解决 goojia_utils.ClickHouseClient.to_sql_plus 在将 DataFrame 写入 ClickHouse 时,
|
||||||
|
# 对含有 list/array 类型的列(如 alias)进行 pd.notna(item) 判定时导致 ValueError 的 bug
|
||||||
|
_orig_notna = pd.notna
|
||||||
|
def patched_notna(obj):
|
||||||
|
if isinstance(obj, (list, tuple, np.ndarray, dict, set)):
|
||||||
|
return True
|
||||||
|
try:
|
||||||
|
res = _orig_notna(obj)
|
||||||
|
if hasattr(res, 'any'):
|
||||||
|
return bool(res.any())
|
||||||
|
elif isinstance(res, (list, np.ndarray)):
|
||||||
|
return any(res)
|
||||||
|
return res
|
||||||
|
except Exception:
|
||||||
|
return True
|
||||||
|
|
||||||
|
pd.notna = patched_notna
|
||||||
|
pd.isna = lambda x: not patched_notna(x)
|
||||||
|
import pandas
|
||||||
|
pandas.notna = patched_notna
|
||||||
|
pandas.isna = lambda x: not patched_notna(x)
|
||||||
|
|
||||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
import os
|
import os
|
||||||
@@ -11,7 +41,8 @@ import sys
|
|||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||||
|
|
||||||
index_name = "community_summary"
|
index_name = "community_summary"
|
||||||
m_date = dateutils.iso_format()
|
m_date = dateutils.iso_format()
|
||||||
@@ -47,7 +78,13 @@ def build_doc(item):
|
|||||||
|
|
||||||
estate_charge = commutils.format_property_fee(item['estate_charge'])
|
estate_charge = commutils.format_property_fee(item['estate_charge'])
|
||||||
if pd.notna(estate_charge):
|
if pd.notna(estate_charge):
|
||||||
|
# 剔除过大物业费导致ES报错
|
||||||
|
if float(estate_charge) <= 1000:
|
||||||
doc_dict['estate_charge'] = float(estate_charge)
|
doc_dict['estate_charge'] = float(estate_charge)
|
||||||
|
# 表示该物业费已经从库中补充
|
||||||
|
doc_dict['estate_charge_flag'] = 0
|
||||||
|
# 先打10分,对于其他的流程在修改分数
|
||||||
|
doc_dict['credit_detail'] = {'estate_charge_score': 10}
|
||||||
|
|
||||||
finish_building_date = commutils.format_finish_building_date(item['finish_building_date'])
|
finish_building_date = commutils.format_finish_building_date(item['finish_building_date'])
|
||||||
if pd.notna(finish_building_date):
|
if pd.notna(finish_building_date):
|
||||||
@@ -57,6 +94,31 @@ def build_doc(item):
|
|||||||
lat = commutils.format_lat_lng(item['lat'])
|
lat = commutils.format_lat_lng(item['lat'])
|
||||||
if pd.notna(lng) and pd.notna(lat):
|
if pd.notna(lng) and pd.notna(lat):
|
||||||
doc_dict['location'] = {'lon': lng, 'lat': lat}
|
doc_dict['location'] = {'lon': lng, 'lat': lat}
|
||||||
|
|
||||||
|
# 新增字段存储
|
||||||
|
all_area = commutils.format_all_area(item['all_area'])
|
||||||
|
if pd.notna(all_area):
|
||||||
|
doc_dict['all_area'] = int(all_area)
|
||||||
|
build_area = commutils.format_build_area(item['build_area'])
|
||||||
|
if pd.notna(build_area):
|
||||||
|
doc_dict['build_area'] = int(build_area)
|
||||||
|
greening_rate = commutils.format_greening_rate(item['virescence_percent'])
|
||||||
|
if pd.notna(greening_rate):
|
||||||
|
doc_dict['greening_rate'] = greening_rate
|
||||||
|
volume_rate = commutils.format_plot_ratio(item['cubage_percent'])
|
||||||
|
if pd.notna(volume_rate):
|
||||||
|
doc_dict['volume_rate'] = volume_rate
|
||||||
|
house_type_area_dic = commutils.format_house_type_area(item['main_type'], item['main_type_area'])
|
||||||
|
main_house_type = house_type_area_dic['house_type']
|
||||||
|
main_house_area = house_type_area_dic['house_area']
|
||||||
|
if pd.notna(main_house_area):
|
||||||
|
try:
|
||||||
|
doc_dict['main_house_area'] = int(float(main_house_area))
|
||||||
|
except Exception as e:
|
||||||
|
pass
|
||||||
|
if pd.notna(main_house_type):
|
||||||
|
doc_dict['main_house_type'] = main_house_type
|
||||||
|
|
||||||
return doc_dict
|
return doc_dict
|
||||||
|
|
||||||
|
|
||||||
@@ -84,21 +146,35 @@ def deal_comm(community_summary_df, seaweed_df):
|
|||||||
def sync(city=None):
|
def sync(city=None):
|
||||||
log.info(f"开始同步小区基本属性至206 {index_name}...")
|
log.info(f"开始同步小区基本属性至206 {index_name}...")
|
||||||
seaweed_query = {'status': 0, 'cat': 'district'}
|
seaweed_query = {'status': 0, 'cat': 'district'}
|
||||||
|
mongo_delete_query = {}
|
||||||
if pd.notna(city):
|
if pd.notna(city):
|
||||||
seaweed_query['city'] = city
|
seaweed_query['city'] = city
|
||||||
|
mongo_delete_query['city'] = city
|
||||||
|
community_summary_conn.delete_many(mongo_delete_query)
|
||||||
|
|
||||||
seaweed_df = pd.DataFrame(
|
seaweed_df = pd.DataFrame(
|
||||||
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1,
|
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1,
|
||||||
'alias': 1, 'estate_type2': 1, 'build_type': 1,
|
'alias': 1, 'estate_type2': 1, 'build_type': 1,
|
||||||
'household_count': 1, 'block': 1,
|
'household_count': 1, 'block': 1,
|
||||||
'estate_charge': 1, 'finish_building_date': 1})
|
'estate_charge': 1, 'finish_building_date': 1,
|
||||||
|
'all_area': 1, 'build_area': 1, 'main_type': 1, 'main_type_area': 1, 'cubage_percent': 1, 'virescence_percent': 1
|
||||||
|
})
|
||||||
.batch_size(100000)))
|
.batch_size(100000)))
|
||||||
|
|
||||||
sql = f"select id from {index_name} "
|
sql = f"select id from {index_name} "
|
||||||
|
|
||||||
if pd.notna(city):
|
if pd.notna(city):
|
||||||
sql += f" where city='{city}'"
|
sql += f" where city='{city}'"
|
||||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||||
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
|
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
|
||||||
|
# 清除别名中逗号
|
||||||
|
def dealAlias(x):
|
||||||
|
if isinstance(x, list) and len(x) > 0:
|
||||||
|
for index, element in enumerate(x):
|
||||||
|
if "'" in element:
|
||||||
|
x[index] = str(element).replace("'", "")
|
||||||
|
return x
|
||||||
|
seaweed_df['alias'] = seaweed_df['alias'].apply(lambda x: dealAlias(x))
|
||||||
|
|
||||||
if len(community_summary_df) > 0:
|
if len(community_summary_df) > 0:
|
||||||
community_summary_df['flag'] = 1
|
community_summary_df['flag'] = 1
|
||||||
@@ -109,6 +185,19 @@ def sync(city=None):
|
|||||||
seaweed_existed_df = pd.DataFrame()
|
seaweed_existed_df = pd.DataFrame()
|
||||||
seaweed_not_existed_df = seaweed_df
|
seaweed_not_existed_df = seaweed_df
|
||||||
|
|
||||||
|
# 写入csv文件,防止小区变动导致的后续过程更新报错,
|
||||||
|
# seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
|
||||||
|
# 写入中间集合 MongoDB, 解决分布式 Worker 节点的共享问题
|
||||||
|
insert_cols = ['_id', 'city', 'region', 'name', 'lng', 'lat', 'alias', 'estate_type2', 'build_type',
|
||||||
|
'household_count', 'block', 'estate_charge', 'finish_building_date']
|
||||||
|
records = seaweed_df[insert_cols].to_dict(orient='records')
|
||||||
|
if len(records) > 0:
|
||||||
|
log.info(f"正在写入 MongoDB community_summary 中间集合,共 {len(records)} 条记录...")
|
||||||
|
batch_size = 5000
|
||||||
|
for i in range(0, len(records), batch_size):
|
||||||
|
community_summary_conn.insert_many(records[i:i + batch_size])
|
||||||
|
log.info("MongoDB community_summary 写入完毕。")
|
||||||
|
|
||||||
actions = list()
|
actions = list()
|
||||||
# 对于存在的更新
|
# 对于存在的更新
|
||||||
count = 0
|
count = 0
|
||||||
@@ -128,6 +217,11 @@ def sync(city=None):
|
|||||||
count += len(actions)
|
count += len(actions)
|
||||||
log.info("更新:" + str(count))
|
log.info("更新:" + str(count))
|
||||||
actions.clear()
|
actions.clear()
|
||||||
|
if len(actions) > 0:
|
||||||
|
helpers.bulk(es206_client.elastic_client, actions)
|
||||||
|
count += len(actions)
|
||||||
|
log.info("更新:" + str(count))
|
||||||
|
actions.clear()
|
||||||
|
|
||||||
# 对于不存在的插入
|
# 对于不存在的插入
|
||||||
count = 0
|
count = 0
|
||||||
@@ -138,7 +232,8 @@ def sync(city=None):
|
|||||||
"_op_type": "index",
|
"_op_type": "index",
|
||||||
"_index": index_name,
|
"_index": index_name,
|
||||||
"_id": item['_id'],
|
"_id": item['_id'],
|
||||||
"_source": doc_info
|
"_source": doc_info,
|
||||||
|
"_retry_on_conflict": 5
|
||||||
}
|
}
|
||||||
|
|
||||||
actions.append(action)
|
actions.append(action)
|
||||||
@@ -149,14 +244,18 @@ def sync(city=None):
|
|||||||
actions.clear()
|
actions.clear()
|
||||||
if len(actions) > 0:
|
if len(actions) > 0:
|
||||||
helpers.bulk(es206_client.elastic_client, actions)
|
helpers.bulk(es206_client.elastic_client, actions)
|
||||||
|
count += len(actions)
|
||||||
|
log.info("插入:" + str(count))
|
||||||
actions.clear()
|
actions.clear()
|
||||||
|
|
||||||
# 写入csv文件,防止小区变动导致的后续过程更新报错
|
|
||||||
seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
|
|
||||||
|
|
||||||
deal_comm(community_summary_df, seaweed_df)
|
deal_comm(community_summary_df, seaweed_df)
|
||||||
log.info(f"完成同步小区基本属性至206 {index_name}")
|
log.info(f"完成同步小区基本属性至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
sync(city)
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
"""
|
"""
|
||||||
月消费潜力、评级计算
|
月消费潜力、评级计算
|
||||||
"""
|
"""
|
||||||
|
import warnings
|
||||||
|
|
||||||
from goojia_utils import ElasticSearchClient, dateutils, nextcloudutils
|
from goojia_utils import ElasticSearchClient, dateutils, nextcloudutils
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@@ -9,13 +11,16 @@ import os
|
|||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
import matplotlib.pyplot as plt
|
import matplotlib.pyplot as plt
|
||||||
import sys
|
import sys
|
||||||
|
from scipy.stats import boxcox
|
||||||
|
|
||||||
|
from sklearn.preprocessing import RobustScaler
|
||||||
from sklearn.mixture import GaussianMixture
|
from sklearn.mixture import GaussianMixture
|
||||||
from sklearn.preprocessing import QuantileTransformer
|
from sklearn.preprocessing import QuantileTransformer
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
# 忽略警告
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
warnings.filterwarnings("ignore", category=UserWarning)
|
||||||
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
index_name = "community_summary"
|
index_name = "community_summary"
|
||||||
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||||
save_path_prefix = "/home/airflow/data/community"
|
save_path_prefix = "/home/airflow/data/community"
|
||||||
@@ -28,18 +33,41 @@ nextcloud_aim_file_path = '共享/财力评级/财力评级图'
|
|||||||
|
|
||||||
reasonable_df = pd.read_excel(path + '/财力评级分布情况20221215.xlsx')
|
reasonable_df = pd.read_excel(path + '/财力评级分布情况20221215.xlsx')
|
||||||
reasonable_dict = reasonable_df.set_index('city')['是否合理'].to_dict()
|
reasonable_dict = reasonable_df.set_index('city')['是否合理'].to_dict()
|
||||||
|
scaler = RobustScaler()
|
||||||
|
|
||||||
|
|
||||||
|
# Min-Max 归一化到 [1, 100]
|
||||||
|
def min_max_scale(df, column, target_column, new_min=1, new_max=100):
|
||||||
|
X_min = df[column].min()
|
||||||
|
X_max = df[column].max()
|
||||||
|
# 如果存在缺失值,则不进行归一化
|
||||||
|
if pd.isna(X_min):
|
||||||
|
return df
|
||||||
|
|
||||||
|
# 防止分母为零(如果所有值相等)
|
||||||
|
if X_max == X_min:
|
||||||
|
df[target_column] = new_min # 如果所有值相同,设为下界
|
||||||
|
else:
|
||||||
|
df[target_column] = (new_min + (df[column] - X_min) * (new_max - new_min) / (X_max - X_min))
|
||||||
|
df[target_column] = df[target_column].astype(int)
|
||||||
|
return df
|
||||||
|
|
||||||
def calc_consumption_potential(city):
|
def calc_consumption_potential(city):
|
||||||
"""
|
"""
|
||||||
计算消费潜力
|
计算消费潜力
|
||||||
"""
|
"""
|
||||||
sql = "select id, city, region, name, total_price, estate_charge, relative_price_ranking, avg_cost from community_summary where city='" + city + "'"
|
sql = "select id, city, region, name, total_price, estate_charge, estate_charge_fix, relative_price_ranking, avg_cost from community_summary where city='" + city + "'"
|
||||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||||
columns = community_summary_df.columns
|
columns = community_summary_df.columns
|
||||||
if 'total_price' not in columns or 'estate_charge' not in columns or 'relative_price_ranking' not in columns or 'avg_cost' not in columns:
|
if 'total_price' not in columns or 'estate_charge' not in columns or 'relative_price_ranking' not in columns or 'avg_cost' not in columns:
|
||||||
return []
|
return []
|
||||||
community_summary_df = community_summary_df.dropna(how='any').copy()
|
# 解决某些城市无修正物业费问题
|
||||||
|
if 'estate_charge_fix' not in community_summary_df.columns:
|
||||||
|
community_summary_df['estate_charge_fix'] = None
|
||||||
|
# 修正的物业费入模
|
||||||
|
community_summary_df['estate_charge_fix'].fillna(community_summary_df['estate_charge'], inplace=True)
|
||||||
|
|
||||||
|
#community_summary_df = community_summary_df.dropna(how='any').copy()
|
||||||
if len(community_summary_df) == 0:
|
if len(community_summary_df) == 0:
|
||||||
log.warning(f"{city}无法评级,缺少因子")
|
log.warning(f"{city}无法评级,缺少因子")
|
||||||
return []
|
return []
|
||||||
@@ -47,13 +75,47 @@ def calc_consumption_potential(city):
|
|||||||
# 户均总价计算时候改成万
|
# 户均总价计算时候改成万
|
||||||
community_summary_df['total_price'] = round(community_summary_df['total_price'] / 10000, 0)
|
community_summary_df['total_price'] = round(community_summary_df['total_price'] / 10000, 0)
|
||||||
|
|
||||||
community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['total_price'].transform(
|
community_summary_df['total_price_fixed'] = community_summary_df['total_price']
|
||||||
np.mean)
|
community_summary_df['avg_cost_fixed'] = community_summary_df['avg_cost']
|
||||||
community_summary_df['total_price_std'] = community_summary_df.groupby(by=['city'])['total_price'].transform(np.std)
|
|
||||||
|
|
||||||
community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(
|
community_summary_df['estate_charge_fixed'] = community_summary_df['estate_charge_fix']
|
||||||
|
# community_summary_df['total_price_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform(
|
||||||
|
# community_summary_df[['total_price']])
|
||||||
|
# community_summary_df['avg_cost_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform(
|
||||||
|
# community_summary_df[['avg_cost']])
|
||||||
|
#
|
||||||
|
# community_summary_df['estate_charge_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform(
|
||||||
|
# community_summary_df[['estate_charge_fix']])
|
||||||
|
# 异常值处理
|
||||||
|
total_price_fixed_threshold = community_summary_df['total_price_fixed'].quantile(0.99)
|
||||||
|
community_summary_df['total_price_fixed'] = np.where(community_summary_df['total_price_fixed'] > total_price_fixed_threshold, total_price_fixed_threshold, community_summary_df['total_price_fixed'])
|
||||||
|
max_total_price_fixed = community_summary_df['total_price_fixed'].max()
|
||||||
|
community_summary_df['total_price_fixed'] = np.log(community_summary_df['total_price_fixed'] + 1) / np.log(max_total_price_fixed + 1)
|
||||||
|
|
||||||
|
# 人均消费极大值处理
|
||||||
|
avg_cost_fixed_threshold = community_summary_df['avg_cost_fixed'].quantile(0.99)
|
||||||
|
community_summary_df['avg_cost_fixed'] = np.where(community_summary_df['avg_cost_fixed'] > avg_cost_fixed_threshold, avg_cost_fixed_threshold, community_summary_df['avg_cost_fixed'])
|
||||||
|
max_avg_cost_fixed = community_summary_df['avg_cost_fixed'].max()
|
||||||
|
try:
|
||||||
|
#community_summary_df['avg_cost_fixed'] = np.log1p(community_summary_df['avg_cost_fixed'])
|
||||||
|
community_summary_df['avg_cost_fixed'] = np.log(community_summary_df['avg_cost_fixed'] + 1) / np.log(max_avg_cost_fixed + 1)
|
||||||
|
except Exception:
|
||||||
|
community_summary_df['avg_cost_fixed'] = None
|
||||||
|
pass
|
||||||
|
#community_summary_df['avg_cost_fixed'] = 1 - np.log(community_summary_df['avg_cost_fixed'] + 1) / np.log(max_avg_cost_fixed + 1)
|
||||||
|
estate_charge_fixed_threshold = community_summary_df['estate_charge_fixed'].quantile(0.99)
|
||||||
|
community_summary_df['estate_charge_fixed'] = np.where(community_summary_df['estate_charge_fixed'] > estate_charge_fixed_threshold, estate_charge_fixed_threshold, community_summary_df['estate_charge_fixed'])
|
||||||
|
max_estate_charge_fixed = community_summary_df['estate_charge_fixed'].max()
|
||||||
|
community_summary_df['estate_charge_fixed'] = np.log(community_summary_df['estate_charge_fixed'] + 1) / np.log(max_estate_charge_fixed + 1)
|
||||||
|
|
||||||
|
#relative_price_ranking_threshold = community_summary_df['relative_price_ranking'].quantile(0.99)
|
||||||
|
#community_summary_df['relative_price_ranking'] = np.where(community_summary_df['relative_price_ranking'] > relative_price_ranking_threshold, relative_price_ranking_threshold, community_summary_df['relative_price_ranking'])
|
||||||
|
max_relative_price_ranking_fixed = community_summary_df['relative_price_ranking'].max()
|
||||||
|
community_summary_df['relative_price_ranking'] = np.log(community_summary_df['relative_price_ranking'] + 1) / np.log(max_relative_price_ranking_fixed + 1)
|
||||||
|
|
||||||
|
community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge_fixed'].transform(
|
||||||
np.mean)
|
np.mean)
|
||||||
community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(
|
community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge_fixed'].transform(
|
||||||
np.std)
|
np.std)
|
||||||
|
|
||||||
community_summary_df['relative_price_ranking_mean'] = community_summary_df.groupby(by=['city'])[
|
community_summary_df['relative_price_ranking_mean'] = community_summary_df.groupby(by=['city'])[
|
||||||
@@ -61,17 +123,25 @@ def calc_consumption_potential(city):
|
|||||||
community_summary_df['relative_price_ranking_std'] = community_summary_df.groupby(by=['city'])[
|
community_summary_df['relative_price_ranking_std'] = community_summary_df.groupby(by=['city'])[
|
||||||
'relative_price_ranking'].transform(np.std)
|
'relative_price_ranking'].transform(np.std)
|
||||||
|
|
||||||
community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.mean)
|
try:
|
||||||
community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.std)
|
community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['avg_cost_fixed'].transform(np.mean)
|
||||||
|
community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['avg_cost_fixed'].transform(np.std)
|
||||||
|
except Exception:
|
||||||
|
community_summary_df['avg_cost_mean'] = None
|
||||||
|
community_summary_df['avg_cost_std'] = None
|
||||||
|
|
||||||
|
community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['total_price_fixed'].transform(
|
||||||
|
np.mean)
|
||||||
|
community_summary_df['total_price_std'] = community_summary_df.groupby(by=['city'])['total_price_fixed'].transform(np.std)
|
||||||
|
|
||||||
# 和田等小城市均值可能为0
|
# 和田等小城市均值可能为0
|
||||||
if community_summary_df['total_price_std'].unique()[0] != 0:
|
if community_summary_df['total_price_std'].unique()[0] != 0:
|
||||||
community_summary_df['total_price_score'] = 286.0134196 * (community_summary_df['total_price'] - community_summary_df['total_price_mean']) / community_summary_df['total_price_std']
|
community_summary_df['total_price_score'] = 286.0134196 * (community_summary_df['total_price_fixed'] - community_summary_df['total_price_mean']) / community_summary_df['total_price_std']
|
||||||
else:
|
else:
|
||||||
community_summary_df['total_price_score'] = 0
|
community_summary_df['total_price_score'] = 0
|
||||||
|
|
||||||
if community_summary_df['estate_charge_std'].unique()[0] != 0:
|
if community_summary_df['estate_charge_std'].unique()[0] != 0:
|
||||||
community_summary_df['estate_charge_score'] = 139.1374524 * (community_summary_df['estate_charge'] - community_summary_df['estate_charge_mean']) / community_summary_df['estate_charge_std']
|
community_summary_df['estate_charge_score'] = 139.1374524 * (community_summary_df['estate_charge_fixed'] - community_summary_df['estate_charge_mean']) / community_summary_df['estate_charge_std']
|
||||||
else:
|
else:
|
||||||
community_summary_df['estate_charge_score'] = 0
|
community_summary_df['estate_charge_score'] = 0
|
||||||
|
|
||||||
@@ -81,18 +151,26 @@ def calc_consumption_potential(city):
|
|||||||
community_summary_df['relative_price_ranking_score'] = 0
|
community_summary_df['relative_price_ranking_score'] = 0
|
||||||
|
|
||||||
if community_summary_df['avg_cost_std'].unique()[0] != 0:
|
if community_summary_df['avg_cost_std'].unique()[0] != 0:
|
||||||
community_summary_df['avg_cost_score'] = 67.0195871700 * (community_summary_df['avg_cost'] - community_summary_df['avg_cost_mean']) / community_summary_df['avg_cost_std']
|
community_summary_df['avg_cost_score'] = 67.0195871700 * (community_summary_df['avg_cost_fixed'] - community_summary_df['avg_cost_mean']) / community_summary_df['avg_cost_std']
|
||||||
else:
|
else:
|
||||||
community_summary_df['avg_cost_score'] = 0
|
community_summary_df['avg_cost_score'] = 0
|
||||||
|
|
||||||
community_summary_df['consumption_potential'] = community_summary_df['total_price_score'] + community_summary_df[
|
community_summary_df['consumption_potential'] = community_summary_df['total_price_score'] + community_summary_df[
|
||||||
'estate_charge_score'] + community_summary_df['relative_price_ranking_score'] + community_summary_df['avg_cost_score'] + community_summary_df['avg_cost_score'] + 4099.978248
|
'estate_charge_score'] + community_summary_df['relative_price_ranking_score'] + community_summary_df['avg_cost_score'] + 4099.978248
|
||||||
return community_summary_df
|
return community_summary_df
|
||||||
|
|
||||||
|
|
||||||
def gmm_train(total_rate, consumption_df, source, target):
|
def gmm_train(total_rate, consumption_df, source, target):
|
||||||
# 高斯混合模型
|
# 高斯混合模型
|
||||||
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(consumption_df[[source]])
|
gmm = GaussianMixture(
|
||||||
|
n_components=total_rate,
|
||||||
|
covariance_type='full', # 更灵活的协方差矩阵
|
||||||
|
max_iter=200,
|
||||||
|
n_init=10,
|
||||||
|
reg_covar=1e-6, # 正则化防止奇异矩阵
|
||||||
|
random_state=42
|
||||||
|
).fit(consumption_df[[source]])
|
||||||
|
consumption_df = consumption_df.copy()
|
||||||
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
|
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
|
||||||
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
|
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
|
||||||
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
|
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
|
||||||
@@ -104,27 +182,86 @@ def calc_consumption_rating(community_summary_df):
|
|||||||
"""
|
"""
|
||||||
计算消费等级
|
计算消费等级
|
||||||
"""
|
"""
|
||||||
df = community_summary_df[['_id', 'city', 'consumption_potential']]
|
df = community_summary_df[['_id', 'city', 'consumption_potential', 'total_price_score', 'estate_charge_score', 'relative_price_ranking_score', 'avg_cost_score']]
|
||||||
df = df.dropna(subset=['consumption_potential'])
|
#df = df.dropna(subset=['consumption_potential'])
|
||||||
source = "normal_consumption_potential"
|
source = "normal_consumption_potential"
|
||||||
target = "consumption_rating"
|
target = "consumption_rating"
|
||||||
actions = list()
|
actions = list()
|
||||||
count = 0
|
count = 0
|
||||||
|
|
||||||
if len(df) < 5:
|
# 总评级
|
||||||
|
total_rate = 6
|
||||||
|
score_df = pd.DataFrame()
|
||||||
|
if len(df.dropna(subset=['consumption_potential'])) < total_rate:
|
||||||
# 对于样本低于5的直接排名
|
# 对于样本低于5的直接排名
|
||||||
|
df = df.dropna(subset=['consumption_potential'])
|
||||||
df[target] = df['consumption_potential'].rank(method='dense', ascending=False).astype(int)
|
df[target] = df['consumption_potential'].rank(method='dense', ascending=False).astype(int)
|
||||||
|
df["total_price_rating"] = df['total_price_score'].rank(method='dense', ascending=False).astype(int)
|
||||||
|
df["estate_charge_rating"] = df['estate_charge_score'].rank(method='dense', ascending=False).astype(int)
|
||||||
|
df["relative_price_rating"] = df['relative_price_ranking_score'].rank(method='dense', ascending=False).astype(int)
|
||||||
|
df["avg_cost_rating"] = df['avg_cost_score'].rank(method='dense', ascending=False).astype(int)
|
||||||
else:
|
else:
|
||||||
# 极值设置
|
# 极值设置
|
||||||
df = filter_min_max(df)
|
score_df = gmm_train(total_rate, df.dropna(subset=['total_price_score']), "total_price_score", "total_price_rating")
|
||||||
df = gmm_train(5, df, source, target)
|
score_df['robust_total_price_score'] = scaler.fit_transform(score_df[['total_price_score']])
|
||||||
# 批量更新
|
score_df = min_max_scale(score_df, 'robust_total_price_score', 'total_price_score_minmax')
|
||||||
|
df = df.copy()
|
||||||
|
df_threshold = community_summary_df['consumption_potential'].quantile(0.99)
|
||||||
|
community_summary_df['consumption_potential'] = np.where(community_summary_df['consumption_potential'] > df_threshold, df_threshold, community_summary_df['consumption_potential'])
|
||||||
|
|
||||||
|
df.loc[:, 'normal_consumption_potential'] = QuantileTransformer(output_distribution="normal").fit_transform(df[['consumption_potential']])
|
||||||
|
if len(df.dropna(subset=['consumption_potential'])) >= total_rate:
|
||||||
|
df = gmm_train(total_rate, df.dropna(subset=['consumption_potential']), source, target)
|
||||||
|
# 新增子因子评级
|
||||||
|
df = gmm_train(total_rate, df, "estate_charge_score", "estate_charge_rating")
|
||||||
|
df = gmm_train(total_rate, df, "relative_price_ranking_score", "relative_price_rating")
|
||||||
|
df = gmm_train(total_rate, df, "avg_cost_score", "avg_cost_rating")
|
||||||
|
|
||||||
|
# 新增子因子评分
|
||||||
|
df = min_max_scale(df, 'consumption_potential', 'consumption_score')
|
||||||
|
df = min_max_scale(df, 'estate_charge_score', 'estate_charge_score_minmax')
|
||||||
|
df = min_max_scale(df, 'relative_price_ranking_score', 'relative_price_ranking_score_minmax')
|
||||||
|
df = min_max_scale(df, 'avg_cost_score', 'avg_cost_score_minmax')
|
||||||
|
if len(score_df) > 0:
|
||||||
|
# 补充财力评级
|
||||||
|
df = pd.merge(score_df[['_id', 'total_price_rating', 'total_price_score_minmax']], df, how='left', on=['_id'])
|
||||||
|
# 填充缺失值
|
||||||
|
if 'consumption_rating' not in df.columns:
|
||||||
|
df["consumption_rating"] = None
|
||||||
|
if 'consumption_score' not in df.columns:
|
||||||
|
df["consumption_score"] = None
|
||||||
|
df["consumption_rating"] = df["consumption_rating"].fillna(df["total_price_rating"])
|
||||||
|
df["consumption_score"] = df["consumption_score"].fillna(df["total_price_score_minmax"])
|
||||||
|
|
||||||
|
# 批量更新 如果财力评级与户均总价评级都没有,不更新
|
||||||
for index, item in df.iterrows():
|
for index, item in df.iterrows():
|
||||||
row = dict()
|
row = dict()
|
||||||
_id = str(item["_id"])
|
_id = str(item["_id"])
|
||||||
|
if 'consumption_rating' in item and pd.notna(item['consumption_rating']):
|
||||||
row['consumption_rating'] = item['consumption_rating']
|
row['consumption_rating'] = item['consumption_rating']
|
||||||
|
if 'consumption_score' in item and pd.notna(item['consumption_score']):
|
||||||
|
row['consumption_score'] = item['consumption_score']
|
||||||
|
if 'consumption_potential' in item and pd.notna(item['consumption_potential']):
|
||||||
row['consumption_potential'] = item['consumption_potential']
|
row['consumption_potential'] = item['consumption_potential']
|
||||||
row['m_date'] = dateutils.iso_format()
|
row['m_date'] = dateutils.iso_format()
|
||||||
|
row['sub_factor'] = {}
|
||||||
|
if 'total_price_rating' in item and pd.notna(item['total_price_rating']):
|
||||||
|
row['sub_factor']['total_price_rating'] = item['total_price_rating']
|
||||||
|
if 'estate_charge_rating' in item and pd.notna(item['estate_charge_rating']):
|
||||||
|
row['sub_factor']['estate_charge_rating'] = item['estate_charge_rating']
|
||||||
|
if 'relative_price_rating' in item and pd.notna(item['relative_price_rating']):
|
||||||
|
row['sub_factor']['relative_price_rating'] = item['relative_price_rating']
|
||||||
|
if 'avg_cost_rating' in item and pd.notna(item['avg_cost_rating']):
|
||||||
|
row['sub_factor']['avg_cost_rating'] = item['avg_cost_rating']
|
||||||
|
|
||||||
|
if 'total_price_score_minmax' in item and pd.notna(item['total_price_score_minmax']):
|
||||||
|
row['sub_factor']['total_price_score'] = item['total_price_score_minmax']
|
||||||
|
if 'estate_charge_score_minmax' in item and pd.notna(item['estate_charge_score_minmax']):
|
||||||
|
row['sub_factor']['estate_charge_score'] = item['estate_charge_score_minmax']
|
||||||
|
if 'relative_price_ranking_score_minmax' in item and pd.notna(item['relative_price_ranking_score_minmax']):
|
||||||
|
row['sub_factor']['relative_price_score'] = item['relative_price_ranking_score_minmax']
|
||||||
|
if 'avg_cost_score_minmax' in item and pd.notna(item['avg_cost_score_minmax']):
|
||||||
|
row['sub_factor']['avg_cost_score'] = item['avg_cost_score_minmax']
|
||||||
|
|
||||||
action = {'_op_type': 'update', # 操作 index update create delete
|
action = {'_op_type': 'update', # 操作 index update create delete
|
||||||
'_index': index_name, # index
|
'_index': index_name, # index
|
||||||
@@ -156,58 +293,58 @@ def sync_by_city(city_now):
|
|||||||
if len(community_summary_df) == 0:
|
if len(community_summary_df) == 0:
|
||||||
log.warning(f"{city_now}无法评级,数量低于5")
|
log.warning(f"{city_now}无法评级,数量低于5")
|
||||||
return
|
return
|
||||||
|
#plot_img(community_summary_df)
|
||||||
save_png(community_summary_df)
|
save_png(community_summary_df)
|
||||||
nextcloudutils.up_to_nextcloud(save_png_path + "/" + city_now + ".png", nextcloud_aim_file_path + "/" + city_now + ".png")
|
nextcloudutils.up_to_nextcloud(save_png_path + "/" + city_now + ".png", nextcloud_aim_file_path + "/" + city_now + ".png")
|
||||||
|
|
||||||
|
|
||||||
|
def clear_rating(temp_df):
|
||||||
|
"""
|
||||||
|
清空评级信息
|
||||||
|
"""
|
||||||
|
actions = list()
|
||||||
|
count = 0
|
||||||
|
for index, item in temp_df.iterrows():
|
||||||
|
_id = str(item["_id"])
|
||||||
|
|
||||||
|
action = {'_op_type': 'update', # 操作 index update create delete
|
||||||
|
'_index': index_name, # index
|
||||||
|
'_id': _id, # 请勿忘记ID
|
||||||
|
'doc': {'consumption_rating': None, 'consumption_potential': None},
|
||||||
|
"_retry_on_conflict": 5
|
||||||
|
}
|
||||||
|
actions.append(action)
|
||||||
|
if len(actions) == 5000:
|
||||||
|
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||||
|
count += len(actions)
|
||||||
|
actions.clear()
|
||||||
|
log.info("清理中: 当前已清理{}".format(count))
|
||||||
|
if len(actions) > 0:
|
||||||
|
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||||
|
count += len(actions)
|
||||||
|
actions.clear()
|
||||||
|
log.info("清理完毕: 共清理{}".format(count))
|
||||||
|
|
||||||
|
|
||||||
def sync(city=None):
|
def sync(city=None):
|
||||||
|
clear_sql = "select city, region, name from community_summary where consumption_potential is not null and (total_price is null or estate_charge is null or avg_cost is null) "
|
||||||
if pd.isna(city):
|
if pd.isna(city):
|
||||||
|
# 清空缺少因子的老潜力
|
||||||
|
temp_df = es206_client.scan_by_sql_to_data_frame(clear_sql)
|
||||||
|
clear_rating(temp_df)
|
||||||
|
run = False
|
||||||
for city_now in reasonable_df['city'].unique():
|
for city_now in reasonable_df['city'].unique():
|
||||||
#if pd.isna(reasonable_dict[city_now]):
|
#if pd.isna(reasonable_dict[city_now]):
|
||||||
|
# if city_now == '昌都':
|
||||||
|
# run = True
|
||||||
|
# if run:
|
||||||
sync_by_city(city_now)
|
sync_by_city(city_now)
|
||||||
else:
|
else:
|
||||||
|
clear_sql += f" and city='{city}' "
|
||||||
|
temp_df = es206_client.scan_by_sql_to_data_frame(clear_sql)
|
||||||
|
clear_rating(temp_df)
|
||||||
sync_by_city(city)
|
sync_by_city(city)
|
||||||
|
|
||||||
|
|
||||||
def filter_min_max(community_summary_df):
|
|
||||||
community_summary_df['normal_consumption_potential'] = QuantileTransformer(output_distribution="normal").fit_transform(community_summary_df[['consumption_potential']])
|
|
||||||
# total_price5 = np.percentile(a=community_summary_df['total_price'], q=2, axis=0)
|
|
||||||
# total_price95 = np.percentile(a=community_summary_df['total_price'], q=98, axis=0)
|
|
||||||
# community_summary_df.loc[community_summary_df['total_price'] > total_price95, 'total_price'] = total_price95
|
|
||||||
# community_summary_df.loc[community_summary_df['total_price'] < total_price5, 'total_price'] = total_price5
|
|
||||||
#
|
|
||||||
# estate_charge5 = np.percentile(a=community_summary_df['estate_charge'], q=2, axis=0)
|
|
||||||
# estate_charge95 = np.percentile(a=community_summary_df['estate_charge'], q=98, axis=0)
|
|
||||||
# community_summary_df.loc[community_summary_df['estate_charge'] > estate_charge95, 'estate_charge'] = estate_charge95
|
|
||||||
# community_summary_df.loc[community_summary_df['estate_charge'] < estate_charge5, 'estate_charge'] = estate_charge5
|
|
||||||
#
|
|
||||||
# avg_cost5 = np.percentile(a=community_summary_df['avg_cost'], q=2, axis=0)
|
|
||||||
# avg_cost95 = np.percentile(a=community_summary_df['avg_cost'], q=98, axis=0)
|
|
||||||
# community_summary_df.loc[community_summary_df['avg_cost'] > avg_cost95, 'avg_cost'] = avg_cost95
|
|
||||||
# community_summary_df.loc[community_summary_df['avg_cost'] < avg_cost5, 'avg_cost'] = avg_cost5
|
|
||||||
|
|
||||||
#if city in factor_dict:
|
|
||||||
# 户均总价按极值设置可靠性不高,当户均总价补充后,图中大于最大值的量非常多
|
|
||||||
# community_summary_df.loc[
|
|
||||||
# community_summary_df['total_price'] < factor_dict[city]['min_total_price'], 'total_price'] = \
|
|
||||||
# factor_dict[city]['min_total_price']
|
|
||||||
# community_summary_df.loc[
|
|
||||||
# community_summary_df['total_price'] > factor_dict[city]['max_total_price'], 'total_price'] = \
|
|
||||||
# factor_dict[city]['max_total_price']
|
|
||||||
|
|
||||||
# community_summary_df.loc[
|
|
||||||
# community_summary_df['estate_charge'] < factor_dict[city]['min_estate_charge'], 'estate_charge'] = \
|
|
||||||
# factor_dict[city]['min_estate_charge']
|
|
||||||
# community_summary_df.loc[
|
|
||||||
# community_summary_df['estate_charge'] > factor_dict[city]['max_estate_charge'], 'estate_charge'] = \
|
|
||||||
# factor_dict[city]['max_estate_charge']
|
|
||||||
# community_summary_df.loc[community_summary_df['avg_cost'] < factor_dict[city]['min_avg_cost'], 'avg_cost'] = \
|
|
||||||
# factor_dict[city]['min_avg_cost']
|
|
||||||
# community_summary_df.loc[community_summary_df['avg_cost'] > factor_dict[city]['max_avg_cost'], 'avg_cost'] = \
|
|
||||||
# factor_dict[city]['max_avg_cost']
|
|
||||||
return community_summary_df
|
|
||||||
|
|
||||||
|
|
||||||
def save_png(community_city):
|
def save_png(community_city):
|
||||||
"""
|
"""
|
||||||
生成财力评级图表
|
生成财力评级图表
|
||||||
@@ -251,6 +388,47 @@ def save_png(community_city):
|
|||||||
plt.close()
|
plt.close()
|
||||||
|
|
||||||
|
|
||||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
def plot_img(community_city):
|
||||||
|
"""
|
||||||
|
生成财力评级图表
|
||||||
|
"""
|
||||||
|
curr_city = community_city['city'].unique()[0]
|
||||||
|
# 解决中文显示问题
|
||||||
|
plt.rcParams['font.sans-serif'] = ['SimHei']
|
||||||
|
plt.rcParams['axes.unicode_minus'] = False
|
||||||
|
plt.title(curr_city)
|
||||||
|
plt.subplot(3, 1, 1)
|
||||||
|
community_city['consumption_score'].hist(bins=100)
|
||||||
|
plt.xlabel('consumption_rating')
|
||||||
|
plt.tight_layout()
|
||||||
|
|
||||||
|
plt.subplot(3, 2, 3)
|
||||||
|
community_city['total_price_score_minmax'].hist(bins=100)
|
||||||
|
plt.xlabel('total_price')
|
||||||
|
plt.tight_layout()
|
||||||
|
|
||||||
|
plt.subplot(3, 2, 4)
|
||||||
|
community_city['estate_charge_score_minmax'].hist(bins=100)
|
||||||
|
plt.xlabel('estate_charge')
|
||||||
|
plt.tight_layout()
|
||||||
|
|
||||||
|
plt.subplot(3, 2, 5)
|
||||||
|
community_city['avg_cost_score_minmax'].hist(bins=100)
|
||||||
|
plt.xlabel('avg_cost')
|
||||||
|
plt.tight_layout()
|
||||||
|
|
||||||
|
plt.subplot(3, 2, 6)
|
||||||
|
community_city['relative_price_ranking_score_minmax'].hist(bins=100)
|
||||||
|
plt.xlabel('relative_price_ranking')
|
||||||
|
plt.tight_layout()
|
||||||
|
# 关窗口
|
||||||
|
plt.show()
|
||||||
|
|
||||||
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
sync(city)
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
#dingutils.send_text("【分布图生成完毕】财力评级已完成, 请点击 https://nextcloud.fangjia.com/s/Jqs5BzXi8xLYeH9 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
|
#dingutils.send_text("【分布图生成完毕】财力评级已完成, 请点击 https://nextcloud.fangjia.com/s/Jqs5BzXi8xLYeH9 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
|
||||||
|
|||||||
@@ -0,0 +1,181 @@
|
|||||||
|
"""
|
||||||
|
财力评级统计
|
||||||
|
"""
|
||||||
|
from elasticsearch import helpers
|
||||||
|
from goojia_utils import ElasticSearchClient, dateutils, nextcloudutils, dingutils, data_utils
|
||||||
|
import pandas as pd
|
||||||
|
import numpy as np
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
index_name = "community_summary"
|
||||||
|
|
||||||
|
save_path_prefix = "/home/airflow/data/community"
|
||||||
|
|
||||||
|
save_png_path = save_path_prefix + "/consumption_rating"
|
||||||
|
if not os.path.exists(save_png_path):
|
||||||
|
os.makedirs(save_png_path)
|
||||||
|
nextcloud_aim_file_path = '财力评级/财力评级/财力评级统计报表'
|
||||||
|
|
||||||
|
|
||||||
|
def format_score_range(min_score, max_score):
|
||||||
|
"""格式化分数范围,防止Excel自动转换"""
|
||||||
|
if pd.isna(min_score) or pd.isna(max_score):
|
||||||
|
return "-"
|
||||||
|
# 使用Excel公式语法强制识别为文本
|
||||||
|
return f'="{int(min_score)}-{int(max_score)}"'
|
||||||
|
|
||||||
|
|
||||||
|
def stat(community_summary_df):
|
||||||
|
df = community_summary_df
|
||||||
|
grouped = df.groupby(["city"])
|
||||||
|
|
||||||
|
results = []
|
||||||
|
for city, group in grouped:
|
||||||
|
# 总数
|
||||||
|
total_num = len(group)
|
||||||
|
# 财力评级不为空数量
|
||||||
|
group = group.dropna(subset=['consumption_rating'])
|
||||||
|
rating_num = len(group)
|
||||||
|
# 计算财力评级指标
|
||||||
|
rating_counts = group["consumption_rating"].value_counts(normalize=True)
|
||||||
|
rating_1 = rating_counts.get(1, 0)
|
||||||
|
rating_2 = rating_counts.get(2, 0)
|
||||||
|
rating_3 = rating_counts.get(3, 0)
|
||||||
|
rating_4 = rating_counts.get(4, 0)
|
||||||
|
rating_5 = rating_counts.get(5, 0)
|
||||||
|
rating_6 = rating_counts.get(6, 0)
|
||||||
|
|
||||||
|
# 计算分数范围
|
||||||
|
rating_1_scores = group[(group["consumption_rating"] == 1) & (group['estate_charge_score'].notna()) & (group['total_price_score'].notna()) & (group['relative_price_score'].notna()) & (group['avg_cost_score'].notna())]["consumption_score"]
|
||||||
|
rating_2_scores = group[(group["consumption_rating"] == 2) & (group['estate_charge_score'].notna()) & (group['total_price_score'].notna()) & (group['relative_price_score'].notna()) & (group['avg_cost_score'].notna())]["consumption_score"]
|
||||||
|
rating_3_scores = group[(group["consumption_rating"] == 3) & (group['estate_charge_score'].notna()) & (group['total_price_score'].notna()) & (group['relative_price_score'].notna()) & (group['avg_cost_score'].notna())]["consumption_score"]
|
||||||
|
rating_4_scores = group[(group["consumption_rating"] == 4) & (group['estate_charge_score'].notna()) & (group['total_price_score'].notna()) & (group['relative_price_score'].notna()) & (group['avg_cost_score'].notna())]["consumption_score"]
|
||||||
|
rating_5_scores = group[(group["consumption_rating"] == 5) & (group['estate_charge_score'].notna()) & (group['total_price_score'].notna()) & (group['relative_price_score'].notna()) & (group['avg_cost_score'].notna())]["consumption_score"]
|
||||||
|
rating_6_scores = group[(group["consumption_rating"] == 6) & (group['estate_charge_score'].notna()) & (group['total_price_score'].notna()) & (group['relative_price_score'].notna()) & (group['avg_cost_score'].notna())]["consumption_score"]
|
||||||
|
|
||||||
|
# 计算置信度占比
|
||||||
|
confidence_counts = group["credit"].value_counts(normalize=True)
|
||||||
|
|
||||||
|
results.append({
|
||||||
|
"城市": city,
|
||||||
|
"楼盘总量": total_num,
|
||||||
|
"有财力评级数量": rating_num,
|
||||||
|
"财力评级1级占比": round(rating_1 * 100, 1),
|
||||||
|
"财力评级1级分数范围": format_score_range(rating_1_scores.min(), rating_1_scores.max()) if not rating_1_scores.empty else "-",
|
||||||
|
"财力评级2级占比": round(rating_2 * 100, 1),
|
||||||
|
"财力评级2级分数范围": format_score_range(rating_2_scores.min(), rating_2_scores.max()) if not rating_2_scores.empty else "-",
|
||||||
|
"财力评级3级占比": round(rating_3 * 100, 1),
|
||||||
|
"财力评级3级分数范围": format_score_range(rating_3_scores.min(), rating_3_scores.max()) if not rating_3_scores.empty else "-",
|
||||||
|
"财力评级4级占比": round(rating_4 * 100, 1),
|
||||||
|
"财力评级4级分数范围": format_score_range(rating_4_scores.min(), rating_4_scores.max()) if not rating_4_scores.empty else "-",
|
||||||
|
"财力评级5级占比": round(rating_5 * 100, 1),
|
||||||
|
"财力评级5级分数范围": format_score_range(rating_5_scores.min(), rating_5_scores.max()) if not rating_5_scores.empty else "-",
|
||||||
|
"财力评级6级占比": round(rating_6 * 100, 1),
|
||||||
|
"财力评级6级分数范围": format_score_range(rating_6_scores.min(), rating_6_scores.max()) if not rating_6_scores.empty else "-",
|
||||||
|
"高置信度占比": round(confidence_counts.get("高", 0) * 100, 1),
|
||||||
|
"中置信度占比": round(confidence_counts.get("中", 0) * 100, 1),
|
||||||
|
"低置信度占比": round(confidence_counts.get("低", 0) * 100, 1)
|
||||||
|
})
|
||||||
|
|
||||||
|
# 生成最终报表
|
||||||
|
report_df = pd.DataFrame(results)
|
||||||
|
aim_file = dateutils.format_date(dateutils.localize()) + '财力评级统计结果.csv'
|
||||||
|
report_df.to_csv(save_png_path + '/' + aim_file, index=False)
|
||||||
|
nextcloudutils.up_to_nextcloud(save_png_path + "/" + aim_file, nextcloud_aim_file_path + "/" + aim_file)
|
||||||
|
|
||||||
|
|
||||||
|
def detail(city):
|
||||||
|
"""
|
||||||
|
计算消费潜力
|
||||||
|
"""
|
||||||
|
if pd.notna(city):
|
||||||
|
if ',' in city:
|
||||||
|
cities = city.split(',')
|
||||||
|
else:
|
||||||
|
cities = [city]
|
||||||
|
query = {
|
||||||
|
"bool": {
|
||||||
|
"must": [
|
||||||
|
{
|
||||||
|
"terms": {
|
||||||
|
"city": cities
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
query = {
|
||||||
|
"bool": {
|
||||||
|
"must": [
|
||||||
|
{
|
||||||
|
"match_all": {}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
community_summary_df = es206_client.scan_to_data_frame_with_source(query={"query": query}, source=[
|
||||||
|
"id",
|
||||||
|
"city",
|
||||||
|
"region",
|
||||||
|
"name",
|
||||||
|
"avg_price",
|
||||||
|
"total_price",
|
||||||
|
"estate_charge",
|
||||||
|
"estate_charge_fix",
|
||||||
|
"estate_charge_flag",
|
||||||
|
"main_house_area",
|
||||||
|
"relative_price_ranking",
|
||||||
|
"avg_cost",
|
||||||
|
"consumption_rating",
|
||||||
|
"price_rating", "avg_price_calc_type",
|
||||||
|
"consumption_score",
|
||||||
|
"sub_factor.total_price_rating",
|
||||||
|
"sub_factor.total_price_score",
|
||||||
|
"sub_factor.estate_charge_score",
|
||||||
|
"sub_factor.relative_price_score",
|
||||||
|
"sub_factor.avg_cost_score",
|
||||||
|
"credit_detail.total_price_score"
|
||||||
|
], index="community_summary")
|
||||||
|
# 减少重复查询
|
||||||
|
community_summary_df = data_utils.flatten_json(community_summary_df, 'credit_detail')
|
||||||
|
community_summary_df = community_summary_df.rename(columns={'total_price_score': 'credit_detail_total_price_score'})
|
||||||
|
community_summary_df = data_utils.flatten_json(community_summary_df, 'sub_factor')
|
||||||
|
community_summary_df['diff'] = np.abs(community_summary_df['consumption_rating'] - community_summary_df['price_rating'])
|
||||||
|
community_summary_df['total_price_diff'] = np.abs(community_summary_df['consumption_rating'] - community_summary_df['total_price_rating'])
|
||||||
|
community_summary_df['estate_charge_type'] = np.where(
|
||||||
|
pd.notna(community_summary_df['estate_charge']), # 仅当 estate_charge 不为空时处理
|
||||||
|
np.where(
|
||||||
|
community_summary_df['estate_charge_flag'] == 1,
|
||||||
|
'周边',
|
||||||
|
'自有'
|
||||||
|
),
|
||||||
|
''
|
||||||
|
)
|
||||||
|
|
||||||
|
community_summary_df['credit'] = np.where(community_summary_df['total_price_diff'] == 0, '高', np.where(community_summary_df['total_price_diff'] == 1, '中', np.where(community_summary_df['total_price_diff'] >= 2, '低', '-')))
|
||||||
|
stat(community_summary_df)
|
||||||
|
community_summary_df = community_summary_df.rename(columns={'city': '城市', 'region': '区域', 'name': '小区名称', 'consumption_score': '财力评分', 'consumption_rating': '财力评级',
|
||||||
|
'total_price_rating': '户均总价评级', 'total_price_diff': '差值',
|
||||||
|
'total_price': '户均总价', 'avg_price': '小区均价', 'main_house_area': '典型面积', 'relative_price_ranking': '相对价格排名',
|
||||||
|
'avg_cost': '人均消费', 'estate_charge': '物业费', 'estate_charge_fix': '物业费修正', 'price_rating': '价格评级', 'estate_charge_type': '物业费获取方式',
|
||||||
|
'credit': '置信度', 'total_price_score': '户均总价评分', 'estate_charge_score': '物业费评分', 'relative_price_score': '相对价格评分', 'avg_cost_score': '人均消费评分'
|
||||||
|
})
|
||||||
|
community_summary_df['计算方式'] = np.where(community_summary_df['avg_price_calc_type'] == 1,'指导价',np.where(community_summary_df['avg_price_calc_type'] == 2,'自身样本',np.where(community_summary_df['avg_price_calc_type'] == 3,'第三方',
|
||||||
|
np.where(community_summary_df['avg_price_calc_type'] == 4,'交易数据',np.where(community_summary_df['avg_price_calc_type'] == 5,'周边楼盘',np.where(community_summary_df['avg_price_calc_type'] == 6,'推导', np.where(community_summary_df['avg_price_calc_type'] == 7,'周边推算','无方式')))))))
|
||||||
|
community_summary_df['获取方式'] = np.where(community_summary_df['credit_detail_total_price_score'] <= 6,'周边','自有')
|
||||||
|
aim_file = dateutils.format_date(dateutils.localize()) + '财力评级结果.csv'
|
||||||
|
community_summary_df.to_csv(save_png_path + '/' + aim_file, index=False, columns=["城市", "区域", "小区名称", "财力评分", "财力评级", "户均总价评级", "差值", "户均总价", "小区均价", "计算方式", "典型面积",
|
||||||
|
"获取方式", "相对价格排名", "人均消费", "物业费", "物业费获取方式", '置信度', '户均总价评分', '物业费评分', '相对价格评分', '人均消费评分'])
|
||||||
|
nextcloudutils.up_to_nextcloud(save_png_path + "/" + aim_file, nextcloud_aim_file_path + "/" + aim_file)
|
||||||
|
|
||||||
|
dingutils.send_text("【统计】财力评级统计完毕,请点击 https://nextcloud.fangjia.com/s/nMbzk6ZKJDimapf 查看", "32400b949005817550937f82a82c064fe5be65c23204306442c15e8b485d7ef2")
|
||||||
|
|
||||||
|
|
||||||
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
detail(sys_city)
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
"""
|
||||||
|
置信度及总分计算
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
import pandas as pd
|
||||||
|
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils, data_utils
|
||||||
|
from elasticsearch import helpers
|
||||||
|
import sys
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||||
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
|
||||||
|
index_name = "community_summary"
|
||||||
|
path = "/home/airflow/data/community"
|
||||||
|
m_date = dateutils.iso_format()
|
||||||
|
avg_price_map = dict()
|
||||||
|
|
||||||
|
|
||||||
|
def sync(city=None):
|
||||||
|
log.info(f"开始置信度打分 {index_name}")
|
||||||
|
|
||||||
|
# 加载库中均价数据
|
||||||
|
query_filter = [{"range": {"consumption_rating": {"gt": 0}}}]
|
||||||
|
if pd.notna(city):
|
||||||
|
query_filter.append({"term": {"city": city}})
|
||||||
|
|
||||||
|
query = {
|
||||||
|
"_source": ["city", "region", "credit_detail"],
|
||||||
|
"query": {
|
||||||
|
"bool": {
|
||||||
|
"filter": query_filter
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
community_summary_df = es206_client.scan_to_data_frame("community_summary", query)
|
||||||
|
community_summary_df = data_utils.flatten_json(community_summary_df, 'credit_detail')
|
||||||
|
community_summary_df['total_price_score'].fillna(0, inplace=True)
|
||||||
|
community_summary_df['relative_price_ranking_score'].fillna(0, inplace=True)
|
||||||
|
community_summary_df['estate_charge_score'].fillna(0, inplace=True)
|
||||||
|
|
||||||
|
community_summary_df['total_score'] = round(0.6 * community_summary_df['total_price_score'] + 0.2 * community_summary_df['relative_price_ranking_score'] + 0.2 * community_summary_df['estate_charge_score'])
|
||||||
|
community_summary_df['credit'] = round(community_summary_df['total_score'] / 10, 2)
|
||||||
|
|
||||||
|
actions = list()
|
||||||
|
count = 0
|
||||||
|
|
||||||
|
for index, item in community_summary_df[community_summary_df['total_score'].notna()].iterrows():
|
||||||
|
doc_info = {
|
||||||
|
'credit': item['credit'],
|
||||||
|
'credit_detail': {
|
||||||
|
'total_score': item['total_score']
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
action = {
|
||||||
|
"_op_type": "update",
|
||||||
|
"_index": index_name,
|
||||||
|
"_id": item['_id'],
|
||||||
|
"doc": doc_info,
|
||||||
|
"_retry_on_conflict": 5
|
||||||
|
}
|
||||||
|
actions.append(action)
|
||||||
|
if len(actions) == 5000:
|
||||||
|
helpers.bulk(es206_client.elastic_client, actions)
|
||||||
|
count += len(actions)
|
||||||
|
logging.info("已更新:" + str(count))
|
||||||
|
actions.clear()
|
||||||
|
if len(actions) > 0:
|
||||||
|
helpers.bulk(es206_client.elastic_client, actions)
|
||||||
|
count += len(actions)
|
||||||
|
logging.info("已更新:" + str(count))
|
||||||
|
actions.clear()
|
||||||
|
log.info(f"完成置信度打分 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,112 @@
|
|||||||
|
"""
|
||||||
|
同步删除小区基本信息至ES206
|
||||||
|
"""
|
||||||
|
# 兼容性补丁:解决 Elasticsearch 8.x/9.x 客户端在老版本 ES 集群上因 compatible-with 头部导致的 ApiError(406)
|
||||||
|
try:
|
||||||
|
import elasticsearch
|
||||||
|
ver = getattr(elasticsearch, '__version__', None)
|
||||||
|
if ver and not str(ver).startswith('7.'):
|
||||||
|
original_init = elasticsearch.Elasticsearch.__init__
|
||||||
|
def patched_init(self, *args, **kwargs):
|
||||||
|
headers = kwargs.get('headers', {})
|
||||||
|
headers['Accept'] = 'application/json'
|
||||||
|
headers['Content-Type'] = 'application/json'
|
||||||
|
kwargs['headers'] = headers
|
||||||
|
original_init(self, *args, **kwargs)
|
||||||
|
elasticsearch.Elasticsearch.__init__ = patched_init
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import pandas as pd
|
||||||
|
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
||||||
|
from elasticsearch import helpers
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||||
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
|
||||||
|
index_name = "community_summary"
|
||||||
|
m_date = dateutils.iso_format()
|
||||||
|
|
||||||
|
path = "/home/airflow/data/community"
|
||||||
|
if not os.path.exists(path):
|
||||||
|
os.makedirs(path)
|
||||||
|
|
||||||
|
def deal_comm(community_summary_df, seaweed_df):
|
||||||
|
"""
|
||||||
|
删除额外小区信息
|
||||||
|
"""
|
||||||
|
actions = list()
|
||||||
|
diff_ids = set(community_summary_df['_id']).difference(set(seaweed_df['_id']))
|
||||||
|
for _id in diff_ids:
|
||||||
|
action = {
|
||||||
|
"_op_type": "delete",
|
||||||
|
"_index": index_name,
|
||||||
|
"_id": _id,
|
||||||
|
}
|
||||||
|
actions.append(action)
|
||||||
|
if len(actions) == 5000:
|
||||||
|
helpers.bulk(es206_client.elastic_client, actions)
|
||||||
|
actions.clear()
|
||||||
|
if len(actions) > 0:
|
||||||
|
helpers.bulk(es206_client.elastic_client, actions)
|
||||||
|
actions.clear()
|
||||||
|
|
||||||
|
|
||||||
|
def sync(city=None):
|
||||||
|
log.info(f"开始删除已删除小区至206 {index_name}...")
|
||||||
|
seaweed_query = {'status': 1, 'cat': 'district'}
|
||||||
|
if pd.notna(city):
|
||||||
|
seaweed_query['city'] = city
|
||||||
|
seaweed_df = pd.DataFrame(
|
||||||
|
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'status': 1}).batch_size(100000)))
|
||||||
|
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
|
||||||
|
|
||||||
|
sql = f"select id from {index_name} "
|
||||||
|
|
||||||
|
if pd.notna(city):
|
||||||
|
sql += f" where city='{city}'"
|
||||||
|
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||||
|
|
||||||
|
if len(community_summary_df) > 0:
|
||||||
|
community_summary_df['flag'] = 1
|
||||||
|
seaweed_new_df = pd.merge(seaweed_df, community_summary_df, how='left', on=['_id'])
|
||||||
|
seaweed_existed_df = seaweed_new_df[seaweed_new_df['flag'] == 1]
|
||||||
|
else:
|
||||||
|
seaweed_existed_df = pd.DataFrame()
|
||||||
|
|
||||||
|
actions = list()
|
||||||
|
# 对于存在的删除
|
||||||
|
count = 0
|
||||||
|
for index, item in seaweed_existed_df.iterrows():
|
||||||
|
action = {
|
||||||
|
"_op_type": "delete",
|
||||||
|
"_index": index_name,
|
||||||
|
"_id": str(item['_id'])
|
||||||
|
}
|
||||||
|
|
||||||
|
actions.append(action)
|
||||||
|
if len(actions) == 5000:
|
||||||
|
helpers.bulk(es206_client.elastic_client, actions)
|
||||||
|
count += len(actions)
|
||||||
|
log.info("删除:" + str(count))
|
||||||
|
actions.clear()
|
||||||
|
if len(actions) > 0:
|
||||||
|
helpers.bulk(es206_client.elastic_client, actions)
|
||||||
|
count += len(actions)
|
||||||
|
log.info("删除:" + str(count))
|
||||||
|
actions.clear()
|
||||||
|
|
||||||
|
log.info(f"完成删除小区基本属性至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
@@ -3,14 +3,15 @@
|
|||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
|
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, MAIN_MONGO_CLIENT
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from threading import Lock
|
from threading import Lock
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||||
|
|
||||||
index_name = "community_summary"
|
index_name = "community_summary"
|
||||||
path = "/home/airflow/data/community"
|
path = "/home/airflow/data/community"
|
||||||
@@ -32,13 +33,15 @@ def deal(actions, item):
|
|||||||
doc_info = dict()
|
doc_info = dict()
|
||||||
|
|
||||||
# 幼儿园
|
# 幼儿园
|
||||||
kindergarten = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "学校", "幼儿园")
|
# kindergarten = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "学校", "幼儿园")
|
||||||
|
kindergarten = poiutils.find_around_by_location_from_mongo(curr_city, 1000, lng, lat, "学校", "幼儿园")
|
||||||
doc_info['kindergarten_num'] = len(kindergarten)
|
doc_info['kindergarten_num'] = len(kindergarten)
|
||||||
if len(kindergarten) > 0:
|
if len(kindergarten) > 0:
|
||||||
doc_info['kindergarten'] = ",".join(kindergarten)
|
doc_info['kindergarten'] = ",".join(kindergarten)
|
||||||
|
|
||||||
# 大学
|
# 大学
|
||||||
university = poiutils.find_around_by_location(curr_city, 5000, [lng, lat], "学校", "大学")
|
# university = poiutils.find_around_by_location(curr_city, 5000, [lng, lat], "学校", "大学")
|
||||||
|
university = poiutils.find_around_by_location_from_mongo(curr_city, 1000, lng, lat, "学校", "大学")
|
||||||
doc_info['university_num'] = len(university)
|
doc_info['university_num'] = len(university)
|
||||||
if len(university) > 0:
|
if len(university) > 0:
|
||||||
doc_info['university'] = ",".join(university)
|
doc_info['university'] = ",".join(university)
|
||||||
@@ -63,10 +66,10 @@ def deal(actions, item):
|
|||||||
|
|
||||||
def sync(city=None):
|
def sync(city=None):
|
||||||
log.info(f"开始同步教育资源【幼儿园、大学】至206 {index_name}")
|
log.info(f"开始同步教育资源【幼儿园、大学】至206 {index_name}")
|
||||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
query = {}
|
||||||
if pd.notna(city):
|
if pd.notna(city):
|
||||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
query['city'] = city
|
||||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
|
||||||
|
|
||||||
actions = list()
|
actions = list()
|
||||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||||
@@ -83,5 +86,10 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步教育资源至206 {index_name}")
|
log.info(f"完成同步教育资源至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
sync(city)
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
@@ -3,14 +3,15 @@
|
|||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
|
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, MAIN_MONGO_CLIENT
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from threading import Lock
|
from threading import Lock
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||||
|
|
||||||
index_name = "community_summary"
|
index_name = "community_summary"
|
||||||
path = "/home/airflow/data/community"
|
path = "/home/airflow/data/community"
|
||||||
@@ -32,8 +33,10 @@ def deal(actions, item):
|
|||||||
doc_info = dict()
|
doc_info = dict()
|
||||||
|
|
||||||
# 小学
|
# 小学
|
||||||
key_primary_school = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "学校", "小学", "重点")
|
# key_primary_school = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "学校", "小学", "重点")
|
||||||
ordinary_primary_school = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "学校", "小学", "普通")
|
key_primary_school = poiutils.find_around_by_location_from_mongo(curr_city, 1000, lng, lat, "学校", "小学", "重点")
|
||||||
|
# ordinary_primary_school = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "学校", "小学", "普通")
|
||||||
|
ordinary_primary_school = poiutils.find_around_by_location_from_mongo(curr_city, 1000, lng, lat, "学校", "小学","普通")
|
||||||
|
|
||||||
doc_info['key_primary_school_num'] = len(key_primary_school)
|
doc_info['key_primary_school_num'] = len(key_primary_school)
|
||||||
if len(key_primary_school) > 0:
|
if len(key_primary_school) > 0:
|
||||||
@@ -44,8 +47,10 @@ def deal(actions, item):
|
|||||||
doc_info['ordinary_primary_school'] = ",".join(ordinary_primary_school)
|
doc_info['ordinary_primary_school'] = ",".join(ordinary_primary_school)
|
||||||
|
|
||||||
# 中学
|
# 中学
|
||||||
key_middle_school = poiutils.find_around_by_location(curr_city, 2000, [lng, lat], "学校", "中学", "重点")
|
# key_middle_school = poiutils.find_around_by_location(curr_city, 2000, [lng, lat], "学校", "中学", "重点")
|
||||||
ordinary_middle_school = poiutils.find_around_by_location(curr_city, 2000, [lng, lat], "学校", "中学", "普通")
|
key_middle_school = poiutils.find_around_by_location_from_mongo(curr_city, 1000, lng, lat, "学校", "中学", "重点")
|
||||||
|
# ordinary_middle_school = poiutils.find_around_by_location(curr_city, 2000, [lng, lat], "学校", "中学", "普通")
|
||||||
|
ordinary_middle_school = poiutils.find_around_by_location_from_mongo(curr_city, 1000, lng, lat, "学校", "中学","普通")
|
||||||
|
|
||||||
doc_info['key_middle_school_num'] = len(key_middle_school)
|
doc_info['key_middle_school_num'] = len(key_middle_school)
|
||||||
if len(key_middle_school) > 0:
|
if len(key_middle_school) > 0:
|
||||||
@@ -76,10 +81,10 @@ def deal(actions, item):
|
|||||||
def sync(city=None):
|
def sync(city=None):
|
||||||
log.info(f"开始同步教育资源【小学、中学】至206 {index_name}")
|
log.info(f"开始同步教育资源【小学、中学】至206 {index_name}")
|
||||||
|
|
||||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
query = {}
|
||||||
if pd.notna(city):
|
if pd.notna(city):
|
||||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
query['city'] = city
|
||||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
|
||||||
|
|
||||||
actions = list()
|
actions = list()
|
||||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||||
@@ -96,5 +101,10 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步教育资源至206 {index_name}")
|
log.info(f"完成同步教育资源至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
sync(city)
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
|
|||||||
@@ -7,18 +7,73 @@ import logging
|
|||||||
import sys
|
import sys
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from threading import Lock
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
index_name = "community_summary"
|
index_name = "community_summary"
|
||||||
|
file_path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||||
|
mutex = Lock()
|
||||||
|
estate_charge_min_max_df = pd.read_excel(file_path + '/全国各区域物业费中位值.xlsx')
|
||||||
|
estate_charge_city_region_dict = dict()
|
||||||
|
|
||||||
|
|
||||||
def save(community_summary_df):
|
# 加载各区域物业费最大最小值
|
||||||
if 'estate_charge' not in community_summary_df.columns:
|
def deal_estate_charge(x):
|
||||||
|
city_now = x['city']
|
||||||
|
region_now = x['region']
|
||||||
|
if pd.notna(city_now) and pd.notna(region_now):
|
||||||
|
estate_charge_city_region_dict[city_now + region_now] = {'min': x['3'], 'max': x['97']}
|
||||||
|
elif pd.notna(city_now):
|
||||||
|
estate_charge_city_region_dict[city_now] = {'min': x['3'], 'max': x['97']}
|
||||||
|
estate_charge_min_max_df.apply(lambda x: deal_estate_charge(x), axis=1)
|
||||||
|
|
||||||
|
|
||||||
|
def save_fix(community_summary_df):
|
||||||
|
if 'estate_charge_fix' not in community_summary_df.columns:
|
||||||
|
log.info("该城市无调整物业费")
|
||||||
|
return
|
||||||
|
df = community_summary_df[['_id', 'city', 'estate_charge_fix', 'estate_charge_score']]
|
||||||
|
df = df.dropna(subset=['estate_charge_fix'])
|
||||||
|
actions = list()
|
||||||
|
count = 0
|
||||||
|
|
||||||
|
# 批量更新
|
||||||
|
m_date = dateutils.iso_format()
|
||||||
|
for index, item in df.iterrows():
|
||||||
|
row = dict()
|
||||||
|
_id = str(item["_id"])
|
||||||
|
row['estate_charge_fix'] = item['estate_charge_fix']
|
||||||
|
row['credit_detail'] = {'estate_charge_score': int(item['estate_charge_score'])}
|
||||||
|
row['m_date'] = m_date
|
||||||
|
|
||||||
|
action = {'_op_type': 'update',
|
||||||
|
'_index': index_name,
|
||||||
|
'_id': _id, # 请勿忘记ID
|
||||||
|
'doc': row,
|
||||||
|
"_retry_on_conflict": 5
|
||||||
|
}
|
||||||
|
actions.append(action)
|
||||||
|
if len(actions) == 5000:
|
||||||
|
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||||
|
count += len(actions)
|
||||||
|
actions.clear()
|
||||||
|
log.info("已更新{}".format(count))
|
||||||
|
if len(actions) > 0:
|
||||||
|
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||||
|
count += len(actions)
|
||||||
|
actions.clear()
|
||||||
|
log.info("更新完成,共更新: {}".format(count))
|
||||||
|
return df
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def save_fill(community_summary_df):
|
||||||
|
if 'estate_charge' not in community_summary_df.columns or 'estate_charge_score' not in community_summary_df.columns:
|
||||||
log.info("该城市无补充物业费")
|
log.info("该城市无补充物业费")
|
||||||
return
|
return
|
||||||
df = community_summary_df[['_id', 'city', 'estate_charge']]
|
df = community_summary_df[['_id', 'city', 'estate_charge', 'estate_charge_score']]
|
||||||
df = df.dropna(subset=['estate_charge'])
|
df = df.dropna(subset=['estate_charge'])
|
||||||
actions = list()
|
actions = list()
|
||||||
count = 0
|
count = 0
|
||||||
@@ -29,6 +84,8 @@ def save(community_summary_df):
|
|||||||
row = dict()
|
row = dict()
|
||||||
_id = str(item["_id"])
|
_id = str(item["_id"])
|
||||||
row['estate_charge'] = item['estate_charge']
|
row['estate_charge'] = item['estate_charge']
|
||||||
|
if pd.notna(item['estate_charge_score']):
|
||||||
|
row['credit_detail'] = {'estate_charge_score': int(item['estate_charge_score'])}
|
||||||
# 1 表示为物业费补充的,在实际应用中不输出
|
# 1 表示为物业费补充的,在实际应用中不输出
|
||||||
row['estate_charge_flag'] = 1
|
row['estate_charge_flag'] = 1
|
||||||
row['m_date'] = m_date
|
row['m_date'] = m_date
|
||||||
@@ -57,8 +114,8 @@ def find_nearest_by_location(city, location):
|
|||||||
docs = list()
|
docs = list()
|
||||||
try:
|
try:
|
||||||
scan = es206_client.scan(index_name, {
|
scan = es206_client.scan(index_name, {
|
||||||
"size": 3,
|
"size": 4,
|
||||||
"_source": ["finish_building_date", "estate_charge"],
|
"_source": ["finish_building_date", "estate_charge", "estate_charge_fix"],
|
||||||
"query": {
|
"query": {
|
||||||
"bool": {
|
"bool": {
|
||||||
"filter": [
|
"filter": [
|
||||||
@@ -88,7 +145,12 @@ def find_nearest_by_location(city, location):
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
}, preserve_order=True)
|
}, preserve_order=True)
|
||||||
|
ignored_first = False
|
||||||
for doc in scan:
|
for doc in scan:
|
||||||
|
# 忽略自己
|
||||||
|
if ignored_first is False:
|
||||||
|
ignored_first = True
|
||||||
|
continue
|
||||||
docs.append(doc['_source'])
|
docs.append(doc['_source'])
|
||||||
if len(docs) == 3:
|
if len(docs) == 3:
|
||||||
return pd.DataFrame(docs)
|
return pd.DataFrame(docs)
|
||||||
@@ -98,7 +160,7 @@ def find_nearest_by_location(city, location):
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def get_estate_charge(finish_building_date, docs):
|
def get_estate_charge_for_adjust(finish_building_date, docs):
|
||||||
aim_year = int(finish_building_date / 10)
|
aim_year = int(finish_building_date / 10)
|
||||||
for index, item in docs.iterrows():
|
for index, item in docs.iterrows():
|
||||||
if pd.notna(item['finish_building_date']) and pd.notna(item['estate_charge']) and aim_year == int(item['finish_building_date'] / 10):
|
if pd.notna(item['finish_building_date']) and pd.notna(item['estate_charge']) and aim_year == int(item['finish_building_date'] / 10):
|
||||||
@@ -106,31 +168,153 @@ def get_estate_charge(finish_building_date, docs):
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def sync_by_city(city_now):
|
def get_estate_charge(finish_building_date, docs):
|
||||||
log.info(f"{city_now}开始补充物业费....")
|
aim_year = int(finish_building_date / 10)
|
||||||
sql = "select id, city, region, name, estate_charge, finish_building_date, location from community_summary where city='" + city_now + "' and (estate_charge is null or estate_charge_flag=1)"
|
for index, item in docs.iterrows():
|
||||||
|
if pd.notna(item['finish_building_date']) and pd.notna(item['estate_charge_fix']) and aim_year == int(item['finish_building_date'] / 10):
|
||||||
|
return item['estate_charge_fix']
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
find_count = 0
|
||||||
|
def deal_adjust(community):
|
||||||
|
global find_count
|
||||||
|
find_count += 1
|
||||||
|
if find_count % 1000 == 0:
|
||||||
|
log.info("已查找" + str(find_count))
|
||||||
|
max_now = 7
|
||||||
|
city_short = community['city']
|
||||||
|
region_short = community['region']
|
||||||
|
if (city_short + region_short) in estate_charge_city_region_dict:
|
||||||
|
max_now = estate_charge_city_region_dict[city_short + region_short]['max']
|
||||||
|
elif city_short in estate_charge_city_region_dict:
|
||||||
|
max_now = estate_charge_city_region_dict[city_short]['max']
|
||||||
|
|
||||||
|
if community['estate_charge'] > max_now:
|
||||||
|
estate_charge, score = find_estate_charge_for_adjust(community, max_now)
|
||||||
|
if pd.notna(estate_charge):
|
||||||
|
mutex.acquire(timeout=60)
|
||||||
|
community['estate_charge_fix'] = estate_charge
|
||||||
|
community['estate_charge_score'] = score
|
||||||
|
mutex.release()
|
||||||
|
|
||||||
|
def adjust_estate_charge(city_now):
|
||||||
|
log.info(f"{city_now}开始调整物业费....")
|
||||||
|
sql = f"select id, city, region, name, estate_charge, finish_building_date, location from community_summary where city='{city_now}' and estate_charge_flag != 1 and estate_type != '别墅' "
|
||||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||||
with ThreadPoolExecutor(max_workers=50) as pool:
|
community_summary_list = list()
|
||||||
def deal(index, item):
|
community_summary_df.apply(lambda x: community_summary_list.append(dict(x)), axis=1)
|
||||||
if index % 1000 == 0 and index > 0:
|
|
||||||
log.info(f"已查找{index}")
|
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||||
if pd.notna(item['location']) and pd.notna(item['location']['lon']):
|
for community_summary in community_summary_list:
|
||||||
location = [item['location']['lon'], item['location']['lat']]
|
pool.submit(deal_adjust, community_summary)
|
||||||
docs = find_nearest_by_location(city_now, location)
|
# for community_summary in community_summary_list:
|
||||||
|
# deal_adjust(community_summary)
|
||||||
|
community_summary_df = pd.DataFrame(community_summary_list)
|
||||||
|
save_fix(community_summary_df)
|
||||||
|
|
||||||
|
log.info(f"{city_now}完成调整物业费....")
|
||||||
|
|
||||||
|
|
||||||
|
def find_estate_charge_for_adjust(community, max_now):
|
||||||
|
"""
|
||||||
|
查找小区周边物业费
|
||||||
|
"""
|
||||||
estate_charge = None
|
estate_charge = None
|
||||||
|
if pd.isna(community['location']) or pd.isna(community['location']['lon']):
|
||||||
|
return estate_charge
|
||||||
|
location = [community['location']['lon'], community['location']['lat']]
|
||||||
|
docs = find_nearest_by_location(community['city'], location)
|
||||||
if len(docs) <= 0 or 'estate_charge' not in docs.columns:
|
if len(docs) <= 0 or 'estate_charge' not in docs.columns:
|
||||||
return
|
return estate_charge
|
||||||
if pd.notna(item['finish_building_date']) and 'finish_building_date' in docs.columns:
|
# 记录打分
|
||||||
|
score = None
|
||||||
|
if pd.notna(community['finish_building_date']) and 'finish_building_date' in docs.columns:
|
||||||
# 根据相近建筑年代赋值
|
# 根据相近建筑年代赋值
|
||||||
estate_charge = get_estate_charge(item['finish_building_date'], docs)
|
estate_charge = get_estate_charge_for_adjust(community['finish_building_date'], docs)
|
||||||
if pd.isna(estate_charge):
|
score = 9
|
||||||
|
if pd.isna(estate_charge) or estate_charge > max_now:
|
||||||
# 取周边均值
|
# 取周边均值
|
||||||
estate_charge = docs['estate_charge'].mean()
|
estate_charge = docs['estate_charge'].mean()
|
||||||
|
score = 8
|
||||||
|
# 如果查找的还大于分位值,不调整
|
||||||
|
if estate_charge > max_now:
|
||||||
|
estate_charge = None
|
||||||
|
score = None
|
||||||
|
return estate_charge, score
|
||||||
|
|
||||||
|
def find_estate_charge(community, max_now):
|
||||||
|
"""
|
||||||
|
查找小区周边物业费
|
||||||
|
"""
|
||||||
|
estate_charge = None
|
||||||
|
if pd.isna(community['location']) or pd.isna(community['location']['lon']):
|
||||||
|
return estate_charge, 5
|
||||||
|
location = [community['location']['lon'], community['location']['lat']]
|
||||||
|
docs = find_nearest_by_location(community['city'], location)
|
||||||
|
if len(docs) <= 0 or 'estate_charge' not in docs.columns:
|
||||||
|
return estate_charge, 5
|
||||||
|
if 'estate_charge_fix' not in docs.columns:
|
||||||
|
docs['estate_charge_fix'] = None
|
||||||
|
# 用修正后的物业费补充
|
||||||
|
docs['estate_charge_fix'].fillna(docs['estate_charge'], inplace=True)
|
||||||
|
score = None
|
||||||
|
if pd.notna(community['finish_building_date']) and 'finish_building_date' in docs.columns:
|
||||||
|
# 根据相近建筑年代赋值
|
||||||
|
estate_charge = get_estate_charge(community['finish_building_date'], docs)
|
||||||
|
score = 7
|
||||||
|
if pd.isna(estate_charge) or estate_charge > max_now:
|
||||||
|
# 取周边均值
|
||||||
|
estate_charge = docs['estate_charge_fix'].mean()
|
||||||
|
score = 6
|
||||||
|
if estate_charge > max_now:
|
||||||
|
estate_charge = None
|
||||||
|
score = None
|
||||||
|
return estate_charge, score
|
||||||
|
|
||||||
|
find_count = 0
|
||||||
|
def deal_fill(community):
|
||||||
|
global find_count
|
||||||
|
find_count += 1
|
||||||
|
if find_count % 1000 == 0:
|
||||||
|
log.info("已查找" + str(find_count))
|
||||||
|
|
||||||
|
max_now = 7
|
||||||
|
city_short = community['city']
|
||||||
|
region_short = community['region']
|
||||||
|
if (city_short + region_short) in estate_charge_city_region_dict:
|
||||||
|
max_now = estate_charge_city_region_dict[city_short + region_short]['max']
|
||||||
|
elif city_short in estate_charge_city_region_dict:
|
||||||
|
max_now = estate_charge_city_region_dict[city_short]['max']
|
||||||
|
if community['estate_type'] == '别墅':
|
||||||
|
# 别墅不限制物业费最大值
|
||||||
|
max_now = 99
|
||||||
|
estate_charge, score = find_estate_charge(community, max_now)
|
||||||
if pd.notna(estate_charge):
|
if pd.notna(estate_charge):
|
||||||
community_summary_df.loc[index, "estate_charge"] = estate_charge
|
mutex.acquire(timeout=60)
|
||||||
for index, item in community_summary_df.iterrows():
|
community["estate_charge"] = estate_charge
|
||||||
pool.submit(deal, index, item)
|
community["estate_charge_score"] = score
|
||||||
save(community_summary_df)
|
mutex.release()
|
||||||
|
|
||||||
|
def fill_estate_charge(city_now):
|
||||||
|
log.info(f"{city_now}开始补充物业费....")
|
||||||
|
sql = "select id, city, region, name, estate_charge, estate_charge_fix, finish_building_date, estate_type, location from community_summary where city='" + city_now + "' and (estate_charge is null or estate_charge_flag=1) "
|
||||||
|
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||||
|
|
||||||
|
community_summary_list = list()
|
||||||
|
community_summary_df.apply(lambda x: community_summary_list.append(dict(x)), axis=1)
|
||||||
|
|
||||||
|
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||||
|
for community_summary in community_summary_list:
|
||||||
|
pool.submit(deal_fill, community_summary)
|
||||||
|
community_summary_df = pd.DataFrame(community_summary_list)
|
||||||
|
save_fill(community_summary_df)
|
||||||
|
log.info(f"{city_now}完成补充物业费....")
|
||||||
|
|
||||||
|
|
||||||
|
def sync_by_city(city_now):
|
||||||
|
adjust_estate_charge(city_now)
|
||||||
|
fill_estate_charge(city_now)
|
||||||
|
|
||||||
|
|
||||||
def sync(city=None):
|
def sync(city=None):
|
||||||
@@ -141,5 +325,10 @@ def sync(city=None):
|
|||||||
sync_by_city(city)
|
sync_by_city(city)
|
||||||
|
|
||||||
|
|
||||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
sync(city)
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
@@ -14,7 +14,7 @@ from sklearn.preprocessing import QuantileTransformer
|
|||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
index_name = "community_summary"
|
index_name = "community_summary"
|
||||||
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||||
|
|
||||||
@@ -82,11 +82,11 @@ def calc_estate_charge_rating(community_summary_df):
|
|||||||
actions = list()
|
actions = list()
|
||||||
count = 0
|
count = 0
|
||||||
|
|
||||||
if len(df) < 5:
|
if len(df) < 6:
|
||||||
# 对于样本低于5的直接排名
|
# 对于样本低于5的直接排名
|
||||||
df[target] = df['estate_charge'].rank(method='dense', ascending=False).astype(int)
|
df[target] = df['estate_charge'].rank(method='dense', ascending=False).astype(int)
|
||||||
else:
|
else:
|
||||||
df = gmm_train(5, df, source, target)
|
df = gmm_train(6, df, source, target)
|
||||||
# 批量更新
|
# 批量更新
|
||||||
for index, item in df.iterrows():
|
for index, item in df.iterrows():
|
||||||
row = dict()
|
row = dict()
|
||||||
@@ -164,6 +164,11 @@ def save_png(community_city):
|
|||||||
plt.close()
|
plt.close()
|
||||||
|
|
||||||
|
|
||||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
sync(city)
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
#dingutils.send_text("【分布图生成完毕】物业费评级已完成, 请点击 https://nextcloud.fangjia.com/s/HbifLoEDc5wi8HR 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
|
#dingutils.send_text("【分布图生成完毕】物业费评级已完成, 请点击 https://nextcloud.fangjia.com/s/HbifLoEDc5wi8HR 查看分布图", "c7ac6c86bfbe49a95e30a68616c4d784ae7781c715430dc9c5f6b46d3e6ebd78")
|
||||||
@@ -17,7 +17,7 @@ from threading import Lock
|
|||||||
import sys
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
|
||||||
index_name = "community_summary"
|
index_name = "community_summary"
|
||||||
path = "/home/airflow/data/community"
|
path = "/home/airflow/data/community"
|
||||||
@@ -141,5 +141,10 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步附加标签至206 {index_name}")
|
log.info(f"完成同步附加标签至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
sync(city)
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
"""
|
||||||
|
省市区简拼转全拼
|
||||||
|
"""
|
||||||
|
from goojia_utils import ElasticSearchClient, dateutils
|
||||||
|
import logging
|
||||||
|
import pandas as pd
|
||||||
|
from elasticsearch import helpers
|
||||||
|
import sys
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
es89_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
|
||||||
|
index_name = "community_summary"
|
||||||
|
path = "/home/airflow/data/community"
|
||||||
|
m_date = dateutils.iso_format()
|
||||||
|
|
||||||
|
count = 0
|
||||||
|
|
||||||
|
short_df = es89_client.scan_by_sql_to_data_frame("select province, city, city_short, district, district_short from administrative_mapping where type='区域' ")
|
||||||
|
short_map = {}
|
||||||
|
def deal_short_map(x):
|
||||||
|
short_map[x['city_short'] + x['district_short']] = x['province'] + '|' + x['city'] + '|' + x['district']
|
||||||
|
short_df.apply(lambda x: deal_short_map(x), axis=1)
|
||||||
|
|
||||||
|
def deal(actions, x):
|
||||||
|
global count
|
||||||
|
doc_info = dict()
|
||||||
|
if pd.notna(x['city']) and pd.notna(x['region']):
|
||||||
|
if x['city'] + x['region'] in short_map:
|
||||||
|
short_info = short_map[x['city'] + x['region']]
|
||||||
|
doc_info['province_full'] = short_info.split('|')[0]
|
||||||
|
doc_info['city_full'] = short_info.split('|')[1]
|
||||||
|
doc_info['region_full'] = short_info.split('|')[2]
|
||||||
|
|
||||||
|
action_doc = {
|
||||||
|
"_op_type": "update",
|
||||||
|
"_index": index_name,
|
||||||
|
"_id": x['_id'],
|
||||||
|
"doc": doc_info
|
||||||
|
}
|
||||||
|
if len(doc_info) > 0:
|
||||||
|
actions.append(action_doc)
|
||||||
|
if len(actions) == 5000:
|
||||||
|
helpers.bulk(es89_client.elastic_client, actions)
|
||||||
|
count += len(actions)
|
||||||
|
log.info(f"已更新{count}")
|
||||||
|
actions.clear()
|
||||||
|
|
||||||
|
|
||||||
|
def sync(city=None):
|
||||||
|
log.info(f"开始转换省市区全拼至206 {index_name}")
|
||||||
|
|
||||||
|
sql = "select city, region from community_summary"
|
||||||
|
if pd.notna(city):
|
||||||
|
sql += " where city='" + city + "'"
|
||||||
|
community_summary_df = es89_client.scan_by_sql_to_data_frame(sql)
|
||||||
|
|
||||||
|
actions = list()
|
||||||
|
community_summary_df.apply(lambda x: deal(actions, x), axis=1)
|
||||||
|
|
||||||
|
if len(actions) > 0:
|
||||||
|
helpers.bulk(es89_client.elastic_client, actions)
|
||||||
|
actions.clear()
|
||||||
|
|
||||||
|
log.info(f"完成转换省市区全拼至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
@@ -3,14 +3,15 @@
|
|||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
|
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, MAIN_MONGO_CLIENT
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from threading import Lock
|
from threading import Lock
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||||
|
|
||||||
index_name = "community_summary"
|
index_name = "community_summary"
|
||||||
path = "/home/airflow/data/community"
|
path = "/home/airflow/data/community"
|
||||||
@@ -77,10 +78,10 @@ def deal(actions, item):
|
|||||||
|
|
||||||
def sync(city=None):
|
def sync(city=None):
|
||||||
log.info(f"开始同步生活配套至206 {index_name}")
|
log.info(f"开始同步生活配套至206 {index_name}")
|
||||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
query = {}
|
||||||
if pd.notna(city):
|
if pd.notna(city):
|
||||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
query['city'] = city
|
||||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
|
||||||
|
|
||||||
actions = list()
|
actions = list()
|
||||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||||
@@ -97,5 +98,10 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步生活配套至206 {index_name}")
|
log.info(f"完成同步生活配套至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
sync(city)
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
@@ -3,7 +3,7 @@
|
|||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils
|
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, poiutils
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from threading import Lock
|
from threading import Lock
|
||||||
@@ -11,7 +11,8 @@ import sys
|
|||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||||
|
|
||||||
index_name = "community_summary"
|
index_name = "community_summary"
|
||||||
path = "/home/airflow/data/community"
|
path = "/home/airflow/data/community"
|
||||||
@@ -32,13 +33,15 @@ def deal(actions, item):
|
|||||||
|
|
||||||
doc_info = dict()
|
doc_info = dict()
|
||||||
# 三甲
|
# 三甲
|
||||||
top_three_hospital = poiutils.find_around_by_location(curr_city, 5000, [lng, lat], "医院", "三甲医院")
|
# top_three_hospital = poiutils.find_around_by_location(curr_city, 5000, [lng, lat], "医院", "三甲医院")
|
||||||
|
top_three_hospital = poiutils.find_around_by_location_from_mongo(curr_city, 5000, lng, lat, "医院", None,"三甲")
|
||||||
doc_info['top_three_hospital_num'] = len(top_three_hospital)
|
doc_info['top_three_hospital_num'] = len(top_three_hospital)
|
||||||
if len(top_three_hospital) > 0:
|
if len(top_three_hospital) > 0:
|
||||||
doc_info['top_three_hospital'] = ",".join(top_three_hospital)
|
doc_info['top_three_hospital'] = ",".join(top_three_hospital)
|
||||||
|
|
||||||
# 非三甲
|
# 非三甲
|
||||||
non_3a_hospital = poiutils.find_around_by_location(curr_city, 5000, [lng, lat], "医院", "普通医院")
|
# non_3a_hospital = poiutils.find_around_by_location(curr_city, 5000, [lng, lat], "医院", "普通医院")
|
||||||
|
non_3a_hospital = poiutils.find_around_by_location_from_mongo(curr_city, 5000, lng, lat, "医院", None,"普通")
|
||||||
doc_info['non_3a_hospital_num'] = len(non_3a_hospital)
|
doc_info['non_3a_hospital_num'] = len(non_3a_hospital)
|
||||||
if len(non_3a_hospital) > 0:
|
if len(non_3a_hospital) > 0:
|
||||||
doc_info['non_3a_hospital'] = ",".join(non_3a_hospital)
|
doc_info['non_3a_hospital'] = ",".join(non_3a_hospital)
|
||||||
@@ -63,11 +66,10 @@ def deal(actions, item):
|
|||||||
|
|
||||||
def sync(city=None):
|
def sync(city=None):
|
||||||
log.info(f"开始同步医院至206 {index_name}")
|
log.info(f"开始同步医院至206 {index_name}")
|
||||||
seaweed_query = {'status': 0, 'cat': 'district'}
|
query = {}
|
||||||
if pd.notna(city):
|
if pd.notna(city):
|
||||||
seaweed_query['city'] = city
|
query['city'] = city
|
||||||
|
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
|
||||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
|
||||||
|
|
||||||
actions = list()
|
actions = list()
|
||||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||||
@@ -84,5 +86,10 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步医院至206 {index_name}")
|
log.info(f"完成同步医院至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
sync(city)
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
@@ -7,7 +7,7 @@ import pandas as pd
|
|||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
import sys
|
import sys
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
|
||||||
index_name = "community_summary"
|
index_name = "community_summary"
|
||||||
path = "/home/airflow/data/community"
|
path = "/home/airflow/data/community"
|
||||||
@@ -19,7 +19,7 @@ count = 0
|
|||||||
def deal(actions, x):
|
def deal(actions, x):
|
||||||
global count
|
global count
|
||||||
doc_info = dict()
|
doc_info = dict()
|
||||||
if pd.notna(x['avg_price']) and pd.notna(x['rent_price']):
|
if 'rent_price' in x and pd.notna(x['avg_price']) and pd.notna(x['rent_price']):
|
||||||
doc_info['sell_to_rent'] = int(x['avg_price'] / x['rent_price'])
|
doc_info['sell_to_rent'] = int(x['avg_price'] / x['rent_price'])
|
||||||
if pd.notna(x['household_count']):
|
if pd.notna(x['household_count']):
|
||||||
doc_info['person_num'] = int(x['household_count'] * 2.3)
|
doc_info['person_num'] = int(x['household_count'] * 2.3)
|
||||||
@@ -57,5 +57,10 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步售租比、人口数至206 {index_name}")
|
log.info(f"完成同步售租比、人口数至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
sync(city)
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
@@ -0,0 +1,129 @@
|
|||||||
|
"""
|
||||||
|
百分位计算 1 最好 100最差
|
||||||
|
"""
|
||||||
|
from goojia_utils import ElasticSearchClient
|
||||||
|
import pandas as pd
|
||||||
|
import logging
|
||||||
|
from elasticsearch import helpers
|
||||||
|
import sys
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
index_name = "community_summary"
|
||||||
|
|
||||||
|
|
||||||
|
def calc_percentile(city=None):
|
||||||
|
"""
|
||||||
|
计算消费等级
|
||||||
|
"""
|
||||||
|
sql = "select id, city, region, name, avg_price, total_price, avg_cost, rent_price, relative_price_ranking, finish_building_date, estate_charge, estate_charge_fix, household_count, build_area from community_summary"
|
||||||
|
if pd.notna(city):
|
||||||
|
sql += " where city='" + city + "'"
|
||||||
|
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||||
|
if 'estate_charge_fix' not in community_summary_df.columns:
|
||||||
|
community_summary_df['estate_charge_fix'] = 0
|
||||||
|
community_summary_df['estate_charge_fix'] = community_summary_df['estate_charge'].combine_first(community_summary_df['estate_charge_fix'])
|
||||||
|
community_summary_df['relative_price_ranking'] = community_summary_df['relative_price_ranking'] * 100
|
||||||
|
if 'rent_price' not in community_summary_df.columns:
|
||||||
|
community_summary_df['rent_price'] = 0
|
||||||
|
consumption_aim_df = community_summary_df[['id', 'city', 'avg_price', 'total_price', 'avg_cost', 'rent_price', 'relative_price_ranking', 'finish_building_date',
|
||||||
|
'estate_charge_fix', 'household_count', 'build_area']]
|
||||||
|
number = []
|
||||||
|
for i in range(1, 101):
|
||||||
|
number.append(i / 100)
|
||||||
|
|
||||||
|
percents_actions = list()
|
||||||
|
# 计算百分位的字段
|
||||||
|
columns = ['avg_price', 'total_price', 'avg_cost', 'rent_price', 'relative_price_ranking', 'finish_building_date',
|
||||||
|
'estate_charge_fix', 'household_count', 'build_area']
|
||||||
|
for city_now, item in consumption_aim_df.groupby(by=['city']).describe(percentiles=number).iterrows():
|
||||||
|
for column in columns:
|
||||||
|
for i in range(1, 101):
|
||||||
|
if str(i) + '%' in item[column]:
|
||||||
|
if pd.notna((item[column])[str(i) + '%']):
|
||||||
|
action = {'city': city_now, 'type': column,
|
||||||
|
'price': int((item[column])[str(i) + '%']), 'percent': i}
|
||||||
|
else:
|
||||||
|
if pd.notna((item[column])[str(i) + '.0%']):
|
||||||
|
action = {'city': city_now, 'type': column,
|
||||||
|
'price': int((item[column])[str(i) + '.0%']), 'percent': i}
|
||||||
|
percents_actions.append(action)
|
||||||
|
|
||||||
|
percents_df = pd.DataFrame(percents_actions)
|
||||||
|
|
||||||
|
cities = consumption_aim_df['city'].unique()
|
||||||
|
actions = list()
|
||||||
|
count = 0
|
||||||
|
|
||||||
|
for city_now in cities:
|
||||||
|
log.info("开始计算:{}".format(city_now))
|
||||||
|
df = consumption_aim_df[consumption_aim_df['city'] == city_now].copy()
|
||||||
|
percents_now_df = percents_df[percents_df['city'] == city_now].copy()
|
||||||
|
try:
|
||||||
|
for i in range(1, 101):
|
||||||
|
if i == 1:
|
||||||
|
for column in columns:
|
||||||
|
df.loc[(df[column] <= percents_now_df[(percents_now_df['type'] == column) & (percents_now_df['percent'] == i)].iloc[0]['price']), column + '_percentile'] = 101 - i
|
||||||
|
else:
|
||||||
|
for column in columns:
|
||||||
|
df.loc[(df[column] > percents_now_df[(percents_now_df['type'] == column) & (percents_now_df['percent'] == i - 1)].iloc[0]['price']) & (df[column] <= percents_now_df[(percents_now_df['type'] == column) & (percents_now_df['percent'] == i)].iloc[0]['price']), column + '_percentile'] = 101 - i
|
||||||
|
except Exception as e:
|
||||||
|
log.error("解析错误{}", e)
|
||||||
|
continue
|
||||||
|
# 批量更新
|
||||||
|
for index, item in df.iterrows():
|
||||||
|
row = dict()
|
||||||
|
_id = str(item["id"])
|
||||||
|
for column in columns:
|
||||||
|
if pd.notna(item[column + '_percentile']):
|
||||||
|
if 'estate_charge_fix' == column:
|
||||||
|
db_column = "estate_charge_percentile"
|
||||||
|
else:
|
||||||
|
db_column = column + '_percentile'
|
||||||
|
row[db_column] = int(item[column + '_percentile'])
|
||||||
|
# 根据百分位计算小区规模,优先户数,然后根据建筑面积
|
||||||
|
if 'household_count_percentile' in item and pd.notna(item['household_count_percentile']):
|
||||||
|
if item['household_count_percentile'] <= 33:
|
||||||
|
row['scale'] = '小'
|
||||||
|
elif item['household_count_percentile'] <= 66:
|
||||||
|
row['scale'] = '中'
|
||||||
|
else:
|
||||||
|
row['scale'] = '大'
|
||||||
|
if 'scale' not in row:
|
||||||
|
if 'build_area_percentile' in item and pd.notna(item['build_area_percentile']):
|
||||||
|
if item['build_area_percentile'] <= 33:
|
||||||
|
row['scale'] = '小'
|
||||||
|
elif item['build_area_percentile'] <= 66:
|
||||||
|
row['scale'] = '中'
|
||||||
|
else:
|
||||||
|
row['scale'] = '大'
|
||||||
|
if 'build_area_percentile' in row:
|
||||||
|
del row['build_area_percentile']
|
||||||
|
|
||||||
|
action = {'_op_type': 'update',
|
||||||
|
'_index': index_name,
|
||||||
|
'_id': _id, # 请勿忘记ID
|
||||||
|
'doc': row,
|
||||||
|
"_retry_on_conflict": 5
|
||||||
|
}
|
||||||
|
actions.append(action)
|
||||||
|
if len(actions) == 5000:
|
||||||
|
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||||
|
count += len(actions)
|
||||||
|
actions.clear()
|
||||||
|
log.info("已更新:{}".format(count))
|
||||||
|
if len(actions) > 0:
|
||||||
|
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||||
|
count += len(actions)
|
||||||
|
actions.clear()
|
||||||
|
log.info("更新完成,共更新: {}".format(count))
|
||||||
|
|
||||||
|
|
||||||
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
|
calc_percentile(city)
|
||||||
|
else:
|
||||||
|
calc_percentile(sys_city)
|
||||||
@@ -0,0 +1,84 @@
|
|||||||
|
"""
|
||||||
|
人口评级
|
||||||
|
"""
|
||||||
|
from goojia_utils import ElasticSearchClient
|
||||||
|
import pandas as pd
|
||||||
|
import logging
|
||||||
|
from elasticsearch import helpers
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from sklearn.mixture import GaussianMixture
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
index_name = "community_summary"
|
||||||
|
|
||||||
|
|
||||||
|
def gmm_train(total_rate, consumption_df, source, target):
|
||||||
|
# 高斯混合模型
|
||||||
|
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(consumption_df[[source]])
|
||||||
|
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
|
||||||
|
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
|
||||||
|
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
|
||||||
|
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
|
||||||
|
return consumption_df
|
||||||
|
|
||||||
|
|
||||||
|
def calc_person_rating(city=None):
|
||||||
|
"""
|
||||||
|
计算消费等级
|
||||||
|
"""
|
||||||
|
sql = "select id, city, region, name, household_count from community_summary"
|
||||||
|
if pd.notna(city):
|
||||||
|
sql += " where city='" + city + "'"
|
||||||
|
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||||
|
|
||||||
|
consumption_aim_df = community_summary_df[['_id', 'city', 'household_count']]
|
||||||
|
consumption_aim_df = consumption_aim_df.dropna(subset=['household_count'])
|
||||||
|
cities = consumption_aim_df['city'].unique()
|
||||||
|
source = "household_count"
|
||||||
|
target = "person_rating"
|
||||||
|
actions = list()
|
||||||
|
count = 0
|
||||||
|
|
||||||
|
for city in cities:
|
||||||
|
df = consumption_aim_df[consumption_aim_df['city'] == city].copy()
|
||||||
|
if len(df) < 5:
|
||||||
|
continue
|
||||||
|
df = gmm_train(5, df, source, target)
|
||||||
|
|
||||||
|
# 批量更新
|
||||||
|
for index, item in df.iterrows():
|
||||||
|
row = dict()
|
||||||
|
_id = str(item["_id"])
|
||||||
|
row['person_rating'] = item['person_rating']
|
||||||
|
|
||||||
|
action = {'_op_type': 'update',
|
||||||
|
'_index': index_name,
|
||||||
|
'_id': _id, # 请勿忘记ID
|
||||||
|
'doc': row}
|
||||||
|
actions.append(action)
|
||||||
|
if len(actions) == 5000:
|
||||||
|
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||||
|
count += len(actions)
|
||||||
|
actions.clear()
|
||||||
|
log.info("评级中: 当前已评级{}".format(count))
|
||||||
|
if len(actions) > 0:
|
||||||
|
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||||
|
count += len(actions)
|
||||||
|
actions.clear()
|
||||||
|
log.info("评级完成,共更新: {}".format(count))
|
||||||
|
|
||||||
|
|
||||||
|
def sync(city=None):
|
||||||
|
calc_person_rating(city)
|
||||||
|
|
||||||
|
|
||||||
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
@@ -3,12 +3,13 @@
|
|||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils
|
from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils, MAIN_MONGO_CLIENT
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||||
|
|
||||||
index_name = "community_summary"
|
index_name = "community_summary"
|
||||||
path = "/home/airflow/data/community"
|
path = "/home/airflow/data/community"
|
||||||
@@ -35,14 +36,15 @@ def build_doc(item):
|
|||||||
doc_dict['rent_price'] = int(item['rent_price'])
|
doc_dict['rent_price'] = int(item['rent_price'])
|
||||||
if 'guide_price' in item and pd.notna(item['guide_price']):
|
if 'guide_price' in item and pd.notna(item['guide_price']):
|
||||||
doc_dict['guide_price'] = int(item['guide_price'])
|
doc_dict['guide_price'] = int(item['guide_price'])
|
||||||
|
if 'avg_price_calc_type' in item and pd.notna(item['avg_price_calc_type']):
|
||||||
|
doc_dict['avg_price_calc_type'] = int(item['avg_price_calc_type'])
|
||||||
return doc_dict
|
return doc_dict
|
||||||
|
|
||||||
|
|
||||||
def sync_by_city(city, total_seaweed_df):
|
def sync_by_city(city):
|
||||||
seaweed_df = total_seaweed_df[total_seaweed_df['city'] == city].copy()
|
seaweed_df = pd.DataFrame(list(community_summary_conn.find({"city": city}, {"_id": 1, "city": 1, "region": 1, "name": 1})))
|
||||||
if len(seaweed_df) == 0:
|
if len(seaweed_df) == 0:
|
||||||
return
|
return
|
||||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
|
||||||
|
|
||||||
# 小区所在城市均价
|
# 小区所在城市均价
|
||||||
city_price_df = commutils.get_city_price(city=city)
|
city_price_df = commutils.get_city_price(city=city)
|
||||||
@@ -61,13 +63,14 @@ def sync_by_city(city, total_seaweed_df):
|
|||||||
# 小区均价
|
# 小区均价
|
||||||
avg_price_df = commutils.get_avg_price(city=city)
|
avg_price_df = commutils.get_avg_price(city=city)
|
||||||
if len(avg_price_df) > 0:
|
if len(avg_price_df) > 0:
|
||||||
avg_price_df = avg_price_df[['city', 'district_name', 'region', 'avg_price']]
|
avg_price_df = avg_price_df[['city', 'district_name', 'region', 'avg_price', 'value_type']]
|
||||||
avg_price_df = avg_price_df.rename(columns={'district_name': 'name'})
|
avg_price_df = avg_price_df.rename(columns={'district_name': 'name', 'value_type': 'avg_price_calc_type'})
|
||||||
seaweed_df = pd.merge(seaweed_df, avg_price_df, how='left', on=['city', 'region', 'name'])
|
seaweed_df = pd.merge(seaweed_df, avg_price_df, how='left', on=['city', 'region', 'name'])
|
||||||
|
|
||||||
# 小区均价区域排名
|
# 小区均价区域排名
|
||||||
seaweed_df['avg_price_region_ranking'] = \
|
seaweed_df['avg_price_region_ranking'] = \
|
||||||
seaweed_df.groupby(by=['city', 'region']).rank(method='first', na_option='bottom', ascending=False)['avg_price']
|
seaweed_df.groupby(by=['city', 'region']).rank(method='first', na_option='bottom', ascending=False)[
|
||||||
|
'avg_price']
|
||||||
# 小区均价城市排名
|
# 小区均价城市排名
|
||||||
seaweed_df['avg_price_ranking'] = \
|
seaweed_df['avg_price_ranking'] = \
|
||||||
seaweed_df.groupby(by=['city']).rank(method='first', na_option='bottom', ascending=False)['avg_price']
|
seaweed_df.groupby(by=['city']).rank(method='first', na_option='bottom', ascending=False)['avg_price']
|
||||||
@@ -78,16 +81,22 @@ def sync_by_city(city, total_seaweed_df):
|
|||||||
avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']]
|
avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']]
|
||||||
avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(
|
avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(
|
||||||
columns={'district_name': 'name', 'up_and_downs': 'year_on_year'})
|
columns={'district_name': 'name', 'up_and_downs': 'year_on_year'})
|
||||||
|
avg_price_up_and_downs_df = avg_price_up_and_downs_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first').reset_index(
|
||||||
|
drop=True).copy()
|
||||||
seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name'])
|
seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name'])
|
||||||
|
|
||||||
|
|
||||||
# 近半年均价涨跌幅
|
# 近半年均价涨跌幅
|
||||||
avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=[city], months=-6)
|
avg_price_up_and_downs_df = commutils.get_avg_price_up_and_downs(cities=[city], months=-6)
|
||||||
if len(avg_price_up_and_downs_df) > 0:
|
if len(avg_price_up_and_downs_df) > 0:
|
||||||
avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']]
|
avg_price_up_and_downs_df = avg_price_up_and_downs_df[['city', 'region', 'district_name', 'up_and_downs']]
|
||||||
avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(
|
avg_price_up_and_downs_df = avg_price_up_and_downs_df.rename(
|
||||||
columns={'district_name': 'name', 'up_and_downs': 'half_on_year'})
|
columns={'district_name': 'name', 'up_and_downs': 'half_on_year'})
|
||||||
|
avg_price_up_and_downs_df = avg_price_up_and_downs_df.drop_duplicates(subset=['city', 'region', 'name'], keep='first').reset_index(
|
||||||
|
drop=True).copy()
|
||||||
seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name'])
|
seaweed_df = pd.merge(seaweed_df, avg_price_up_and_downs_df, how='left', on=['city', 'region', 'name'])
|
||||||
|
|
||||||
|
|
||||||
# 平米租金
|
# 平米租金
|
||||||
rent_price_df = commutils.get_rent_price(city=city)
|
rent_price_df = commutils.get_rent_price(city=city)
|
||||||
if len(rent_price_df) > 0:
|
if len(rent_price_df) > 0:
|
||||||
@@ -101,7 +110,6 @@ def sync_by_city(city, total_seaweed_df):
|
|||||||
gov_price_df = gov_price_df[['city', 'region', 'name', 'avg_price']]
|
gov_price_df = gov_price_df[['city', 'region', 'name', 'avg_price']]
|
||||||
gov_price_df = gov_price_df.rename(columns={'avg_price': 'guide_price'})
|
gov_price_df = gov_price_df.rename(columns={'avg_price': 'guide_price'})
|
||||||
seaweed_df = pd.merge(seaweed_df, gov_price_df, how='left', on=['city', 'region', 'name'])
|
seaweed_df = pd.merge(seaweed_df, gov_price_df, how='left', on=['city', 'region', 'name'])
|
||||||
|
|
||||||
actions = list()
|
actions = list()
|
||||||
count = 0
|
count = 0
|
||||||
for index, item in seaweed_df.iterrows():
|
for index, item in seaweed_df.iterrows():
|
||||||
@@ -113,9 +121,11 @@ def sync_by_city(city, total_seaweed_df):
|
|||||||
"_id": item['_id'],
|
"_id": item['_id'],
|
||||||
"doc": doc_info
|
"doc": doc_info
|
||||||
}
|
}
|
||||||
|
|
||||||
actions.append(action)
|
actions.append(action)
|
||||||
|
|
||||||
if len(actions) == 5000:
|
if len(actions) == 5000:
|
||||||
|
print(index)
|
||||||
|
print(item)
|
||||||
helpers.bulk(es206_client.elastic_client, actions)
|
helpers.bulk(es206_client.elastic_client, actions)
|
||||||
count += len(actions)
|
count += len(actions)
|
||||||
log.info("更新:" + str(count))
|
log.info("更新:" + str(count))
|
||||||
@@ -125,20 +135,24 @@ def sync_by_city(city, total_seaweed_df):
|
|||||||
actions.clear()
|
actions.clear()
|
||||||
|
|
||||||
|
|
||||||
def sync(city=None):
|
def sync(curr_city=None):
|
||||||
log.info(f"开始同步价格信息至206 {index_name}")
|
log.info(f"开始同步价格信息至206 {index_name}")
|
||||||
if pd.notna(city):
|
if pd.notna(curr_city):
|
||||||
cities = [city]
|
cities = [curr_city]
|
||||||
else:
|
else:
|
||||||
cities = areautils.get_all_cities()
|
cities = areautils.get_all_cities()
|
||||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
for city_now in cities:
|
||||||
for city in cities:
|
log.info(f"开始同步{city_now}")
|
||||||
log.info(f"开始同步{city}")
|
sync_by_city(city_now)
|
||||||
sync_by_city(city, seaweed_df)
|
log.info(f"完成同步{city_now}")
|
||||||
log.info(f"完成同步{city}")
|
|
||||||
|
|
||||||
log.info(f"完成同步价格信息至206 {index_name}")
|
log.info(f"完成同步价格信息至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
sync(city)
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from sklearn.mixture import GaussianMixture
|
|||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
index_name = "community_summary"
|
index_name = "community_summary"
|
||||||
|
|
||||||
|
|
||||||
@@ -66,7 +66,8 @@ def calc_price_rating(city=None):
|
|||||||
action = {'_op_type': 'update',
|
action = {'_op_type': 'update',
|
||||||
'_index': index_name,
|
'_index': index_name,
|
||||||
'_id': _id, # 请勿忘记ID
|
'_id': _id, # 请勿忘记ID
|
||||||
'doc': row}
|
'doc': row,
|
||||||
|
"_retry_on_conflict": 5}
|
||||||
actions.append(action)
|
actions.append(action)
|
||||||
if len(actions) == 5000:
|
if len(actions) == 5000:
|
||||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||||
@@ -84,5 +85,10 @@ def sync(city=None):
|
|||||||
calc_price_rating(city)
|
calc_price_rating(city)
|
||||||
|
|
||||||
|
|
||||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
sync(city)
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
|
|||||||
@@ -4,20 +4,23 @@
|
|||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from goojia_utils import ElasticSearchClient, commutils, dateutils
|
from goojia_utils import ElasticSearchClient, commutils, dateutils, MAIN_MONGO_CLIENT
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
import sys
|
import sys
|
||||||
|
from threading import Lock
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||||
|
|
||||||
index_name = "community_summary"
|
index_name = "community_summary"
|
||||||
path = "/home/airflow/data/community"
|
path = "/home/airflow/data/community"
|
||||||
m_date = dateutils.iso_format()
|
m_date = dateutils.iso_format()
|
||||||
|
mutex = Lock()
|
||||||
|
|
||||||
|
|
||||||
def find_around_seaweed(lng, lat):
|
def find_around_seaweed(city_now, lng, lat):
|
||||||
"""
|
"""
|
||||||
查找周边均价
|
查找周边均价
|
||||||
"""
|
"""
|
||||||
@@ -28,6 +31,13 @@ def find_around_seaweed(lng, lat):
|
|||||||
"query": {
|
"query": {
|
||||||
"bool": {
|
"bool": {
|
||||||
"filter": [
|
"filter": [
|
||||||
|
{
|
||||||
|
"term": {
|
||||||
|
"city": {
|
||||||
|
"value": city_now
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"range": {
|
"range": {
|
||||||
"avg_price": {
|
"avg_price": {
|
||||||
@@ -50,68 +60,106 @@ def find_around_seaweed(lng, lat):
|
|||||||
|
|
||||||
def build_doc(item):
|
def build_doc(item):
|
||||||
doc_dict = dict()
|
doc_dict = dict()
|
||||||
if 'relative_price_ranking' in item and pd.notna(item['relative_price_ranking']):
|
if 'relative_price_ranking' in item:
|
||||||
|
if pd.notna(item['relative_price_ranking']):
|
||||||
doc_dict['relative_price_ranking'] = item['relative_price_ranking']
|
doc_dict['relative_price_ranking'] = item['relative_price_ranking']
|
||||||
|
else:
|
||||||
|
doc_dict['relative_price_ranking'] = None
|
||||||
|
if 'credit_detail' in item:
|
||||||
|
doc_dict['credit_detail'] = item['credit_detail']
|
||||||
return doc_dict
|
return doc_dict
|
||||||
|
|
||||||
|
|
||||||
|
find_count = 0
|
||||||
|
actions = list()
|
||||||
|
|
||||||
|
|
||||||
|
def find_around_price_item(seaweed):
|
||||||
|
if 'lng' not in seaweed:
|
||||||
|
return
|
||||||
|
lng = commutils.format_lat_lng(seaweed['lng'])
|
||||||
|
lat = commutils.format_lat_lng(seaweed['lat'])
|
||||||
|
if pd.notna(lng):
|
||||||
|
city_now = seaweed['city']
|
||||||
|
around_seaweed_df = find_around_seaweed(city_now, lng, lat)
|
||||||
|
# 某些经纬度根本不在该城市下,数据问题
|
||||||
|
if len(around_seaweed_df) == 0:
|
||||||
|
return
|
||||||
|
global find_count
|
||||||
|
find_count += 1
|
||||||
|
if find_count % 1000 == 0:
|
||||||
|
log.info("已查找" + str(find_count))
|
||||||
|
|
||||||
|
mutex.acquire()
|
||||||
|
if len(around_seaweed_df) > 1:
|
||||||
|
around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True)
|
||||||
|
temp_df = around_seaweed_df[around_seaweed_df['id'] == seaweed['_id']]
|
||||||
|
if len(temp_df) > 0:
|
||||||
|
seaweed['relative_price_ranking'] = temp_df.iloc[0]['relative_price_ranking']
|
||||||
|
else:
|
||||||
|
seaweed['relative_price_ranking'] = None
|
||||||
|
elif len(around_seaweed_df) == 1:
|
||||||
|
seaweed['relative_price_ranking'] = 0.5
|
||||||
|
|
||||||
|
around_community_num = len(around_seaweed_df)
|
||||||
|
if around_community_num >= 20:
|
||||||
|
relative_price_ranking_score = 10
|
||||||
|
elif 5 <= around_community_num < 20:
|
||||||
|
relative_price_ranking_score = 9
|
||||||
|
elif 1 < around_community_num < 5:
|
||||||
|
relative_price_ranking_score = 8
|
||||||
|
elif around_community_num == 1:
|
||||||
|
relative_price_ranking_score = 7
|
||||||
|
|
||||||
|
seaweed['credit_detail'] = {'around_community_num': around_community_num, 'relative_price_ranking_score': relative_price_ranking_score}
|
||||||
|
|
||||||
|
doc_info = build_doc(seaweed)
|
||||||
|
|
||||||
|
action = {
|
||||||
|
"_op_type": "update",
|
||||||
|
"_index": index_name,
|
||||||
|
"_id": seaweed['_id'],
|
||||||
|
"doc": doc_info,
|
||||||
|
"_retry_on_conflict": 5
|
||||||
|
}
|
||||||
|
|
||||||
|
actions.append(action)
|
||||||
|
if len(actions) == 5000:
|
||||||
|
helpers.bulk(es206_client.elastic_client, actions)
|
||||||
|
log.info("更新:" + str(find_count))
|
||||||
|
actions.clear()
|
||||||
|
mutex.release()
|
||||||
|
|
||||||
|
|
||||||
def sync(city=None):
|
def sync(city=None):
|
||||||
log.info("开始同步相对价格排名数据...")
|
log.info("开始同步相对价格排名数据...")
|
||||||
|
|
||||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
query = {}
|
||||||
if pd.notna(city):
|
if pd.notna(city):
|
||||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
query['city'] = city
|
||||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
|
||||||
|
|
||||||
def find_around_price_item(index, item):
|
seaweed_list = list()
|
||||||
if index % 1000 == 0:
|
seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1)
|
||||||
log.info("已处理" + str(index))
|
|
||||||
if 'lng' not in item:
|
|
||||||
return
|
|
||||||
lng = commutils.format_lat_lng(item['lng'])
|
|
||||||
lat = commutils.format_lat_lng(item['lat'])
|
|
||||||
if pd.notna(lng):
|
|
||||||
around_seaweed_df = find_around_seaweed(lng, lat)
|
|
||||||
if len(around_seaweed_df) > 1:
|
|
||||||
around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True)
|
|
||||||
seaweed_df.loc[index, 'relative_price_ranking'] = around_seaweed_df.loc[around_seaweed_df['id'] == item['_id']]['relative_price_ranking'].tolist()[0]
|
|
||||||
elif len(around_seaweed_df) == 1:
|
|
||||||
seaweed_df.loc[index, 'relative_price_ranking'] = 0.5
|
|
||||||
|
|
||||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||||
for index, item in seaweed_df.iterrows():
|
for seaweed in seaweed_list:
|
||||||
pool.submit(find_around_price_item, index, item)
|
pool.submit(find_around_price_item, seaweed)
|
||||||
|
|
||||||
actions = list()
|
# for seaweed in seaweed_list:
|
||||||
count = 0
|
# find_around_price_item(seaweed)
|
||||||
for index, item in seaweed_df.iterrows():
|
|
||||||
doc_info = build_doc(item)
|
|
||||||
|
|
||||||
action = {
|
|
||||||
"_op_type": "update",
|
|
||||||
"_index": index_name,
|
|
||||||
"_id": item['_id'],
|
|
||||||
"doc": doc_info
|
|
||||||
}
|
|
||||||
|
|
||||||
actions.append(action)
|
|
||||||
if len(actions) == 5000:
|
|
||||||
try:
|
|
||||||
helpers.bulk(es206_client.elastic_client, actions)
|
|
||||||
count += len(actions)
|
|
||||||
log.info("更新:" + str(count))
|
|
||||||
except Exception as e:
|
|
||||||
log.error(e)
|
|
||||||
actions.clear()
|
|
||||||
if len(actions) > 0:
|
if len(actions) > 0:
|
||||||
try:
|
|
||||||
helpers.bulk(es206_client.elastic_client, actions)
|
helpers.bulk(es206_client.elastic_client, actions)
|
||||||
except Exception as e:
|
|
||||||
log.error(e)
|
|
||||||
actions.clear()
|
actions.clear()
|
||||||
|
|
||||||
log.info("完成同步相对价格排名数据...")
|
log.info("完成同步相对价格排名数据...")
|
||||||
|
|
||||||
|
|
||||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
sync(city)
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
"""
|
||||||
|
修正相对价格排名
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
import pandas as pd
|
||||||
|
from goojia_utils import ElasticSearchClient, dateutils, data_utils
|
||||||
|
from elasticsearch import helpers
|
||||||
|
import sys
|
||||||
|
from threading import Lock
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
|
||||||
|
index_name = "community_summary"
|
||||||
|
path = "/home/airflow/data/community"
|
||||||
|
m_date = dateutils.iso_format()
|
||||||
|
mutex = Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def sync(city=None):
|
||||||
|
log.info("开始调整相对价格排名数据...")
|
||||||
|
|
||||||
|
query_filter = []
|
||||||
|
if pd.notna(city):
|
||||||
|
query_filter.append({"term": {"city": city}})
|
||||||
|
|
||||||
|
query = {
|
||||||
|
"_source": ["city", "region", "relative_price_ranking", "credit_detail"],
|
||||||
|
"query": {
|
||||||
|
"bool": {
|
||||||
|
"filter": query_filter
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
community_summary_origin_df = es206_client.scan_to_data_frame("community_summary", query)
|
||||||
|
community_summary_df = community_summary_origin_df[community_summary_origin_df['credit_detail'].notna()].reset_index(drop=True)
|
||||||
|
community_summary_df = data_utils.flatten_json(community_summary_df, 'credit_detail')
|
||||||
|
community_summary_df['around_community_max_num'] = community_summary_df.groupby(by=['city'])['around_community_num'].transform('max')
|
||||||
|
community_summary_df['relative_price_ranking_fix'] = community_summary_df['around_community_num'] / community_summary_df['around_community_max_num'] * community_summary_df['relative_price_ranking']
|
||||||
|
|
||||||
|
actions = list()
|
||||||
|
count = 0
|
||||||
|
for index, item in community_summary_df[community_summary_df['relative_price_ranking_fix'].notna()].iterrows():
|
||||||
|
doc_info = {'relative_price_ranking_fix': round(item['relative_price_ranking_fix'], 6)}
|
||||||
|
|
||||||
|
action = {
|
||||||
|
"_op_type": "update",
|
||||||
|
"_index": index_name,
|
||||||
|
"_id": item['_id'],
|
||||||
|
"doc": doc_info,
|
||||||
|
"_retry_on_conflict": 5
|
||||||
|
}
|
||||||
|
|
||||||
|
actions.append(action)
|
||||||
|
if len(actions) == 5000:
|
||||||
|
helpers.bulk(es206_client.elastic_client, actions)
|
||||||
|
count += len(actions)
|
||||||
|
log.info("更新:" + str(count))
|
||||||
|
actions.clear()
|
||||||
|
if len(actions) > 0:
|
||||||
|
helpers.bulk(es206_client.elastic_client, actions)
|
||||||
|
count += len(actions)
|
||||||
|
log.info("更新:" + str(count))
|
||||||
|
actions.clear()
|
||||||
|
|
||||||
|
log.info("完成调整相对价格排名数据...")
|
||||||
|
|
||||||
|
|
||||||
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
@@ -7,14 +7,17 @@ from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, date
|
|||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
import sys
|
import sys
|
||||||
|
from threading import Lock
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||||
|
|
||||||
index_name = "community_summary"
|
index_name = "community_summary"
|
||||||
path = "/home/airflow/data/community"
|
path = "/home/airflow/data/community"
|
||||||
m_date = dateutils.iso_format()
|
m_date = dateutils.iso_format()
|
||||||
|
mutex = Lock()
|
||||||
|
|
||||||
|
|
||||||
def build_doc(item):
|
def build_doc(item):
|
||||||
@@ -30,10 +33,10 @@ def build_doc(item):
|
|||||||
def sync(city=None):
|
def sync(city=None):
|
||||||
log.info(f"开始同步出售信息至206 {index_name}")
|
log.info(f"开始同步出售信息至206 {index_name}")
|
||||||
|
|
||||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
query = {}
|
||||||
if pd.notna(city):
|
if pd.notna(city):
|
||||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
query['city'] = city
|
||||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
|
||||||
|
|
||||||
with ThreadPoolExecutor(30) as pool:
|
with ThreadPoolExecutor(30) as pool:
|
||||||
def deal(index, item):
|
def deal(index, item):
|
||||||
@@ -50,12 +53,15 @@ def sync(city=None):
|
|||||||
names_str = "'" + names_str + "'"
|
names_str = "'" + names_str + "'"
|
||||||
|
|
||||||
# 近三个月挂牌量
|
# 近三个月挂牌量
|
||||||
# sample_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=93)
|
sample_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=93)
|
||||||
# seaweed_df.loc[index, 'three_months_sell_num'] = sample_num
|
|
||||||
|
|
||||||
# 近半年挂牌量
|
# 近半年挂牌量
|
||||||
sample_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=186)
|
sample_half_year_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=186)
|
||||||
seaweed_df.loc[index, 'half_on_year_sell_num'] = sample_num
|
|
||||||
|
mutex.acquire()
|
||||||
|
seaweed_df.loc[index, 'three_months_sell_num'] = sample_num
|
||||||
|
seaweed_df.loc[index, 'half_on_year_sell_num'] = sample_half_year_num
|
||||||
|
mutex.release()
|
||||||
|
|
||||||
for index, item in seaweed_df.iterrows():
|
for index, item in seaweed_df.iterrows():
|
||||||
pool.submit(deal, index, item)
|
pool.submit(deal, index, item)
|
||||||
@@ -85,5 +91,10 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步出售信息至206 {index_name}")
|
log.info(f"完成同步出售信息至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
sync(city)
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
@@ -0,0 +1,231 @@
|
|||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import logging
|
||||||
|
import pandas as pd
|
||||||
|
from datetime import datetime
|
||||||
|
from pymongo import UpdateOne
|
||||||
|
|
||||||
|
# 兼容性补丁:解决 Elasticsearch 8.x/9.x 客户端在老版本 ES 集群上因 compatible-with 头部导致的 ApiError(406)
|
||||||
|
os.environ["ELASTIC_CLIENT_APIVERSIONING"] = "0"
|
||||||
|
try:
|
||||||
|
import elasticsearch
|
||||||
|
ver = getattr(elasticsearch, '__version__', None)
|
||||||
|
if ver and not str(ver).startswith('7.'):
|
||||||
|
original_init = elasticsearch.Elasticsearch.__init__
|
||||||
|
def patched_init(self, *args, **kwargs):
|
||||||
|
headers = kwargs.get('headers', {})
|
||||||
|
headers['Accept'] = 'application/json'
|
||||||
|
headers['Content-Type'] = 'application/json'
|
||||||
|
kwargs['headers'] = headers
|
||||||
|
original_init(self, *args, **kwargs)
|
||||||
|
elasticsearch.Elasticsearch.__init__ = patched_init
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# 初始化 ES 206 客户端与 85 上的 MongoDB 集合
|
||||||
|
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||||
|
average_statistics_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'average_statistics')
|
||||||
|
|
||||||
|
index_name = "community"
|
||||||
|
|
||||||
|
# 映射关系:将 ES 中的 root 主域映射为 Mongo 表中的对应二手房价格 (_esf_price) 字段
|
||||||
|
ROOT_FIELD_MAP = {
|
||||||
|
"anjuke.com": "anjuke_esf_price", # 安居客
|
||||||
|
"lianjia.com": "lianjia_esf_price", # 链家
|
||||||
|
"ke.com": "beike_esf_price", # 贝壳
|
||||||
|
"beike.com": "beike_esf_price", # 贝壳
|
||||||
|
"fang.com": "soufang_esf_price", # 房天下/搜房
|
||||||
|
"soufang.com": "soufang_esf_price", # 搜房
|
||||||
|
"5i5j.com": "5i5j_esf_price", # 我爱我家
|
||||||
|
"centanet.com": "zhongyuan_esf_price", # 中原地产
|
||||||
|
"qfang.com": "qfang_esf_price", # Q房网
|
||||||
|
"leyoujia.com": "leyoujia_esf_price", # 乐有家
|
||||||
|
"zhuge.com": "zhugezhaofang_esf_price", # 诸葛找房
|
||||||
|
"zhugezhaofang.com": "zhugezhaofang_esf_price",
|
||||||
|
"jiwu.com": "jiwu_esf_price", # 吉屋
|
||||||
|
"house365.com": "365house_esf_price", # 365淘房
|
||||||
|
"365.com": "365house_esf_price",
|
||||||
|
"fanggugu.com": "fanggugu_esf_price", # 房估估
|
||||||
|
"yungujia.com": "yungujia_esf_price", # 云估价
|
||||||
|
"tuboshi.com": "tuboshi_esf_price", # 兔博士
|
||||||
|
"xiaozijia.com": "xiaozijia_esf_price", # 小资家
|
||||||
|
"fangdoudou.com": "fangdoudou_esf_price",# 房兜兜
|
||||||
|
"fangtan.com": "fangtan_esf_price", # 房探
|
||||||
|
"guoxinda.com": "gxd_esf_price", # 国信达
|
||||||
|
"gxd.com": "gxd_esf_price",
|
||||||
|
"xingfuli.com": "xingfuli_esf_price", # 幸福里
|
||||||
|
"maitian.cn": "maitian_esf_price", # 麦田房产
|
||||||
|
"maitian.com": "maitian_esf_price",
|
||||||
|
"sina.com.cn": "sina_esf_price", # 新浪二手房
|
||||||
|
"sina.com": "sina_esf_price",
|
||||||
|
"cric.com": "cric_esf_price", # 克而瑞
|
||||||
|
"ganji.com": "ganji_esf_price", # 赶集网
|
||||||
|
}
|
||||||
|
|
||||||
|
def parse_s_date(m_date_str):
|
||||||
|
"""
|
||||||
|
将 ES 抓取时间 '2022-06-17T13:12:47.148Z' 转化为整型 s_date YYYYMM (例如 202206)
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
if pd.isna(m_date_str) or not isinstance(m_date_str, str):
|
||||||
|
return None
|
||||||
|
return int(m_date_str[:7].replace('-', ''))
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def sync_data(target_month=None):
|
||||||
|
"""
|
||||||
|
数据流式拉取、转换与批量 Upsert
|
||||||
|
"""
|
||||||
|
log.info("开始从 ES 206 的 community 索引拉取小区价格数据...")
|
||||||
|
|
||||||
|
# 构造 SQL,默认仅同步近 3 个月(90 天)内更新的数据,降低传输和存储开销
|
||||||
|
if target_month is not None:
|
||||||
|
try:
|
||||||
|
month_str = str(target_month)
|
||||||
|
year = int(month_str[:4])
|
||||||
|
month = int(month_str[4:6])
|
||||||
|
start_date = f"{year:04d}-{month:02d}-01T00:00:00"
|
||||||
|
if month == 12:
|
||||||
|
end_date = f"{year + 1:04d}-01-01T00:00:00"
|
||||||
|
else:
|
||||||
|
end_date = f"{year:04d}-{month + 1:02d}-01T00:00:00"
|
||||||
|
|
||||||
|
date_filter = f"and m_date >= '{start_date}' and m_date < '{end_date}'"
|
||||||
|
log.info(f"指定历史月份跑数,过滤条件: {start_date} 至 {end_date}")
|
||||||
|
except Exception as e:
|
||||||
|
log.error(f"解析指定月份失败,退回到默认的近3个月同步。Error: {e}")
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
three_months_ago = (datetime.now() - timedelta(days=90)).strftime('%Y-%m-%dT%H:%M:%S')
|
||||||
|
date_filter = f"and m_date >= '{three_months_ago}'"
|
||||||
|
else:
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
three_months_ago = (datetime.now() - timedelta(days=90)).strftime('%Y-%m-%dT%H:%M:%S')
|
||||||
|
date_filter = f"and m_date >= '{three_months_ago}'"
|
||||||
|
log.info(f"增量运行,过滤近3个月内有更新的数据 (m_date >= '{three_months_ago}')")
|
||||||
|
|
||||||
|
sql = f"select city, region, name, avg_price, root, m_date from {index_name} where avg_price is not null and avg_price > 0 {date_filter}"
|
||||||
|
|
||||||
|
# 城市手动过滤(可选,通过外部参数调度)
|
||||||
|
sys_city = sys.argv[1] if len(sys.argv) > 1 and not sys.argv[1].isdigit() else None
|
||||||
|
if pd.notna(sys_city):
|
||||||
|
sql += f" and city = '{sys_city}'"
|
||||||
|
log.info(f"正在过滤目标城市: {sys_city}")
|
||||||
|
|
||||||
|
df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||||
|
if df is None or len(df) == 0:
|
||||||
|
log.info("未从 ES 中获取到可用数据。")
|
||||||
|
return
|
||||||
|
|
||||||
|
log.info(f"成功读取到 {len(df)} 条原始记录。开始解析字段,并根据唯一键 (city, region, name, s_date) 进行内存合并...")
|
||||||
|
|
||||||
|
# 聚合内存合并,以支持多个不同 root (渠道来源) 的价格在同一个 (城市-行政区-小区名-月份) 唯一键文档中合并设置
|
||||||
|
grouped = {}
|
||||||
|
|
||||||
|
for idx, row in df.iterrows():
|
||||||
|
city = row.get('city')
|
||||||
|
region = row.get('region')
|
||||||
|
name = row.get('name')
|
||||||
|
avg_price = row.get('avg_price')
|
||||||
|
root = row.get('root')
|
||||||
|
m_date = row.get('m_date')
|
||||||
|
|
||||||
|
if pd.isna(city) or pd.isna(region) or pd.isna(name) or pd.isna(avg_price) or pd.isna(root):
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
price_val = int(float(avg_price))
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 过滤价格 <= 0 的异常/无效数据
|
||||||
|
if price_val <= 0:
|
||||||
|
continue
|
||||||
|
|
||||||
|
s_date = parse_s_date(m_date)
|
||||||
|
if s_date is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 若指定了目标月份过滤(如仅同步指定月份的数据)
|
||||||
|
if target_month is not None and s_date != int(target_month):
|
||||||
|
continue
|
||||||
|
|
||||||
|
field_name = ROOT_FIELD_MAP.get(root)
|
||||||
|
if not field_name:
|
||||||
|
continue
|
||||||
|
|
||||||
|
key = (city, region, name, s_date)
|
||||||
|
if key not in grouped:
|
||||||
|
grouped[key] = {}
|
||||||
|
|
||||||
|
# 多个相同 root 冲突时,此处保留最新或较高价格(以整型存储)
|
||||||
|
current_val = grouped[key].get(field_name)
|
||||||
|
if current_val is None or price_val > current_val:
|
||||||
|
grouped[key][field_name] = price_val
|
||||||
|
|
||||||
|
log.info(f"内存聚合清洗完毕。共获得 {len(grouped)} 组唯一索引项。开始构建批量 MongoDB UpdateOne upsert 任务...")
|
||||||
|
|
||||||
|
# 构建 Bulk Write Upsert 任务
|
||||||
|
actions = []
|
||||||
|
now = datetime.now()
|
||||||
|
|
||||||
|
for (city, region, name, s_date), fields_dict in grouped.items():
|
||||||
|
# 需要 $set 写入的所有第三方价格字段及最近修改时间
|
||||||
|
update_doc = {**fields_dict}
|
||||||
|
update_doc["m_date"] = now # ModificationDate 对应实体属性
|
||||||
|
|
||||||
|
query = {
|
||||||
|
"city": city,
|
||||||
|
"region": region,
|
||||||
|
"name": name,
|
||||||
|
"s_date": s_date
|
||||||
|
}
|
||||||
|
|
||||||
|
# Upsert 动作
|
||||||
|
action = UpdateOne(
|
||||||
|
query,
|
||||||
|
{
|
||||||
|
"$set": update_doc,
|
||||||
|
"$setOnInsert": {
|
||||||
|
"category": "district", # 默认对应小区类别 district
|
||||||
|
"date": now # 对应首建/大日期 longDate
|
||||||
|
}
|
||||||
|
},
|
||||||
|
upsert=True
|
||||||
|
)
|
||||||
|
actions.append(action)
|
||||||
|
|
||||||
|
total_actions = len(actions)
|
||||||
|
if total_actions == 0:
|
||||||
|
log.info("过滤处理后无数据需写入 MongoDB。")
|
||||||
|
return
|
||||||
|
|
||||||
|
log.info(f"开始向 192.168.0.85 数据库的 average_statistics 集合中分批写入 {total_actions} 条大批处理记录...")
|
||||||
|
|
||||||
|
# 5000条一笔批量执行写入,防止网络长连接超时或内存剧增
|
||||||
|
batch_size = 5000
|
||||||
|
for i in range(0, total_actions, batch_size):
|
||||||
|
batch_actions = actions[i:i + batch_size]
|
||||||
|
result = average_statistics_conn.bulk_write(batch_actions)
|
||||||
|
log.info(f"批次 {i // batch_size + 1} 写入情况: "
|
||||||
|
f"Matched={result.matched_count}, "
|
||||||
|
f"Upserted={result.upserted_count}, "
|
||||||
|
f"Modified={result.modified_count}")
|
||||||
|
|
||||||
|
log.info("数据同步流任务全部成功结束!")
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
# 支持传入参数如 `python sync_es_price_to_mongo.py 202401` 进行指定月份跑数
|
||||||
|
month_arg = None
|
||||||
|
if len(sys.argv) > 1:
|
||||||
|
arg = sys.argv[1]
|
||||||
|
if arg.isdigit() and len(arg) == 6:
|
||||||
|
month_arg = int(arg)
|
||||||
|
log.info(f"读取到指定月份运行参数: {month_arg}")
|
||||||
|
|
||||||
|
sync_data(target_month=month_arg)
|
||||||
@@ -3,15 +3,13 @@
|
|||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
|
||||||
import re
|
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
|
||||||
index_name = "community_summary"
|
index_name = "community_summary"
|
||||||
path = "/home/airflow/data/community"
|
path = "/home/airflow/data/community"
|
||||||
@@ -26,59 +24,20 @@ def build_doc(item):
|
|||||||
return doc_dict
|
return doc_dict
|
||||||
|
|
||||||
|
|
||||||
# def avg_price_map_deal(x):
|
|
||||||
# avg_price_map[x['city'] + x['region'] + x['name']] = x['avg_price']
|
|
||||||
# return x
|
|
||||||
|
|
||||||
|
|
||||||
def deal(seaweed_df, index, item):
|
|
||||||
if index % 1000 == 0:
|
|
||||||
log.info("已处理" + str(index))
|
|
||||||
curr_city = item['city']
|
|
||||||
curr_region = item['region']
|
|
||||||
curr_name = item['name']
|
|
||||||
names = [curr_name]
|
|
||||||
|
|
||||||
if pd.notna(item['alias']) and '[]' != item['alias']:
|
|
||||||
alias_list = re.sub("\[|\]|'", '', item['alias']).split(',')
|
|
||||||
if len(alias_list) > 0:
|
|
||||||
for alias_item in alias_list:
|
|
||||||
names.append(alias_item)
|
|
||||||
|
|
||||||
# 户均总价
|
|
||||||
# if (curr_city + curr_region + curr_name) in avg_price_map:
|
|
||||||
# avg_price = avg_price_map[curr_city + curr_region + curr_name]
|
|
||||||
# if pd.notna(avg_price):
|
|
||||||
# 由于覆盖率较低,暂不过滤
|
|
||||||
# min_avg_price = 0.8 * avg_price
|
|
||||||
# max_avg_rice = 1.2 * avg_price
|
|
||||||
total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=366)
|
|
||||||
seaweed_df.loc[index, 'total_price'] = total_price
|
|
||||||
|
|
||||||
|
|
||||||
def sync(city=None):
|
def sync(city=None):
|
||||||
log.info(f"开始同步户均总价信息至206 {index_name}")
|
log.info(f"开始同步户均总价信息至206 {index_name}")
|
||||||
|
|
||||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
|
||||||
if pd.notna(city):
|
|
||||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
|
||||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
|
||||||
|
|
||||||
# 加载库中均价数据
|
# 加载库中均价数据
|
||||||
# query_sql = "SELECT city, region, name, avg_price FROM community_summary "
|
query_sql = "SELECT city, region, name, avg_price, house_avg_area FROM community_summary WHERE avg_price > 0 and house_avg_area > 0 "
|
||||||
# if pd.notna(city):
|
if pd.notna(city):
|
||||||
# query_sql += " WHERE city='" + city + "' "
|
query_sql += " AND city='" + city + "' "
|
||||||
# community_summary_df = es206_client.scan_by_sql_to_data_frame(query_sql)
|
community_summary_df = es206_client.scan_by_sql_to_data_frame(query_sql)
|
||||||
|
|
||||||
# community_summary_df.apply(lambda x: avg_price_map_deal(x), axis=1)
|
|
||||||
|
|
||||||
with ThreadPoolExecutor(30) as pool:
|
|
||||||
for index, item in seaweed_df.iterrows():
|
|
||||||
pool.submit(deal, seaweed_df, index, item)
|
|
||||||
|
|
||||||
actions = list()
|
actions = list()
|
||||||
count = 0
|
count = 0
|
||||||
for index, item in seaweed_df.iterrows():
|
community_summary_df['total_price'] = community_summary_df['avg_price'] * community_summary_df['house_avg_area']
|
||||||
|
community_summary_df['total_price'] = community_summary_df['total_price'].astype(int)
|
||||||
|
for index, item in community_summary_df.iterrows():
|
||||||
doc_info = build_doc(item)
|
doc_info = build_doc(item)
|
||||||
|
|
||||||
action = {
|
action = {
|
||||||
@@ -107,5 +66,10 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步户均总价信息至206 {index_name}")
|
log.info(f"完成同步户均总价信息至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
sync(city)
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
|
|||||||
@@ -6,19 +6,24 @@ import pandas as pd
|
|||||||
from goojia_utils import ElasticSearchClient, dateutils
|
from goojia_utils import ElasticSearchClient, dateutils
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
import sys
|
import sys
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from threading import Lock
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
|
||||||
index_name = "community_summary"
|
index_name = "community_summary"
|
||||||
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||||
m_date = dateutils.iso_format()
|
m_date = dateutils.iso_format()
|
||||||
|
|
||||||
|
mutex = Lock()
|
||||||
|
|
||||||
def build_doc(item):
|
def build_doc(item):
|
||||||
doc_dict = dict()
|
doc_dict = dict()
|
||||||
if 'total_price' in item and pd.notna(item['total_price']):
|
if 'total_price' in item and pd.notna(item['total_price']):
|
||||||
doc_dict['total_price'] = int(item['total_price'])
|
doc_dict['total_price'] = int(item['total_price'])
|
||||||
|
if 'total_price_score' in item and pd.notna(item['total_price_score']):
|
||||||
|
doc_dict['credit_detail'] = {'total_price_score': item['total_price_score']}
|
||||||
return doc_dict
|
return doc_dict
|
||||||
|
|
||||||
|
|
||||||
@@ -49,13 +54,6 @@ def save(community_factor_df):
|
|||||||
actions.clear()
|
actions.clear()
|
||||||
|
|
||||||
|
|
||||||
def complete_by_area(community_df):
|
|
||||||
factor_df = pd.read_csv(path + '/小区户均总价面积因子已去极值.csv', sep='@', encoding='UTF-8')
|
|
||||||
community_factor_df = pd.merge(community_df, factor_df, how='left', on=['_id', 'city', 'region', 'name'])
|
|
||||||
community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor']
|
|
||||||
return community_factor_df
|
|
||||||
|
|
||||||
|
|
||||||
def complete_by_build_year_area(community_factor_df):
|
def complete_by_build_year_area(community_factor_df):
|
||||||
build_year_area_df = pd.read_excel(path + '/全国各区各建筑年代户均面积已去极值.xlsx')
|
build_year_area_df = pd.read_excel(path + '/全国各区各建筑年代户均面积已去极值.xlsx')
|
||||||
build_year_area_dict = dict()
|
build_year_area_dict = dict()
|
||||||
@@ -93,6 +91,8 @@ def complete_by_build_year_area(community_factor_df):
|
|||||||
item['city'] + "|" + item['region'] + "|2020及以后"] if item['city'] + "|" + item[
|
item['city'] + "|" + item['region'] + "|2020及以后"] if item['city'] + "|" + item[
|
||||||
'region'] + "|2020及以后" in build_year_area_dict else None
|
'region'] + "|2020及以后" in build_year_area_dict else None
|
||||||
community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor']
|
community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor']
|
||||||
|
# 按各区各建筑年代户均面积补充, 得6分
|
||||||
|
community_factor_df.loc[community_factor_df['total_price'].notna(), 'total_price_score'] = 6
|
||||||
|
|
||||||
|
|
||||||
def complete_by_region_area(community_factor_df):
|
def complete_by_region_area(community_factor_df):
|
||||||
@@ -105,22 +105,78 @@ def complete_by_region_area(community_factor_df):
|
|||||||
community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor']
|
community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor']
|
||||||
|
|
||||||
|
|
||||||
|
def find_around_avg_area(city_now, region, lng, lat):
|
||||||
|
ret_es = es206_client.elastic_client.search({
|
||||||
|
"query": {
|
||||||
|
"bool": {
|
||||||
|
"filter": [
|
||||||
|
{"term": {"city": {"value": city_now}}},
|
||||||
|
{"term": {"region": {"value": region}}},
|
||||||
|
{"geo_distance": {"distance": 3000, "location": [lng, lat]}},
|
||||||
|
{"range": {"house_avg_area": {"gt": 0}}}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"size": 0,
|
||||||
|
"aggs": {
|
||||||
|
"house_avg_area": {
|
||||||
|
"avg": {
|
||||||
|
"field": "house_avg_area"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, index='community_summary')
|
||||||
|
return ret_es['aggregations']['house_avg_area']['value']
|
||||||
|
|
||||||
|
|
||||||
|
def complete_by_round_area(community_factor_df):
|
||||||
|
"""
|
||||||
|
按周边三公里小区户均面积均值补充
|
||||||
|
"""
|
||||||
|
def deal_complete_by_round_area(index, item):
|
||||||
|
avg_price = item['avg_price']
|
||||||
|
house_avg_area = find_around_avg_area(item['city'], item['region'], item['location']['lon'],
|
||||||
|
item['location']['lat'])
|
||||||
|
mutex.acquire(timeout=60)
|
||||||
|
community_factor_df.loc[index, 'factor'] = house_avg_area
|
||||||
|
if pd.notna(avg_price) and pd.notna(house_avg_area):
|
||||||
|
community_factor_df.loc[index, 'total_price'] = avg_price * house_avg_area
|
||||||
|
# 按周边三公里小区户均面积均值补充,得5分
|
||||||
|
community_factor_df.loc[index, 'total_price_score'] = 5
|
||||||
|
mutex.release()
|
||||||
|
|
||||||
|
# for index, item in community_factor_df[community_factor_df['total_price'].isna()].iterrows():
|
||||||
|
# deal_complete_by_round_area(index, item)
|
||||||
|
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||||
|
for index, item in community_factor_df[community_factor_df['total_price'].isna()].iterrows():
|
||||||
|
pool.submit(deal_complete_by_round_area, index, item)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def sync(city=None):
|
def sync(city=None):
|
||||||
log.info(f"开始补全户均总价信息至206 {index_name}")
|
log.info(f"开始补全户均总价信息至206 {index_name}")
|
||||||
|
|
||||||
sql = "select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null "
|
sql = "select city, region, name, avg_price, finish_building_date, location from community_summary where avg_price is not null and total_price is null "
|
||||||
if pd.notna(city):
|
if pd.notna(city):
|
||||||
sql += " and city='" + city + "'"
|
sql += " and city='" + city + "'"
|
||||||
community_df = es206_client.scan_by_sql_to_data_frame(sql)
|
community_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||||
|
|
||||||
if len(community_df) > 0:
|
if len(community_df) > 0:
|
||||||
community_factor_df = complete_by_area(community_df)
|
community_df['total_price'] = None
|
||||||
complete_by_build_year_area(community_factor_df)
|
community_df['total_price_score'] = None
|
||||||
complete_by_region_area(community_factor_df)
|
complete_by_build_year_area(community_df)
|
||||||
save(community_factor_df)
|
#complete_by_region_area(community_df)
|
||||||
|
complete_by_round_area(community_df)
|
||||||
|
save(community_df)
|
||||||
|
|
||||||
log.info(f"完成补全户均总价信息至206 {index_name}")
|
log.info(f"完成补全户均总价信息至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
sync(city)
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
|
|||||||
@@ -4,11 +4,12 @@
|
|||||||
import logging
|
import logging
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import sys
|
import sys
|
||||||
from goojia_utils import ElasticSearchClient, commutils, dateutils
|
from goojia_utils import ElasticSearchClient, commutils, dateutils, MAIN_MONGO_CLIENT
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||||
|
|
||||||
index_name = "community_summary"
|
index_name = "community_summary"
|
||||||
path = "/home/airflow/data/community"
|
path = "/home/airflow/data/community"
|
||||||
@@ -26,11 +27,11 @@ def sync(city=None):
|
|||||||
log.info(f"开始同步成交信息至206 {index_name}")
|
log.info(f"开始同步成交信息至206 {index_name}")
|
||||||
|
|
||||||
cities = None
|
cities = None
|
||||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
|
||||||
|
query = {}
|
||||||
if pd.notna(city):
|
if pd.notna(city):
|
||||||
cities = [city]
|
query['city'] = city
|
||||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
|
||||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
|
||||||
|
|
||||||
deal_price_df = commutils.get_deal_avg_price(cities, 3)
|
deal_price_df = commutils.get_deal_avg_price(cities, 3)
|
||||||
if len(deal_price_df) <= 0:
|
if len(deal_price_df) <= 0:
|
||||||
@@ -75,5 +76,10 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步成交信息至206 {index_name}")
|
log.info(f"完成同步成交信息至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
sync(city)
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
|
|||||||
@@ -3,14 +3,15 @@
|
|||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils
|
from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils, MAIN_MONGO_CLIENT
|
||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from threading import Lock
|
from threading import Lock
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||||
|
community_summary_conn = MAIN_MONGO_CLIENT.get_collection('tmp', 'community_summary')
|
||||||
|
|
||||||
index_name = "community_summary"
|
index_name = "community_summary"
|
||||||
path = "/home/airflow/data/community"
|
path = "/home/airflow/data/community"
|
||||||
@@ -71,10 +72,10 @@ def deal(actions, item):
|
|||||||
|
|
||||||
def sync(city=None):
|
def sync(city=None):
|
||||||
log.info(f"开始同步交通至206 {index_name}")
|
log.info(f"开始同步交通至206 {index_name}")
|
||||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
query = {}
|
||||||
if pd.notna(city):
|
if pd.notna(city):
|
||||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
query['city'] = city
|
||||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
seaweed_df = pd.DataFrame(list(community_summary_conn.find(query)))
|
||||||
|
|
||||||
actions = list()
|
actions = list()
|
||||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||||
@@ -87,5 +88,10 @@ def sync(city=None):
|
|||||||
log.info(f"完成同步交通至206 {index_name}")
|
log.info(f"完成同步交通至206 {index_name}")
|
||||||
|
|
||||||
|
|
||||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
sys_city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(sys_city) and "," in sys_city:
|
||||||
|
cities = sys_city.split(",")
|
||||||
|
for city in cities:
|
||||||
sync(city)
|
sync(city)
|
||||||
|
else:
|
||||||
|
sync(sys_city)
|
||||||
@@ -1,76 +1,82 @@
|
|||||||
import logging
|
# import logging
|
||||||
import sys
|
# import sys
|
||||||
from goojia_utils import MAIN_MONGO_CLIENT, DIDATA_MONGO_CLIENT, areautils
|
# from goojia_utils import MAIN_MONGO_CLIENT, DIDATA_MONGO_CLIENT, areautils
|
||||||
from pymongo import UpdateOne
|
# from pymongo import UpdateOne
|
||||||
from bson.objectid import ObjectId
|
# from bson.objectid import ObjectId
|
||||||
import pandas as pd
|
# import pandas as pd
|
||||||
import time
|
# import time
|
||||||
|
#
|
||||||
log = logging.getLogger(__name__)
|
# log = logging.getLogger(__name__)
|
||||||
|
#
|
||||||
type = sys.argv[1] if len(sys.argv) > 1 else 'all'
|
# type = sys.argv[1] if len(sys.argv) > 1 else 'all'
|
||||||
log.info("同步类型:" + type)
|
# log.info("同步类型:" + type)
|
||||||
|
#
|
||||||
|
#
|
||||||
def sync_district_office(type):
|
# def sync_district_office(type):
|
||||||
if type != 'district' and type != 'office':
|
# if type != 'district' and type != 'office':
|
||||||
return
|
# return
|
||||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
# seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||||
seaweed_didata_conn = DIDATA_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
# seaweed_didata_conn = DIDATA_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||||
|
#
|
||||||
cities = areautils.get_all_cities()
|
# cities = areautils.get_all_cities()
|
||||||
update_date = int(round(time.time() * 1000))
|
# update_date = int(round(time.time() * 1000))
|
||||||
for city in cities:
|
# for city in cities:
|
||||||
actions = list()
|
# actions = list()
|
||||||
id_list = list()
|
# id_list = list()
|
||||||
for doc in seaweed_conn.find({'city': city, 'cat': type, 'status': 0}, {'city': 1, 'region': 1, 'block': 1, 'cat': 1, 'name': 1, 'address': 1,
|
# for doc in seaweed_conn.find({'city': city, 'cat': type, 'status': 0},
|
||||||
'estate_type2': 1, 'township': 1, 'alias': 1, 'addr_alias': 1, 'lng': 1, 'lat': 1, 'status': 1})\
|
# {'city': 1, 'region': 1, 'block': 1, 'cat': 1, 'name': 1, 'address': 1,
|
||||||
.batch_size(10000):
|
# 'estate_type2': 1, 'township': 1, 'alias': 1, 'addr_alias': 1, 'lng': 1, 'lat': 1,
|
||||||
updateDict = {'city': doc['city'], 'region': doc['region'], 'category': doc['cat'], 'name': doc['name'], 'status': doc['status']}
|
# 'status': 1}) \
|
||||||
if 'estate_type2' in doc:
|
# .batch_size(10000):
|
||||||
updateDict['estate_type'] = doc['estate_type2']
|
# updateDict = {'city': doc['city'], 'region': doc['region'], 'category': doc['cat'], 'name': doc['name'],
|
||||||
|
# 'status': doc['status']}
|
||||||
id_list.append({'_id': doc['_id']})
|
# if 'estate_type2' in doc:
|
||||||
if 'block' in doc:
|
# updateDict['estate_type'] = doc['estate_type2']
|
||||||
updateDict['block'] = doc['block']
|
#
|
||||||
if 'address' in doc:
|
# id_list.append({'_id': doc['_id']})
|
||||||
updateDict['address'] = doc['address']
|
# if 'block' in doc:
|
||||||
if 'township' in doc:
|
# updateDict['block'] = doc['block']
|
||||||
updateDict['township'] = doc['township']
|
# if 'address' in doc:
|
||||||
if 'alias' in doc:
|
# updateDict['address'] = doc['address']
|
||||||
updateDict['alias'] = doc['alias']
|
# if 'township' in doc:
|
||||||
if 'addr_alias' in doc:
|
# updateDict['township'] = doc['township']
|
||||||
updateDict['addr_alias'] = doc['addr_alias']
|
# if 'alias' in doc:
|
||||||
if 'lng' in doc and doc['lng'] is not None and str(doc['lng']) != 'nan':
|
# updateDict['alias'] = doc['alias']
|
||||||
updateDict['location'] = {'lon': doc['lng'] / 10**10, 'lat': doc['lat'] / 10**10}
|
# if 'addr_alias' in doc:
|
||||||
updateDict['update_date'] = update_date
|
# updateDict['addr_alias'] = doc['addr_alias']
|
||||||
action = UpdateOne({'_id': doc['_id']}, {'$set': updateDict}, upsert=True)
|
# if 'lng' in doc and doc['lng'] is not None and str(doc['lng']) != 'nan':
|
||||||
|
# updateDict['location'] = {'lon': doc['lng'] / 10 ** 10, 'lat': doc['lat'] / 10 ** 10}
|
||||||
actions.append(action)
|
# updateDict['update_date'] = update_date
|
||||||
if len(actions) == 10000:
|
# action = UpdateOne({'_id': doc['_id']}, {'$set': updateDict}, upsert=True)
|
||||||
seaweed_didata_conn.bulk_write(actions)
|
#
|
||||||
actions.clear()
|
# actions.append(action)
|
||||||
|
# if len(actions) == 10000:
|
||||||
didata_id_df = pd.DataFrame(list(seaweed_didata_conn.find({'city': city, 'category': type}, {'_id': 1}).batch_size(10000)))
|
# seaweed_didata_conn.bulk_write(actions)
|
||||||
if len(didata_id_df) > 0:
|
# actions.clear()
|
||||||
didata_id_df = didata_id_df[['_id']]
|
#
|
||||||
id_df = pd.DataFrame(id_list)
|
# didata_id_df = pd.DataFrame(
|
||||||
deleted_seaweed_df = pd.concat([didata_id_df, id_df, id_df]).drop_duplicates(keep=False)
|
# list(seaweed_didata_conn.find({'city': city, 'category': type}, {'_id': 1}).batch_size(10000)))
|
||||||
|
# if len(didata_id_df) > 0:
|
||||||
for deleted_seaweed_index, deleted_seaweed_item in deleted_seaweed_df.iterrows():
|
# didata_id_df = didata_id_df[['_id']]
|
||||||
action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])}, {'$set': {'status': 1, 'update_date': update_date}}, upsert=False)
|
# id_df = pd.DataFrame(id_list)
|
||||||
actions.append(action)
|
# deleted_seaweed_df = pd.concat([didata_id_df, id_df, id_df]).drop_duplicates(keep=False)
|
||||||
if len(actions) == 10000:
|
#
|
||||||
seaweed_didata_conn.bulk_write(actions)
|
# for deleted_seaweed_index, deleted_seaweed_item in deleted_seaweed_df.iterrows():
|
||||||
actions.clear()
|
# action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])},
|
||||||
|
# {'$set': {'status': 1, 'update_date': update_date}}, upsert=False)
|
||||||
if len(actions) > 0:
|
# actions.append(action)
|
||||||
seaweed_didata_conn.bulk_write(actions)
|
# if len(actions) == 10000:
|
||||||
actions.clear()
|
# seaweed_didata_conn.bulk_write(actions)
|
||||||
log.info(city + "-" + type + "-同步完成")
|
# actions.clear()
|
||||||
|
#
|
||||||
if type == 'all':
|
# if len(actions) > 0:
|
||||||
sync_district_office("district")
|
# seaweed_didata_conn.bulk_write(actions)
|
||||||
sync_district_office("office")
|
# actions.clear()
|
||||||
else:
|
# log.info(city + "-" + type + "-同步完成")
|
||||||
sync_district_office(type)
|
#
|
||||||
|
#
|
||||||
|
# if type == 'all':
|
||||||
|
# sync_district_office("district")
|
||||||
|
# sync_district_office("office")
|
||||||
|
# else:
|
||||||
|
# sync_district_office(type)
|
||||||
|
|||||||
@@ -0,0 +1,161 @@
|
|||||||
|
from goojia_utils import MAIN_MONGO_CLIENT, dateutils
|
||||||
|
from goojia_utils import areautils
|
||||||
|
from datetime import datetime, date
|
||||||
|
import pandas as pd
|
||||||
|
from dateutil.relativedelta import relativedelta
|
||||||
|
import logging
|
||||||
|
|
||||||
|
deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price')
|
||||||
|
district_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_sell_stat')
|
||||||
|
evaluate_ratio_conn = MAIN_MONGO_CLIENT.get_collection('business', 'evaluate_ratio')
|
||||||
|
|
||||||
|
m_date = dateutils.iso_format()
|
||||||
|
|
||||||
|
datetimeNow = date.today()
|
||||||
|
datetimeTwoYearAgo = datetimeNow - relativedelta(years=2)
|
||||||
|
|
||||||
|
ago_year = datetimeTwoYearAgo.year
|
||||||
|
ago_month = datetimeTwoYearAgo.month
|
||||||
|
ago_day = datetimeTwoYearAgo.day
|
||||||
|
|
||||||
|
year = datetimeNow.year
|
||||||
|
month = datetimeNow.month
|
||||||
|
day = datetimeNow.day
|
||||||
|
|
||||||
|
|
||||||
|
def sync(city):
|
||||||
|
# 查询2年内成交价格
|
||||||
|
dear_price_r = deal_price_conn.find({
|
||||||
|
'city': city,
|
||||||
|
'trade_date':
|
||||||
|
{
|
||||||
|
"$gte": datetime(ago_year, ago_month, ago_day, 0, 0),
|
||||||
|
"$lt": datetime(year, month, day, 0, 0)
|
||||||
|
}
|
||||||
|
}, {'city': 1, 'region': 1, 'district_name': 1, 'avg_price': 1, 'area': 1, 'trade_date': 1, 'match_id': 1,
|
||||||
|
'type': 1}).batch_size(10000)
|
||||||
|
|
||||||
|
df1 = pd.DataFrame(list(dear_price_r))
|
||||||
|
if df1.empty:
|
||||||
|
return
|
||||||
|
if 'type' in df1:
|
||||||
|
df1 = df1[df1.type != "车位"]
|
||||||
|
df1 = df1[df1.area > 20]
|
||||||
|
df1 = df1[df1.avg_price < 500000]
|
||||||
|
df1 = df1[~df1.match_id.isna()]
|
||||||
|
if df1.empty:
|
||||||
|
return
|
||||||
|
# 去重
|
||||||
|
df1.drop_duplicates(subset=['city', 'region', 'district_name', 'avg_price', 'area', 'trade_date', 'match_id'],
|
||||||
|
keep='first', inplace=True)
|
||||||
|
df1.reset_index(drop=True, inplace=True)
|
||||||
|
|
||||||
|
# 过滤数据(同小区成交价与小区成交均价偏差较大)
|
||||||
|
df1['中位值'] = df1.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform("median")
|
||||||
|
df1['ratio'] = round((df1['avg_price'] - df1['中位值']) / df1['中位值'], 2)
|
||||||
|
df1 = df1.drop(df1.index[df1['ratio'] > 0.25])
|
||||||
|
df1 = df1.drop(df1.index[df1['ratio'] < -0.25])
|
||||||
|
|
||||||
|
# 从日期中提取年月,进行拼接
|
||||||
|
df1['Time'] = pd.to_datetime(df1['trade_date'])
|
||||||
|
df1['Year'] = df1['Time'].dt.strftime('%Y').map(str)
|
||||||
|
df1['Month'] = df1['Time'].dt.strftime('%m').map(str)
|
||||||
|
df1['date'] = df1['Year'] + df1['Month']
|
||||||
|
convert_dict = {'date': int}
|
||||||
|
df1 = df1.astype(convert_dict)
|
||||||
|
|
||||||
|
year_month = int(dateutils.format_date(datetimeNow, fm='%Y%m'))
|
||||||
|
two_year_ago_year_month = int(dateutils.format_date(datetimeTwoYearAgo, fm='%Y%m'))
|
||||||
|
# 查询2年内挂牌价格
|
||||||
|
district_stat_r = district_sell_stat_conn.find({
|
||||||
|
'city': city,
|
||||||
|
'date':
|
||||||
|
{
|
||||||
|
"$gte": two_year_ago_year_month,
|
||||||
|
"$lt": year_month
|
||||||
|
}
|
||||||
|
}, {'city': 1, 'region': 1, 'district_name': 1, 'avg_price': 1, 'date': 1})
|
||||||
|
logging.info(f"2C价格查询范围{two_year_ago_year_month}-{year_month}")
|
||||||
|
df2 = pd.DataFrame(list(district_stat_r))
|
||||||
|
|
||||||
|
# 过滤数据(同小区挂牌价与小区挂牌均价偏差较大)
|
||||||
|
df2['中位值'] = df2.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform("median")
|
||||||
|
df2['ratio'] = round((df2['avg_price'] - df2['中位值']) / df2['中位值'], 2)
|
||||||
|
df2 = df2.drop(df2.index[df2['ratio'] > 0.25])
|
||||||
|
df2 = df2.drop(df2.index[df2['ratio'] < -0.25])
|
||||||
|
|
||||||
|
# 合并表
|
||||||
|
df1_2 = pd.merge(df1, df2, how='left', on=['city', 'region', 'district_name', 'date'])
|
||||||
|
|
||||||
|
# 过滤数据(成交价与挂牌价偏差较大)
|
||||||
|
df1_2['ratio'] = round((df1_2['avg_price_x'] - df1_2['avg_price_y']) / df1_2['avg_price_y'], 2)
|
||||||
|
|
||||||
|
df1_2 = df1_2.drop(df1_2.index[df1_2['ratio'] > 0.25])
|
||||||
|
df1_2 = df1_2.drop(df1_2.index[df1_2['ratio'] < -0.25])
|
||||||
|
df1_2 = df1_2.dropna(axis='index', how='all', subset=['ratio'])
|
||||||
|
|
||||||
|
# 城市挂牌价与成交价计算价格系数
|
||||||
|
deal_price_city_avgprice = df1_2.groupby(by=['city'])['avg_price_x'].mean().reset_index()
|
||||||
|
district_stat_city_avgprice = df1_2.groupby(by=['city'])['avg_price_y'].mean().reset_index()
|
||||||
|
|
||||||
|
df = pd.merge(deal_price_city_avgprice, district_stat_city_avgprice, how='inner', on=['city'])
|
||||||
|
df['ratio'] = round((df['avg_price_x'] - df['avg_price_y']) / df['avg_price_y'], 2)
|
||||||
|
|
||||||
|
# df.to_excel('data/city_ratio.xlsx', index=False)
|
||||||
|
|
||||||
|
for index in df.index:
|
||||||
|
city = df['city'].get(index)
|
||||||
|
ratio = df['ratio'].get(index)
|
||||||
|
if ratio > 0.25 or ratio < -0.25:
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
query = {"city": city, "category": "city"}
|
||||||
|
update = {"$set": {"ratio": ratio, "m_date": m_date}}
|
||||||
|
evaluate_ratio_conn.update_one(query, update, upsert=True)
|
||||||
|
|
||||||
|
# 区域挂牌价与成交价计算价格系数
|
||||||
|
deal_price_region_avgprice = df1_2.groupby(by=['city', 'region'])['avg_price_x'].mean().reset_index()
|
||||||
|
district_stat_region_avgprice = df1_2.groupby(by=['city', 'region'])['avg_price_y'].mean().reset_index()
|
||||||
|
df = pd.merge(deal_price_region_avgprice, district_stat_region_avgprice, how='inner', on=['city', 'region'])
|
||||||
|
df['ratio'] = round((df['avg_price_x'] - df['avg_price_y']) / df['avg_price_y'], 2)
|
||||||
|
|
||||||
|
for index in df.index:
|
||||||
|
city = df['city'].get(index)
|
||||||
|
ratio = df['ratio'].get(index)
|
||||||
|
region = df['region'].get(index)
|
||||||
|
if ratio > 0.25 or ratio < -0.25:
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
query = {"city": city, "region": region, "category": "region"}
|
||||||
|
update = {"$set": {"ratio": ratio, "region": region, "m_date": m_date}}
|
||||||
|
evaluate_ratio_conn.update_one(query, update, upsert=True)
|
||||||
|
|
||||||
|
# 小区挂牌价与成交价计算价格系数
|
||||||
|
deal_price_district_avgprice = df1_2.groupby(by=['city', 'region', 'district_name'])[
|
||||||
|
'avg_price_x'].mean().reset_index()
|
||||||
|
district_stat_district_avgprice = df1_2.groupby(by=['city', 'region', 'district_name'])[
|
||||||
|
'avg_price_y'].mean().reset_index()
|
||||||
|
df = pd.merge(deal_price_district_avgprice, district_stat_district_avgprice, how='inner',
|
||||||
|
on=['city', 'region', 'district_name'])
|
||||||
|
df['ratio'] = round((df['avg_price_x'] - df['avg_price_y']) / df['avg_price_y'], 2)
|
||||||
|
df = df[['city', 'region', 'district_name', 'ratio']]
|
||||||
|
|
||||||
|
for index in df.index:
|
||||||
|
city = df['city'].get(index)
|
||||||
|
ratio = df['ratio'].get(index)
|
||||||
|
region = df['region'].get(index)
|
||||||
|
district_name = df['district_name'].get(index)
|
||||||
|
if ratio > 0 or ratio < -0.25:
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
query = {"city": city, "region": region, "district_name": district_name, "category": "district"}
|
||||||
|
update = {
|
||||||
|
"$set": {"ratio": ratio, "m_date": m_date}}
|
||||||
|
evaluate_ratio_conn.update_one(query, update, upsert=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
for city in areautils.get_all_cities():
|
||||||
|
logging.info(f"开始计算{city}市估值系数")
|
||||||
|
sync(city)
|
||||||
|
logging.info(f"{city}市估值系数计算结束")
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
import logging
|
||||||
|
from goojia_utils import ElasticSearchClient, dingutils
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
|
||||||
|
# 获取当前时间
|
||||||
|
now = datetime.now()
|
||||||
|
# 计算当前时间以前2分钟的时间
|
||||||
|
two_minutes_ago = now - timedelta(minutes=2)
|
||||||
|
# 将时间转换为时间戳(以毫秒为单位)
|
||||||
|
# 注意:timestamp() 方法返回的是秒,所以我们需要乘以1000来得到毫秒
|
||||||
|
timestamp_milliseconds = int(two_minutes_ago.timestamp() * 1000)
|
||||||
|
|
||||||
|
count = 0
|
||||||
|
for doc in es206_client.scan_by_sql("select code from fangjia_log where code >= 500 and creationDate > " + str(timestamp_milliseconds)):
|
||||||
|
count += 1
|
||||||
|
if count > 2:
|
||||||
|
print('【OPEN日志监控】最近两分钟超过两条错误,请检查日志!')
|
||||||
|
dingutils.send_text('【OPEN日志监控】最近两分钟超过两条错误,请检查OPEN日志!', 'ab4231000995a8c90d8d1e91dfd6930c3896dd1c3bbf1f0af19e19004cf6c98c')
|
||||||
|
break
|
||||||
@@ -0,0 +1,151 @@
|
|||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
from goojia_utils import ElasticSearchClient, commutils
|
||||||
|
import pandas as pd
|
||||||
|
from elasticsearch import helpers
|
||||||
|
import transbigdata as tbd
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
city_name = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if pd.notna(city_name):
|
||||||
|
log.info("同步城市:" + city_name)
|
||||||
|
|
||||||
|
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||||
|
# 中国地理位置边界[lon1,lat1,lon2,lat2] (WGS84坐标系) 其中,lon1,lat1是左下角坐标,lon2,lat2是右上角坐标,
|
||||||
|
china_bounds = [73.66, 3.86, 135.05, 53.55]
|
||||||
|
# 栅格大小(米)
|
||||||
|
accuracy = 1000
|
||||||
|
# 生成栅格参数
|
||||||
|
params_rect = tbd.area_to_params(china_bounds, accuracy=accuracy)
|
||||||
|
|
||||||
|
|
||||||
|
# def find_comm_by_grid(city, geo_shape):
|
||||||
|
# comm_info_df = es206_client.scan_to_data_frame('community_summary', {
|
||||||
|
# "_source": ["id", "city", "region", "name", "estate_type"],
|
||||||
|
# "query": {
|
||||||
|
# "bool": {
|
||||||
|
# "filter": [
|
||||||
|
# {
|
||||||
|
# "term": {
|
||||||
|
# "city": {
|
||||||
|
# "value": city
|
||||||
|
# }
|
||||||
|
# }
|
||||||
|
# },
|
||||||
|
# {
|
||||||
|
# "geo_polygon": {
|
||||||
|
# "location": {
|
||||||
|
# "points": geo_shape
|
||||||
|
# }
|
||||||
|
# }
|
||||||
|
# }
|
||||||
|
# ]
|
||||||
|
# }
|
||||||
|
# }
|
||||||
|
# })
|
||||||
|
# return comm_info_df
|
||||||
|
|
||||||
|
estate_type_list = ["普通住宅", "别墅", "商住"]
|
||||||
|
|
||||||
|
|
||||||
|
def save_to_community_summary(comm_df):
|
||||||
|
actions = list()
|
||||||
|
count = 0
|
||||||
|
for index, item in comm_df.iterrows():
|
||||||
|
if pd.isna(item['grid_id']):
|
||||||
|
continue
|
||||||
|
doc_info = {'grid_id': item['grid_id']}
|
||||||
|
|
||||||
|
action = {
|
||||||
|
"_op_type": "update",
|
||||||
|
"_index": "community_summary",
|
||||||
|
"_id": item['_id'],
|
||||||
|
"doc": doc_info,
|
||||||
|
"_retry_on_conflict": 5
|
||||||
|
}
|
||||||
|
|
||||||
|
actions.append(action)
|
||||||
|
if len(actions) == 5000:
|
||||||
|
helpers.bulk(es206_client.elastic_client, actions)
|
||||||
|
count += len(actions)
|
||||||
|
log.info("更新:" + str(count))
|
||||||
|
actions.clear()
|
||||||
|
if len(actions) > 0:
|
||||||
|
helpers.bulk(es206_client.elastic_client, actions)
|
||||||
|
actions.clear()
|
||||||
|
|
||||||
|
|
||||||
|
def sync(city):
|
||||||
|
sql = f"select * from grid "
|
||||||
|
if pd.notna(city):
|
||||||
|
sql += f" where city='{city}' "
|
||||||
|
origin_grid_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||||
|
cities = origin_grid_df['city'].unique()
|
||||||
|
|
||||||
|
for curr_city in cities:
|
||||||
|
grid_df = origin_grid_df[origin_grid_df['city'] == curr_city].copy()
|
||||||
|
avg_price_df = commutils.get_avg_price(curr_city)
|
||||||
|
if len(avg_price_df) <= 0:
|
||||||
|
logging.info(curr_city + "无价格信息")
|
||||||
|
continue
|
||||||
|
avg_price_df = avg_price_df[['city', 'region', 'district_name', 'avg_price']]
|
||||||
|
avg_price_df = avg_price_df.rename(columns={'district_name': 'name'})
|
||||||
|
|
||||||
|
comm_df = es206_client.scan_by_sql_to_data_frame(f"select id, city, region, name, estate_type, location from community_summary where city='{curr_city}' ")
|
||||||
|
|
||||||
|
def deal_lng_lat(x):
|
||||||
|
location = x['location']
|
||||||
|
if pd.notna(location):
|
||||||
|
lon_lat_col = tbd.GPS_to_grid(location['lon'], location['lat'], params_rect)
|
||||||
|
x['grid_id'] = str(lon_lat_col[0]) + "_" + str(lon_lat_col[1])
|
||||||
|
return x
|
||||||
|
comm_df = comm_df.apply(lambda x: deal_lng_lat(x), axis=1)
|
||||||
|
# 格子信息写入community_summary
|
||||||
|
save_to_community_summary(comm_df)
|
||||||
|
# 求解格子均价
|
||||||
|
actions = list()
|
||||||
|
count = 0
|
||||||
|
log.info(f"开始计算{curr_city}格子价格")
|
||||||
|
for index, item in grid_df.iterrows():
|
||||||
|
comm_info_df = comm_df[comm_df['grid_id'] == item['_id']]
|
||||||
|
if len(comm_info_df) <= 0:
|
||||||
|
continue
|
||||||
|
grid_info = dict()
|
||||||
|
price_list = list()
|
||||||
|
for estate_type in estate_type_list:
|
||||||
|
# 计算不同物业类型的小区均价
|
||||||
|
comm_info_estate_type_df = comm_info_df[comm_info_df['estate_type'] == estate_type].copy()
|
||||||
|
if len(comm_info_estate_type_df) > 0:
|
||||||
|
comm_info_estate_type_df = pd.merge(comm_info_estate_type_df, avg_price_df, how='left', on=['city', 'region', 'name'])
|
||||||
|
comm_info_estate_type_df = comm_info_estate_type_df[comm_info_estate_type_df['avg_price'].notna()]
|
||||||
|
if len(comm_info_estate_type_df) > 0:
|
||||||
|
price_info = dict()
|
||||||
|
price_info['type'] = 'district'
|
||||||
|
price_info['secondary_type'] = estate_type
|
||||||
|
price_info['price'] = int(comm_info_estate_type_df['avg_price'].mean())
|
||||||
|
price_list.append(price_info)
|
||||||
|
if len(price_list) > 0:
|
||||||
|
grid_info['price'] = price_list
|
||||||
|
action = {
|
||||||
|
"_op_type": "update",
|
||||||
|
"_index": "grid",
|
||||||
|
"_id": item['_id'],
|
||||||
|
"doc": grid_info,
|
||||||
|
"_retry_on_conflict": 5
|
||||||
|
}
|
||||||
|
actions.append(action)
|
||||||
|
if len(actions) == 5000:
|
||||||
|
helpers.bulk(es206_client.elastic_client, actions)
|
||||||
|
count += len(actions)
|
||||||
|
log.info("更新格子数:" + str(count))
|
||||||
|
actions.clear()
|
||||||
|
if len(actions) > 0:
|
||||||
|
helpers.bulk(es206_client.elastic_client, actions)
|
||||||
|
count += len(actions)
|
||||||
|
log.info("更新格子数:" + str(count))
|
||||||
|
actions.clear()
|
||||||
|
log.info(f"完成计算{curr_city}格子价格")
|
||||||
|
|
||||||
|
|
||||||
|
sync(city_name)
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
from goojia_utils import KafkaClient
|
||||||
|
KAFKA_CLIENT = KafkaClient()
|
||||||
|
KAFKA_CLIENT.send_message("email-data-topic","sendEmail")
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
from goojia_utils import KafkaClient
|
||||||
|
KAFKA_CLIENT = KafkaClient()
|
||||||
|
KAFKA_CLIENT.send_message("sms-data-topic","sendEndTimeSms")
|
||||||
@@ -1,13 +1,13 @@
|
|||||||
from airflow.operators.bash import BashOperator
|
from airflow.providers.standard.operators.bash import BashOperator
|
||||||
import pendulum
|
import pendulum
|
||||||
from airflow import DAG
|
from airflow import DAG
|
||||||
|
|
||||||
with DAG(
|
with DAG(
|
||||||
"amap_key_generate_dag",
|
dag_id="amap_key_generate_dag",
|
||||||
# 每天运行
|
# 每天运行
|
||||||
schedule_interval="0 2 * * *",
|
schedule="0/1 14 * * *",
|
||||||
start_date=pendulum.datetime(2022, 7, 18, tz="Asia/Shanghai"),
|
start_date=pendulum.datetime(2023, 3, 18, tz="Asia/Shanghai"),
|
||||||
catchup=True,
|
catchup=True
|
||||||
) as dag:
|
) as dag:
|
||||||
t1 = BashOperator(
|
t1 = BashOperator(
|
||||||
task_id='amap_key_generate',
|
task_id='amap_key_generate',
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
from airflow.providers.standard.operators.bash import BashOperator
|
||||||
|
import pendulum
|
||||||
|
from airflow import DAG
|
||||||
|
|
||||||
|
with DAG(
|
||||||
|
dag_id="apiuser_notice_dag",
|
||||||
|
# 每小时运行
|
||||||
|
schedule="0 * * * * ",
|
||||||
|
start_date=pendulum.datetime(2023, 7, 10, tz="Asia/Shanghai"),
|
||||||
|
catchup=True,
|
||||||
|
description="API接口监控提醒"
|
||||||
|
) as dag:
|
||||||
|
t1 = BashOperator(
|
||||||
|
task_id='apiuser_notice',
|
||||||
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/apiuser_notice.py',
|
||||||
|
)
|
||||||
|
t1
|
||||||
+3
-3
@@ -1,11 +1,11 @@
|
|||||||
from airflow.operators.bash import BashOperator
|
from airflow.providers.standard.operators.bash import BashOperator
|
||||||
import pendulum
|
import pendulum
|
||||||
from airflow import DAG
|
from airflow import DAG
|
||||||
|
|
||||||
with DAG(
|
with DAG(
|
||||||
"change_new_to_old_seaweed_dag",
|
dag_id="change_new_to_old_seaweed_dag",
|
||||||
# 每月1日1点运行
|
# 每月1日1点运行
|
||||||
schedule_interval="0 1 1 * *",
|
schedule="0 1 1 * *",
|
||||||
start_date=pendulum.datetime(2023, 2, 10, tz="Asia/Shanghai"),
|
start_date=pendulum.datetime(2023, 2, 10, tz="Asia/Shanghai"),
|
||||||
catchup=True,
|
catchup=True,
|
||||||
description="新房变二手房定时脚本"
|
description="新房变二手房定时脚本"
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
# from airflow.providers.docker.operators.docker import DockerOperator
|
||||||
|
# from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator
|
||||||
|
# import pendulum
|
||||||
|
# from airflow import DAG
|
||||||
|
# from docker.types import Mount
|
||||||
|
# from kubernetes.client import models as k8s
|
||||||
|
#
|
||||||
|
# with DAG(
|
||||||
|
# dag_id="comm_add_dag",
|
||||||
|
# # 每月20日1点整分运行
|
||||||
|
# schedule="0 1 20 * *",
|
||||||
|
# start_date=pendulum.datetime(2023, 5, 8, tz="Asia/Shanghai"),
|
||||||
|
# catchup=True,
|
||||||
|
# description="小区自动化入库"
|
||||||
|
# ) as dag:
|
||||||
|
# # k = KubernetesPodOperator(
|
||||||
|
# # name="comm-add",
|
||||||
|
# # image="192.168.0.19:9012/comm-add:1.0.1",
|
||||||
|
# # cmds=["bash", "-cx"],
|
||||||
|
# # arguments=["echo", "10"],
|
||||||
|
# # labels={"foo": "bar"},
|
||||||
|
# # task_id="comm_add",
|
||||||
|
# # do_xcom_push=True,
|
||||||
|
# # )
|
||||||
|
# # k.dry_run()
|
||||||
|
#
|
||||||
|
# production_task = KubernetesPodOperator(namespace='airflow-plus',
|
||||||
|
# # image="my-production-job:release-1.0.1", <-- old release
|
||||||
|
# image="192.168.0.19:9012/comm-add:1.0.1",
|
||||||
|
# image_pull_policy="Always",
|
||||||
|
# image_pull_secrets=[k8s.V1LocalObjectReference("nexus-registry-secret")],
|
||||||
|
# cmds=["python", "main.py"],
|
||||||
|
# arguments=["--city", '{{dag_run.conf["city"] if "city" in dag_run.conf else "all"}}'],
|
||||||
|
# name="comm_add",
|
||||||
|
# task_id="comm_add",
|
||||||
|
# get_logs=True
|
||||||
|
# )
|
||||||
|
#
|
||||||
|
# production_task
|
||||||
|
# # t1 = DockerOperator(
|
||||||
|
# # # 刚才定义的conn id
|
||||||
|
# # docker_conn_id="docker_repo",
|
||||||
|
# # # WebServer中点击Admin -> Providers -> Docker对应的Version,如果是auto,可能会报错
|
||||||
|
# # api_version="2.2.0",
|
||||||
|
# # auto_remove=True,
|
||||||
|
# # docker_url='unix://var/run/docker.sock',
|
||||||
|
# # network_mode='bridge',
|
||||||
|
# # image='192.168.0.19:9012/comm-add:1.0.1',
|
||||||
|
# # # 临沂为我们的传参,python代码中, 镜像中部分代码
|
||||||
|
# # # @ click.command()
|
||||||
|
# # # @ click.option('--city', default="all", help='要计算的城市')
|
||||||
|
# # #command='python main.py --city {{dag_run.conf["city"] if "city" in dag_run.conf}} else all',
|
||||||
|
# # command='echo "this is a test message shown from within the container',
|
||||||
|
# # mounts=[
|
||||||
|
# # Mount(source="/var/run/docker.sock", target="/var/run/docker.sock", type="bind")
|
||||||
|
# # ],
|
||||||
|
# # task_id='comm_add'
|
||||||
|
# # )
|
||||||
|
# # t1
|
||||||
@@ -1,15 +1,19 @@
|
|||||||
from airflow.operators.bash import BashOperator
|
from airflow.providers.standard.operators.bash import BashOperator
|
||||||
import pendulum
|
import pendulum
|
||||||
from airflow import DAG
|
from airflow import DAG
|
||||||
|
|
||||||
with DAG(
|
with DAG(
|
||||||
"community_summary_dag",
|
dag_id="community_summary_dag",
|
||||||
# 每月2日1点整分运行
|
# 每月2日1点整分运行
|
||||||
schedule_interval="0 1 2 * *",
|
schedule="0 1 2 * *",
|
||||||
start_date=pendulum.datetime(2022, 10, 12, tz="Asia/Shanghai"),
|
start_date=pendulum.datetime(2026, 8, 16, tz="Asia/Shanghai"),
|
||||||
catchup=True,
|
catchup=True,
|
||||||
description="地址输出信息分类数据定时更新"
|
description="地址输出信息分类数据定时更新"
|
||||||
) as dag:
|
) as dag:
|
||||||
|
del_comm = BashOperator(
|
||||||
|
task_id='del_comm',
|
||||||
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/del_comm.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
|
)
|
||||||
comm = BashOperator(
|
comm = BashOperator(
|
||||||
task_id='comm',
|
task_id='comm',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
@@ -58,6 +62,10 @@ with DAG(
|
|||||||
task_id='relative_price_ranking',
|
task_id='relative_price_ranking',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
|
# relative_price_ranking_fix = BashOperator(
|
||||||
|
# task_id='relative_price_ranking_fix',
|
||||||
|
# bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking_fix.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
|
# )
|
||||||
estate_charge_complete = BashOperator(
|
estate_charge_complete = BashOperator(
|
||||||
task_id='estate_charge_complete',
|
task_id='estate_charge_complete',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
@@ -70,10 +78,30 @@ with DAG(
|
|||||||
task_id='consumption',
|
task_id='consumption',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
|
city_level_and_price_rating = BashOperator(
|
||||||
|
task_id='city_level_and_price_rating',
|
||||||
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/city_level_and_price_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
|
)
|
||||||
|
credit = BashOperator(
|
||||||
|
task_id='credit',
|
||||||
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/credit.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
|
)
|
||||||
|
person_rating = BashOperator(
|
||||||
|
task_id='person_rating',
|
||||||
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/person_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
|
)
|
||||||
price_rating = BashOperator(
|
price_rating = BashOperator(
|
||||||
task_id='price_rating',
|
task_id='price_rating',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
|
consumption_stat = BashOperator(
|
||||||
|
task_id='consumption_stat',
|
||||||
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption_stat.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
|
)
|
||||||
|
percentile = BashOperator(
|
||||||
|
task_id='percentile',
|
||||||
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/percentile.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
|
)
|
||||||
education = BashOperator(
|
education = BashOperator(
|
||||||
task_id='education',
|
task_id='education',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
@@ -90,5 +118,12 @@ with DAG(
|
|||||||
task_id='other',
|
task_id='other',
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
)
|
)
|
||||||
comm >> price >> avg_cost >> total_price >> total_price_complete >> relative_price_ranking >> estate_charge_complete >> estate_charge_ranking \
|
# 统一输出为省市区全拼
|
||||||
>> consumption >> price_rating >> sell >> trade >> traffic >> business_district >> medical >> life >> education >> education2 >> extra >> other
|
full_name = BashOperator(
|
||||||
|
task_id='full_name',
|
||||||
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/full_name.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
|
)
|
||||||
|
# 人均消费并行,值为空
|
||||||
|
del_comm >> comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> consumption >> city_level_and_price_rating >> credit >> [estate_charge_ranking, price_rating, person_rating, percentile] >> \
|
||||||
|
consumption_stat >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other \
|
||||||
|
>> full_name
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
from airflow.providers.standard.operators.bash import BashOperator
|
||||||
|
import pendulum
|
||||||
|
from airflow import DAG
|
||||||
|
|
||||||
|
# with DAG(
|
||||||
|
# dag_id="didata_sync_dag",
|
||||||
|
# # 每月28日3点整分运行
|
||||||
|
# schedule="0 3 28 * *",
|
||||||
|
# start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"),
|
||||||
|
# catchup=True,
|
||||||
|
# description="地数相关数据同步type: district或office, 不填默认同步全部"
|
||||||
|
# ) as dag:
|
||||||
|
# t1 = BashOperator(
|
||||||
|
# task_id='didata_sync_dag',
|
||||||
|
# bash_command='python /opt/airflow/dags/repo/dags/business/business/didata_sync.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
||||||
|
# )
|
||||||
|
# t1
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
from airflow.providers.standard.operators.bash import BashOperator
|
||||||
|
import pendulum
|
||||||
|
from airflow import DAG
|
||||||
|
|
||||||
|
with DAG(
|
||||||
|
dag_id="evaluate_ratio_dag",
|
||||||
|
# 每月1日0点运行
|
||||||
|
schedule="0 0 1 * *",
|
||||||
|
start_date=pendulum.datetime(2023, 7, 5, tz="Asia/Shanghai"),
|
||||||
|
catchup=True,
|
||||||
|
description="挂牌价与成交价计算价格系数"
|
||||||
|
) as dag:
|
||||||
|
t1 = BashOperator(
|
||||||
|
task_id='evaluate_ratio',
|
||||||
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/evaluate_ratio.py',
|
||||||
|
)
|
||||||
|
t1
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
from datetime import timedelta
|
||||||
|
|
||||||
|
from airflow.providers.standard.operators.bash import BashOperator
|
||||||
|
import pendulum
|
||||||
|
from airflow import DAG
|
||||||
|
|
||||||
|
with DAG(
|
||||||
|
dag_id="fangjia_log_dag",
|
||||||
|
# 每2分钟监控
|
||||||
|
schedule=timedelta(minutes=2),
|
||||||
|
start_date=pendulum.datetime(2024, 8, 14, tz="Asia/Shanghai"),
|
||||||
|
catchup=True,
|
||||||
|
description="OPEN日志监控"
|
||||||
|
) as dag:
|
||||||
|
t1 = BashOperator(
|
||||||
|
task_id='fangjia_log',
|
||||||
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/fangjia_log.py',
|
||||||
|
)
|
||||||
|
t1
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
from airflow.providers.standard.operators.bash import BashOperator
|
||||||
|
import pendulum
|
||||||
|
from airflow import DAG
|
||||||
|
|
||||||
|
with DAG(
|
||||||
|
dag_id="grid_price_dag",
|
||||||
|
# 每周二运行
|
||||||
|
schedule="0 3 * * 2",
|
||||||
|
start_date=pendulum.datetime(2023, 5, 9, tz="Asia/Shanghai"),
|
||||||
|
catchup=True,
|
||||||
|
description="全国格子价格计算"
|
||||||
|
) as dag:
|
||||||
|
t1 = BashOperator(
|
||||||
|
task_id='grid_price',
|
||||||
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/grid_price.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||||
|
)
|
||||||
|
t1
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
from airflow.providers.standard.operators.bash import BashOperator
|
||||||
|
import pendulum
|
||||||
|
from airflow import DAG
|
||||||
|
|
||||||
|
with DAG(
|
||||||
|
dag_id="send_email",
|
||||||
|
# 每月1日8点运行
|
||||||
|
schedule="0 8 1 * *",
|
||||||
|
start_date=pendulum.datetime(2024, 8, 30, tz="Asia/Shanghai"),
|
||||||
|
catchup=True,
|
||||||
|
description="API用户使用及明细的邮件发送"
|
||||||
|
) as dag:
|
||||||
|
t1 = BashOperator(
|
||||||
|
task_id='send_email',
|
||||||
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/send_email.py',
|
||||||
|
)
|
||||||
|
t1
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
from airflow.providers.standard.operators.bash import BashOperator
|
||||||
|
import pendulum
|
||||||
|
from airflow import DAG
|
||||||
|
|
||||||
|
with DAG(
|
||||||
|
dag_id="send_sms",
|
||||||
|
# 每月1日8点运行
|
||||||
|
schedule="0 8 * * *",
|
||||||
|
start_date=pendulum.datetime(2024, 9, 25, tz="Asia/Shanghai"),
|
||||||
|
catchup=True,
|
||||||
|
description="API用户使用及明细的邮件发送"
|
||||||
|
) as dag:
|
||||||
|
t1 = BashOperator(
|
||||||
|
task_id='send_sms',
|
||||||
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/send_sms.py',
|
||||||
|
)
|
||||||
|
t1
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
from airflow.providers.standard.operators.bash import BashOperator
|
||||||
|
import pendulum
|
||||||
|
from airflow import DAG
|
||||||
|
|
||||||
|
with DAG(
|
||||||
|
dag_id="sync_es_price_to_mongo_dag",
|
||||||
|
# 每月3日凌晨2点整执行(在大部分上游爬取和ES清洗完毕后进行数据同步归档)
|
||||||
|
schedule="0 2 3 * *",
|
||||||
|
start_date=pendulum.datetime(2026, 8, 16, tz="Asia/Shanghai"),
|
||||||
|
catchup=False,
|
||||||
|
description="每月定时把 ES 206 上的 community 均价数据同步到 85 Mongo 的 average_statistics 表"
|
||||||
|
) as dag:
|
||||||
|
|
||||||
|
sync_es_price_to_mongo = BashOperator(
|
||||||
|
task_id='sync_es_price_to_mongo',
|
||||||
|
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sync_es_price_to_mongo.py {{dag_run.conf["month"] if "month" in dag_run.conf else ""}}',
|
||||||
|
)
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
from airflow.operators.bash import BashOperator
|
|
||||||
import pendulum
|
|
||||||
from airflow import DAG
|
|
||||||
|
|
||||||
with DAG(
|
|
||||||
"didata_sync_dag",
|
|
||||||
# 每月28日3点整分运行
|
|
||||||
schedule_interval="0 3 28 * *",
|
|
||||||
start_date=pendulum.datetime(2022, 8, 22, tz="Asia/Shanghai"),
|
|
||||||
catchup=True,
|
|
||||||
description="地数相关数据同步type: district或office, 不填默认同步全部"
|
|
||||||
) as dag:
|
|
||||||
t1 = BashOperator(
|
|
||||||
task_id='didata_sync_dag',
|
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/didata_sync.py {{dag_run.conf["type"] if "type" in dag_run.conf}}',
|
|
||||||
)
|
|
||||||
t1
|
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
"""
|
||||||
|
第一步: 将百度经纬度转为高德经纬度(只需跑批一次即可) TODO
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
# 1. 尝试环境变量兼容
|
||||||
|
os.environ["ELASTIC_CLIENT_APIVERSIONING"] = "0"
|
||||||
|
|
||||||
|
from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, threads, geohashutils, areautils
|
||||||
|
import pandas as pd
|
||||||
|
from elasticsearch import helpers
|
||||||
|
import logging
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# ================== 🛡️ 安全运行入口 ==================
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
flag = False
|
||||||
|
for province in areautils.get_all_province():
|
||||||
|
print(f"开始处理{province}数据")
|
||||||
|
# 百度经纬度转高德经纬度
|
||||||
|
df = DIDATA_DEV_ELASTIC_CLIENT.scan_to_data_frame(index='didata_enterprise', query={
|
||||||
|
"query": {
|
||||||
|
"bool": {
|
||||||
|
"must": [
|
||||||
|
{
|
||||||
|
"term": {
|
||||||
|
"province": {
|
||||||
|
"value": "" + province + ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"must_not": [
|
||||||
|
{
|
||||||
|
"exists": {
|
||||||
|
"field": "gaode_location"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if len(df) == 0:
|
||||||
|
continue
|
||||||
|
|
||||||
|
def deal_location(item):
|
||||||
|
if pd.isna(item['location']):
|
||||||
|
return item
|
||||||
|
gd_location = geohashutils.bd_to_gd(item['location']['lon'], item['location']['lat'])
|
||||||
|
item['gaode_location'] = {'lon': gd_location[0], 'lat': gd_location[1]}
|
||||||
|
return item
|
||||||
|
|
||||||
|
my_df = threads.process_dataframe_multithread(df, deal_location, num_threads=100)
|
||||||
|
|
||||||
|
actions = list()
|
||||||
|
count = 0
|
||||||
|
for index, item in my_df[my_df['gaode_location'].notna()].iterrows():
|
||||||
|
doc_info = {'gaode_location': item['gaode_location']}
|
||||||
|
|
||||||
|
action = {
|
||||||
|
"_op_type": "update",
|
||||||
|
"_index": "didata_enterprise",
|
||||||
|
"_id": item['_id'],
|
||||||
|
"doc": doc_info,
|
||||||
|
"_retry_on_conflict": 5
|
||||||
|
}
|
||||||
|
|
||||||
|
actions.append(action)
|
||||||
|
if len(actions) == 5000:
|
||||||
|
for success, info in helpers.parallel_bulk(
|
||||||
|
# 此时这里的底层 elastic_client 将自动套用强行重置后的 JSON 请求头
|
||||||
|
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
|
||||||
|
actions,
|
||||||
|
thread_count=5, # 并行线程数
|
||||||
|
chunk_size=1000 # 批量大小
|
||||||
|
):
|
||||||
|
if not success:
|
||||||
|
print(f"索引失败: {info}")
|
||||||
|
count += len(actions)
|
||||||
|
print("更新:" + str(count))
|
||||||
|
actions.clear()
|
||||||
|
|
||||||
|
if len(actions) > 0:
|
||||||
|
helpers.bulk(DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions)
|
||||||
|
count += len(actions)
|
||||||
|
print("更新:" + str(count))
|
||||||
|
actions.clear()
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
"""
|
||||||
|
通过居委围栏更新隶属关系
|
||||||
|
"""
|
||||||
|
from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, areautils
|
||||||
|
import pandas as pd
|
||||||
|
from elasticsearch import helpers
|
||||||
|
import logging
|
||||||
|
# 配置日志
|
||||||
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
def sync(province):
|
||||||
|
logger.info(f"开始同步{province}企业数据的居委村委数据")
|
||||||
|
shape_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(
|
||||||
|
f"select city_name, area_name, street_name, name, shape from community where province_name='{province}'")
|
||||||
|
if len(shape_df) == 0 or 'shape' not in shape_df.columns or len(shape_df[shape_df['shape'].notna()]) == 0:
|
||||||
|
logger.info(f"{province}无围栏数据")
|
||||||
|
return
|
||||||
|
logger.info(f"已加载{len(shape_df)}条围栏")
|
||||||
|
# 先临时禁用索引刷新
|
||||||
|
DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings(
|
||||||
|
index="didata_company",
|
||||||
|
body={"index": {"refresh_interval": "-1"}}
|
||||||
|
)
|
||||||
|
ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name, committee from didata_company where province='{province}'")
|
||||||
|
#ids = pd.DataFrame({})
|
||||||
|
if len(ids) > 0:
|
||||||
|
company_ids = ids['_id'].to_list()
|
||||||
|
else:
|
||||||
|
company_ids = list()
|
||||||
|
actions = list()
|
||||||
|
count = 0
|
||||||
|
errors = list()
|
||||||
|
for index, item in shape_df[shape_df['shape'].notna()].iterrows():
|
||||||
|
logger.info(f"开始查找{item['city_name'], item['area_name'], item['name']}下企业数据")
|
||||||
|
ret = DIDATA_DEV_ELASTIC_CLIENT.fetch_all_geo_points('didata_enterprise', item['shape']['coordinates'][0], field='gaode_location')
|
||||||
|
com_count = 0
|
||||||
|
for info in ret:
|
||||||
|
com_count += 1
|
||||||
|
source = info['_source']
|
||||||
|
if 'district' in source and (source['district'] != item['area_name']):
|
||||||
|
errors.append(source)
|
||||||
|
continue
|
||||||
|
if ('registered_address' not in source) or ('company_name' not in source) or ('district' not in source) or ('city' not in source):
|
||||||
|
continue
|
||||||
|
doc_info = {'province': source['province'], 'city': source['city'], 'district': source['district'],
|
||||||
|
'company_name': source['company_name'], 'registered_address': source['registered_address'],
|
||||||
|
'location': source['gaode_location'], 'township': item['street_name'],
|
||||||
|
'committee': item['name']}
|
||||||
|
if info['_id'] not in company_ids:
|
||||||
|
action = {
|
||||||
|
"_op_type": "index",
|
||||||
|
"_index": "didata_company",
|
||||||
|
"_id": info['_id'],
|
||||||
|
"_source": doc_info,
|
||||||
|
"_retry_on_conflict": 5
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
action = {
|
||||||
|
"_op_type": "update",
|
||||||
|
"_index": "didata_company",
|
||||||
|
"_id": info['_id'],
|
||||||
|
"doc": doc_info,
|
||||||
|
"_retry_on_conflict": 5
|
||||||
|
}
|
||||||
|
|
||||||
|
actions.append(action)
|
||||||
|
#pd.DataFrame(errors)[['registered_address', 'company_name', 'province', 'city', 'district', 'gaode_location']].to_csv(f'{province}经纬度错误地址.csv', index=False)
|
||||||
|
logger.info(f"已查找到该围栏下{com_count}个企业数据")
|
||||||
|
for success, info in helpers.parallel_bulk(
|
||||||
|
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
|
||||||
|
actions,
|
||||||
|
thread_count=5, # 并行线程数
|
||||||
|
refresh=False,
|
||||||
|
chunk_size=1000 # 批量大小
|
||||||
|
):
|
||||||
|
if not success:
|
||||||
|
logger.info(f"索引失败: {info}")
|
||||||
|
count += len(actions)
|
||||||
|
logger.info("更新:" + str(count))
|
||||||
|
actions.clear()
|
||||||
|
if len(actions) > 0:
|
||||||
|
for success, info in helpers.parallel_bulk(
|
||||||
|
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
|
||||||
|
actions,
|
||||||
|
thread_count=5, # 并行线程数
|
||||||
|
refresh=False,
|
||||||
|
chunk_size=1000 # 批量大小
|
||||||
|
):
|
||||||
|
if not success:
|
||||||
|
logger.error(f"索引失败: {info}")
|
||||||
|
count += len(actions)
|
||||||
|
logger.info("更新:" + str(count))
|
||||||
|
actions.clear()
|
||||||
|
#pd.DataFrame(errors)[['registered_address', 'company_name', 'province', 'city', 'district', 'gaode_location']].to_csv(f'{province}经纬度错误地址.csv', index=False)
|
||||||
|
logger.info(f"完成同步{province}企业数据的居委村委数据")
|
||||||
|
# 恢复索引刷新
|
||||||
|
DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings(
|
||||||
|
index="didata_company",
|
||||||
|
body={"index": {"refresh_interval": "5s"}}
|
||||||
|
)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
flag = False
|
||||||
|
# for province in areautils.get_all_province():
|
||||||
|
# print(province)
|
||||||
|
for province in areautils.get_all_province():
|
||||||
|
sync(province)
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, areautils
|
||||||
|
import pandas as pd
|
||||||
|
from elasticsearch import helpers
|
||||||
|
import numpy as np
|
||||||
|
import re
|
||||||
|
import logging
|
||||||
|
# 配置日志
|
||||||
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
"""
|
||||||
|
通过开发区围栏更新企业开发区 TODO
|
||||||
|
"""
|
||||||
|
def sync(province):
|
||||||
|
logger.info(f"开始同步{province}企业数据的开发区数据")
|
||||||
|
# 先临时禁用索引刷新
|
||||||
|
DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings(
|
||||||
|
index="didata_company",
|
||||||
|
body={"index": {"refresh_interval": "-1"}}
|
||||||
|
)
|
||||||
|
|
||||||
|
shape_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select name, shape from development_zone where province='{province}'")
|
||||||
|
if len(shape_df) == 0 or 'shape' not in shape_df.columns or len(shape_df[shape_df['shape'].notna()]) == 0:
|
||||||
|
logger.info(f"{province}无围栏数据")
|
||||||
|
return
|
||||||
|
actions = list()
|
||||||
|
count = 0
|
||||||
|
for index, item in shape_df[shape_df['shape'].notna()].iterrows():
|
||||||
|
logger.info(f"开始查找{item['name']}下企业数据")
|
||||||
|
ret = DIDATA_DEV_ELASTIC_CLIENT.fetch_all_geo_points('didata_company', item['shape']['coordinates'][0],
|
||||||
|
field='location')
|
||||||
|
com_count = 0
|
||||||
|
for info in ret:
|
||||||
|
com_count += 1
|
||||||
|
doc_info = {'dev': item['name']}
|
||||||
|
action = {
|
||||||
|
"_op_type": "update",
|
||||||
|
"_index": "didata_company",
|
||||||
|
"_id": info['_id'],
|
||||||
|
"doc": doc_info,
|
||||||
|
"_retry_on_conflict": 5
|
||||||
|
}
|
||||||
|
|
||||||
|
actions.append(action)
|
||||||
|
logger.info(f"已查找到该围栏下{com_count}个企业数据")
|
||||||
|
if len(actions) > 0:
|
||||||
|
for success, info in helpers.parallel_bulk(
|
||||||
|
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
|
||||||
|
actions,
|
||||||
|
thread_count=5, # 并行线程数
|
||||||
|
chunk_size=10000 # 批量大小
|
||||||
|
):
|
||||||
|
if not success:
|
||||||
|
print(f"索引失败: {info}")
|
||||||
|
count += len(actions)
|
||||||
|
print("更新:" + str(count))
|
||||||
|
actions.clear()
|
||||||
|
if len(actions) > 0:
|
||||||
|
for success, info in helpers.parallel_bulk(
|
||||||
|
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
|
||||||
|
actions,
|
||||||
|
thread_count=5, # 并行线程数
|
||||||
|
chunk_size=10000 # 批量大小
|
||||||
|
):
|
||||||
|
if not success:
|
||||||
|
print(f"索引失败: {info}")
|
||||||
|
count += len(actions)
|
||||||
|
print("更新:" + str(count))
|
||||||
|
actions.clear()
|
||||||
|
logger.info(f"完成同步{province}企业数据的开发区数据")
|
||||||
|
# 恢复索引刷新
|
||||||
|
DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings(
|
||||||
|
index="didata_company",
|
||||||
|
body={"index": {"refresh_interval": "5s"}}
|
||||||
|
)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
for province in areautils.get_all_province():
|
||||||
|
sync(province)
|
||||||
@@ -0,0 +1,134 @@
|
|||||||
|
from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, areautils
|
||||||
|
import pandas as pd
|
||||||
|
from elasticsearch import helpers
|
||||||
|
import re
|
||||||
|
import logging
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
def sync():
|
||||||
|
"""
|
||||||
|
对于没有围栏结果的,通过match_phrase匹配出来的开发区一定正确,如果有围栏出来的开发区,则企业经纬度不正确
|
||||||
|
"""
|
||||||
|
|
||||||
|
dev_alias = {'福建永定工业园区': '永定区永定工业园', '福建漳州古雷港经济开发区': '古雷港经济开发区', '福建莆田湄洲湾北岸经济开发区': '湄洲湾北岸经济开发区',
|
||||||
|
'福建南平工业园区': '南平工业园区', '泉州台商投资区': '泉州市台商投资区', '福建洛江经济开发区': '洛江园区', '福建武平工业园区': '武平工业园',
|
||||||
|
'福建晋江经济开发区': '晋江市经济开发区', '漳州台商投资区': '龙池开发区', '莆田高新技术产业开发区': '莆田市高新技术',
|
||||||
|
'莆田高新技术产业开发区': '莆田市涵江区高新', '绥安工业开发区': '绥安工业园', '绥安工业开发区': '黄仓开发区', '绥安工业开发区': '黄仓工业园',
|
||||||
|
'绥安工业开发区': '旧镇工业园', '绥安工业开发区': '大南坂开发区', '绥安工业开发区': '大南坂工业开发区', '福建龙州工业园区': '龙州工业园',
|
||||||
|
'长汀经济开发区':'腾飞开发区', '福建福州金山工业园区': '金山投资区', '厦门象屿综合保税区': '厦门现代物流园', '福清融侨经济技术开发区': '融侨经济',
|
||||||
|
'福建洛江经济开发区': '万安开发区'}
|
||||||
|
dev_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame("select province, city, district, name, alias from development_zone")
|
||||||
|
# 不要去重,有可能存在别名, 如福建云霄常山经济开发区
|
||||||
|
#dev_df = dev_df[dev_df['name'] == '福建政和经济开发区']
|
||||||
|
actions = list()
|
||||||
|
for dev in dev_df.itertuples():
|
||||||
|
if dev.name in ['中国(福建)自由贸易试验区厦门片区', '中国(福建)自由贸易试验区福州片区', '中国(福建)自由贸易试验区平潭片区']:
|
||||||
|
# 中国(福建)自由贸易试验区 已包含,防止反复更新
|
||||||
|
continue
|
||||||
|
if dev.name in ['政和经济开发区']:
|
||||||
|
continue
|
||||||
|
|
||||||
|
count = 0
|
||||||
|
names = list()
|
||||||
|
names.append(dev.name)
|
||||||
|
if str(dev.alias) != 'nan':
|
||||||
|
names.extend(dev.alias)
|
||||||
|
#东侨经济技术开发区 福建省宁德市东侨经济开发区塔山村塔兴路25号
|
||||||
|
if '经济技术开发区' in dev.name:
|
||||||
|
names.append(re.sub('经济技术开发区', '经济开发区', dev.name))
|
||||||
|
# 福州经济技术开发区 福州开发区君竹路83号科技发展中心大楼第四层R472室(自贸试验区内)
|
||||||
|
names.append(re.sub('经济技术开发区', '开发区', dev.name))
|
||||||
|
# 长汀经济开发区 长汀县经济开发区工业新区河梁大山体
|
||||||
|
if '经济开发区' in dev.name:
|
||||||
|
if pd.notna(dev.city) and pd.notna(dev.district):
|
||||||
|
district_short = areautils.get_district_short(dev.city, dev.district)
|
||||||
|
if pd.notna(district_short):
|
||||||
|
names.append(re.sub(district_short, dev.district, dev.name))
|
||||||
|
# 漳州高新技术产业开发区 福建省漳州高新区九湖镇木棉村后壁山401-1号
|
||||||
|
if '高新技术产业开发区' in dev.name:
|
||||||
|
names.append(re.sub('高新技术产业开发区', '高新区', dev.name))
|
||||||
|
# 厦门火炬高技术产业开发区 厦门火炬高新区软件园三期溪西山尾路63号1303室-3单元之三
|
||||||
|
if '高技术产业开发区' in dev.name:
|
||||||
|
names.append(re.sub('高技术产业开发区', '高新区', dev.name))
|
||||||
|
#areautils.get_district_display()
|
||||||
|
# 福建诏安工业园区 福建省漳州市诏安工业园区丹诏大道15号
|
||||||
|
if pd.notna(dev.city):
|
||||||
|
province_short = areautils.get_province(dev.city)
|
||||||
|
if pd.notna(province_short):
|
||||||
|
new_name = re.sub(province_short, '', dev.name)
|
||||||
|
if pd.notna(dev.city) and pd.notna(dev.district):
|
||||||
|
city_short = areautils.get_city_short(dev.city)
|
||||||
|
district_short = areautils.get_district_short(dev.city, dev.district)
|
||||||
|
if pd.notna(city_short) and city_short in new_name:
|
||||||
|
names.append(new_name)
|
||||||
|
if pd.notna(district_short) and district_short in new_name:
|
||||||
|
names.append(new_name)
|
||||||
|
# 福建省漳州市长泰县经济开发区林墩办事处帝君路 福建长泰经济开发区
|
||||||
|
new_new_name = re.sub(district_short, district_short + '县', new_name)
|
||||||
|
names.append(new_new_name)
|
||||||
|
new_new_name = re.sub(district_short, district_short + '区', new_name)
|
||||||
|
names.append(new_new_name)
|
||||||
|
|
||||||
|
for key in dev_alias:
|
||||||
|
if dev.name == key:
|
||||||
|
names.append(dev_alias[key])
|
||||||
|
for name in names:
|
||||||
|
query = {
|
||||||
|
"query": {
|
||||||
|
"bool": {
|
||||||
|
"must_not": [
|
||||||
|
{
|
||||||
|
"term": {
|
||||||
|
"dev": {
|
||||||
|
"value": dev.name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"term": {
|
||||||
|
"dev_status": {
|
||||||
|
"value": 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"must": [
|
||||||
|
{
|
||||||
|
"term": {
|
||||||
|
"province": {
|
||||||
|
"value": dev.province
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"match_phrase": {
|
||||||
|
"registered_address": name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
count_data = DIDATA_DEV_ELASTIC_CLIENT.elastic_client.count(index='didata_company',body=query)
|
||||||
|
if count_data['count'] > 0:
|
||||||
|
for doc in DIDATA_DEV_ELASTIC_CLIENT.scan(index='didata_company',query=query):
|
||||||
|
action = {
|
||||||
|
"_op_type": "update",
|
||||||
|
"_index": "didata_company",
|
||||||
|
"_id": doc['_id'],
|
||||||
|
"doc": {'dev': dev.name, 'dev_status': 0}
|
||||||
|
}
|
||||||
|
actions.append(action)
|
||||||
|
if len(actions) == 5000:
|
||||||
|
helpers.bulk(DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions)
|
||||||
|
count += len(actions)
|
||||||
|
print(f"已更新 {dev.name} 开发区" + str(count) + "记录")
|
||||||
|
actions.clear()
|
||||||
|
if len(actions) > 0:
|
||||||
|
helpers.bulk(DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions)
|
||||||
|
count += len(actions)
|
||||||
|
print(f"已更新 {dev.name} 开发区" + str(count) + "记录")
|
||||||
|
actions.clear()
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
sync()
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
from datetime import timedelta
|
||||||
|
|
||||||
|
from airflow.providers.standard.operators.bash import BashOperator
|
||||||
|
import pendulum
|
||||||
|
from airflow import DAG
|
||||||
|
|
||||||
|
with DAG(
|
||||||
|
dag_id="didata_company",
|
||||||
|
schedule=None,
|
||||||
|
start_date=pendulum.datetime(2025, 6, 5, tz="Asia/Shanghai"),
|
||||||
|
catchup=True,
|
||||||
|
description="企业涉农处理"
|
||||||
|
) as dag:
|
||||||
|
# t1 = BashOperator(
|
||||||
|
# task_id='fill_commpany_comm',
|
||||||
|
# bash_command='python /opt/airflow/dags/repo/dags/company/company/fill_company_comm.py',
|
||||||
|
# )
|
||||||
|
# t2 = BashOperator(
|
||||||
|
# task_id='fill_commpany_dev',
|
||||||
|
# bash_command='python /opt/airflow/dags/repo/dags/company/company/fill_company_dev.py',
|
||||||
|
# )
|
||||||
|
t3 = BashOperator(
|
||||||
|
task_id='mark_right_dev',
|
||||||
|
bash_command='python /opt/airflow/dags/repo/dags/company/company/mark_right_dev.py',
|
||||||
|
)
|
||||||
|
t3
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
CREATE TABLE etl.company (
|
||||||
|
id Int64,
|
||||||
|
district String,
|
||||||
|
ent_name String,
|
||||||
|
reg_addr String,
|
||||||
|
unified_code String,
|
||||||
|
authority String
|
||||||
|
) ENGINE MySQL('192.168.0.20:3306', 'database', 'td_gov_company_basic', 'root', '12345678')
|
||||||
|
SETTINGS
|
||||||
|
connection_wait_timeout=9999999,
|
||||||
|
connect_timeout=9999999,
|
||||||
|
read_write_timeout=9999999,
|
||||||
|
connection_auto_close=false;
|
||||||
|
|
||||||
|
CREATE TABLE etl.dwd_company (
|
||||||
|
district String,
|
||||||
|
ent_name String,
|
||||||
|
reg_addr String,
|
||||||
|
unified_code String,
|
||||||
|
authority String,
|
||||||
|
region_code String,
|
||||||
|
reg_addr1 String,
|
||||||
|
province_code String,
|
||||||
|
city_code String,
|
||||||
|
province_name String,
|
||||||
|
city_name String,
|
||||||
|
region_name String,
|
||||||
|
m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间'
|
||||||
|
) ENGINE ReplacingMergeTree(m_date)
|
||||||
|
PARTITION BY province_code PRIMARY KEY (unified_code) ORDER BY (unified_code);
|
||||||
|
|
||||||
|
CREATE TABLE etl.dwm_company (
|
||||||
|
unified_code String,
|
||||||
|
province String,
|
||||||
|
city String,
|
||||||
|
region String,
|
||||||
|
name String,
|
||||||
|
address String,
|
||||||
|
m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间'
|
||||||
|
) ENGINE ReplacingMergeTree(m_date)
|
||||||
|
PARTITION BY province PRIMARY KEY (unified_code) ORDER BY (unified_code);
|
||||||
|
|
||||||
|
INSERT INTO etl.dwm_company(unified_code, province, city, region, name, address) SELECT unified_code, province_name, city_name, region_name, ent_name, reg_addr1 FROM etl.dwd_company WHERE city_name !='' and region_name != '' and reg_addr1 != '';
|
||||||
|
|
||||||
|
# 导入company.csv
|
||||||
|
clickhouse-client --format_csv_delimiter="|" --input_format_with_names_use_header=0 --query="INSERT INTO etl.dwd_company FORMAT CSVWithNames" --host=192.168.0.216 --password=goojia123456 < company.csv
|
||||||
|
|
||||||
|
# 导出company.csv
|
||||||
|
clickhouse-client --password goojia123456 -d etl -q "select * from company FORMAT CSVWithNames" --format_csv_delimiter='|' > company.csv
|
||||||
|
|
||||||
|
# 执行sql
|
||||||
|
|
||||||
|
clickhouse-client --password goojia123456 -d etl -d dwd_company -m < multi.sql
|
||||||
|
|
||||||
|
CREATE TABLE etl.company_local ON CLUSTER fangjia
|
||||||
|
(
|
||||||
|
district String,
|
||||||
|
ent_name String,
|
||||||
|
reg_addr String,
|
||||||
|
unified_code String,
|
||||||
|
authority String
|
||||||
|
) ENGINE=MergeTree PARTITION BY substring(unified_code, 2, 2) PRIMARY KEY(unified_code) ORDER BY (unified_code)
|
||||||
|
|
||||||
|
CREATE TABLE etl.company_shard ON CLUSTER fangjia
|
||||||
|
(
|
||||||
|
district String,
|
||||||
|
ent_name String,
|
||||||
|
reg_addr String,
|
||||||
|
unified_code String,
|
||||||
|
authority String
|
||||||
|
) ENGINE=Distributed(fangjia, etl, company, hiveHash(substring(unified_code, 2, 2)))
|
||||||
|
|
||||||
|
clickhouse-client --format_csv_delimiter="|" --input_format_with_names_use_header=0 --query="INSERT INTO etl.company_shard FORMAT CSVWithNames" --input_format_allow_errors_ratio=0.3 --host=192.168.0.216 --password=goojia123456 < td_gov_company_basic.csv
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
CREATE TABLE etl.company_report (
|
||||||
|
id String,
|
||||||
|
company_id String,
|
||||||
|
unified_code String,
|
||||||
|
year String,
|
||||||
|
company String,
|
||||||
|
mail_address String,
|
||||||
|
m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间'
|
||||||
|
) ENGINE ReplacingMergeTree(m_date)
|
||||||
|
PARTITION BY substring(unified_code, 2, 2) PRIMARY KEY (id) ORDER BY (id);
|
||||||
|
|
||||||
|
# 导入数据
|
||||||
|
clickhouse-client --format_csv_delimiter="@" --input_format_with_names_use_header=0 --query="INSERT INTO etl.company_report FORMAT CSV" --host=192.168.0.216 --password=goojia123456 < part-r-00000
|
||||||
|
|
||||||
|
# 关联导入,可能DataGrip客户端超时,就在ClickHouse-Client命令行运行即可
|
||||||
|
INSERT INTO etl.dwd_company(district, ent_name, reg_addr, unified_code, authority, region_code, reg_addr1, province_code, city_code, province_name, city_name, region_name, mail_address)
|
||||||
|
SELECT district, ent_name, reg_addr, unified_code, authority, region_code, reg_addr1, province_code, city_code, province_name, city_name, region_name, cr.mail_address
|
||||||
|
FROM etl.dwd_company c
|
||||||
|
LEFT JOIN (
|
||||||
|
SELECT unified_code, argMax(mail_address, year) mail_address, argMax(year, year) new_year FROM etl.company_report GROUP BY unified_code
|
||||||
|
) cr ON c.unified_code=cr.unified_code
|
||||||
|
WHERE cr.mail_address!='' and cr.mail_address is not null;
|
||||||
|
|
||||||
|
# 合并
|
||||||
|
optimize table etl.dwd_company final;
|
||||||
|
|
||||||
|
# 清洗企业通信地址
|
||||||
|
ALTER TABLE etl.dwd_company update mail_address1=replaceRegexpAll(mail_address, '^(.{2,}(省|自治区))?(.{2,}市)?(.{2,}(区|县))?', '') WHERE 1=1
|
||||||
|
|
||||||
|
INSERT INTO etl.dwm_company(unified_code, province, city, region, name, address) SELECT unified_code, province_name, city_name, region_name, ent_name, reg_addr1 FROM etl.dwd_company WHERE city_name !='' and region_name != '' and reg_addr1 != '';
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
create table etl.region_local on cluster fangjia (
|
||||||
|
province_code String,
|
||||||
|
fangjia_province String,
|
||||||
|
province_name String,
|
||||||
|
city_code String,
|
||||||
|
city_name String,
|
||||||
|
fangjia_city String,
|
||||||
|
region_code String,
|
||||||
|
fangjia_area String,
|
||||||
|
region_name String
|
||||||
|
)
|
||||||
|
ENGINE MergeTree()
|
||||||
|
PARTITION BY province_code PRIMARY KEY (region_code) ORDER BY (region_code);
|
||||||
|
|
||||||
|
|
||||||
|
CREATE TABLE etl.region_shard ON CLUSTER fangjia
|
||||||
|
(
|
||||||
|
province_code String,
|
||||||
|
fangjia_province String,
|
||||||
|
province_name String,
|
||||||
|
city_code String,
|
||||||
|
city_name String,
|
||||||
|
fangjia_city String,
|
||||||
|
region_code String,
|
||||||
|
fangjia_area String,
|
||||||
|
region_name String
|
||||||
|
) ENGINE=Distributed(fangjia, etl, region_local, hiveHash(province_code))
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
create table fangjia_stat.community_coverage_stat
|
||||||
|
(
|
||||||
|
city String,
|
||||||
|
city_level Int32,
|
||||||
|
community_count Int32,
|
||||||
|
avg_price_count Int32,
|
||||||
|
avg_price_coverage Float32,
|
||||||
|
sqmetre_rent_count Int32,
|
||||||
|
sqmetre_rent_coverage Float32,
|
||||||
|
weekend Date,
|
||||||
|
alias_count Nullable(Int32),
|
||||||
|
addr_alias_count Nullable(Int32),
|
||||||
|
one_year_deal Nullable(Int32),
|
||||||
|
exist_alias_community_count Nullable(Int32),
|
||||||
|
main_type_coverage Nullable(Float32),
|
||||||
|
main_type_area_coverage Nullable(Float32),
|
||||||
|
estate_charge_coverage Nullable(Float32),
|
||||||
|
finish_building_date_coverage Nullable(Float32),
|
||||||
|
all_area_coverage Nullable(Float32),
|
||||||
|
build_area_coverage Nullable(Float32),
|
||||||
|
greening_rate_coverage Nullable(Float32),
|
||||||
|
volume_rate_coverage Nullable(Float32),
|
||||||
|
consumption_rating_coverage Nullable(Float32)
|
||||||
|
)
|
||||||
|
engine = Memory;
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
CREATE TABLE tmp.community_summary
|
||||||
|
(
|
||||||
|
_id String COMMENT '小区ID',
|
||||||
|
city String COMMENT '城市',
|
||||||
|
region String COMMENT '区域',
|
||||||
|
name String COMMENT '名称',
|
||||||
|
lng Nullable(Float32) COMMENT '经度',
|
||||||
|
lat Nullable(Float32) COMMENT '纬度',
|
||||||
|
alias Array(String) COMMENT '别名',
|
||||||
|
estate_type2 String COMMENT '物业类型',
|
||||||
|
build_type String COMMENT '建筑类型',
|
||||||
|
household_count String COMMENT '户数',
|
||||||
|
block String COMMENT '板块',
|
||||||
|
estate_charge String COMMENT '物业费',
|
||||||
|
finish_building_date String COMMENT '建成年代',
|
||||||
|
m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间',
|
||||||
|
avg_price Nullable(Int32) COMMENT '均价',
|
||||||
|
total_price Nullable(Int32) COMMENT '户均总价',
|
||||||
|
price_rating Nullable(Int32) COMMENT '价格评级',
|
||||||
|
consumption_rating Nullable(Int32) COMMENT '财力评级'
|
||||||
|
) ENGINE ReplacingMergeTree(m_date)
|
||||||
|
PARTITION BY city PRIMARY KEY (city, region, name) ORDER BY (city, region, name);
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
CREATE TABLE tmp.mt_community
|
||||||
|
(
|
||||||
|
id String COMMENT '小区ID',
|
||||||
|
city String COMMENT '城市',
|
||||||
|
region String COMMENT '区域',
|
||||||
|
name String COMMENT '名称',
|
||||||
|
alias String COMMENT '别名',
|
||||||
|
address String COMMENT '地址',
|
||||||
|
addr_alias String COMMENT '地址别名',
|
||||||
|
avg_price Nullable(INT) COMMENT '均价',
|
||||||
|
rent_price Nullable(INT) COMMENT '平米租金',
|
||||||
|
lng Nullable(Float32) COMMENT '经度',
|
||||||
|
lat Nullable(Float32) COMMENT '纬度',
|
||||||
|
block String COMMENT '板块',
|
||||||
|
loop_line_info String COMMENT '环线',
|
||||||
|
business_quarter String COMMENT '商圈',
|
||||||
|
households String COMMENT '户数',
|
||||||
|
property_fee String COMMENT '物业费',
|
||||||
|
car_site_count String COMMENT '停车位',
|
||||||
|
property_life String COMMENT '产权年限',
|
||||||
|
house_property String COMMENT '产权性质',
|
||||||
|
all_area String COMMENT '占地面积',
|
||||||
|
build_area String COMMENT '建筑面积',
|
||||||
|
plot_ratio String COMMENT '容积率',
|
||||||
|
landscaping_ratio String COMMENT '绿化率',
|
||||||
|
property_type String COMMENT '物业类型',
|
||||||
|
build_type String COMMENT '建筑类型',
|
||||||
|
build_structure String COMMENT '建筑结构',
|
||||||
|
finish_building_date String COMMENT '建成年代',
|
||||||
|
developer String COMMENT '开发商',
|
||||||
|
house_type String COMMENT '户型',
|
||||||
|
house_area String COMMENT '房型建筑面积',
|
||||||
|
deal_num Nullable(INT) COMMENT '近一年成交量',
|
||||||
|
district_facilitys String COMMENT '周边配套数量字段',
|
||||||
|
school Nullable(INT) COMMENT '学校数量',
|
||||||
|
market Nullable(INT) COMMENT '商场数量',
|
||||||
|
subway Nullable(INT) COMMENT '地铁数量',
|
||||||
|
bank Nullable(INT) COMMENT '银行数量',
|
||||||
|
hospital Nullable(INT) COMMENT '医院数量',
|
||||||
|
status INT COMMENT '0 正常 1 删除',
|
||||||
|
m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间'
|
||||||
|
) ENGINE ReplacingMergeTree(m_date)
|
||||||
|
PARTITION BY city PRIMARY KEY (city, region, name) ORDER BY (city, region, name);
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
pip uninstall goojia_utils
|
||||||
|
pip install goojia_utils -i https://repo.fangjia.com/repository/pypi-public/simple/
|
||||||
|
pip install -r /opt/airflow/dags/repo/dags/requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
"""
|
|
||||||
建行脚本
|
|
||||||
"""
|
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
from airflow.providers.standard.operators.bash import BashOperator
|
||||||
|
import pendulum
|
||||||
|
from airflow import DAG
|
||||||
|
|
||||||
|
with DAG(
|
||||||
|
dag_id="evaluate_dag",
|
||||||
|
schedule="30 16 27 * *",
|
||||||
|
start_date=pendulum.datetime(2023, 3, 28, tz="Asia/Shanghai"),
|
||||||
|
catchup=True,
|
||||||
|
description="每月27日估值文件检测"
|
||||||
|
) as dag:
|
||||||
|
t1 = BashOperator(
|
||||||
|
task_id='evaluate',
|
||||||
|
bash_command='python /opt/airflow/dags/repo/dags/jh/merge/evaluate.py {{dag_run.conf["day"] if "day" in dag_run.conf}}',
|
||||||
|
)
|
||||||
|
t1
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
from airflow.providers.standard.operators.bash import BashOperator
|
||||||
|
import pendulum
|
||||||
|
from airflow import DAG
|
||||||
|
|
||||||
|
with DAG(
|
||||||
|
dag_id="merge_dag",
|
||||||
|
# 每日下午4点
|
||||||
|
schedule="0 16 * * *",
|
||||||
|
start_date=pendulum.datetime(2023, 3, 18, tz="Asia/Shanghai"),
|
||||||
|
catchup=True,
|
||||||
|
description="每日融合文件检测, 若想指定检测某一天如20230314,则传参day='20230314'"
|
||||||
|
) as dag:
|
||||||
|
t1 = BashOperator(
|
||||||
|
task_id='merge',
|
||||||
|
bash_command='python /opt/airflow/dags/repo/dags/jh/merge/merge.py {{dag_run.conf["day"] if "day" in dag_run.conf}}',
|
||||||
|
)
|
||||||
|
t1
|
||||||
@@ -1,12 +1,12 @@
|
|||||||
from airflow.operators.bash import BashOperator
|
from airflow.providers.standard.operators.bash import BashOperator
|
||||||
import pendulum
|
import pendulum
|
||||||
from airflow import DAG
|
from airflow import DAG
|
||||||
|
|
||||||
with DAG(
|
with DAG(
|
||||||
dag_id="price_exception_dag",
|
dag_id="price_exception_dag",
|
||||||
# 每月23日运行
|
# 每月23日运行
|
||||||
schedule_interval="0 10 23 * *",
|
schedule="0 10 23 * *",
|
||||||
start_date=pendulum.datetime(2022, 8, 23, tz="Asia/Shanghai"),
|
start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"),
|
||||||
catchup=True,
|
catchup=True,
|
||||||
description="建行每月价格异常检测脚本,默认检测本月与上月差异, 若对比上月与前月价格,设置year_month=\"202210\"(前月日期即可) 若输出样本存在与否,设置year_month=sample"
|
description="建行每月价格异常检测脚本,默认检测本月与上月差异, 若对比上月与前月价格,设置year_month=\"202210\"(前月日期即可) 若输出样本存在与否,设置year_month=sample"
|
||||||
) as dag:
|
) as dag:
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
from airflow.providers.standard.operators.bash import BashOperator
|
||||||
|
import pendulum
|
||||||
|
from airflow import DAG
|
||||||
|
|
||||||
|
with DAG(
|
||||||
|
dag_id="samples_notice_dag",
|
||||||
|
# 每月10日1点运行
|
||||||
|
schedule="0 0 10 * *",
|
||||||
|
start_date=pendulum.datetime(2025, 6, 9, tz="Asia/Shanghai"),
|
||||||
|
catchup=True,
|
||||||
|
description="样本数量监控提醒"
|
||||||
|
) as dag:
|
||||||
|
t1 = BashOperator(
|
||||||
|
task_id='samples_notice',
|
||||||
|
bash_command='python /opt/airflow/dags/repo/dags/jh/sample/samples_notice.py',
|
||||||
|
)
|
||||||
|
t1
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
from airflow.providers.standard.operators.bash import BashOperator
|
||||||
|
import pendulum
|
||||||
|
from airflow import DAG
|
||||||
|
|
||||||
|
with DAG(
|
||||||
|
dag_id="third_gxd_dag",
|
||||||
|
schedule=None,
|
||||||
|
start_date=pendulum.datetime(2024, 7, 22, tz="Asia/Shanghai"),
|
||||||
|
catchup=True,
|
||||||
|
description="第三方均价及国信达价格输出"
|
||||||
|
) as dag:
|
||||||
|
t1 = BashOperator(
|
||||||
|
task_id='gxd_dag',
|
||||||
|
bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_third_gxd.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}} ',
|
||||||
|
)
|
||||||
|
t1
|
||||||
@@ -1,12 +1,12 @@
|
|||||||
from airflow.operators.bash import BashOperator
|
from airflow.providers.standard.operators.bash import BashOperator
|
||||||
import pendulum
|
import pendulum
|
||||||
from airflow import DAG
|
from airflow import DAG
|
||||||
|
|
||||||
with DAG(
|
with DAG(
|
||||||
dag_id="third_price_dag",
|
dag_id="third_price_dag",
|
||||||
# 每月23日运行
|
# 每月23日运行
|
||||||
schedule_interval="0 16 18 * *",
|
schedule="0 16 18 * *",
|
||||||
start_date=pendulum.datetime(2022, 11, 21, tz="Asia/Shanghai"),
|
start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"),
|
||||||
catchup=True,
|
catchup=True,
|
||||||
description="建行过滤第三方均价"
|
description="建行过滤第三方均价"
|
||||||
) as dag:
|
) as dag:
|
||||||
@@ -0,0 +1,131 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
# coding: utf-8
|
||||||
|
|
||||||
|
from goojia_utils import dateutils, ftputils, tarutils
|
||||||
|
import datetime
|
||||||
|
import shutil
|
||||||
|
import os
|
||||||
|
import requests
|
||||||
|
import logging
|
||||||
|
|
||||||
|
# #### 检测传输压缩文件是否完整
|
||||||
|
|
||||||
|
# ##### 2.估值每月27日检测
|
||||||
|
|
||||||
|
# In[60]:
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
date = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '28'
|
||||||
|
|
||||||
|
path = '/home/airflow/data/gz'
|
||||||
|
local_aim_dir = path + '/' + date
|
||||||
|
if not os.path.exists(path):
|
||||||
|
os.makedirs(path)
|
||||||
|
if not os.path.exists(local_aim_dir):
|
||||||
|
os.makedirs(local_aim_dir)
|
||||||
|
|
||||||
|
|
||||||
|
client = ftputils.SftpClient()
|
||||||
|
client.sftp_connect(username="ccb", password="ccb!@#", host="192.168.0.45", port=17495)
|
||||||
|
client.get(remote="/home/ccb_evaluation/output/" + date + '/WS_GJ_000000_' + date + '_0001.tar', local=local_aim_dir)
|
||||||
|
client.get(remote="/home/ccb_evaluation/output/" + date + '/' + date + '.ok', local=path + '/' + date)
|
||||||
|
client.get(remote="/home/ccb_evaluation/output/" + date + '/WS_GJ_000000_' + date + '_0002.tar', local=local_aim_dir)
|
||||||
|
client.disconnect()
|
||||||
|
|
||||||
|
|
||||||
|
# In[57]:
|
||||||
|
|
||||||
|
|
||||||
|
## 解压
|
||||||
|
result = []
|
||||||
|
try:
|
||||||
|
dirs = os.listdir(local_aim_dir)
|
||||||
|
if date + '.ok' not in dirs:
|
||||||
|
result.append(date + '.ok' + '文件不存在')
|
||||||
|
tar_file = 'WS_GJ_000000_' + date + '_0001.tar'
|
||||||
|
has_two_tar = False
|
||||||
|
if 'WS_GJ_000000_' + date + '_0002.tar' in dirs:
|
||||||
|
has_two_tar = True
|
||||||
|
shutil.rmtree(local_aim_dir + '/' + date + '_2')
|
||||||
|
tarutils.untar_file(local_aim_dir + '/WS_GJ_000000_' + date + '_0002.tar', local_aim_dir + '/')
|
||||||
|
if tar_file not in dirs:
|
||||||
|
result.append(tar_file + '文件不存在')
|
||||||
|
else:
|
||||||
|
if has_two_tar is True:
|
||||||
|
shutil.rmtree(local_aim_dir + '/' + date + '_1')
|
||||||
|
tarutils.untar_file(local_aim_dir + "/" + tar_file, local_aim_dir + '/')
|
||||||
|
if has_two_tar is False:
|
||||||
|
sub_dirs = os.listdir(local_aim_dir)
|
||||||
|
else:
|
||||||
|
sub_dirs = os.listdir(local_aim_dir + '/' + date + '_1')
|
||||||
|
sub_dirs.extend(os.listdir(local_aim_dir + '/' + date + '_2'))
|
||||||
|
if 'MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'MODEL_000000_000000_BIGDATA_POI_GJ_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('MODEL_000000_000000_BIGDATA_POI_GJ_ALL_' + date + '_0001.dat文件不存在')
|
||||||
|
|
||||||
|
if 'MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'MODEL_000000_000000_CITY_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('MODEL_000000_000000_CITY_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||||
|
|
||||||
|
if 'MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||||
|
|
||||||
|
if len(result) == 0:
|
||||||
|
result.append('估值检测完毕未发现异常文件')
|
||||||
|
shutil.rmtree(local_aim_dir)
|
||||||
|
else:
|
||||||
|
result.append(" @高娟 ")
|
||||||
|
|
||||||
|
except Exception as ex:
|
||||||
|
result.append(" @高娟 ")
|
||||||
|
result.append(str(ex))
|
||||||
|
|
||||||
|
infos = "【" + date + "自动检测】"
|
||||||
|
for item in result:
|
||||||
|
infos += item + "\n"
|
||||||
|
data = {
|
||||||
|
"msgtype": "text",
|
||||||
|
"text": {
|
||||||
|
"content": infos
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.info(data)
|
||||||
|
|
||||||
|
# 发送消息至群
|
||||||
|
requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736',
|
||||||
|
json=data)
|
||||||
|
|
||||||
+26
-35
@@ -1,30 +1,23 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
# coding: utf-8
|
|
||||||
|
|
||||||
# In[56]:
|
|
||||||
|
|
||||||
|
|
||||||
from goojia_utils import dateutils, ftputils, tarutils
|
from goojia_utils import dateutils, ftputils, tarutils
|
||||||
import datetime
|
import datetime
|
||||||
import calendar
|
|
||||||
import requests
|
import requests
|
||||||
import json
|
|
||||||
import shutil
|
import shutil
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import sys
|
||||||
|
import logging
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
result = []
|
result = []
|
||||||
|
|
||||||
# #### 检测传输压缩文件是否完整
|
|
||||||
|
|
||||||
# ##### 1. 融合每天检测
|
# ##### 1. 融合每天检测
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
date = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
if date is None:
|
||||||
today = dateutils.add_months(datetime.datetime.now(), days=1)
|
today = dateutils.add_months(datetime.datetime.now(), days=1)
|
||||||
date = dateutils.format_date(today, '%Y%m%d')
|
date = dateutils.format_date(today, '%Y%m%d')
|
||||||
print('当前月:' + date)
|
print('当前月:' + date)
|
||||||
curr_month_days = calendar.monthrange(today.year, today.month)[1]
|
# curr_month_days = calendar.monthrange(today.year, today.month)[1]
|
||||||
print('当月天数:' + str(curr_month_days))
|
# print('当月天数:' + str(curr_month_days))
|
||||||
|
|
||||||
|
|
||||||
# In[79]:
|
# In[79]:
|
||||||
@@ -121,41 +114,38 @@ try:
|
|||||||
result.append('FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_' + date + '_0001.dat文件不存在')
|
result.append('FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_' + date + '_0001.dat文件不存在')
|
||||||
if 'FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
|
if 'FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
result.append('FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat文件不存在')
|
result.append('FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat文件不存在')
|
||||||
if 'FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
if 'FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
result.append('FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在')
|
result.append('FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_' + date + '_0001.dat文件不存在')
|
||||||
|
# if 'FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
# result.append('FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||||
|
|
||||||
if 'FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_' + date + '_0001.dat' not in sub_dirs:
|
if date.endswith('28'):
|
||||||
result.append('FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_' + date + '_0001.dat文件不存在')
|
|
||||||
if 'FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_' + date + '_0001.dat' not in sub_dirs:
|
if 'FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
result.append('FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_' + date + '_0001.dat文件不存在')
|
result.append('FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||||
|
if 'FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
|
result.append('FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_' + date + '_0001.dat文件不存在')
|
||||||
if 'FJW_000000_000000_GUIDE_STATISTICS_ADD_' + date + '_0001.dat' not in sub_dirs:
|
if 'FJW_000000_000000_GUIDE_STATISTICS_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||||
result.append('FJW_000000_000000_GUIDE_STATISTICS_ADD_' + date + '_0001.dat文件不存在')
|
result.append('FJW_000000_000000_GUIDE_STATISTICS_ADD_' + date + '_0001.dat文件不存在')
|
||||||
if 'FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
|
||||||
result.append('FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_' + date + '_0001.dat文件不存在')
|
|
||||||
if 'FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_' + date + '_0001.dat' not in sub_dirs:
|
|
||||||
result.append('FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_' + date + '_0001.dat文件不存在')
|
|
||||||
|
|
||||||
if 'FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
|
|
||||||
result.append('FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_' + date + '_0001.dat文件不存在')
|
|
||||||
|
|
||||||
if len(result) == 0:
|
if len(result) == 0:
|
||||||
result.append('融合检测完毕未发现异常文件')
|
result.append('融合检测完毕未发现异常文件')
|
||||||
shutil.rmtree(path + "/" + date)
|
shutil.rmtree(path + "/" + date)
|
||||||
|
else:
|
||||||
|
result.append(" @高娟 ")
|
||||||
|
|
||||||
except Exception as ex:
|
except Exception as ex:
|
||||||
|
result.append(" @高娟 ")
|
||||||
result.append(str(ex))
|
result.append(str(ex))
|
||||||
|
|
||||||
|
|
||||||
# In[82]:
|
|
||||||
|
|
||||||
|
|
||||||
print(result)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# In[76]:
|
|
||||||
|
|
||||||
|
|
||||||
infos = "【" + date + "自动检测】"
|
infos = "【" + date + "自动检测】"
|
||||||
for item in result:
|
for item in result:
|
||||||
infos += item + "\n"
|
infos += item + "\n"
|
||||||
@@ -165,6 +155,7 @@ data = {
|
|||||||
"content": infos
|
"content": infos
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
log.info(data)
|
||||||
# 发送消息至群
|
# 发送消息至群
|
||||||
requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736',
|
requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736',
|
||||||
json=data)
|
json=data)
|
||||||
|
|||||||
@@ -1,17 +0,0 @@
|
|||||||
from airflow.operators.bash import BashOperator
|
|
||||||
import pendulum
|
|
||||||
from airflow import DAG
|
|
||||||
|
|
||||||
with DAG(
|
|
||||||
dag_id="merge_dag",
|
|
||||||
# 每日下午4点
|
|
||||||
schedule_interval="0 16 * * *",
|
|
||||||
start_date=pendulum.datetime(2022, 11, 15, tz="Asia/Shanghai"),
|
|
||||||
catchup=True,
|
|
||||||
description="每日融合文件检测"
|
|
||||||
) as dag:
|
|
||||||
t1 = BashOperator(
|
|
||||||
task_id='merge',
|
|
||||||
bash_command='python /opt/airflow/dags/repo/dags/jh/merge/merge.py',
|
|
||||||
)
|
|
||||||
t1
|
|
||||||
@@ -0,0 +1,109 @@
|
|||||||
|
import pandas as pd
|
||||||
|
city_name_id_map = {'七台河': '230900', '万宁': '469006', '三亚': '460200', '东莞': '441900', '三明': '350400', '三沙': '460300', '三门峡': '411200',
|
||||||
|
'上海': '310000', '上饶': '361100', '东方': '469007', '中山': '442000', '东营': '370500', '中卫': '640500', '临夏': '622900',
|
||||||
|
'临汾': '141000', '临沂': '371300', '临沧': '530900', '临高': '469024', '丹东': '210600', '丽水': '331100', '丽江': '530700',
|
||||||
|
'乌兰察布': '150900', '乌海': '150300', '乌鲁木齐': '650100', '乐东': '469027', '乐山': '511100', '九江': '360400',
|
||||||
|
'云浮': '445300', '五家渠': '659004', '五指山': '469001', '亳州': '341600', '仙桃': '429004', '伊春': '230700',
|
||||||
|
'伊犁': '654000', '佛山': '440600', '佳木斯': '230800', '保亭': '469029', '保定': '130600', '保山': '530500', '信阳': '411500',
|
||||||
|
'儋州': '460400', '克州': '653000', '克拉玛依': '650200', '六安': '341500', '六盘水': '520200', '兰州': '620100',
|
||||||
|
'兴安': '152200', '内江': '511000', '凉山': '513400', '包头': '150200', '北京': '110000', '北屯': '659005', '北海': '450500',
|
||||||
|
'十堰': '420300', '南京': '320100', '南充': '511300', '南宁': '450100', '南平': '350700', '南昌': '360100', '南通': '320600',
|
||||||
|
'南阳': '411300', '博州': '652700', '厦门': '350200', '双河': '659007', '双鸭山': '230500', '可克达拉': '659008',
|
||||||
|
'台州': '331000', '合肥': '340100', '吉安': '360800', '吉林': '220200', '吐鲁番': '650400', '吕梁': '141100',
|
||||||
|
'吴忠': '640300', '周口': '411600', '呼伦贝尔': '150700', '呼和浩特': '150100', '和田': '653200', '咸宁': '421200',
|
||||||
|
'咸阳': '610400', '哈密': '650500', '哈尔滨': '230100', '唐山': '130200', '商丘': '411400', '商洛': '611000',
|
||||||
|
'喀什': '653100', '嘉兴': '330400', '嘉峪关': '620200', '四平': '220300', '固原': '640400', '图木舒克': '659003',
|
||||||
|
'塔城': '654200', '大兴安岭': '232700', '大同': '140200', '大庆': '230600', '大理': '532900', '大连': '210200',
|
||||||
|
'天水': '620500', '天津': '120000', '天门': '429006', '太原': '140100', '威海': '371000', '娄底': '431300', '孝感': '420900',
|
||||||
|
'宁德': '350900', '宁波': '330200', '安庆': '340800', '安康': '610900', '安阳': '410500', '安顺': '520400', '定安': '469021',
|
||||||
|
'定西': '621100', '宜宾': '511500', '宜昌': '420500', '宜春': '360900', '宝鸡': '610300', '宣城': '341800', '宿州': '341300',
|
||||||
|
'宿迁': '321300', '屯昌': '469022', '山南': '540500', '岳阳': '430600', '崇左': '451400', '巴中': '511900',
|
||||||
|
'巴彦淖尔': '150800', '巴州': '652800', '常州': '320400', '常德': '430700', '平凉': '620800', '平顶山': '410400',
|
||||||
|
'广元': '510800', '广安': '511600', '广州': '440100', '庆阳': '621000', '廊坊': '131000', '延安': '610600',
|
||||||
|
'延边': '222400', '开封': '410200', '张家口': '130700', '张家界': '430800', '张掖': '620700', '徐州': '320300',
|
||||||
|
'德宏': '533100', '德州': '371400', '德阳': '510600', '忻州': '140900', '怀化': '431200', '怒江': '533300', '恩施': '422800',
|
||||||
|
'惠州': '441300', '成都': '510100', '扬州': '321000', '承德': '130800', '抚州': '361000', '抚顺': '210400', '拉萨': '540100',
|
||||||
|
'揭阳': '445200', '攀枝花': '510400', '文山': '532600', '文昌': '469005', '新乡': '410700', '新余': '360500',
|
||||||
|
'无锡': '320200', '日喀则': '540200', '日照': '371100', '昆明': '530100', '昆玉': '659009', '昌吉': '652300',
|
||||||
|
'昌江': '469026', '昌都': '540300', '昭通': '530600', '晋中': '140700', '晋城': '140500', '普洱': '530800', '景德镇': '360200',
|
||||||
|
'曲靖': '530300', '朔州': '140600', '朝阳': '211300', '本溪': '210500', '来宾': '451300', '杭州': '330100', '松原': '220700',
|
||||||
|
'林芝': '540400', '果洛': '632600', '枣庄': '370400', '柳州': '450200', '株洲': '430200', '桂林': '450300', '梅州': '441400',
|
||||||
|
'梧州': '450400', '楚雄': '532300', '榆林': '610800', '武威': '620600', '武汉': '420100', '毕节': '520500', '永州': '431100',
|
||||||
|
'汉中': '610700', '汕头': '440500', '汕尾': '441500', '江门': '440700', '池州': '341700', '沈阳': '210100', '沧州': '130900',
|
||||||
|
'河池': '451200', '河源': '441600', '泉州': '350500', '泰安': '370900', '泰州': '321200', '泸州': '510500', '洛阳': '410300',
|
||||||
|
'济南': '370100', '济宁': '370800', '济源': '419001', '海东': '630200', '海北': '632200', '海南州': '632500',
|
||||||
|
'海口': '460100', '海西': '632800', '淄博': '370300', '淮北': '340600', '淮南': '340400', '淮安': '320800', '深圳': '440300',
|
||||||
|
'清远': '441800', '温州': '330300', '渭南': '610500', '湖州': '330500', '湘潭': '430300', '湘西': '433100', '湛江': '440800',
|
||||||
|
'滁州': '341100', '滨州': '371600', '漯河': '411100', '漳州': '350600', '潍坊': '370700', '潜江': '429005', '潮州': '445100',
|
||||||
|
'澄迈': '469023', '濮阳': '410900', '烟台': '370600', '焦作': '410800', '牡丹江': '231000', '玉林': '450900',
|
||||||
|
'玉树': '632700', '玉溪': '530400', '珠海': '440400', '琼中': '469030', '琼海': '469002', '甘南': '623000', '甘孜': '513300',
|
||||||
|
'白城': '220800', '白山': '220600', '白沙': '469025', '白银': '620400', '百色': '451000', '益阳': '430900', '盐城': '320900',
|
||||||
|
'盘锦': '211100', '眉山': '511400', '石嘴山': '640200', '石家庄': '130100', '石河子': '659001', '神农架': '429021',
|
||||||
|
'福州': '350100', '秦皇岛': '130300', '红河': '532500', '绍兴': '330600', '绥化': '231200', '绵阳': '510700',
|
||||||
|
'聊城': '371500', '肇庆': '441200', '自贡': '510300', '舟山': '330900', '芜湖': '340200', '苏州': '320500', '茂名': '440900',
|
||||||
|
'荆州': '421000', '荆门': '420800', '菏泽': '371700', '莆田': '350300', '萍乡': '360300', '营口': '210800',
|
||||||
|
'葫芦岛': '211400', '蚌埠': '340300', '衡水': '131100', '衡阳': '430400', '衢州': '330800', '襄阳': '420600', '西双版纳': '532800',
|
||||||
|
'西宁': '630100', '西安': '610100', '许昌': '411000', '贵港': '450800', '贵阳': '520100', '贺州': '451100', '资阳': '512000',
|
||||||
|
'赣州': '360700', '赤峰': '150400', '辽源': '220400', '辽阳': '211000', '达州': '511700', '运城': '140800', '连云港': '320700',
|
||||||
|
'迪庆': '533400', '通化': '220500', '通辽': '150500', '遂宁': '510900', '遵义': '520300', '邢台': '130500', '那曲': '540600',
|
||||||
|
'邯郸': '130400', '邵阳': '430500', '郑州': '410100', '郴州': '431000', '鄂尔多斯': '150600', '鄂州': '420700', '酒泉': '620900',
|
||||||
|
'重庆': '500000', '金华': '330700', '金昌': '620300', '钦州': '450700', '铁岭': '211200', '铁门关': '659006', '铜仁': '520600',
|
||||||
|
'铜川': '610200', '铜陵': '340700', '银川': '640100', '锡林郭勒': '152500', '锦州': '210700', '镇江': '321100', '长春': '220100',
|
||||||
|
'长沙': '430100', '长治': '140400', '阜新': '210900', '阜阳': '341200', '防城港': '450600', '阳江': '441700', '阳泉': '140300',
|
||||||
|
'阿克苏': '652900', '阿勒泰': '654300', '阿坝': '513200', '阿拉善': '152900', '阿拉尔': '659002', '阿里': '542500', '陇南': '621200',
|
||||||
|
'陵水': '469028', '随州': '421300', '雅安': '511800', '青岛': '370200', '鞍山': '210300', '韶关': '440200',
|
||||||
|
'驻马店': '411700', '鸡西': '230300', '鹤壁': '410600', '鹤岗': '230400', '鹰潭': '360600', '黄冈': '421100', '黄南': '632300',
|
||||||
|
'黄山': '341000', '黄石': '420200', '黑河': '231100', '黔东南': '522600', '黔南': '522700', '黔西南': '522300', '齐齐哈尔': '230200',
|
||||||
|
'龙岩': '350800', "马鞍山": "340500"}
|
||||||
|
third_data_source_id_map = {'01': '诸葛找房','02': '安居客','03': '房天下','04': '链家','05': '城市居家',
|
||||||
|
'06': '麦田','07': '贝壳','08': '淘房356','09': '中原','10': '兔博士',
|
||||||
|
'11': 'Q房','12': '房探','13': '房多多','14': '我爱我家','15': '禧泰数据',
|
||||||
|
'16': '吉屋网','17': '乐居','18': '21世纪','19': '房产超市','20': '58同城'}
|
||||||
|
city_id_name_map = {k: v for v, k in city_name_id_map.items()}
|
||||||
|
|
||||||
|
|
||||||
|
def get_city_name(city_id):
|
||||||
|
"""
|
||||||
|
通过城市ID获取城市名
|
||||||
|
:param city_id:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
return city_id_name_map[city_id] if city_id in city_id_name_map else None
|
||||||
|
|
||||||
|
|
||||||
|
def get_city_id(city_name):
|
||||||
|
"""
|
||||||
|
通过城市名获取城市ID
|
||||||
|
:param city_name:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
return city_name_id_map[city_name] if city_name in city_name_id_map else None
|
||||||
|
|
||||||
|
|
||||||
|
def is_city_id(city):
|
||||||
|
"""
|
||||||
|
判断是不是合法的cityid
|
||||||
|
:param city:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
return city in city_id_name_map
|
||||||
|
|
||||||
|
|
||||||
|
def is_city_name(city):
|
||||||
|
"""
|
||||||
|
判断是不是合法的城市名
|
||||||
|
:param city:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
return city in city_name_id_map
|
||||||
|
|
||||||
|
|
||||||
|
def get_city_data():
|
||||||
|
"""
|
||||||
|
返回所有的城市数据DataFrame[cityid,cityname]
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
data = list()
|
||||||
|
for city_id in city_id_name_map.keys():
|
||||||
|
data.append({"cityid": city_id, "cityname": city_id_name_map[city_id]})
|
||||||
|
return pd.DataFrame(data)
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
from pymongo import MongoClient
|
||||||
|
def getMongo(*args):
|
||||||
|
if len(args) == 6:
|
||||||
|
con = MongoClient(args[0], args[1]) # 获取链接\
|
||||||
|
db_auth = con.fangjia_craw
|
||||||
|
if args[2] =="vip_site":
|
||||||
|
db_auth.authenticate(args[4], args[5],"fangjia_craw")
|
||||||
|
else:
|
||||||
|
db_auth.authenticate(args[4], args[5],"admin")
|
||||||
|
db = con[args[2]] # 关联数据库
|
||||||
|
collection = db[args[3]] # 关联数据表
|
||||||
|
elif len(args) == 4:
|
||||||
|
con = MongoClient(args[0], args[1]) # 获取链接
|
||||||
|
db = con[args[2]] # 关联数据库
|
||||||
|
collection = db[args[3]] # 关联数据表
|
||||||
|
return collection
|
||||||
|
#线上小区库
|
||||||
|
seaweed_on = getMongo("192.168.0.43",28017,"fangjia","seaweed","developer","G00IDCDEV12385")
|
||||||
|
sz_price_on = getMongo("192.168.0.43",28017,"fangjia","sz_price","developer","G00IDCDEV12385")
|
||||||
|
district_stat_on = getMongo("192.168.0.43",28017,"fangjia_stat","district_stat","developer","G00IDCDEV12385")
|
||||||
|
seaweed_map = getMongo("192.168.0.235",27018,"fangjia","seaweedmap")
|
||||||
|
web_community_check = getMongo("192.168.0.235",27018,"fangjia","web_community_check")
|
||||||
|
web_invalid_comm = getMongo("192.168.0.235",27018,"fangjia","web_invalid_comm")
|
||||||
|
addr_nine_level = getMongo("192.168.0.206",27020,"etl","nine_level")
|
||||||
|
gxd_price = getMongo("192.168.0.43",28017,"fangjia","confirm_price_data","developer","G00IDCDEV12385")
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
from mongo_client import seaweed_on,seaweed_map,addr_nine_level,district_stat_on,gxd_price
|
||||||
|
import pandas as pd
|
||||||
|
import datetime
|
||||||
|
import pytz
|
||||||
|
pd.set_option ('display.max_colwidth',5)
|
||||||
|
|
||||||
|
def parse_date(time, fm='%Y-%m-%d'):
|
||||||
|
pacific = pytz.timezone('Asia/Shanghai')
|
||||||
|
return pacific.localize(datetime.datetime.strptime(time, fm))
|
||||||
|
def get_last_weekend():
|
||||||
|
sunday = datetime.datetime.utcnow()
|
||||||
|
sunday = sunday.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||||
|
one_day = datetime.timedelta(days=1)
|
||||||
|
while sunday.weekday() != 6:
|
||||||
|
sunday -= one_day
|
||||||
|
pacific = pytz.timezone('Asia/Shanghai')
|
||||||
|
return pacific.localize(sunday)
|
||||||
|
|
||||||
|
confirm_date = parse_date("2024-05-20")
|
||||||
|
def get_seaweed(city=None,status=0):
|
||||||
|
seaweed_fields = {"city": 1, "region": 1, "name": 1,'estate_type2':1,'cat':1,"address":1,"alias":1,"lat":1,"lng":1,"addr_alias":1,"quality":1,"name_cat":1,"mapping_alias":1,"m_date":1,"is_new":1}
|
||||||
|
# seaweed_fields = {"_id": 0}
|
||||||
|
if city:
|
||||||
|
if city in ['阿坝','玉树','甘孜','德宏','迪庆','海北','海西','黄南','海南','怒江','文山','西双版纳','凉山','果洛']:
|
||||||
|
city = city+'州'
|
||||||
|
elif city in ['锡林郭勒','兴安','阿拉善']:
|
||||||
|
city = city+'盟'
|
||||||
|
elif city=='克州':
|
||||||
|
city = '克孜勒苏'
|
||||||
|
elif city=='博州':
|
||||||
|
city = '博尔塔拉'
|
||||||
|
elif city=='巴州':
|
||||||
|
city = '巴音郭楞'
|
||||||
|
seaweed_condition = {"cat": "district", "status": status, 'city':city}
|
||||||
|
else:
|
||||||
|
seaweed_condition = {"cat": "district", "status": status}
|
||||||
|
seaweed_data = seaweed_on.find(seaweed_condition, seaweed_fields)
|
||||||
|
seaweed_df = pd.DataFrame(seaweed_data)
|
||||||
|
seaweed_df = seaweed_df.rename(columns={'estate_type2': 'property_type'})
|
||||||
|
return seaweed_df
|
||||||
|
|
||||||
|
def get_map(city=None):
|
||||||
|
# seaweed_fields = {"city": 1, "region": 1, "name": 1,"address":1,"alias":1,"lat":1,"lng":1,"estate_type2":1,"addr_alias":1}
|
||||||
|
seaweed_fields = {"_id": 0}
|
||||||
|
if city:
|
||||||
|
seaweed_condition = {'city':city}
|
||||||
|
else:
|
||||||
|
seaweed_condition = {}
|
||||||
|
seaweed_map_data = seaweed_map.find(seaweed_condition, seaweed_fields)
|
||||||
|
seaweed_map_df = pd.DataFrame(seaweed_map_data)
|
||||||
|
return seaweed_map_df
|
||||||
|
|
||||||
|
def get_addr_nine_level(count):
|
||||||
|
_fields = {"address":1,"province":1,"city": 1, "district": 1,"township": 1, "name": 1,"":1}
|
||||||
|
_condition = {'address':{"$exists": True, "$ne": None},"remark":{"$exists": False}}
|
||||||
|
addr_nine_data = addr_nine_level.find(_condition,_fields).limit(count)
|
||||||
|
addr_nine_df = pd.DataFrame(addr_nine_data)
|
||||||
|
return addr_nine_df
|
||||||
|
|
||||||
|
def get_district_stat_on(city=None):
|
||||||
|
price_date = get_last_weekend()
|
||||||
|
stat_fields = {"_id": 0, "city": 1, "region": 1, "district_name": 1, "avg_price": 1,"avg_price_2b":1}
|
||||||
|
if city:
|
||||||
|
stat_condition = {'city':city,"weekend": price_date}
|
||||||
|
else:
|
||||||
|
stat_condition = {"weekend": price_date}
|
||||||
|
stat_data = district_stat_on.find(stat_condition, stat_fields)
|
||||||
|
stat_df = pd.DataFrame(stat_data)
|
||||||
|
stat_df = stat_df.rename(columns={'district_name': 'name'})
|
||||||
|
return stat_df
|
||||||
|
|
||||||
|
def get_gxd_price():
|
||||||
|
gxd_fields = {"_id": 0}
|
||||||
|
gxd_condition = {'source':'GXD'}
|
||||||
|
gxd_price_data = gxd_price.find(gxd_condition, gxd_fields)
|
||||||
|
gxd_price_df = pd.DataFrame(gxd_price_data)
|
||||||
|
return gxd_price_df
|
||||||
|
if __name__ == '__main__':
|
||||||
|
city = '厦门'
|
||||||
|
df = get_gxd_price()
|
||||||
|
print(df.head())
|
||||||
@@ -3,12 +3,7 @@ import logging
|
|||||||
import pandas as pd
|
import pandas as pd
|
||||||
import requests
|
import requests
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from goojia_utils import dateutils, oracle
|
from goojia_utils import dateutils, pgsql
|
||||||
from sqlalchemy import create_engine
|
|
||||||
|
|
||||||
os.environ["NLS_LANG"] = "American_America.AL32UTF8"
|
|
||||||
os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7"
|
|
||||||
engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False)
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
path = "/home/airflow/data/jh/price"
|
path = "/home/airflow/data/jh/price"
|
||||||
@@ -17,9 +12,9 @@ if not os.path.exists(path):
|
|||||||
year_month = int(sys.argv[1]) if len(sys.argv) > 1 else int(dateutils.format_date(datetime.utcnow(), "%Y%m"))
|
year_month = int(sys.argv[1]) if len(sys.argv) > 1 else int(dateutils.format_date(datetime.utcnow(), "%Y%m"))
|
||||||
log.info(f"测试月份:{year_month}")
|
log.info(f"测试月份:{year_month}")
|
||||||
|
|
||||||
avm_df = pd.read_sql(f"select urbn_id,urbn_nm,gujia_bds_comm_num,gujiabdsvalmnycommnum from t_app_p9pha_plda.avm_evaluation_gj_h where gujia_bds_yrmo={year_month} ", con=engine)
|
avm_df = pgsql.scan_by_sql_to_data_frame(f"select urbn_id,urbn_nm,gujia_bds_comm_num,gujiabdsvalmnycommnum from price_stat.avm_evaluation_gj_h where gujia_bds_yrmo={year_month} ")
|
||||||
comm_df = pd.read_sql(f"select urbn_id,gujia_admndiv_nm,gujia_comm_id,comm_nm from base.t0831_bigdata_community_gj_h where gujia_bds_bds_st=0 ", con=engine)
|
comm_df = pgsql.scan_by_sql_to_data_frame(f"select urbn_id,gujia_admndiv_nm,gujia_comm_id,comm_nm from base.bigdata_community where gujia_bds_bds_st=0 ")
|
||||||
price_df = pd.read_sql(f"select gujia_bds_id as gujia_comm_id,gujia_comm_basic_prc from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={year_month} ", con=engine)
|
price_df = pgsql.scan_by_sql_to_data_frame(f"select gujia_bds_id as gujia_comm_id,gujia_comm_basic_prc from price_stat.avm_community_base_price where gujiabds_clc_yrmo_val={year_month} ")
|
||||||
comm_df = comm_df.merge(avm_df, on="urbn_id", how="left")
|
comm_df = comm_df.merge(avm_df, on="urbn_id", how="left")
|
||||||
comm_df = comm_df.merge(price_df, on="gujia_comm_id", how="left")
|
comm_df = comm_df.merge(price_df, on="gujia_comm_id", how="left")
|
||||||
no_price_comm_df = comm_df[comm_df.gujia_comm_basic_prc.isna()][['urbn_id','urbn_nm','gujia_admndiv_nm','gujia_comm_id','comm_nm']]
|
no_price_comm_df = comm_df[comm_df.gujia_comm_basic_prc.isna()][['urbn_id','urbn_nm','gujia_admndiv_nm','gujia_comm_id','comm_nm']]
|
||||||
|
|||||||
@@ -0,0 +1,42 @@
|
|||||||
|
import os
|
||||||
|
import pandas as pd
|
||||||
|
from pandas import DataFrame
|
||||||
|
from retrying import retry
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
|
||||||
|
# os.environ["NLS_LANG"] = "AMERICAN_AMERICA.AL32UTF8"
|
||||||
|
|
||||||
|
|
||||||
|
class PgsqlDb:
|
||||||
|
"""
|
||||||
|
PgsqlDb Database Base Class
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self,db):
|
||||||
|
if db == 'warehouse':
|
||||||
|
url=f"postgresql+psycopg2://postgres:postgres@192.168.0.236:5433/{db}"
|
||||||
|
elif db == 'fangjia':
|
||||||
|
url = f"postgresql+psycopg2://postgres:Goojia123456@192.168.0.235:5433/{db}"
|
||||||
|
self.engine = create_engine(url, echo=False,client_encoding='utf-8')
|
||||||
|
@retry(stop_max_attempt_number=3, wait_random_min=1000, wait_random_max=3000)
|
||||||
|
def read(self, sql) -> DataFrame:
|
||||||
|
"""
|
||||||
|
读取数据
|
||||||
|
:param sql:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
data_df = pd.read_sql(sql, con=self.engine)
|
||||||
|
return data_df
|
||||||
|
|
||||||
|
@retry(stop_max_attempt_number=3, wait_random_min=1000, wait_random_max=3000)
|
||||||
|
def count(self, sql) -> int:
|
||||||
|
"""
|
||||||
|
执行统计类SQL
|
||||||
|
:param sql: eg:select count(1) from t
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
return pd.read_sql_query(sql, con=self.engine).iat[0, 0]
|
||||||
|
def sql_execute(self,sql):
|
||||||
|
self.engine.execute(sql)
|
||||||
|
if __name__ == '__main__':
|
||||||
|
pgdb = PgsqlDb()
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
import pandas as pd
|
||||||
|
import os
|
||||||
|
from pgsql_client import PgsqlDb
|
||||||
|
os.environ['NLS_LANG'] = 'SAMPLIFIED CHINESE_CHINA.UTF8'
|
||||||
|
pgdb = PgsqlDb('warehouse')
|
||||||
|
pgdb_new = PgsqlDb('fangjia')
|
||||||
|
def get_community_base_price(cityid,yearmonth):
|
||||||
|
"""
|
||||||
|
查询提供建行小区数据
|
||||||
|
:param cr:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
sql = f"select gujia_admndiv_nm,gujia_bds_id,urbn_nm,comm_nm,gujiabds_clc_yrmo_val,gujia_comm_basic_prc,gujia_bds_clmtd from price_stat.avm_community_base_price where urbn_id='{cityid}' and gujiabds_clc_yrmo_val={yearmonth}"
|
||||||
|
data_df = pgdb.read(sql)
|
||||||
|
data_df = data_df.rename(columns={'gujia_admndiv_nm':'district','gujia_bds_id': '_id','urbn_nm':'city','comm_nm':'name','gujiabds_clc_yrmo_val':'yearmonth','gujia_comm_basic_prc':'price','gujia_bds_clmtd':'type'})
|
||||||
|
return data_df
|
||||||
|
|
||||||
|
def get_community_base_price_new(cityid,yearmonth):
|
||||||
|
"""
|
||||||
|
查询提供建行小区数据
|
||||||
|
:param cr:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
sql = f"select gujia_bds_id,urbn_nm,comm_nm,gujiabds_clc_yrmo_val,gujia_comm_basic_prc,gujia_bds_clmtd from price_stat.avm_community_base_price where urbn_id='{cityid}' and gujiabds_clc_yrmo_val={yearmonth}"
|
||||||
|
data_df = pgdb_new.read(sql)
|
||||||
|
data_df = data_df.rename(columns={'gujia_bds_id': '_id','urbn_nm':'city_copy','comm_nm':'name_copy','gujiabds_clc_yrmo_val':'yearmonth','gujia_comm_basic_prc':'price_new','gujia_bds_clmtd':'type_new'})
|
||||||
|
return data_df
|
||||||
|
def get_district_polyline(cityid=None,cr=None):
|
||||||
|
"""
|
||||||
|
查询提供建行小区数据
|
||||||
|
:param cr:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
if cityid:
|
||||||
|
sql = f"select * from bigdata_district_polyline_gd where cityid='{cityid}'"
|
||||||
|
else:
|
||||||
|
sql = f"select distinct city,distname from bigdata_district_polyline_gd"
|
||||||
|
queryse = cr.execute(sql)
|
||||||
|
columns = [i[0].lower() for i in queryse.description]
|
||||||
|
rs = cr.fetchall()
|
||||||
|
data_df = pd.DataFrame(rs,columns=columns)
|
||||||
|
return data_df
|
||||||
|
def get_community_ccb(cityid):
|
||||||
|
sql = f"select gujia_comm_id,gujia_admndiv_nm,comm_nm from base.bigdata_community where urbn_id='{cityid}' and gujia_bds_bds_st=0"
|
||||||
|
data_df = pgdb.read(sql)
|
||||||
|
data_df = data_df.rename(columns={'gujia_comm_id': '_id','gujia_admndiv_nm':'region','comm_nm':'name'})
|
||||||
|
return data_df
|
||||||
|
|
||||||
|
def get_bak_community(cityid,cr):
|
||||||
|
"""
|
||||||
|
查询小区数据
|
||||||
|
:param cr:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
sql = f"select a.*,b.longitude,b.latitude,c.distname from bigdata_community_bak a left join (select cityid, objectid, LONGITUDE, LATITUDE from bigdata_position_bak where objecttype = '5' and positiontype = '1') b on a.cityid = b.cityid and a.commid = b.objectid left join bigdata_district_bak c on a.cityid=c.cityid and a.distid=c.distid where a.cityid = '{cityid}'"
|
||||||
|
queryse = cr.execute(sql)
|
||||||
|
columns = [i[0].lower() for i in queryse.description]
|
||||||
|
rs = cr.fetchall()
|
||||||
|
data_df = pd.DataFrame(rs,columns=columns)
|
||||||
|
return data_df
|
||||||
|
def avm_price_copy(cityid,yearmonth):
|
||||||
|
sql = f"insert into price_stat.avm_community_base_price_gj_h_copy select * from price_stat.avm_community_base_price where urbn_id='{cityid}' and gujiabds_clc_yrmo_val={yearmonth}"
|
||||||
|
pgdb.sql_execute(sql)
|
||||||
|
|
||||||
|
def get_sample(cityid):
|
||||||
|
sql = f"select urbn_id,gujia_comm_id,comm_nm,gujia_bds_unitprc,guiabconstructionarea from base.bigdata_forsale_gj_h where urbn_id='{cityid}' and gujia_hs_src_dtsrc in ('1','4') and anc_tm >= TO_DATE('2023-04-01','YYYY-MM-DD')"
|
||||||
|
data_df = pgdb.read(sql)
|
||||||
|
return data_df
|
||||||
|
|
||||||
|
def get_deal(cityid):
|
||||||
|
sql = f"select urbn_id,gujia_comm_id,comm_nm,guiabconstructionarea,gujia_bds_trdtckt_prc from base.bigdata_deal where urbn_id='{cityid}' and txn_dt >= TO_DATE('2024-03-01','YYYY-MM-DD') and txn_dt < TO_DATE('2024-04-01','YYYY-MM-DD')"
|
||||||
|
data_df = pgdb.read(sql)
|
||||||
|
data_df = data_df.rename(columns={'urbn_id': 'cityid','gujia_comm_id': '_id','comm_nm': 'name', 'guiabconstructionarea': 'area','gujia_bds_trdtckt_prc': 'deal_price'})
|
||||||
|
return data_df
|
||||||
|
|
||||||
|
def get_deal_data(cityid):
|
||||||
|
sql = f"select * from base.bigdata_deal where urbn_id='{cityid}' and txn_dt >= TO_DATE('2024-03-01','YYYY-MM-DD') and txn_dt < TO_DATE('2024-04-01','YYYY-MM-DD')"
|
||||||
|
data_df = pgdb.read(sql)
|
||||||
|
return data_df
|
||||||
|
|
||||||
|
def get_region_price(cityid,yearmonth):
|
||||||
|
sql = f"select urbn_id,gujia_admndiv_id,gxda_admndiv_nm,gujia_bds_urbn_avprc from price_stat.avm_district_price_gj_h where urbn_id='{cityid}' and gujiabds_clc_yrmo_val={yearmonth}"
|
||||||
|
data_df = pgdb.read(sql)
|
||||||
|
data_df = data_df.rename(columns={'urbn_id': 'cityid','gujia_admndiv_id': 'district_id','gxda_admndiv_nm': 'district_name', 'gujia_bds_urbn_avprc': 'district_price'})
|
||||||
|
return data_df
|
||||||
|
def get_evaluation_third_price(cityid,yearmonth,type='第三方'):
|
||||||
|
sql = f"select urbn_id,gujia_comm_id,gujiabds_to_mkprc_mny from price_stat.avm_evaluation_item_gj_h where urbn_id='{cityid}' and gujiabds_clc_yrmo_val={yearmonth} and gujia_data_tp='{type}' "
|
||||||
|
data_df = pgdb.read(sql)
|
||||||
|
data_df = data_df.rename(columns={'urbn_id': 'cityid','gujiabds_to_mkprc_mny': 'third_price','gujia_comm_id':'_id'})
|
||||||
|
return data_df
|
||||||
|
|
||||||
|
def get_third_price(cityid,yearmonth):
|
||||||
|
sql = f"select urbn_id,gujia_bds_id,comm_nm,gujia_dtsr_dsc,gujia_bds_comm_avprc from base.bigdata_third_party_price where urbn_id='{cityid}' and gujiabds_avprc_dt_val={yearmonth} and gujia_dtsr_dsc in ('链家','安居客') "
|
||||||
|
data_df = pgdb.read(sql)
|
||||||
|
data_df = data_df.rename(columns={'urbn_id': 'cityid','gujia_bds_id': '_id','gujia_dtsr_dsc': 'source','gujia_bds_comm_avprc': 'price'})
|
||||||
|
return data_df
|
||||||
|
if __name__ == '__main__':
|
||||||
|
city='常州'
|
||||||
|
yearmonth=202302
|
||||||
|
get_community_base_price(city=city,yearmonth=yearmonth)
|
||||||
@@ -2,13 +2,9 @@ import os, sys
|
|||||||
import logging
|
import logging
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import requests
|
import requests
|
||||||
from datetime import datetime
|
from goojia_utils import dateutils, pgsql
|
||||||
from goojia_utils import dateutils, oracle
|
from datetime import datetime, timedelta
|
||||||
from sqlalchemy import create_engine
|
from goojia_utils import MAIN_MONGO_CLIENT
|
||||||
|
|
||||||
os.environ["NLS_LANG"] = "American_America.AL32UTF8"
|
|
||||||
os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7"
|
|
||||||
engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False)
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
path = "/home/airflow/data/jh/price"
|
path = "/home/airflow/data/jh/price"
|
||||||
@@ -27,13 +23,32 @@ if type == month_last:
|
|||||||
month_last = month_before
|
month_last = month_before
|
||||||
log.info(f"测试月份:{month_now}")
|
log.info(f"测试月份:{month_now}")
|
||||||
|
|
||||||
price_df = pd.read_sql(f"select urbn_id,urbn_nm,gujia_admndiv_nm,gujia_bds_id as gujia_comm_id,comm_nm,gujia_bds_clmtd,gujia_comm_basic_prc from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={month_now} ", con=engine)
|
price_df = pgsql.scan_by_sql_to_data_frame(f"select urbn_id,urbn_nm,gujia_admndiv_nm,gujia_bds_id as gujia_comm_id,comm_nm,gujia_bds_clmtd,gujia_comm_basic_prc from price_stat.avm_community_base_price where gujiabds_clc_yrmo_val={month_now} ")
|
||||||
deal_df = pd.read_sql(f"select gujia_bds_mdl_id,gujia_comm_id,gujia_bds_trdtckt_prc from base.t0831_bigdata_deal_gj_h where to_char(txn_dt,'yyyyMM')='{month_last}' ", con=engine)
|
deal_df = pgsql.scan_by_sql_to_data_frame(f"select gujia_bds_mdl_id,gujia_comm_id,gujia_bds_trdtckt_prc from base.bigdata_deal where to_char(txn_dt,'yyyyMM')='{month_last}' ")
|
||||||
deal_price_df = deal_df.merge(price_df, on="gujia_comm_id", how="left")
|
deal_price_df = deal_df.merge(price_df, on="gujia_comm_id", how="left")
|
||||||
deal_price_df["ppe"] = round((deal_price_df["gujia_comm_basic_prc"] - deal_price_df['gujia_bds_trdtckt_prc']) / deal_price_df['gujia_bds_trdtckt_prc'], 4)
|
deal_price_df["ppe"] = round((deal_price_df["gujia_comm_basic_prc"] - deal_price_df['gujia_bds_trdtckt_prc']) / deal_price_df['gujia_bds_trdtckt_prc'], 4)
|
||||||
deal_price_df = deal_price_df[abs(deal_price_df['ppe']) > 0.15]
|
#deal_price_df = deal_price_df[abs(deal_price_df['ppe']) > 0.15]
|
||||||
|
|
||||||
|
third_df = pgsql.scan_by_sql_to_data_frame(f"select gujia_bds_id as gujia_comm_id,gujia_dtsr_dsc,gujia_bds_comm_avprc from base.bigdata_third_party_price where gujiabds_avprc_dt_val={month_last} ")
|
||||||
|
third_df = third_df[third_df.gujia_dtsr_dsc.isin(['贝壳','链家'])]
|
||||||
|
# third_df = third_df[['gujia_comm_id','gujia_dtsr_dsc','gujia_bds_comm_avprc']]
|
||||||
|
pivot_df = third_df.pivot(index='gujia_comm_id', columns='gujia_dtsr_dsc', values='gujia_bds_comm_avprc').reset_index()
|
||||||
|
deal_price_df = pd.merge(deal_price_df, pivot_df, how='left', left_on='gujia_comm_id', right_on='gujia_comm_id')
|
||||||
|
|
||||||
|
# 获取上周日
|
||||||
|
today = datetime.today()
|
||||||
|
last_sunday = today - timedelta(days=today.weekday() + 1)
|
||||||
|
last_sunday = last_sunday.strftime("%Y-%m-%d")
|
||||||
|
stat = pd.DataFrame(list(MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "district_stat").find({"weekend": dateutils.parse_date(last_sunday)},{"city":1,"region":1,"district_name":1,"avg_price":1,"avg_price_2b":1,"_id":0})))
|
||||||
|
seaweed = pd.DataFrame(list(MAIN_MONGO_CLIENT.get_collection("fangjia", "seaweed").find({"status":0, "cat":"district"},{'city':1, 'region':1,'name':1,'_id':1})))
|
||||||
|
comm_price = pd.merge(seaweed, stat, how='left', left_on=['city','region','name'], right_on=['city','region','district_name'])
|
||||||
|
comm_price = comm_price[['_id','avg_price','avg_price_2b']]
|
||||||
|
deal_price_df['gujia_comm_id'] = deal_price_df['gujia_comm_id'].astype('str')
|
||||||
|
comm_price['_id'] = comm_price['_id'].astype('str')
|
||||||
|
deal_price_df = pd.merge(deal_price_df, comm_price, how='left', left_on='gujia_comm_id', right_on='_id')
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# third_df = pd.read_sql(f"select gujia_trdpt_avprc_id,gujia_bds_id as gujia_comm_id,gujia_dtsr_dsc,gujia_bds_comm_avprc from base.t0831_bigdata_third_party_price_gj_h where gujiabds_avprc_dt_val={month_last} ", con=engine)
|
|
||||||
# third_price_df = third_df.merge(price_df, on="gujia_comm_id", how="left")
|
# third_price_df = third_df.merge(price_df, on="gujia_comm_id", how="left")
|
||||||
# third_price_df["ppe"] = round((third_price_df["gujia_comm_basic_prc"] - third_price_df['gujia_bds_comm_avprc']) / third_price_df['gujia_bds_comm_avprc'], 4)
|
# third_price_df["ppe"] = round((third_price_df["gujia_comm_basic_prc"] - third_price_df['gujia_bds_comm_avprc']) / third_price_df['gujia_bds_comm_avprc'], 4)
|
||||||
# third_price_df = third_price_df[abs(third_price_df['ppe']) > 0.15]
|
# third_price_df = third_price_df[abs(third_price_df['ppe']) > 0.15]
|
||||||
@@ -42,7 +57,7 @@ deal_price_df = deal_price_df[abs(deal_price_df['ppe']) > 0.15]
|
|||||||
file_name = f"{month_now}测试偏差较大案例统计数据.xlsx"
|
file_name = f"{month_now}测试偏差较大案例统计数据.xlsx"
|
||||||
aim_file_name = path + '/' + file_name
|
aim_file_name = path + '/' + file_name
|
||||||
with pd.ExcelWriter(aim_file_name) as writer:
|
with pd.ExcelWriter(aim_file_name) as writer:
|
||||||
deal_price_df.to_excel(writer, sheet_name="成交", columns=['urbn_id','urbn_nm','gujia_bds_mdl_id','gujia_admndiv_nm','gujia_comm_id','comm_nm','gujia_bds_clmtd','gujia_comm_basic_prc','gujia_bds_trdtckt_prc','ppe'], header=["城市id", "城市名称","成交id","区域","小区id","小区名称",'计算类型','计算价格','成交价格','PPE'], index=False)
|
deal_price_df.to_excel(writer, sheet_name="成交", columns=['urbn_id','urbn_nm','gujia_bds_mdl_id','gujia_admndiv_nm','gujia_comm_id','comm_nm','gujia_bds_clmtd','gujia_comm_basic_prc','gujia_bds_trdtckt_prc','ppe','贝壳','链家','avg_price','avg_price_2b'], header=["城市id", "城市名称","成交id","区域","小区id","小区名称",'计算类型','计算价格','成交价格','PPE','贝壳均价','链家均价','房价网均价','房价网均价2b'], index=False)
|
||||||
# third_price_df.to_excel(writer, sheet_name="第三方", columns=['urbn_id','urbn_nm','gujia_trdpt_avprc_id','gujia_admndiv_nm','gujia_comm_id','comm_nm','gujia_bds_clmtd','gujia_comm_basic_prc','gujia_bds_comm_avprc','PPE','gujia_dtsr_dsc'], header=["城市id", "城市名称","第三方id","区域","小区id","小区名称",'计算类型','计算价格','第三方价格','PPE','来源'],index=False)
|
# third_price_df.to_excel(writer, sheet_name="第三方", columns=['urbn_id','urbn_nm','gujia_trdpt_avprc_id','gujia_admndiv_nm','gujia_comm_id','comm_nm','gujia_bds_clmtd','gujia_comm_basic_prc','gujia_bds_comm_avprc','PPE','gujia_dtsr_dsc'], header=["城市id", "城市名称","第三方id","区域","小区id","小区名称",'计算类型','计算价格','第三方价格','PPE','来源'],index=False)
|
||||||
|
|
||||||
log.info("偏差较大案例统计数据结束")
|
log.info("偏差较大案例统计数据结束")
|
||||||
|
|||||||
@@ -3,12 +3,7 @@ import logging
|
|||||||
import pandas as pd
|
import pandas as pd
|
||||||
import requests
|
import requests
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from goojia_utils import dateutils, oracle
|
from goojia_utils import dateutils, pgsql
|
||||||
from sqlalchemy import create_engine
|
|
||||||
|
|
||||||
os.environ["NLS_LANG"] = "American_America.AL32UTF8"
|
|
||||||
os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7"
|
|
||||||
engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False)
|
|
||||||
|
|
||||||
important_cities =["北京","上海","广州","深圳","重庆","杭州","苏州","厦门","大连","呼和浩特"]
|
important_cities =["北京","上海","广州","深圳","重庆","杭州","苏州","厦门","大连","呼和浩特"]
|
||||||
city_order = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx")
|
city_order = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx")
|
||||||
@@ -56,14 +51,14 @@ year_month = int(sys.argv[1]) if len(sys.argv) > 1 else int(dateutils.format_dat
|
|||||||
log.info(f"测试月份:{year_month}")
|
log.info(f"测试月份:{year_month}")
|
||||||
avm_evaluation_column_map = {'gujia_fsid': 'id', 'urbn_id': 'cityid', 'urbn_nm': 'cityname', 'gujia_bds_yrmo': 'year_month', 'gujia_bds_comm_num': 'community_num', 'gujiabdsvalmnycommnum': 'has_price_community_num', 'gujia_prc_cvr_rate': 'price_coverage', 'gujiabdsmnldpccommnum': 'artificial_calc_num', 'gujia_mnl_drc_prc_pct': 'artificial_calc_coverage', 'gujiabdssamp_comm_num': 'sample_calc_num', 'gujiaext_samp_clc_pct': 'sample_calc_coverage', 'gujiabdsnervetncomnum': 'ann_calc_num', 'gujianervethrunclcpct': 'ann_calc_coverage', 'gujiabds_tst_case_num': 'test_num', 'gujiabdsmdltstcommnum': 'deal_test_comm_num', 'gujiatst_comm_num_pct': 'test_comm_coverage', 'gujia_ppe15_cvr_rate': 'ppe15_coverage', 'gujia_rto_mdn': 'rate_median', 'gujia_rto_lv_san_ceft': 'coefficient', 'gujiabdsmdltstcasenum': 'deal_test_case_num', 'gujiamdltstcommnumpct': 'deal_test_comm_coverage', 'gujiamdltst_ppe15_pct': 'deal_ppe15_coverage', 'gujiamdlrtolvsan_ceft': 'deal_coefficient', 'gujia_mdl_md_bit_rto': 'deal_rate_median', 'gujiabdstrdpttcasenum': 'third_test_case_num', 'gujiabdstrdpttcommnum': 'third_test_comm_num', 'gujiatrdpttcommnumpct': 'third_test_comm_coverage', 'gujiatrdpttstppe15pct': 'third_ppe15_coverage', 'gujiatrdptrolvsanceft': 'third_coefficient', 'gujiatrdpt_md_bit_rto': 'third_rate_median', 'data_crt_tm': 'createtime', 'data_udt_tm': 'updatetime', 'gujia_src_co': 'sourcecode'}
|
avm_evaluation_column_map = {'gujia_fsid': 'id', 'urbn_id': 'cityid', 'urbn_nm': 'cityname', 'gujia_bds_yrmo': 'year_month', 'gujia_bds_comm_num': 'community_num', 'gujiabdsvalmnycommnum': 'has_price_community_num', 'gujia_prc_cvr_rate': 'price_coverage', 'gujiabdsmnldpccommnum': 'artificial_calc_num', 'gujia_mnl_drc_prc_pct': 'artificial_calc_coverage', 'gujiabdssamp_comm_num': 'sample_calc_num', 'gujiaext_samp_clc_pct': 'sample_calc_coverage', 'gujiabdsnervetncomnum': 'ann_calc_num', 'gujianervethrunclcpct': 'ann_calc_coverage', 'gujiabds_tst_case_num': 'test_num', 'gujiabdsmdltstcommnum': 'deal_test_comm_num', 'gujiatst_comm_num_pct': 'test_comm_coverage', 'gujia_ppe15_cvr_rate': 'ppe15_coverage', 'gujia_rto_mdn': 'rate_median', 'gujia_rto_lv_san_ceft': 'coefficient', 'gujiabdsmdltstcasenum': 'deal_test_case_num', 'gujiamdltstcommnumpct': 'deal_test_comm_coverage', 'gujiamdltst_ppe15_pct': 'deal_ppe15_coverage', 'gujiamdlrtolvsan_ceft': 'deal_coefficient', 'gujia_mdl_md_bit_rto': 'deal_rate_median', 'gujiabdstrdpttcasenum': 'third_test_case_num', 'gujiabdstrdpttcommnum': 'third_test_comm_num', 'gujiatrdpttcommnumpct': 'third_test_comm_coverage', 'gujiatrdpttstppe15pct': 'third_ppe15_coverage', 'gujiatrdptrolvsanceft': 'third_coefficient', 'gujiatrdpt_md_bit_rto': 'third_rate_median', 'data_crt_tm': 'createtime', 'data_udt_tm': 'updatetime', 'gujia_src_co': 'sourcecode'}
|
||||||
avm_evaluation_item_column_map = {'gujia_fsid': 'id', 'urbn_id': 'cityid', 'urbn_nm': 'cityname', 'gujia_rgon_id': 'distid', 'gujia_rgon_nm': 'distname', 'gujia_comm_id': 'commid', 'gujia_comm_nm': 'commname', 'gujiabds_clc_yrmo_val': 'year_month', 'gujia_bds_clc_avprc': 'calc_avg_price', 'gujia_bds_avprc': 'avg_price', 'gujia_bds_clc_tp': 'calc_type', 'gujia_bds_rlb': 'confidence', 'gujiabds_to_mkprc_mny': 'target_avg_price', 'mdl_dt': 'dealdate', 'gujiabds_valt_unitprc': 'unit_price', 'gujia_area': 'buildarea', 'gujia_dcrt_val': 'fitment', 'gujia_twrd_val': 'houseaspect', 'gujia_bds_floor_num': 'floornum', 'gujiabdstot_floor_num': 'floorcnt', 'gujia_data_tp': 'data_type', 'gujia_rto': 'rate', 'gujia_mns_rto': 'pe', 'data_crt_tm': 'createtime'}
|
avm_evaluation_item_column_map = {'gujia_fsid': 'id', 'urbn_id': 'cityid', 'urbn_nm': 'cityname', 'gujia_rgon_id': 'distid', 'gujia_rgon_nm': 'distname', 'gujia_comm_id': 'commid', 'gujia_comm_nm': 'commname', 'gujiabds_clc_yrmo_val': 'year_month', 'gujia_bds_clc_avprc': 'calc_avg_price', 'gujia_bds_avprc': 'avg_price', 'gujia_bds_clc_tp': 'calc_type', 'gujia_bds_rlb': 'confidence', 'gujiabds_to_mkprc_mny': 'target_avg_price', 'mdl_dt': 'dealdate', 'gujiabds_valt_unitprc': 'unit_price', 'gujia_area': 'buildarea', 'gujia_dcrt_val': 'fitment', 'gujia_twrd_val': 'houseaspect', 'gujia_bds_floor_num': 'floornum', 'gujiabdstot_floor_num': 'floorcnt', 'gujia_data_tp': 'data_type', 'gujia_rto': 'rate', 'gujia_mns_rto': 'pe', 'data_crt_tm': 'createtime'}
|
||||||
avm_evaluation = pd.read_sql(f"SELECT * FROM t_app_p9pha_plda.avm_evaluation_gj_h where gujia_bds_yrmo={year_month}", con=engine)
|
avm_evaluation = pgsql.scan_by_sql_to_data_frame(f"SELECT * FROM price_stat.avm_evaluation_gj_h where gujia_bds_yrmo={year_month}")
|
||||||
avm_evaluation_item = pd.read_sql(f"SELECT * FROM t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val={year_month}", con=engine)
|
avm_evaluation_item = pgsql.scan_by_sql_to_data_frame(f"SELECT * FROM price_stat.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val={year_month}")
|
||||||
avm_evaluation.rename(columns=avm_evaluation_column_map, inplace=True)
|
avm_evaluation.rename(columns=avm_evaluation_column_map, inplace=True)
|
||||||
avm_evaluation_item.rename(columns=avm_evaluation_item_column_map, inplace=True)
|
avm_evaluation_item.rename(columns=avm_evaluation_item_column_map, inplace=True)
|
||||||
community_base_price = pd.read_sql(f'''SELECT gujia_bds_id as commid,urbn_nm as cityname,gujia_admndiv_nm as distname,comm_nm as commname,
|
community_base_price = pgsql.scan_by_sql_to_data_frame(f'''SELECT gujia_bds_id as commid,urbn_nm as cityname,gujia_admndiv_nm as distname,comm_nm as commname,
|
||||||
gujia_comm_basic_prc as comm_avg_price,gujia_bds_cfdc_dgr as confidence,gujia_bds_clmtd as calc_type from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={year_month}''',
|
gujia_comm_basic_prc as comm_avg_price,gujia_bds_cfdc_dgr as confidence,gujia_bds_clmtd as calc_type from price_stat.avm_community_base_price where gujiabds_clc_yrmo_val={year_month}'''
|
||||||
con=engine)
|
)
|
||||||
communities = pd.read_sql(f"select gujia_comm_id as COMMID from base.t0831_bigdata_community_gj_h", con=engine)
|
communities = pgsql.scan_by_sql_to_data_frame(f"select gujia_comm_id as COMMID from base.bigdata_community")
|
||||||
|
|
||||||
community_base_price = community_base_price[community_base_price.commid.isin(communities["commid"])]
|
community_base_price = community_base_price[community_base_price.commid.isin(communities["commid"])]
|
||||||
avm_evaluation_item["fitment"] = avm_evaluation_item["fitment"].map(convert_fitment_code)
|
avm_evaluation_item["fitment"] = avm_evaluation_item["fitment"].map(convert_fitment_code)
|
||||||
@@ -96,7 +91,7 @@ import_avm_evaluation_item_third.sort_values(by="cityrank", inplace=True)
|
|||||||
file_name = f"{year_month}估值模型测试.xlsx"
|
file_name = f"{year_month}估值模型测试.xlsx"
|
||||||
aim_file_name = path + '/' + file_name
|
aim_file_name = path + '/' + file_name
|
||||||
with pd.ExcelWriter(aim_file_name) as writer:
|
with pd.ExcelWriter(aim_file_name) as writer:
|
||||||
avm_evaluation.to_excel(writer, sheet_name="92个城市估值概况", columns=["cityname","community_num","price_coverage","sample_calc_coverage","ann_calc_coverage","artificial_calc_coverage","deal_test_case_num","deal_ppe15_coverage","deal_test_comm_num","deal_test_comm_coverage","deal_rate_median","deal_coefficient","third_test_comm_num","third_test_comm_coverage","third_ppe15_coverage","third_rate_median","third_coefficient"], header=["城市名称","小区量","可输出均价小区占比","样本计算占比","神经网络计算占比","人工指导价占比","成交测试案例总量","成交测试PPE15占比","成交测试小区量","成交测试小区量占比","成交比率中位数","成交比率离散系数","第三方均价测试小区量","第三方均价测试小区量占比","第三方均价测试PPE15占比","第三方比率中位数","第三方比率离散系数"], index=False)
|
avm_evaluation.to_excel(writer, encoding="UTF-8", sheet_name="92个城市估值概况", columns=["cityname","community_num","price_coverage","sample_calc_coverage","ann_calc_coverage","artificial_calc_coverage","deal_test_case_num","deal_ppe15_coverage","deal_test_comm_num","deal_test_comm_coverage","deal_rate_median","deal_coefficient","third_test_comm_num","third_test_comm_coverage","third_ppe15_coverage","third_rate_median","third_coefficient"], header=["城市名称","小区量","可输出均价小区占比","样本计算占比","神经网络计算占比","人工指导价占比","成交测试案例总量","成交测试PPE15占比","成交测试小区量","成交测试小区量占比","成交比率中位数","成交比率离散系数","第三方均价测试小区量","第三方均价测试小区量占比","第三方均价测试PPE15占比","第三方比率中位数","第三方比率离散系数"], index=False)
|
||||||
import_community_base_price.to_excel(writer, sheet_name="测试10个城市估值数据", columns=["cityname","distname","commname","comm_avg_price","confidence","calc_type"], header=["城市","区域","小区名","估值小区均价","置信度","估值方式"],index=False)
|
import_community_base_price.to_excel(writer, sheet_name="测试10个城市估值数据", columns=["cityname","distname","commname","comm_avg_price","confidence","calc_type"], header=["城市","区域","小区名","估值小区均价","置信度","估值方式"],index=False)
|
||||||
import_avm_evaluation_item_deal.to_excel(writer, sheet_name="10个城市成交测试案例", columns=['cityname','distname', 'commname', 'unit_price', 'confidence', 'calc_type','total_price', 'deal_total_price', 'target_avg_price','pe',"dealdate","fitment","houseaspect","floornum","floorcnt","buildarea"], header=["城市","区域","小区名","修正均价","置信度","估值方式","小区估价","对标成交价","对标均价","PPE","dealdate","fitment","houseaspect","floornum","floorcnt","buildarea"], index=False)
|
import_avm_evaluation_item_deal.to_excel(writer, sheet_name="10个城市成交测试案例", columns=['cityname','distname', 'commname', 'unit_price', 'confidence', 'calc_type','total_price', 'deal_total_price', 'target_avg_price','pe',"dealdate","fitment","houseaspect","floornum","floorcnt","buildarea"], header=["城市","区域","小区名","修正均价","置信度","估值方式","小区估价","对标成交价","对标均价","PPE","dealdate","fitment","houseaspect","floornum","floorcnt","buildarea"], index=False)
|
||||||
import_avm_evaluation_item_third.to_excel(writer, sheet_name="10个城市第三方均价测试", columns=["cityname","distname","commname","avg_price","confidence","calc_type","target_avg_price","pe"], header=["城市","区域","小区名","小区均价","置信度","估值方式","对标第三方均价","PPE"],index=False)
|
import_avm_evaluation_item_third.to_excel(writer, sheet_name="10个城市第三方均价测试", columns=["cityname","distname","commname","avg_price","confidence","calc_type","target_avg_price","pe"], header=["城市","区域","小区名","小区均价","置信度","估值方式","对标第三方均价","PPE"],index=False)
|
||||||
|
|||||||
@@ -1,16 +1,19 @@
|
|||||||
"""
|
"""
|
||||||
价格异常检测脚本
|
价格异常检测脚本
|
||||||
"""
|
"""
|
||||||
from goojia_utils import dateutils, oracle
|
from goojia_utils import dateutils, pgsql, MAIN_MONGO_CLIENT, data_utils
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import requests
|
import requests
|
||||||
from sqlalchemy import create_engine
|
|
||||||
|
|
||||||
os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7"
|
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||||
engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False)
|
# 获取独家小区
|
||||||
|
exclusive_seaweed_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0, 'mark_map.exclusive': '是'}, {'city': 1, 'region': 1, 'name': 1, 'mark_map.exclusive': 1}).batch_size(100000)))
|
||||||
|
exclusive_seaweed_df = data_utils.flatten_json(exclusive_seaweed_df, 'mark_map')
|
||||||
|
exclusive_seaweed_df = exclusive_seaweed_df.rename(columns={'name': 'commname', 'exclusive': '独家小区'})
|
||||||
|
|
||||||
def simple_calc(result_df):
|
def simple_calc(result_df):
|
||||||
"""
|
"""
|
||||||
新增是否有样本列
|
新增是否有样本列
|
||||||
@@ -26,9 +29,9 @@ def simple_calc(result_df):
|
|||||||
if len(actions) == 800:
|
if len(actions) == 800:
|
||||||
ins = "','".join(actions)
|
ins = "','".join(actions)
|
||||||
ins = "'" + ins + "'"
|
ins = "'" + ins + "'"
|
||||||
simple_temp_df = pd.read_sql(
|
simple_temp_df = pgsql.scan_by_sql_to_data_frame(
|
||||||
"select distinct gujia_comm_id as commid from base.t0831_bigdata_forsale_gj_h where " +
|
"select distinct gujia_comm_id as commid from base.bigdata_forsale_gj_h where " +
|
||||||
" anc_tm> TO_DATE('" + month_before + "','YYYY-MM-DD') and gujia_comm_id in (" + ins + ")", con=engine)
|
" anc_tm> TO_DATE('" + month_before + "','YYYY-MM-DD') and gujia_comm_id in (" + ins + ")")
|
||||||
|
|
||||||
simple_df = pd.concat([simple_df, simple_temp_df])
|
simple_df = pd.concat([simple_df, simple_temp_df])
|
||||||
actions.clear()
|
actions.clear()
|
||||||
@@ -36,9 +39,9 @@ def simple_calc(result_df):
|
|||||||
if len(actions) > 0:
|
if len(actions) > 0:
|
||||||
ins = "','".join(actions)
|
ins = "','".join(actions)
|
||||||
ins = "'" + ins + "'"
|
ins = "'" + ins + "'"
|
||||||
simple_temp_df = pd.read_sql(
|
simple_temp_df = pgsql.scan_by_sql_to_data_frame(
|
||||||
"select distinct gujia_comm_id as commid from base.t0831_bigdata_forsale_gj_h where " +
|
"select distinct gujia_comm_id as commid from base.bigdata_forsale_gj_h where " +
|
||||||
" anc_tm> TO_DATE('" + month_before + "','YYYY-MM-DD') and gujia_comm_id in (" + ins + ")", con=engine)
|
" anc_tm> TO_DATE('" + month_before + "','YYYY-MM-DD') and gujia_comm_id in (" + ins + ")")
|
||||||
simple_df = pd.concat([simple_df, simple_temp_df])
|
simple_df = pd.concat([simple_df, simple_temp_df])
|
||||||
actions.clear()
|
actions.clear()
|
||||||
simple_df['has_sample'] = 1
|
simple_df['has_sample'] = 1
|
||||||
@@ -88,8 +91,8 @@ if type == month_last:
|
|||||||
month_now = month_last
|
month_now = month_last
|
||||||
month_last = month_before
|
month_last = month_before
|
||||||
|
|
||||||
df = pd.read_sql("SELECT gujia_bss_prc_id as id, urbn_nm as cityname, gujia_admndiv_nm as distname, gujia_bds_id as commid, comm_nm as commname, gujia_bds_clmtd as calc_type, gujia_bds_cfdc_dgr as confidence, gujia_comm_basic_prc as avg_price, gujiabds_clc_yrmo_val as year_month FROM t_app_p9pha_plda.avm_community_base_price_gj_h WHERE gujiabds_clc_yrmo_val IN('"+month_now+"', '"+month_last+"') " +
|
df = pgsql.scan_by_sql_to_data_frame("SELECT gujia_bss_prc_id as id, urbn_nm as cityname, gujia_admndiv_nm as distname, gujia_bds_id as commid, comm_nm as commname, gujia_bds_clmtd as calc_type, gujia_bds_cfdc_dgr as confidence, gujia_comm_basic_prc as avg_price, gujiabds_clc_yrmo_val as year_month FROM price_stat.avm_community_base_price WHERE gujiabds_clc_yrmo_val IN('"+month_now+"', '"+month_last+"') " +
|
||||||
" ORDER BY urbn_nm, gujia_admndiv_nm, gujia_bds_id, comm_nm, gujia_bds_clmtd", con=engine)
|
" ORDER BY urbn_nm, gujia_admndiv_nm, gujia_bds_id, comm_nm, gujia_bds_clmtd")
|
||||||
|
|
||||||
six_map = dict()
|
six_map = dict()
|
||||||
five_map = dict()
|
five_map = dict()
|
||||||
@@ -113,7 +116,7 @@ for key in six_map:
|
|||||||
if abs(six_price - five_price) * 100 / five_price > 10:
|
if abs(six_price - five_price) * 100 / five_price > 10:
|
||||||
actions.append({'city': info[0], 'region': info[1], 'commid': info[2], 'commname': info[3],
|
actions.append({'city': info[0], 'region': info[1], 'commid': info[2], 'commname': info[3],
|
||||||
'five_price': five_price,'five_calc_type': five_map[key].split("|")[1],
|
'five_price': five_price,'five_calc_type': five_map[key].split("|")[1],
|
||||||
'six_price': six_price,'six_calc_type': six_map[key].split("|")[1],
|
'six_price': six_price,'six_calc_type': six_map[key].split("|")[1], '差异值': abs(six_price - five_price) * 100 / five_price,
|
||||||
'涨跌幅': (six_price - five_price) * 100 / five_price, 'confidence': six_map[key].split("|")[2]})
|
'涨跌幅': (six_price - five_price) * 100 / five_price, 'confidence': six_map[key].split("|")[2]})
|
||||||
if info[0] not in city_seaweed_dict:
|
if info[0] not in city_seaweed_dict:
|
||||||
city_seaweed_dict[info[0]] = 1
|
city_seaweed_dict[info[0]] = 1
|
||||||
@@ -134,6 +137,7 @@ excel_writer = pd.ExcelWriter(aim_file_name)
|
|||||||
if type == 'sample':
|
if type == 'sample':
|
||||||
result_df = simple_calc(result_df)
|
result_df = simple_calc(result_df)
|
||||||
|
|
||||||
|
result_df = pd.merge(result_df, exclusive_seaweed_df, on=['city', 'region', 'commname'], how='left')
|
||||||
result_df = result_df.drop_duplicates(subset=['commid'], keep='first')
|
result_df = result_df.drop_duplicates(subset=['commid'], keep='first')
|
||||||
result_df = result_df.rename(columns={'city': '城市', 'region': '区域', 'commid': '小区ID', 'commname': '小区名', 'five_price': '上一月价格',
|
result_df = result_df.rename(columns={'city': '城市', 'region': '区域', 'commid': '小区ID', 'commname': '小区名', 'five_price': '上一月价格',
|
||||||
'five_calc_type': '上一月计算方式', 'six_price': '当月价格', 'six_calc_type': '当月计算方式', 'confidence': '置信度'})
|
'five_calc_type': '上一月计算方式', 'six_price': '当月价格', 'six_calc_type': '当月计算方式', 'confidence': '置信度'})
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user