diff --git a/main.py b/main.py index d68dde5..a3b1fc0 100644 --- a/main.py +++ b/main.py @@ -15,17 +15,28 @@ sys.path.append('./src') from src.acquire_fund_snapshot import get_fund_list from src.acquire_fund_base import acquire_fund_base +from src.fund_info.supplement import FundSupplement +from src.acquire_fund_quarter import acquire_fund_quarter + def main(): input_value = input("请输入下列序号执行操作:\n \ 1.“快照” \n \ 2.“新基入库”\n \ + 3.“季度信息”\n \ + 4.“基金状态归档”\n \ 输入:") if input_value == '1' or input_value == '快照': page_index = 1 get_fund_list(page_index) # 执行申万行业信息入库 elif input_value == '2' or input_value == '新基入库': acquire_fund_base() # 执行行业股票信息入库 + elif input_value == '3' or input_value == "季度信息": + acquire_fund_quarter() + elif input_value == '4' or input_value == "基金状态归档": + fund_supplement = FundSupplement() + # 补充基金清算维度信息 + fund_supplement.update_archive_status() if __name__ == '__main__': diff --git a/src/acquire_fund_quarter.py b/src/acquire_fund_quarter.py index 55aa3cb..126b9b4 100644 --- a/src/acquire_fund_quarter.py +++ b/src/acquire_fund_quarter.py @@ -9,9 +9,8 @@ Author: luxuemin2108@gmail.com Copyright (c) 2020 Camel Lu ''' -import math -from threading import Thread, Lock, current_thread -from time import sleep, time +from threading import Lock, current_thread +from time import sleep from pprint import pprint from fund_info.crawler import FundSpider from fund_info.api import FundApier @@ -36,13 +35,12 @@ def get_total_asset(fund_code, platform): total_asset = each_fund.get_total_asset() return total_asset - -if __name__ == '__main__': +def acquire_fund_quarter(): lock = Lock() each_fund_query = FundQuery() record_total = each_fund_query.get_crawler_quarter_fund_total() # 获取记录条数 print('record_total', record_total) - IdWorker = IdWorker() + idWorker = IdWorker() result_dir = './output/' fund_csv = FundCSV(result_dir) fund_csv.write_season_catch_fund(True) @@ -50,7 +48,7 @@ if __name__ == '__main__': def crawlData(start, end): login_url = 'https://www.morningstar.cn/membership/signin.aspx' - chrome_driver = login_morning_star(login_url, True) + chrome_driver = login_morning_star(login_url, False) page_start = start page_limit = 10 while(page_start < end): @@ -96,7 +94,7 @@ if __name__ == '__main__': fund_csv.write_season_catch_fund(False, output_line) # 入库 lock.acquire() - snow_flake_id = IdWorker.get_id() + snow_flake_id = idWorker.get_id() lock.release() # 开始存入数据 fund_insert = FundInsert() @@ -189,3 +187,6 @@ if __name__ == '__main__': bootstrap_thread(crawlData, record_total, 4) exit() + +if __name__ == '__main__': + acquire_fund_quarter() diff --git a/src/sql_model/fund_query.py b/src/sql_model/fund_query.py index cfe9849..5141286 100644 --- a/src/sql_model/fund_query.py +++ b/src/sql_model/fund_query.py @@ -78,13 +78,12 @@ class FundQuery(BaseQuery): # 过滤没有股票持仓的基金 sql_count = "SELECT COUNT(1) FROM fund_morning_base as a \ WHERE a.fund_cat NOT LIKE '%%货币%%' \ - AND a.fund_cat NOT LIKE '%%纯债基金%%' \ - AND a.fund_cat NOT LIKE '目标日期' \ AND a.is_archive = 0 \ AND a.found_date <= %s \ AND a.fund_name NOT LIKE '%%C' \ AND a.fund_name NOT LIKE '%%B' \ - AND a.fund_cat NOT LIKE '%%短债基金%%' \ + AND a.fund_cat NOT IN ('利率债', '短债基金', '短债型', '短债基金(封闭)', '纯债基金', '纯债基金(封闭)', \ + '普通债券型', '普通债券型基金','普通债券型基金(封闭)', '信用债', '信用债(封闭)','目标日期' ) \ AND a.fund_code NOT IN( SELECT fund_code FROM fund_morning_quarter as b \ WHERE b.quarter_index = %s);" self.cursor.execute(sql_count, [self.quarter_date, self.quarter_index]) @@ -98,9 +97,8 @@ class FundQuery(BaseQuery): t.morning_star_code, t.fund_name, t.fund_cat \ FROM fund_morning_base as t \ WHERE t.fund_cat NOT LIKE '%%货币%%' \ - AND t.fund_cat NOT LIKE '%%纯债基金%%' \ - AND t.fund_cat NOT LIKE '目标日期' \ - AND t.fund_cat NOT LIKE '%%短债基金%%' \ + AND t.fund_cat NOT IN ('利率债', '短债基金', '短债型', '短债基金(封闭)', '纯债基金', '纯债基金(封闭)', \ + '普通债券型', '普通债券型基金','普通债券型基金(封闭)', '信用债', '信用债(封闭)','目标日期' ) \ AND t.found_date <= %s \ AND t.is_archive = 0 \ AND t.fund_name NOT LIKE '%%C' \