feat: add update archive filed

src/acquire_fund_quarter.py

@@ -44,7 +44,6 @@ if __name__ == '__main__':
     each_fund_query = FundQuery()
     record_total = each_fund_query.get_crawler_quarter_fund_total()    # 获取记录条数
     print('record_total', record_total)
-    exit()
     IdWorker = IdWorker()
     page_start = 0
     # error_funds = []
@@ -198,19 +197,19 @@ if __name__ == '__main__':
         chrome_driver.close()
     threaders = []
     start_time = time()
-    steps = [{
-        "start": 0,
-        "end": 1000
-    }, {
-        "start": 1000,
-        "end": 2000
-    }, {
-        "start": 2000,
-        "end": 3000
-    }, {
-        "start": 3500,
-        "end": record_total
-    }]
+    # steps = [{
+    #     "start": 0,
+    #     "end": 1000
+    # }, {
+    #     "start": 1000,
+    #     "end": 2000
+    # }, {
+    #     "start": 2000,
+    #     "end": 3000
+    # }, {
+    #     "start": 3500,
+    #     "end": record_total
+    # }]
     thread_count = 4
     step_num = record_total / thread_count
     for i in range(thread_count):

src/acquire_fund_snapshot.py

@@ -96,8 +96,7 @@ def get_fund_list():
         data = chrome_driver.page_source
         # 利用BeautifulSoup解析网页源代码
         bs = BeautifulSoup(data, 'lxml')
-        class_list = ['gridItem', 'gridAlternateItem']  # 数据在这两个类下面
-
+        class_list = ['gridItem', 'gridAlternateItem']  # 数据在这两个类下
         # 取出所有类的信息，并保存到对应的列表里
         for i in range(len(class_list)):
             for tr in bs.find_all('tr', {'class': class_list[i]}):

src/fund_info/api.py

@@ -42,7 +42,7 @@ class FundApier:
         if self.platform == 'ai_fund':
             return self.get_base_info_ai()
         elif self.platform == 'zh_fund':
-            return self.get_base_info_hz()
+            return self.get_base_info_zh()
 
     def get_asset_from_json(self):
         with open(self.file_path) as json_file:
@@ -88,7 +88,7 @@ class FundApier:
             raise('中断')
 
     # 基金信息--来源展恒基金
-    def get_base_info_hz(self):
+    def get_base_info_zh(self):
         url = "https://www.myfund.com/webinterface/Bamboo.ashx?command={0}".format(
             'fundInfoHead_NEW')
         headers = {
@@ -125,7 +125,7 @@ class FundApier:
             print('code:3', self.fund_code)
             raise('中断')
 
-    def get_analyse_info_hz(self):
+    def get_analyse_info_zh(self):
             url = "https://www.myfund.com/webinterface/Bamboo.ashx?command={0}".format(
                 'singlefundAnalyse')
             headers = {
@@ -142,7 +142,6 @@ class FundApier:
                     res_json = res.json()
                     buy_status = res_json.get('BuyStatus')
                     sell_status = res_json.get('SellStatus')
-                    pprint(res_json)
                     if res_json.get('Msg') == 'OK' and buy_status != None:
                         self.sell_status = sell_status
                         self.buy_status = buy_status
@@ -170,5 +169,5 @@ class FundApier:
 
 if __name__ == '__main__':
     fund_api = FundApier('000421', end_date='2021-05-31',)
-    # fund_api.get_analyse_info_hz()
+    # fund_api.get_analyse_info_zh()
     # print("fund_api", fund_api)

src/fund_info/supplement.py

@@ -0,0 +1,37 @@
+'''
+Desc: 基金数据的补充，更新
+File: /supplement.py
+Project: fund_info
+File Created: Thursday, 3rd June 2021 3:19:21 pm
+Author: luxuemin2108@gmail.com
+-----
+Copyright (c) 2021 Camel Lu
+'''
+
+import time
+from utils.index import get_last_quarter_str
+from sql_model.fund_query import FundQuery
+from sql_model.fund_update import FundUpdate
+from fund_info.api import FundApier
+
+class FundSupplement:
+    def __init__(self, code=None):
+      self.fund_code = code
+      # 动态计算季度信息
+      self.quarter_index = get_last_quarter_str()
+    
+    def update_archive_status(self):
+      fund_query = FundQuery()
+      print("fund_query", fund_query)
+      each_fund_update = FundUpdate()
+      funds = fund_query.select_quarter_fund(0, 10000)
+      for fund_item in funds:
+        print("fund_item", fund_item)
+        fund_code = fund_item[0]
+        fund_api = FundApier(fund_code, platform='zh_fund')
+        fund_api.get_analyse_info_zh()
+        buy_status = fund_api.buy_status
+        if buy_status == '已清盘':
+          each_fund_update.update_archive_status(1, fund_code=fund_code)
+          continue
+        print('fund_api', fund_api.buy_status, fund_api.sell_status)

src/fund_info_supplement.py

@@ -0,0 +1,17 @@
+'''
+Desc: 更新基金一些信息，例如是否清盘
+File: /fund_update.py
+Project: sql_model
+File Created: Thursday, 3rd June 2021 3:13:40 pm
+Author: luxuemin2108@gmail.com
+-----
+Copyright (c) 2021 Camel Lu
+'''
+
+
+from fund_info.supplement import FundSupplement
+
+
+if __name__ == '__main__':
+  fund_supplement = FundSupplement()
+  fund_supplement.update_archive_status()

src/sql_model/fund_query.py

@@ -9,7 +9,7 @@ Copyright (c) 2021 Camel Lu
 '''
 import time
 from threading import Lock
-from utils.index import get_last_quarter_str
+from utils.index import get_last_quarter_str, get_quarter_date
 from db.connect import connect
 
 
@@ -38,6 +38,7 @@ class FundQuery:
 
     def __init__(self):
         self.quarter_index = get_last_quarter_str()
+        self.quarter_date = get_quarter_date(self.quarter_index)
         connect_instance = connect()
         self.connect_instance = connect_instance
         self.cursor = connect_instance.cursor()
@@ -51,12 +52,13 @@ class FundQuery:
         AND a.fund_cat NOT LIKE '%%纯债基金%%' \
         AND a.fund_cat NOT LIKE '目标日期' \
         AND a.is_archive = 0 \
+        AND a.found_date <= %s \
         AND a.fund_name NOT LIKE '%%C' \
         AND a.fund_name NOT LIKE '%%B' \
         AND a.fund_cat NOT LIKE '%%短债基金%%' \
         AND a.fund_code	NOT IN( SELECT fund_code FROM fund_morning_quarter as b \
         WHERE b.quarter_index = %s);"
-        self.cursor.execute(sql_count, [self.quarter_index])
+        self.cursor.execute(sql_count, [self.quarter_date, self.quarter_index])
         count = self.cursor.fetchone()
         return count[0]
 
@@ -69,6 +71,7 @@ class FundQuery:
             AND t.fund_cat NOT LIKE '%%纯债基金%%' \
             AND t.fund_cat NOT LIKE '目标日期' \
             AND t.fund_cat NOT LIKE '%%短债基金%%' \
+            AND t.found_date <= %s \
             AND t.is_archive = 0 \
             AND t.fund_name NOT LIKE '%%C' \
             AND t.fund_name NOT LIKE '%%B' \
@@ -76,7 +79,7 @@ class FundQuery:
             WHERE b.quarter_index = %s) LIMIT %s, %s;"
         self.lock.acquire()
         self.cursor.execute(
-            sql, [self.quarter_index, page_start, page_limit])    # 执行sql语句
+            sql, [self.quarter_date, self.quarter_index, page_start, page_limit])    # 执行sql语句
         results = self.cursor.fetchall()    # 获取查询的所有记录
         self.lock.release()
         return results

src/sql_model/fund_update.py

@@ -0,0 +1,36 @@
+'''
+Desc: 更新基金一些信息，例如是否清盘
+File: /fund_update.py
+Project: sql_model
+File Created: Thursday, 3rd June 2021 3:13:40 pm
+Author: luxuemin2108@gmail.com
+-----
+Copyright (c) 2021 Camel Lu
+'''
+import time
+from utils.index import get_quarter_index
+from db.connect import connect
+
+class FundUpdate:
+    def __init__(self, code=None):
+      self.fund_code = code
+      last_quarter_time = time.localtime(time.time() - 3 * 30 * 24 * 3600)
+      year = time.strftime("%Y", last_quarter_time)
+      date = time.strftime("%m-%d", last_quarter_time)
+      index = get_quarter_index(date)
+      quarter_index = year + '-Q' + str(index)
+      self.quarter_index = quarter_index
+      connect_instance = connect()
+      self.connect_instance = connect_instance
+      self.cursor = connect_instance.cursor()
+
+    def update_archive_status(self, archive_value=0, *, fund_code=None):
+      code = fund_code
+      if fund_code == None:
+          code = self.fund_code
+      if archive_value == 0:
+        print(code, '本来为0')
+        return
+      sql_update_archive = "UPDATE fund_morning_base SET is_archive = %s WHERE fund_code = %s;"
+      self.cursor.execute(sql_update_archive, (archive_value, code))
+      self.connect_instance.commit()

src/utils/index.py

@@ -69,3 +69,9 @@ def get_last_quarter_str():
     index = get_quarter_index(date)
     quarter_index_str = year + '-Q' + str(index)
     return quarter_index_str
+
+def get_quarter_date(quarter_index_str):
+    year = quarter_index_str.split('-')[0]
+    boundary_date_list = ['03-31', '06-30', '09-30', '12-31']
+    quarter_index = quarter_index_str.split('-')[1][1:]
+    return year + '-' + boundary_date_list[int(quarter_index) - 1]