feat: add update fund total assets

src/fund_info/api.py

@@ -0,0 +1,141 @@
+'''
+Desc: 从基金平台api获取基金信息
+File: /fund_info_api.py
+File Created: Wednesday, 5th May 2021 4:17:44 pm
+Author: luxuemin2108@gmail.com
+-----
+Copyright (c) 2021 Camel Lu
+'''
+
+import requests
+import json
+import time
+import os
+import sys
+from pprint import pprint
+sys.path.append('../')
+sys.path.append(os.getcwd() + '/src')
+from utils.file_op import write_fund_json_data
+
+
+class FundApier:
+    def __init__(self, code, end_date, platform='ai_fund'):
+        self.fund_code = code
+        self.cur_date = time.strftime("%Y-%m-%d", time.localtime(time.time()))
+        self.end_date = end_date
+        self.platform = platform
+        self.total_asset = None
+        # 默认的爱基金
+        self.file_dir = os.getcwd() + '/output/json/' + self.platform + \
+            '/' + self.cur_date + '/'
+        self.file_path = '{file_dir}{fund_code}-{end_date}-base.json'.format(
+            file_dir=self.file_dir,
+            fund_code=code,
+            end_date=self.end_date
+        )
+
+    def get_total_asset(self):
+        if self.base_info_is_exist():
+            return self.get_asset_from_json()
+        if self.platform == 'ai_fund':
+            return self.get_base_info_ai()
+        elif self.platform == 'zh_fund':
+            return self.get_base_info_hz()
+
+    def get_asset_from_json(self):
+        with open(self.file_path) as json_file:
+            my_data = json.load(json_file)
+            asset_str = my_data.get(
+                'asset') if self.platform == 'ai_fund' else my_data.get('FundScope')[0:-1]
+            try:
+                self.total_asset = float(asset_str)
+                return self.total_asset
+            except ValueError:
+                print(asset_str, "not a number")
+
+    # 基金信息--来源爱基金
+
+    def get_base_info_ai(self):
+        url = "http://fund.10jqka.com.cn/data/client/myfund/{0}".format(
+            self.fund_code)
+
+        res = requests.get(url)  # 自动编码
+        time.sleep(1)
+        try:
+            if res.status_code == 200:
+                res_json = res.json()
+                if res_json.get('error').get('id') == 0:
+                    base_info = res.json().get('data')[0]
+                    end_date = base_info.get('enddate')
+                    total_asset = base_info.get('asset')
+                    self.write_info_in_json(end_date, base_info)
+                    try:
+                        self.total_asset = float(total_asset)
+                        return self.total_asset
+                    except ValueError:
+                        print(total_asset, "not a number")
+                else:
+                    pprint(res_json)
+                    print('code:1', self.fund_code)
+            else:
+                pprint(res.raw)
+                print('code:2', self.fund_code)
+                raise('中断')
+        except:
+            pprint(res.raw)
+            print('code:3', self.fund_code)
+            raise('中断')
+
+    # 基金信息--来源展恒基金
+    def get_base_info_hz(self):
+        url = "https://www.myfund.com/webinterface/Bamboo.ashx?command={0}".format(
+            'fundInfoHead_NEW')
+        headers = {
+            'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8'
+        }
+        payload = {
+            'fundcode': self.fund_code,
+        }
+        res = requests.post(url, headers=headers, data=payload)
+        res.encoding = "utf-8"
+        time.sleep(1)
+        try:
+            if res.status_code == 200:
+                res_json = res.json()
+                fund_scope = res_json.get('FundScope')
+                pprint(res_json)
+                if res_json.get('Msg') == 'OK' and fund_scope != None:
+                    end_date = res_json.get('DealDate')
+                    fund_scope = res_json.get('FundScope')
+                    total_asset = fund_scope[0:-1]
+                    self.write_info_in_json(end_date, res_json)
+                    try:
+                        self.total_asset = float(total_asset)
+                        return self.total_asset
+                    except ValueError:
+                        print(total_asset, "not a number")
+                else:
+                    pprint(res_json)
+                    print('code:1', self.fund_code)
+            else:
+                pprint(res.content)
+                print('code:2', self.fund_code)
+                raise('中断')
+        except:
+            print('code:3', self.fund_code)
+            raise('中断')
+
+    def write_info_in_json(self, end_date, json_data):
+        filename = '{fund_code}{end_date}-base.json'.format(
+            fund_code=self.fund_code,
+            end_date='-' + end_date if end_date else ''
+        )
+        write_fund_json_data(json_data, filename, self.file_dir)
+
+    def base_info_is_exist(self):
+        return os.path.exists(self.file_path)
+
+
+if __name__ == '__main__':
+    fund_api = FundApier('675113', '2021-05-07', 'zh_fund')
+    fund_api.get_base_info_hz()

src/fund_info_api.py

@@ -1,112 +0,0 @@
-'''
-Desc: 从基金平台api获取基金信息
-File: /fund_info_api.py
-File Created: Wednesday, 5th May 2021 4:17:44 pm
-Author: luxuemin2108@gmail.com
------
-Copyright (c) 2021 Camel Lu
-'''
-
-import requests
-import json
-import time
-import os
-from pprint import pprint
-from utils import write_json_data
-
-
-class FundApier:
-    def __init__(self, code):
-        self.fund_code = code
-        self.cur_date = time.strftime("%Y-%m-%d", time.localtime(time.time()))
-        self.end_date = '2021-04-30'
-        self.total_asset = None
-        file_dir = os.getcwd() + '/output/json/' + self.cur_date + '/'
-        self.file_path = '{file_dir}{fund_code}-{end_date}-base.json'.format(
-            file_dir=file_dir,
-            fund_code=code,
-            end_date=self.end_date
-        )
-
-    # 基金信息--来源爱基金
-    def get_base_info(self):
-        if self.base_info_is_exist():
-            with open(self.file_path) as json_file:
-                my_data = json.load(json_file)
-                asset_str = my_data.get('asset')
-                self.total_asset = float(asset_str) if asset_str else None
-        else:
-            url = "http://fund.10jqka.com.cn/data/client/myfund/{0}".format(
-                self.fund_code)
-
-            res = requests.get(url)  # 自动编码
-            time.sleep(1)
-            try:
-                if res.status_code == 200:
-                    res_json = res.json()
-                    if res_json.get('error').get('id') == 0:
-                        base_info = res.json().get('data')[0]
-                        end_date = base_info.get('enddate')
-                        self.total_asset = base_info.get('asset')
-
-                        filename = '{fund_code}{end_date}-base.json'.format(
-                            fund_code=self.fund_code,
-                            end_date='-' + end_date if end_date else ''
-                        )
-                        write_json_data(base_info, filename)
-                    else:
-                        pprint(res_json)
-                        print('code:1', self.fund_code)
-                else:
-                    pprint(res.raw)
-                    print('code:2', self.fund_code)
-                    raise('中断')
-            except:
-                pprint(res.raw)
-                print('code:3', self.fund_code)
-                raise('中断')
-
-    def get_base_info_hz(self):
-            if self.base_info_is_exist():
-                with open(self.file_path) as json_file:
-                    my_data = json.load(json_file)
-                    asset_str = my_data.get('asset')
-                    self.total_asset = float(asset_str) if asset_str else None
-            else:
-                url = "http://fund.10jqka.com.cn/data/client/myfund/{0}".format(
-                    self.fund_code)
-
-                res = requests.get(url)  # 自动编码
-                time.sleep(1)
-                try:
-                    if res.status_code == 200:
-                        res_json = res.json()
-                        if res_json.get('error').get('id') == 0:
-                            base_info = res.json().get('data')[0]
-                            end_date = base_info.get('enddate')
-                            self.total_asset = base_info.get('asset')
-
-                            filename = '{fund_code}{end_date}-base.json'.format(
-                                fund_code=self.fund_code,
-                                end_date='-' + end_date if end_date else ''
-                            )
-                            write_json_data(base_info, filename)
-                        else:
-                            pprint(res_json)
-                            print('code:1', self.fund_code)
-                    else:
-                        pprint(res.raw)
-                        print('code:2', self.fund_code)
-                        raise('中断')
-                except:
-                    pprint(res.raw)
-                    print('code:3', self.fund_code)
-                    raise('中断')
-
-        def base_info_is_exist(self):
-            return os.path.exists(self.file_path)
-
-
-if __name__ == '__main__':
-    fund_api = FundApier('003834')
-    fund_api.get_base_info()

src/sql_model/fund_query.py

@@ -20,7 +20,7 @@ class FundQuery:
         self.lock = Lock()
 
     # 需要爬取季度性信息的基金(B,C类基金除外，因为B、C基金大部分信息与A类一致)
-    def get_crawler_quarter_total(self, ):
+    def get_crawler_quarter_fund_total(self):
         # 过滤没有股票持仓的基金
         sql_count = "SELECT COUNT(1) FROM fund_morning_base \
         LEFT JOIN fund_morning_snapshot ON fund_morning_snapshot.fund_code = fund_morning_base.fund_code \
@@ -33,3 +33,38 @@ class FundQuery:
         self.cursor.execute(sql_count)
         count = self.cursor.fetchone()
         return count[0]
+
+    # 筛选基金季度性信息的基金
+    def select_quarter_fund(self, page_start, page_limit):
+        sql = "SELECT t.fund_code,\
+            t.morning_star_code, t.fund_name, t.fund_cat \
+            FROM fund_morning_base as t \
+            LEFT JOIN fund_morning_snapshot as f ON f.fund_code = t.fund_code \
+            WHERE t.fund_cat NOT LIKE '%%货币%%' \
+            AND t.fund_cat NOT LIKE '%%纯债基金%%' \
+            AND t.fund_cat NOT LIKE '目标日期' \
+            AND t.fund_cat NOT LIKE '%%短债基金%%' \
+            AND t.fund_name NOT LIKE '%%C' \
+            AND t.fund_name NOT LIKE '%%B' \
+            ORDER BY f.fund_rating_5 DESC,f.fund_rating_3 DESC, \
+            t.fund_cat, t.fund_code LIMIT %s, %s"
+        self.lock.acquire()
+        self.cursor.execute(
+            sql, [page_start, page_limit])    # 执行sql语句
+        results = self.cursor.fetchall()    # 获取查询的所有记录
+        self.lock.release()
+        return results
+
+    # 筛选同类基金，除了A类
+    def select_similar_fund(self, similar_name):
+        sql_similar = "SELECT t.fund_code,\
+                t.morning_star_code, t.fund_name \
+                FROM fund_morning_base as t \
+                LEFT JOIN fund_morning_snapshot as f ON f.fund_code = t.fund_code \
+                WHERE t.fund_name LIKE %s \
+                AND t.fund_name NOT LIKE '%%A';"
+        self.lock.acquire()
+        self.cursor.execute(sql_similar, [similar_name + '%'])
+        results = self.cursor.fetchall()    # 获取查询的所有记录
+        self.lock.release()
+        return results

src/update_fund_total_asset.py

@@ -11,7 +11,7 @@ from pprint import pprint
 from db.connect import connect
 from time import sleep
 import os
-from fund_info_api import FundApier
+from fund_info.api import FundApier
 
 connect_instance = connect()
 cursor = connect_instance.cursor()
@@ -33,12 +33,15 @@ if __name__ == '__main__':
         if c_class_result:
             fund_code = c_class_result[0]
             fund_name = c_class_result[1]
-            if '封闭' in fund_name:
+            platform = 'zh_fund' if '封闭' in fund_name else 'ai_fund'
-                pprint(c_class_result)
+            each_fund = FundApier(fund_code, '2021-05-07', platform)
-            else:
+
-                each_fund = FundApier(fund_code)
+            total_asset = each_fund.get_total_asset()
-                each_fund.get_base_info()
+            # 如果在爱基金平台找不到，则到展恒基金找
-                total_asset = each_fund.total_asset
+            if total_asset == None and platform == 'ai_fund':
-                sql_update = "UPDATE fund_morning_quarter SET total_asset = %s WHERE fund_code = %s;"
+                print("fund_code", i, fund_name, fund_code)
-                cursor.execute(sql_update, [total_asset, fund_code])
+                each_fund = FundApier(fund_code, '2021-05-07', 'zh_fund')
-                connect_instance.commit()
+                total_asset = each_fund.get_total_asset()
+            sql_update = "UPDATE fund_morning_quarter SET total_asset = %s WHERE fund_code = %s;"
+            cursor.execute(sql_update, [total_asset, fund_code])
+            connect_instance.commit()

src/utils/file_op.py

@@ -0,0 +1,26 @@
+'''
+Desc: 操作文件工具函数
+File: /file_op.py
+Project: utils
+File Created: Sunday, 9th May 2021 5:04:11 pm
+Author: luxuemin2108@gmail.com
+-----
+Copyright (c) 2021 Camel Lu
+'''
+import time
+import datetime
+import os
+
+
+# 写json文件
+def write_fund_json_data(data, filename, file_dir=None):
+    import json
+    if not file_dir:
+        cur_date = time.strftime("%Y-%m-%d", time.localtime(time.time()))
+        file_dir = os.getcwd() + '/output/json/ai_fund/' + cur_date + '/'
+    if not os.path.exists(file_dir):
+        os.makedirs(file_dir)
+        print("目录新建成功：%s" % file_dir)
+    with open(file_dir + filename, 'w', encoding='utf-8') as f:
+        json.dump(data, f, ensure_ascii=False, indent=2)
+        f.close()

src/utils/index.py

@@ -175,18 +175,3 @@ def get_season_index(input_date):
             index = idx + 1
             break
     return index
-
-# 写文件
-
-
-def write_json_data(data, filename, file_dir=None):
-    import json
-    if not file_dir:
-        cur_date = time.strftime("%Y-%m-%d", time.localtime(time.time()))
-        file_dir = os.getcwd() + '/output/json/' + cur_date + '/'
-    if not os.path.exists(file_dir):
-        os.makedirs(file_dir)
-        print("目录新建成功：%s" % file_dir)
-    with open(file_dir + filename, 'w', encoding='utf-8') as f:
-        json.dump(data, f, ensure_ascii=False, indent=2)
-        f.close()