diff --git a/log/crawler.log b/log/crawler.log index e69de29..a037df3 100644 --- a/log/crawler.log +++ b/log/crawler.log @@ -0,0 +1,50 @@ +2022-11-09 00:28:21,537 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError(0, 'Error'))': /webinterface/Bamboo.ashx?command=fundInfoHead_NEW +2022-11-09 00:29:09,526 WARNING:Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/9edf4fbe845b3cf051afd681e2482a4e/window +2022-11-09 00:29:09,556 WARNING:Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/efdac2e5435c85fb25e8a455916e2d84/window +2022-11-09 00:29:09,564 WARNING:Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/c3772efd61d4d7cfde1b096ad95dab43/window +2022-11-09 00:29:09,565 WARNING:Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/a57eadffd072edd5570b8fa9b63d3569/window +2022-11-09 00:29:09,568 WARNING:Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/c3772efd61d4d7cfde1b096ad95dab43/window +2022-11-09 00:29:09,570 WARNING:Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/efdac2e5435c85fb25e8a455916e2d84/window +2022-11-09 00:29:09,570 WARNING:Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/9edf4fbe845b3cf051afd681e2482a4e/window +2022-11-09 00:29:09,571 WARNING:Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/a57eadffd072edd5570b8fa9b63d3569/window +2022-11-09 00:29:09,571 WARNING:Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/c3772efd61d4d7cfde1b096ad95dab43/window +2022-11-09 00:29:09,602 WARNING:Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/efdac2e5435c85fb25e8a455916e2d84/window +2022-11-09 00:29:09,606 WARNING:Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/a57eadffd072edd5570b8fa9b63d3569/window +2022-11-09 00:29:09,606 WARNING:Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/9edf4fbe845b3cf051afd681e2482a4e/window +2022-11-09 00:29:11,468 WARNING:Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/4adbac5e5f9a66dd7e9f4c97ee4fb80b/url +2022-11-09 00:29:11,493 WARNING:Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/4adbac5e5f9a66dd7e9f4c97ee4fb80b/url +2022-11-09 00:29:11,523 WARNING:Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/4adbac5e5f9a66dd7e9f4c97ee4fb80b/url +2022-11-09 00:29:11,574 WARNING:Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/4adbac5e5f9a66dd7e9f4c97ee4fb80b/window +2022-11-09 00:29:11,615 WARNING:Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/4adbac5e5f9a66dd7e9f4c97ee4fb80b/window +2022-11-09 00:29:11,626 WARNING:Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/4adbac5e5f9a66dd7e9f4c97ee4fb80b/window +2022-11-09 00:44:22,070 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError(0, 'Error'))': /djapi/fund/002096 +2022-11-09 07:15:18,033 WARNING:Retrying (Retry(total=9, connect=2, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known')': /djapi/fund/010555 +2022-11-09 07:15:18,035 WARNING:Retrying (Retry(total=9, connect=4, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known')': /sitedataapi/CryptogramAPI.asmx/GetData?data=2eHwS6xvghs%3D +2022-11-09 07:15:19,307 WARNING:Retrying (Retry(total=8, connect=1, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known')': /djapi/fund/010555 +2022-11-09 07:15:19,334 WARNING:Retrying (Retry(total=8, connect=3, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known')': /sitedataapi/CryptogramAPI.asmx/GetData?data=2eHwS6xvghs%3D +2022-11-09 07:15:21,395 WARNING:Retrying (Retry(total=7, connect=0, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known')': /djapi/fund/010555 +2022-11-09 07:15:21,395 WARNING:Retrying (Retry(total=7, connect=2, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known')': /sitedataapi/CryptogramAPI.asmx/GetData?data=2eHwS6xvghs%3D +2022-11-09 07:20:20,567 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response'))': /djapi/fund/010554 +2022-11-09 08:09:31,596 WARNING:Retrying (Retry(total=9, connect=5, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response'))': /sitedataapi/CryptogramAPI.asmx/GetData?data=6dBuCc30bdc%3D +2022-11-09 08:11:25,531 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response'))': /djapi/fund/013595 +2022-11-09 08:18:57,411 WARNING:Retrying (Retry(total=9, connect=5, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response'))': /sitedataapi/CryptogramAPI.asmx/GetData?data=tOmATFnYWjM%3D +2022-11-09 08:48:20,869 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response'))': /djapi/fund/002561 +2022-11-09 08:48:27,155 WARNING:Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/3f5ab656c20236cea575c0987fc8eee3/window +2022-11-09 08:48:27,157 WARNING:Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/3f5ab656c20236cea575c0987fc8eee3/window +2022-11-09 08:48:27,159 WARNING:Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/3f5ab656c20236cea575c0987fc8eee3/window +2022-11-09 21:08:39,250 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response'))': /djapi/fund/013357 +2022-11-09 21:08:59,569 WARNING:Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/23899b106d883fb73061cf0af60beb2a/window +2022-11-09 21:08:59,574 WARNING:Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/75786692b791ac7f7d3070a322d73f99/window +2022-11-09 21:08:59,580 WARNING:Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/75786692b791ac7f7d3070a322d73f99/window +2022-11-09 21:08:59,582 WARNING:Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/23899b106d883fb73061cf0af60beb2a/window +2022-11-09 21:08:59,586 WARNING:Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/75786692b791ac7f7d3070a322d73f99/window +2022-11-09 21:08:59,592 WARNING:Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError(': Failed to establish a new connection: [Errno 61] Connection refused')': /session/23899b106d883fb73061cf0af60beb2a/window +2022-11-09 21:35:00,312 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError(0, 'Error'))': /djapi/fund/006440 +2022-11-09 21:38:49,299 WARNING:Retrying (Retry(total=9, connect=5, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response'))': /sitedataapi/CryptogramAPI.asmx/GetData?data=OtgCHL1S71k%3D +2022-11-09 21:39:37,443 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError(0, 'Error'))': /webinterface/Bamboo.ashx?command=fundInfoHead_NEW +2022-11-09 21:48:25,412 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError(0, 'Error'))': /djapi/fund/006536 +2022-11-09 21:57:06,514 WARNING:Retrying (Retry(total=9, connect=5, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError(0, 'Error'))': /sitedataapi/CryptogramAPI.asmx/GetData?data=%2FNc6LAhYm%2Bw%3D +2022-11-09 22:05:22,159 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response'))': /webinterface/Bamboo.ashx?command=fundInfoHead_NEW +2022-11-09 22:13:55,786 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError(0, 'Error'))': /webinterface/Bamboo.ashx?command=fundInfoHead_NEW +2022-11-09 22:18:57,639 WARNING:Retrying (Retry(total=9, connect=5, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response'))': /sitedataapi/CryptogramAPI.asmx/GetData?data=g26DjlCJLxA%3D +2022-11-09 22:24:01,732 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError(0, 'Error'))': /djapi/fund/012734 diff --git a/src/acquire_fund_quarter.py b/src/acquire_fund_quarter.py index d223681..480f616 100644 --- a/src/acquire_fund_quarter.py +++ b/src/acquire_fund_quarter.py @@ -69,7 +69,7 @@ def acquire_fund_quarter(): for record in results: fund_code = record[0] if fund_code in error_funds_with_page or fund_code in error_funds_with_found_date or fund_code in error_funds_with_unmatch: - print('error fund: ', fund_code) + print('exist error fund: ', fund_code) continue each_fund = FundSpider(fund_code, record[1], record[2], chrome_driver) each_fund.set_found_data(record[3]) @@ -184,16 +184,15 @@ def acquire_fund_quarter(): stock_dict[portion_key] = temp_stock['stock_portion'] market_key = prefix + 'market' stock_dict[market_key] = temp_stock['stock_market'] - # 获取同类基金,再获取同类基金的总资产 if each_fund.fund_name.endswith('A') or each_fund.fund_name.endswith('B') or each_fund.fund_name.endswith('C'): similar_name = each_fund.fund_name[0:-1] - results = each_fund_query.select_similar_fund( + similar_results = each_fund_query.select_similar_fund( similar_name) # 获取查询的所有记录 # platform = 'zh_fund' if '封闭' in similar_name else 'ai_fund' platform = 'danjuan' - for i in range(0, len(results)): - item = results[i] + for i in range(0, len(similar_results)): + item = similar_results[i] item_code = item[0] if item_code == each_fund.fund_code: continue diff --git a/src/sql_model/fund_query.py b/src/sql_model/fund_query.py index 9cb9d59..a025de0 100644 --- a/src/sql_model/fund_query.py +++ b/src/sql_model/fund_query.py @@ -188,7 +188,6 @@ class FundQuery(BaseQuery): sql_similar = "SELECT t.fund_code,\ t.morning_star_code, t.fund_name \ FROM fund_morning_base as t \ - LEFT JOIN fund_morning_snapshot as f ON f.fund_code = t.fund_code \ WHERE t.fund_name LIKE %s;" # AND t.fund_name NOT LIKE '%%A';" diff --git a/src/utils/file_op.py b/src/utils/file_op.py index 2ccb254..23a280c 100644 --- a/src/utils/file_op.py +++ b/src/utils/file_op.py @@ -124,6 +124,7 @@ def read_error_code_from_json(): my_data = json.load(json_file) error_funds_with_page = my_data.get('error_funds_with_page') error_funds_with_found_date = my_data.get('error_funds_with_found_date') + error_funds_with_unmatch = my_data.get('error_funds_with_unmatch') return { "file_dir": file_dir, "filename": filename,