Solin
/
fund-morning-star-crawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: 🐛 complete error_funds_with_unmatch

Browse Source
main
jackluson 2 years ago
parent a729b2c51e
commit da060db601
4 changed files with 55 additions and 6 deletions
Show Stats Download Patch File Download Diff File

50
log/crawler.log
Unescape Escape View File

@ -0,0 +1,50 @@
2022-11-09 00:28:21,537 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError(0, 'Error'))': /webinterface/Bamboo.ashx?command=fundInfoHead_NEW
2022-11-09 00:29:09,526 WARNING:Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15c9f57f0>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/9edf4fbe845b3cf051afd681e2482a4e/window
2022-11-09 00:29:09,556 WARNING:Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15e47b128>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/efdac2e5435c85fb25e8a455916e2d84/window
2022-11-09 00:29:09,564 WARNING:Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15e47b9b0>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/c3772efd61d4d7cfde1b096ad95dab43/window
2022-11-09 00:29:09,565 WARNING:Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15e47bd68>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/a57eadffd072edd5570b8fa9b63d3569/window
2022-11-09 00:29:09,568 WARNING:Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15e56ad30>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/c3772efd61d4d7cfde1b096ad95dab43/window
2022-11-09 00:29:09,570 WARNING:Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15e56a390>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/efdac2e5435c85fb25e8a455916e2d84/window
2022-11-09 00:29:09,570 WARNING:Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15c9ddd30>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/9edf4fbe845b3cf051afd681e2482a4e/window
2022-11-09 00:29:09,571 WARNING:Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15e56ad68>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/a57eadffd072edd5570b8fa9b63d3569/window
2022-11-09 00:29:09,571 WARNING:Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15e56a978>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/c3772efd61d4d7cfde1b096ad95dab43/window
2022-11-09 00:29:09,602 WARNING:Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15e46a160>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/efdac2e5435c85fb25e8a455916e2d84/window
2022-11-09 00:29:09,606 WARNING:Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15e424ac8>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/a57eadffd072edd5570b8fa9b63d3569/window
2022-11-09 00:29:09,606 WARNING:Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15e46a048>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/9edf4fbe845b3cf051afd681e2482a4e/window
2022-11-09 00:29:11,468 WARNING:Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15ca77160>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/4adbac5e5f9a66dd7e9f4c97ee4fb80b/url
2022-11-09 00:29:11,493 WARNING:Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15e47bcc0>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/4adbac5e5f9a66dd7e9f4c97ee4fb80b/url
2022-11-09 00:29:11,523 WARNING:Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15e45bc50>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/4adbac5e5f9a66dd7e9f4c97ee4fb80b/url
2022-11-09 00:29:11,574 WARNING:Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15ca94fd0>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/4adbac5e5f9a66dd7e9f4c97ee4fb80b/window
2022-11-09 00:29:11,615 WARNING:Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15e46f898>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/4adbac5e5f9a66dd7e9f4c97ee4fb80b/window
2022-11-09 00:29:11,626 WARNING:Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x11445c0b8>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/4adbac5e5f9a66dd7e9f4c97ee4fb80b/window
2022-11-09 00:44:22,070 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError(0, 'Error'))': /djapi/fund/002096
2022-11-09 07:15:18,033 WARNING:Retrying (Retry(total=9, connect=2, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x1592e12b0>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known')': /djapi/fund/010555
2022-11-09 07:15:18,035 WARNING:Retrying (Retry(total=9, connect=4, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x15ad73f98>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known')': /sitedataapi/CryptogramAPI.asmx/GetData?data=2eHwS6xvghs%3D
2022-11-09 07:15:19,307 WARNING:Retrying (Retry(total=8, connect=1, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x15aca4b38>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known')': /djapi/fund/010555
2022-11-09 07:15:19,334 WARNING:Retrying (Retry(total=8, connect=3, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x15acbda20>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known')': /sitedataapi/CryptogramAPI.asmx/GetData?data=2eHwS6xvghs%3D
2022-11-09 07:15:21,395 WARNING:Retrying (Retry(total=7, connect=0, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x1592e1780>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known')': /djapi/fund/010555
2022-11-09 07:15:21,395 WARNING:Retrying (Retry(total=7, connect=2, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x1592e1080>: Failed to establish a new connection: [Errno 8] nodename nor servname provided, or not known')': /sitedataapi/CryptogramAPI.asmx/GetData?data=2eHwS6xvghs%3D
2022-11-09 07:20:20,567 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response'))': /djapi/fund/010554
2022-11-09 08:09:31,596 WARNING:Retrying (Retry(total=9, connect=5, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response'))': /sitedataapi/CryptogramAPI.asmx/GetData?data=6dBuCc30bdc%3D
2022-11-09 08:11:25,531 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response'))': /djapi/fund/013595
2022-11-09 08:18:57,411 WARNING:Retrying (Retry(total=9, connect=5, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response'))': /sitedataapi/CryptogramAPI.asmx/GetData?data=tOmATFnYWjM%3D
2022-11-09 08:48:20,869 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response'))': /djapi/fund/002561
2022-11-09 08:48:27,155 WARNING:Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15acef5f8>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/3f5ab656c20236cea575c0987fc8eee3/window
2022-11-09 08:48:27,157 WARNING:Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15ace6d30>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/3f5ab656c20236cea575c0987fc8eee3/window
2022-11-09 08:48:27,159 WARNING:Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15ace6978>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/3f5ab656c20236cea575c0987fc8eee3/window
2022-11-09 21:08:39,250 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response'))': /djapi/fund/013357
2022-11-09 21:08:59,569 WARNING:Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15bc61160>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/23899b106d883fb73061cf0af60beb2a/window
2022-11-09 21:08:59,574 WARNING:Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15bc61978>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/75786692b791ac7f7d3070a322d73f99/window
2022-11-09 21:08:59,580 WARNING:Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15baf2e10>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/75786692b791ac7f7d3070a322d73f99/window
2022-11-09 21:08:59,582 WARNING:Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15bafe198>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/23899b106d883fb73061cf0af60beb2a/window
2022-11-09 21:08:59,586 WARNING:Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15baf21d0>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/75786692b791ac7f7d3070a322d73f99/window
2022-11-09 21:08:59,592 WARNING:Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'NewConnectionError('<urllib3.connection.HTTPConnection object at 0x15baf2eb8>: Failed to establish a new connection: [Errno 61] Connection refused')': /session/23899b106d883fb73061cf0af60beb2a/window
2022-11-09 21:35:00,312 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError(0, 'Error'))': /djapi/fund/006440
2022-11-09 21:38:49,299 WARNING:Retrying (Retry(total=9, connect=5, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response'))': /sitedataapi/CryptogramAPI.asmx/GetData?data=OtgCHL1S71k%3D
2022-11-09 21:39:37,443 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError(0, 'Error'))': /webinterface/Bamboo.ashx?command=fundInfoHead_NEW
2022-11-09 21:48:25,412 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError(0, 'Error'))': /djapi/fund/006536
2022-11-09 21:57:06,514 WARNING:Retrying (Retry(total=9, connect=5, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError(0, 'Error'))': /sitedataapi/CryptogramAPI.asmx/GetData?data=%2FNc6LAhYm%2Bw%3D
2022-11-09 22:05:22,159 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response'))': /webinterface/Bamboo.ashx?command=fundInfoHead_NEW
2022-11-09 22:13:55,786 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError(0, 'Error'))': /webinterface/Bamboo.ashx?command=fundInfoHead_NEW
2022-11-09 22:18:57,639 WARNING:Retrying (Retry(total=9, connect=5, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response'))': /sitedataapi/CryptogramAPI.asmx/GetData?data=g26DjlCJLxA%3D
2022-11-09 22:24:01,732 WARNING:Retrying (Retry(total=9, connect=3, read=None, redirect=None, status=None)) after connection broken by 'ProxyError('Cannot connect to proxy.', OSError(0, 'Error'))': /djapi/fund/012734

9
src/acquire_fund_quarter.py
Unescape Escape View File

@ -69,7 +69,7 @@ def acquire_fund_quarter():
for record in results:
fund_code = record[0]
if fund_code in error_funds_with_page or fund_code in error_funds_with_found_date or fund_code in error_funds_with_unmatch:
print('error fund: ', fund_code)
print('exist error fund: ', fund_code)
continue
each_fund = FundSpider(fund_code, record[1], record[2], chrome_driver)
each_fund.set_found_data(record[3])
@ -184,16 +184,15 @@ def acquire_fund_quarter():
stock_dict[portion_key] = temp_stock['stock_portion']
market_key = prefix + 'market'
stock_dict[market_key] = temp_stock['stock_market']
# 获取同类基金，再获取同类基金的总资产
if each_fund.fund_name.endswith('A') or each_fund.fund_name.endswith('B') or each_fund.fund_name.endswith('C'):
similar_name = each_fund.fund_name[0:-1]
results = each_fund_query.select_similar_fund(
similar_results = each_fund_query.select_similar_fund(
similar_name) # 获取查询的所有记录
# platform = 'zh_fund' if '封闭' in similar_name else 'ai_fund'
platform = 'danjuan'
for i in range(0, len(results)):
item = results[i]
for i in range(0, len(similar_results)):
item = similar_results[i]
item_code = item[0]
if item_code == each_fund.fund_code:
continue

1
src/sql_model/fund_query.py
Unescape Escape View File

@ -188,7 +188,6 @@ class FundQuery(BaseQuery):
sql_similar = "SELECT t.fund_code,\
t.morning_star_code, t.fund_name \
FROM fund_morning_base as t \
LEFT JOIN fund_morning_snapshot as f ON f.fund_code = t.fund_code \
WHERE t.fund_name LIKE %s;"
# AND t.fund_name NOT LIKE '%%A';"

1
src/utils/file_op.py
Unescape Escape View File

@ -124,6 +124,7 @@ def read_error_code_from_json():
my_data = json.load(json_file)
error_funds_with_page = my_data.get('error_funds_with_page')
error_funds_with_found_date = my_data.get('error_funds_with_found_date')
error_funds_with_unmatch = my_data.get('error_funds_with_unmatch')
return {
"file_dir": file_dir,
"filename": filename,

Write Preview
Loading…
Cancel
Save