How to use log method in ava

Best JavaScript code snippet using ava

save_tdx.py

Source:save_tdx.py

1# coding:utf-82#3# The MIT License (MIT)4#5# Copyright (c) 2016-2021 yutiansut/QUANTAXIS6#7# Permission is hereby granted, free of charge, to any person obtaining a copy8# of this software and associated documentation files (the "Software"), to deal9# in the Software without restriction, including without limitation the rights10# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell11# copies of the Software, and to permit persons to whom the Software is12# furnished to do so, subject to the following conditions:13#14# The above copyright notice and this permission notice shall be included in all15# copies or substantial portions of the Software.16#17# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR18# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,19# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE20# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER21# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,22# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE23# SOFTWARE.24import concurrent25import datetime26from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor27import json28import pandas as pd29import pymongo30from QUANTAXIS.QAFetch import QA_fetch_get_stock_block31from QUANTAXIS.QAFetch.QATdx import (32 QA_fetch_get_option_day,33 QA_fetch_get_option_min,34 QA_fetch_get_index_day,35 QA_fetch_get_index_min,36 QA_fetch_get_stock_day,37 QA_fetch_get_stock_info,38 QA_fetch_get_stock_list,39 QA_fetch_get_future_list,40 QA_fetch_get_index_list,41 QA_fetch_get_future_day,42 QA_fetch_get_future_min,43 QA_fetch_get_stock_min,44 QA_fetch_get_stock_transaction,45 QA_fetch_get_index_transaction,46 QA_fetch_get_stock_xdxr,47 QA_fetch_get_bond_day,48 QA_fetch_get_bond_list,49 QA_fetch_get_bond_min,50 select_best_ip,51 QA_fetch_get_hkstock_day,52 QA_fetch_get_hkstock_list,53 QA_fetch_get_hkstock_min,54 QA_fetch_get_usstock_list,55 QA_fetch_get_usstock_day,56 QA_fetch_get_usstock_min,57)58from QUANTAXIS.QAFetch.QATdx import (59 QA_fetch_get_commodity_option_AL_contract_time_to_market,60 QA_fetch_get_commodity_option_AU_contract_time_to_market,61 QA_fetch_get_commodity_option_CU_contract_time_to_market,62 QA_fetch_get_commodity_option_SR_contract_time_to_market,63 QA_fetch_get_commodity_option_M_contract_time_to_market,64 QA_fetch_get_commodity_option_RU_contract_time_to_market,65 QA_fetch_get_commodity_option_CF_contract_time_to_market,66 QA_fetch_get_commodity_option_C_contract_time_to_market,67 QA_fetch_get_option_50etf_contract_time_to_market,68 QA_fetch_get_option_300etf_contract_time_to_market,69 QA_fetch_get_option_all_contract_time_to_market,70 QA_fetch_get_option_list,71)72from QUANTAXIS.QAUtil import (73 DATABASE,74 QA_util_get_next_day,75 QA_util_get_real_date,76 QA_util_log_info,77 QA_util_to_json_from_pandas,78 trade_date_sse79)80from QUANTAXIS.QAData.data_fq import _QA_data_stock_to_fq81from QUANTAXIS.QAFetch.QAQuery import QA_fetch_stock_day82from QUANTAXIS.QAUtil import Parallelism83from QUANTAXIS.QAFetch.QATdx import ping, get_ip_list_by_multi_process_ping, stock_ip_list84from multiprocessing import cpu_count85# ip=select_best_ip()86def now_time():87 return str(QA_util_get_real_date(str(datetime.date.today() - datetime.timedelta(days=1)), trade_date_sse, -1)) + \88 ' 17:00:00' if datetime.datetime.now().hour < 15 else str(QA_util_get_real_date(89 str(datetime.date.today()), trade_date_sse, -1)) + ' 15:00:00'90def QA_SU_save_single_stock_day(code : str, client= DATABASE, ui_log=None):91 '''92 save single stock_day93 保存单个股票日线数据94 :param code: 要保存数据的股票代码95 :param client:96 :param ui_log: 给GUI qt 界面使用97 :param ui_progress: 给GUI qt 界面使用98 '''99 #stock_list = QA_fetch_get_stock_list().code.unique().tolist()100 coll_stock_day = client.stock_day101 coll_stock_day.create_index(102 [("code",103 pymongo.ASCENDING),104 ("date_stamp",105 pymongo.ASCENDING)]106 )107 err = []108 def __saving_work(code, coll_stock_day):109 try:110 QA_util_log_info(111 '##JOB01 Now Saving STOCK_DAY==== {}'.format(str(code)),112 ui_log113 )114 # 首选查找数据库 是否 有 这个代码的数据115 ref = coll_stock_day.find({'code': str(code)[0:6]})116 end_date = str(now_time())[0:10]117 # 当前数据库已经包含了这个代码的数据， 继续增量更新118 # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现119 if ref.count() > 0:120 # 接着上次获取的日期继续更新121 start_date = ref[ref.count() - 1]['date']122 QA_util_log_info(123 'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'124 .format(code,125 start_date,126 end_date),127 ui_log128 )129 if start_date != end_date:130 coll_stock_day.insert_many(131 QA_util_to_json_from_pandas(132 QA_fetch_get_stock_day(133 str(code),134 QA_util_get_next_day(start_date),135 end_date,136 '00'137 )138 )139 )140 # 当前数据库中没有这个代码的股票数据， 从1990-01-01 开始下载所有的数据141 else:142 start_date = '1990-01-01'143 QA_util_log_info(144 'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'145 .format(code,146 start_date,147 end_date),148 ui_log149 )150 if start_date != end_date:151 coll_stock_day.insert_many(152 QA_util_to_json_from_pandas(153 QA_fetch_get_stock_day(154 str(code),155 start_date,156 end_date,157 '00'158 )159 )160 )161 except Exception as error0:162 print(error0)163 err.append(str(code))164 __saving_work(code, coll_stock_day)165 if len(err) < 1:166 QA_util_log_info('SUCCESS save stock day ^_^', ui_log)167 else:168 QA_util_log_info('ERROR CODE \n ', ui_log)169 QA_util_log_info(err, ui_log)170def QA_SU_save_stock_day(client=DATABASE, ui_log=None, ui_progress=None):171 '''172 save stock_day173 保存日线数据174 :param client:175 :param ui_log: 给GUI qt 界面使用176 :param ui_progress: 给GUI qt 界面使用177 :param ui_progress_int_value: 给GUI qt 界面使用178 '''179 stock_list = QA_fetch_get_stock_list().code.unique().tolist()180 coll_stock_day = client.stock_day181 coll_stock_day.create_index(182 [("code",183 pymongo.ASCENDING),184 ("date_stamp",185 pymongo.ASCENDING)]186 )187 err = []188 def __saving_work(code, coll_stock_day):189 try:190 QA_util_log_info(191 '##JOB01 Now Saving STOCK_DAY==== {}'.format(str(code)),192 ui_log193 )194 # 首选查找数据库 是否 有 这个代码的数据195 ref = coll_stock_day.find({'code': str(code)[0:6]})196 end_date = str(now_time())[0:10]197 # 当前数据库已经包含了这个代码的数据， 继续增量更新198 # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现199 if ref.count() > 0:200 # 接着上次获取的日期继续更新201 start_date = ref[ref.count() - 1]['date']202 QA_util_log_info(203 'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'204 .format(code,205 start_date,206 end_date),207 ui_log208 )209 if start_date != end_date:210 coll_stock_day.insert_many(211 QA_util_to_json_from_pandas(212 QA_fetch_get_stock_day(213 str(code),214 QA_util_get_next_day(start_date),215 end_date,216 '00'217 )218 )219 )220 # 当前数据库中没有这个代码的股票数据， 从1990-01-01 开始下载所有的数据221 else:222 start_date = '1990-01-01'223 QA_util_log_info(224 'UPDATE_STOCK_DAY \n Trying updating {} from {} to {}'225 .format(code,226 start_date,227 end_date),228 ui_log229 )230 if start_date != end_date:231 coll_stock_day.insert_many(232 QA_util_to_json_from_pandas(233 QA_fetch_get_stock_day(234 str(code),235 start_date,236 end_date,237 '00'238 )239 )240 )241 except Exception as error0:242 print(error0)243 err.append(str(code))244 for item in range(len(stock_list)):245 QA_util_log_info('The {} of Total {}'.format(item, len(stock_list)))246 strProgressToLog = 'DOWNLOAD PROGRESS {} {}'.format(247 str(float(item / len(stock_list) * 100))[0:4] + '%',248 ui_log249 )250 intProgressToLog = int(float(item / len(stock_list) * 100))251 QA_util_log_info(252 strProgressToLog,253 ui_log=ui_log,254 ui_progress=ui_progress,255 ui_progress_int_value=intProgressToLog256 )257 __saving_work(stock_list[item], coll_stock_day)258 if len(err) < 1:259 QA_util_log_info('SUCCESS save stock day ^_^', ui_log)260 else:261 QA_util_log_info('ERROR CODE \n ', ui_log)262 QA_util_log_info(err, ui_log)263def gen_param(codelist, start_date=None, end_date=None, if_fq='00', frequence='day', IPList=[]):264 # 生成QA.QAFetch.QATdx.QQA_fetch_get_stock_day多进程处理的参数265 count = len(IPList)266 my_iterator = iter(range(len(codelist)))267 start_date = str(start_date)[0:10]268 end_date = str(end_date)[0:10]269 return [(code, start_date, end_date, if_fq, frequence, IPList[i % count]['ip'], IPList[i % count]['port'])270 for code, i in [(code, next(my_iterator) % count) for code in codelist]]271def QA_SU_save_stock_week(client=DATABASE, ui_log=None, ui_progress=None):272 """save stock_week273 Keyword Arguments:274 client {[type]} -- [description] (default: {DATABASE})275 """276 stock_list = QA_fetch_get_stock_list().code.unique().tolist()277 coll_stock_week = client.stock_week278 coll_stock_week.create_index(279 [("code",280 pymongo.ASCENDING),281 ("date_stamp",282 pymongo.ASCENDING)]283 )284 err = []285 def __saving_work(code, coll_stock_week):286 try:287 QA_util_log_info(288 '##JOB01 Now Saving STOCK_WEEK==== {}'.format(str(code)),289 ui_log=ui_log290 )291 ref = coll_stock_week.find({'code': str(code)[0:6]})292 end_date = str(now_time())[0:10]293 if ref.count() > 0:294 # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现295 start_date = ref[ref.count() - 1]['date']296 QA_util_log_info(297 'UPDATE_STOCK_WEEK \n Trying updating {} from {} to {}'298 .format(code,299 start_date,300 end_date),301 ui_log=ui_log302 )303 if start_date != end_date:304 coll_stock_week.insert_many(305 QA_util_to_json_from_pandas(306 QA_fetch_get_stock_day(307 str(code),308 QA_util_get_next_day(start_date),309 end_date,310 '00',311 frequence='week'312 )313 )314 )315 else:316 start_date = '1990-01-01'317 QA_util_log_info(318 'UPDATE_STOCK_WEEK \n Trying updating {} from {} to {}'319 .format(code,320 start_date,321 end_date),322 ui_log=ui_log323 )324 if start_date != end_date:325 coll_stock_week.insert_many(326 QA_util_to_json_from_pandas(327 QA_fetch_get_stock_day(328 str(code),329 start_date,330 end_date,331 '00',332 frequence='week'333 )334 )335 )336 except:337 err.append(str(code))338 for item in range(len(stock_list)):339 QA_util_log_info(340 'The {} of Total {}'.format(item,341 len(stock_list)),342 ui_log=ui_log343 )344 strProgress = 'DOWNLOAD PROGRESS {} '.format(345 str(float(item / len(stock_list) * 100))[0:4] + '%'346 )347 intProgress = int(float(item / len(stock_list) * 100))348 QA_util_log_info(349 strProgress,350 ui_log=ui_log,351 ui_progress=ui_progress,352 ui_progress_int_value=intProgress353 )354 __saving_work(stock_list[item], coll_stock_week)355 if len(err) < 1:356 QA_util_log_info('SUCCESS', ui_log=ui_log)357 else:358 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)359 QA_util_log_info(err, ui_log=ui_log)360def QA_SU_save_stock_month(client=DATABASE, ui_log=None, ui_progress=None):361 """save stock_month362 Keyword Arguments:363 client {[type]} -- [description] (default: {DATABASE})364 """365 stock_list = QA_fetch_get_stock_list().code.unique().tolist()366 coll_stock_month = client.stock_month367 coll_stock_month.create_index(368 [("code",369 pymongo.ASCENDING),370 ("date_stamp",371 pymongo.ASCENDING)]372 )373 err = []374 def __saving_work(code, coll_stock_month):375 try:376 QA_util_log_info(377 '##JOB01 Now Saving STOCK_MONTH==== {}'.format(str(code)),378 ui_log=ui_log379 )380 ref = coll_stock_month.find({'code': str(code)[0:6]})381 end_date = str(now_time())[0:10]382 if ref.count() > 0:383 # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现384 start_date = ref[ref.count() - 1]['date']385 QA_util_log_info(386 'UPDATE_STOCK_MONTH \n Trying updating {} from {} to {}'387 .format(code,388 start_date,389 end_date),390 ui_log=ui_log391 )392 if start_date != end_date:393 coll_stock_month.insert_many(394 QA_util_to_json_from_pandas(395 QA_fetch_get_stock_day(396 str(code),397 QA_util_get_next_day(start_date),398 end_date,399 '00',400 frequence='month'401 )402 )403 )404 else:405 start_date = '1990-01-01'406 QA_util_log_info(407 'UPDATE_STOCK_MONTH \n Trying updating {} from {} to {}'408 .format(code,409 start_date,410 end_date),411 ui_log=ui_log412 )413 if start_date != end_date:414 coll_stock_month.insert_many(415 QA_util_to_json_from_pandas(416 QA_fetch_get_stock_day(417 str(code),418 start_date,419 end_date,420 '00',421 frequence='month'422 )423 )424 )425 except:426 err.append(str(code))427 for item in range(len(stock_list)):428 QA_util_log_info(429 'The {} of Total {}'.format(item,430 len(stock_list)),431 ui_log=ui_log432 )433 strProgress = 'DOWNLOAD PROGRESS {} '.format(434 str(float(item / len(stock_list) * 100))[0:4] + '%'435 )436 intProgress = int(float(item / len(stock_list) * 100))437 QA_util_log_info(438 strProgress,439 ui_log=ui_log,440 ui_progress=ui_progress,441 ui_progress_int_value=intProgress442 )443 __saving_work(stock_list[item], coll_stock_month)444 if len(err) < 1:445 QA_util_log_info('SUCCESS', ui_log=ui_log)446 else:447 QA_util_log_info('ERROR CODE \n ', ui_log=ui_log)448 QA_util_log_info(err, ui_log=ui_log)449def QA_SU_save_stock_year(client=DATABASE, ui_log=None, ui_progress=None):450 """save stock_year451 Keyword Arguments:452 client {[type]} -- [description] (default: {DATABASE})453 """454 stock_list = QA_fetch_get_stock_list().code.unique().tolist()455 coll_stock_year = client.stock_year456 coll_stock_year.create_index(457 [("code",458 pymongo.ASCENDING),459 ("date_stamp",460 pymongo.ASCENDING)]461 )462 err = []463 def __saving_work(code, coll_stock_year):464 try:465 QA_util_log_info(466 '##JOB01 Now Saving STOCK_YEAR==== {}'.format(str(code)),467 ui_log=ui_log468 )469 ref = coll_stock_year.find({'code': str(code)[0:6]})470 end_date = str(now_time())[0:10]471 if ref.count() > 0:472 # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现473 start_date = ref[ref.count() - 1]['date']474 QA_util_log_info(475 'UPDATE_STOCK_YEAR \n Trying updating {} from {} to {}'476 .format(code,477 start_date,478 end_date),479 ui_log=ui_log480 )481 if start_date != end_date:482 coll_stock_year.insert_many(483 QA_util_to_json_from_pandas(484 QA_fetch_get_stock_day(485 str(code),486 QA_util_get_next_day(start_date),487 end_date,488 '00',489 frequence='year'490 )491 )492 )493 else:494 start_date = '1990-01-01'495 QA_util_log_info(496 'UPDATE_STOCK_YEAR \n Trying updating {} from {} to {}'497 .format(code,498 start_date,499 end_date),500 ui_log=ui_log501 )502 if start_date != end_date:503 coll_stock_year.insert_many(504 QA_util_to_json_from_pandas(505 QA_fetch_get_stock_day(506 str(code),507 start_date,508 end_date,509 '00',510 frequence='year'511 )512 )513 )514 except:515 err.append(str(code))516 for item in range(len(stock_list)):517 QA_util_log_info(518 'The {} of Total {}'.format(item,519 len(stock_list)),520 ui_log=ui_log521 )522 strProgress = 'DOWNLOAD PROGRESS {} '.format(523 str(float(item / len(stock_list) * 100))[0:4] + '%'524 )525 intProgress = int(float(item / len(stock_list) * 100))526 QA_util_log_info(527 strProgress,528 ui_log=ui_log,529 ui_progress=ui_progress,530 ui_progress_int_value=intProgress531 )532 __saving_work(stock_list[item], coll_stock_year)533 if len(err) < 1:534 QA_util_log_info('SUCCESS', ui_log=ui_log)535 else:536 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)537 QA_util_log_info(err, ui_log=ui_log)538def QA_SU_save_stock_xdxr(client=DATABASE, ui_log=None, ui_progress=None):539 """[summary]540 Keyword Arguments:541 client {[type]} -- [description] (default: {DATABASE})542 """543 stock_list = QA_fetch_get_stock_list().code.unique().tolist()544 # client.drop_collection('stock_xdxr')545 try:546 coll = client.stock_xdxr547 coll.create_index(548 [('code',549 pymongo.ASCENDING),550 ('date',551 pymongo.ASCENDING)],552 unique=True553 )554 coll_adj = client.stock_adj555 coll_adj.create_index(556 [('code',557 pymongo.ASCENDING),558 ('date',559 pymongo.ASCENDING)],560 unique=True561 )562 except:563 client.drop_collection('stock_xdxr')564 coll = client.stock_xdxr565 coll.create_index(566 [('code',567 pymongo.ASCENDING),568 ('date',569 pymongo.ASCENDING)],570 unique=True571 )572 client.drop_collection('stock_adj')573 coll_adj = client.stock_adj574 coll_adj.create_index(575 [('code',576 pymongo.ASCENDING),577 ('date',578 pymongo.ASCENDING)],579 unique=True580 )581 err = []582 def __saving_work(code, coll):583 QA_util_log_info(584 '##JOB02 Now Saving XDXR INFO ==== {}'.format(str(code)),585 ui_log=ui_log586 )587 try:588 xdxr = QA_fetch_get_stock_xdxr(str(code))589 try:590 coll.insert_many(591 QA_util_to_json_from_pandas(xdxr),592 ordered=False593 )594 except:595 pass596 try:597 data = QA_fetch_stock_day(str(code), '1990-01-01',str(datetime.date.today()), 'pd')598 qfq = _QA_data_stock_to_fq(data, xdxr, 'qfq')599 qfq = qfq.assign(date=qfq.date.apply(lambda x: str(x)[0:10]))600 adjdata = QA_util_to_json_from_pandas(qfq.loc[:, ['date','code', 'adj']])601 coll_adj.delete_many({'code': code})602 #print(adjdata)603 coll_adj.insert_many(adjdata)604 except Exception as e:605 print(e)606 except Exception as e:607 print(e)608 err.append(str(code))609 for i_ in range(len(stock_list)):610 QA_util_log_info(611 'The {} of Total {}'.format(i_,612 len(stock_list)),613 ui_log=ui_log614 )615 strLogInfo = 'DOWNLOAD PROGRESS {} '.format(616 str(float(i_ / len(stock_list) * 100))[0:4] + '%'617 )618 intLogProgress = int(float(i_ / len(stock_list) * 100))619 QA_util_log_info(620 strLogInfo,621 ui_log=ui_log,622 ui_progress=ui_progress,623 ui_progress_int_value=intLogProgress624 )625 __saving_work(stock_list[i_], coll)626def QA_SU_save_stock_min(client=DATABASE, ui_log=None, ui_progress=None):627 """save stock_min628 Keyword Arguments:629 client {[type]} -- [description] (default: {DATABASE})630 """631 stock_list = QA_fetch_get_stock_list().code.unique().tolist()632 coll = client.stock_min633 coll.create_index(634 [635 ('code',636 pymongo.ASCENDING),637 ('time_stamp',638 pymongo.ASCENDING),639 ('date_stamp',640 pymongo.ASCENDING)641 ]642 )643 err = []644 def __saving_work(code, coll):645 QA_util_log_info(646 '##JOB03 Now Saving STOCK_MIN ==== {}'.format(str(code)),647 ui_log=ui_log648 )649 try:650 for type in ['1min', '5min', '15min', '30min', '60min']:651 ref_ = coll.find({'code': str(code)[0:6], 'type': type})652 end_time = str(now_time())[0:19]653 if ref_.count() > 0:654 start_time = ref_[ref_.count() - 1]['datetime']655 QA_util_log_info(656 '##JOB03.{} Now Saving {} from {} to {} =={} '.format(657 ['1min',658 '5min',659 '15min',660 '30min',661 '60min'].index(type),662 str(code),663 start_time,664 end_time,665 type666 ),667 ui_log=ui_log668 )669 if start_time != end_time:670 __data = QA_fetch_get_stock_min(671 str(code),672 start_time,673 end_time,674 type675 )676 if len(__data) > 1:677 coll.insert_many(678 QA_util_to_json_from_pandas(__data)[1::]679 )680 else:681 start_time = '2015-01-01'682 QA_util_log_info(683 '##JOB03.{} Now Saving {} from {} to {} =={} '.format(684 ['1min',685 '5min',686 '15min',687 '30min',688 '60min'].index(type),689 str(code),690 start_time,691 end_time,692 type693 ),694 ui_log=ui_log695 )696 if start_time != end_time:697 __data = QA_fetch_get_stock_min(698 str(code),699 start_time,700 end_time,701 type702 )703 if len(__data) > 1:704 coll.insert_many(705 QA_util_to_json_from_pandas(__data)706 )707 except Exception as e:708 QA_util_log_info(e, ui_log=ui_log)709 err.append(code)710 QA_util_log_info(err, ui_log=ui_log)711 executor = ThreadPoolExecutor(max_workers=4)712 # executor.map((__saving_work, stock_list[i_], coll),URLS)713 res = {714 executor.submit(__saving_work,715 stock_list[i_],716 coll)717 for i_ in range(len(stock_list))718 }719 count = 0720 for i_ in concurrent.futures.as_completed(res):721 QA_util_log_info(722 'The {} of Total {}'.format(count,723 len(stock_list)),724 ui_log=ui_log725 )726 strProgress = 'DOWNLOAD PROGRESS {} '.format(727 str(float(count / len(stock_list) * 100))[0:4] + '%'728 )729 intProgress = int(count / len(stock_list) * 10000.0)730 QA_util_log_info(731 strProgress,732 ui_log,733 ui_progress=ui_progress,734 ui_progress_int_value=intProgress735 )736 count = count + 1737 if len(err) < 1:738 QA_util_log_info('SUCCESS', ui_log=ui_log)739 else:740 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)741 QA_util_log_info(err, ui_log=ui_log)742def QA_SU_save_single_stock_min(code : str, client=DATABASE, ui_log=None, ui_progress=None):743 """save single stock_min744 Keyword Arguments:745 client {[type]} -- [description] (default: {DATABASE})746 """747 #stock_list = QA_fetch_get_stock_list().code.unique().tolist()748 stock_list = [code]749 coll = client.stock_min750 coll.create_index(751 [752 ('code',753 pymongo.ASCENDING),754 ('time_stamp',755 pymongo.ASCENDING),756 ('date_stamp',757 pymongo.ASCENDING)758 ]759 )760 err = []761 def __saving_work(code, coll):762 QA_util_log_info(763 '##JOB03 Now Saving STOCK_MIN ==== {}'.format(str(code)),764 ui_log=ui_log765 )766 try:767 for type in ['1min', '5min', '15min', '30min', '60min']:768 ref_ = coll.find({'code': str(code)[0:6], 'type': type})769 end_time = str(now_time())[0:19]770 if ref_.count() > 0:771 start_time = ref_[ref_.count() - 1]['datetime']772 QA_util_log_info(773 '##JOB03.{} Now Saving {} from {} to {} =={} '.format(774 ['1min',775 '5min',776 '15min',777 '30min',778 '60min'].index(type),779 str(code),780 start_time,781 end_time,782 type783 ),784 ui_log=ui_log785 )786 if start_time != end_time:787 __data = QA_fetch_get_stock_min(788 str(code),789 start_time,790 end_time,791 type792 )793 if len(__data) > 1:794 coll.insert_many(795 QA_util_to_json_from_pandas(__data)[1::]796 )797 else:798 start_time = '2015-01-01'799 QA_util_log_info(800 '##JOB03.{} Now Saving {} from {} to {} =={} '.format(801 ['1min',802 '5min',803 '15min',804 '30min',805 '60min'].index(type),806 str(code),807 start_time,808 end_time,809 type810 ),811 ui_log=ui_log812 )813 if start_time != end_time:814 __data = QA_fetch_get_stock_min(815 str(code),816 start_time,817 end_time,818 type819 )820 if len(__data) > 1:821 coll.insert_many(822 QA_util_to_json_from_pandas(__data)823 )824 except Exception as e:825 QA_util_log_info(e, ui_log=ui_log)826 err.append(code)827 QA_util_log_info(err, ui_log=ui_log)828 executor = ThreadPoolExecutor(max_workers=4)829 # executor.map((__saving_work, stock_list[i_], coll),URLS)830 res = {831 executor.submit(__saving_work,832 stock_list[i_],833 coll)834 for i_ in range(len(stock_list))835 }836 count = 1837 for i_ in concurrent.futures.as_completed(res):838 QA_util_log_info(839 'The {} of Total {}'.format(count,840 len(stock_list)),841 ui_log=ui_log842 )843 strProgress = 'DOWNLOAD PROGRESS {} '.format(844 str(float(count / len(stock_list) * 100))[0:4] + '%'845 )846 intProgress = int(count / len(stock_list) * 10000.0)847 QA_util_log_info(848 strProgress,849 ui_log,850 ui_progress=ui_progress,851 ui_progress_int_value=intProgress852 )853 count = count + 1854 if len(err) < 1:855 QA_util_log_info('SUCCESS', ui_log=ui_log)856 else:857 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)858 QA_util_log_info(err, ui_log=ui_log)859def QA_SU_save_single_index_day(code : str, client=DATABASE, ui_log=None):860 """save index_day861 Keyword Arguments:862 code : single index code863 client {[type]} -- [description] (default: {DATABASE})864 """865 #__index_list = QA_fetch_get_stock_list('index')866 coll = client.index_day867 coll.create_index(868 [('code',869 pymongo.ASCENDING),870 ('date_stamp',871 pymongo.ASCENDING)]872 )873 err = []874 def __saving_work(code, coll):875 try:876 ref_ = coll.find({'code': str(code)[0:6]})877 end_time = str(now_time())[0:10]878 if ref_.count() > 0:879 start_time = ref_[ref_.count() - 1]['date']880 QA_util_log_info(881 '##JOB04 Now Saving INDEX_DAY==== \n Trying updating {} from {} to {}'882 .format(code,883 start_time,884 end_time),885 ui_log=ui_log886 )887 if start_time != end_time:888 coll.insert_many(889 QA_util_to_json_from_pandas(890 QA_fetch_get_index_day(891 str(code),892 QA_util_get_next_day(start_time),893 end_time894 )895 )896 )897 else:898 try:899 start_time = '1990-01-01'900 QA_util_log_info(901 '##JOB04 Now Saving INDEX_DAY==== \n Trying updating {} from {} to {}'902 .format(code,903 start_time,904 end_time),905 ui_log=ui_log906 )907 coll.insert_many(908 QA_util_to_json_from_pandas(909 QA_fetch_get_index_day(910 str(code),911 start_time,912 end_time913 )914 )915 )916 except:917 start_time = '2009-01-01'918 QA_util_log_info(919 '##JOB04 Now Saving INDEX_DAY==== \n Trying updating {} from {} to {}'920 .format(code,921 start_time,922 end_time),923 ui_log=ui_log924 )925 coll.insert_many(926 QA_util_to_json_from_pandas(927 QA_fetch_get_index_day(928 str(code),929 start_time,930 end_time931 )932 )933 )934 except Exception as e:935 QA_util_log_info(e, ui_log=ui_log)936 err.append(str(code))937 QA_util_log_info(err, ui_log=ui_log)938 __saving_work(code, coll)939 if len(err) < 1:940 QA_util_log_info('SUCCESS', ui_log=ui_log)941 else:942 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)943 QA_util_log_info(err, ui_log=ui_log)944def QA_SU_save_index_day(client=DATABASE, ui_log=None, ui_progress=None):945 """save index_day946 Keyword Arguments:947 client {[type]} -- [description] (default: {DATABASE})948 """949 __index_list = QA_fetch_get_stock_list('index')950 coll = client.index_day951 coll.create_index(952 [('code',953 pymongo.ASCENDING),954 ('date_stamp',955 pymongo.ASCENDING)]956 )957 err = []958 def __saving_work(code, coll):959 try:960 ref_ = coll.find({'code': str(code)[0:6]})961 end_time = str(now_time())[0:10]962 if ref_.count() > 0:963 start_time = ref_[ref_.count() - 1]['date']964 QA_util_log_info(965 '##JOB04 Now Saving INDEX_DAY==== \n Trying updating {} from {} to {}'966 .format(code,967 start_time,968 end_time),969 ui_log=ui_log970 )971 if start_time != end_time:972 coll.insert_many(973 QA_util_to_json_from_pandas(974 QA_fetch_get_index_day(975 str(code),976 QA_util_get_next_day(start_time),977 end_time978 )979 )980 )981 else:982 try:983 start_time = '1990-01-01'984 QA_util_log_info(985 '##JOB04 Now Saving INDEX_DAY==== \n Trying updating {} from {} to {}'986 .format(code,987 start_time,988 end_time),989 ui_log=ui_log990 )991 coll.insert_many(992 QA_util_to_json_from_pandas(993 QA_fetch_get_index_day(994 str(code),995 start_time,996 end_time997 )998 )999 )1000 except:1001 start_time = '2009-01-01'1002 QA_util_log_info(1003 '##JOB04 Now Saving INDEX_DAY==== \n Trying updating {} from {} to {}'1004 .format(code,1005 start_time,1006 end_time),1007 ui_log=ui_log1008 )1009 coll.insert_many(1010 QA_util_to_json_from_pandas(1011 QA_fetch_get_index_day(1012 str(code),1013 start_time,1014 end_time1015 )1016 )1017 )1018 except Exception as e:1019 QA_util_log_info(e, ui_log=ui_log)1020 err.append(str(code))1021 QA_util_log_info(err, ui_log=ui_log)1022 for i_ in range(len(__index_list)):1023 # __saving_work('000001')1024 QA_util_log_info(1025 'The {} of Total {}'.format(i_,1026 len(__index_list)),1027 ui_log=ui_log1028 )1029 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(1030 str(float(i_ / len(__index_list) * 100))[0:4] + '%'1031 )1032 intLogProgress = int(float(i_ / len(__index_list) * 10000.0))1033 QA_util_log_info(1034 strLogProgress,1035 ui_log=ui_log,1036 ui_progress=ui_progress,1037 ui_progress_int_value=intLogProgress1038 )1039 __saving_work(__index_list.index[i_][0], coll)1040 if len(err) < 1:1041 QA_util_log_info('SUCCESS', ui_log=ui_log)1042 else:1043 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)1044 QA_util_log_info(err, ui_log=ui_log)1045def QA_SU_save_index_min(client=DATABASE, ui_log=None, ui_progress=None):1046 """save index_min1047 Keyword Arguments:1048 client {[type]} -- [description] (default: {DATABASE})1049 """1050 __index_list = QA_fetch_get_stock_list('index')1051 coll = client.index_min1052 coll.create_index(1053 [1054 ('code',1055 pymongo.ASCENDING),1056 ('time_stamp',1057 pymongo.ASCENDING),1058 ('date_stamp',1059 pymongo.ASCENDING)1060 ]1061 )1062 err = []1063 def __saving_work(code, coll):1064 QA_util_log_info(1065 '##JOB05 Now Saving Index_MIN ==== {}'.format(str(code)),1066 ui_log=ui_log1067 )1068 try:1069 for type in ['1min', '5min', '15min', '30min', '60min']:1070 ref_ = coll.find({'code': str(code)[0:6], 'type': type})1071 end_time = str(now_time())[0:19]1072 if ref_.count() > 0:1073 start_time = ref_[ref_.count() - 1]['datetime']1074 QA_util_log_info(1075 '##JOB05.{} Now Saving {} from {} to {} =={} '.format(1076 ['1min',1077 '5min',1078 '15min',1079 '30min',1080 '60min'].index(type),1081 str(code),1082 start_time,1083 end_time,1084 type1085 ),1086 ui_log=ui_log1087 )1088 if start_time != end_time:1089 __data = QA_fetch_get_index_min(1090 str(code),1091 start_time,1092 end_time,1093 type1094 )1095 if len(__data) > 1:1096 coll.insert_many(1097 QA_util_to_json_from_pandas(__data[1::])1098 )1099 else:1100 start_time = '2015-01-01'1101 QA_util_log_info(1102 '##JOB05.{} Now Saving {} from {} to {} =={} '.format(1103 ['1min',1104 '5min',1105 '15min',1106 '30min',1107 '60min'].index(type),1108 str(code),1109 start_time,1110 end_time,1111 type1112 ),1113 ui_log=ui_log1114 )1115 if start_time != end_time:1116 __data = QA_fetch_get_index_min(1117 str(code),1118 start_time,1119 end_time,1120 type1121 )1122 if len(__data) > 1:1123 coll.insert_many(1124 QA_util_to_json_from_pandas(__data)1125 )1126 except:1127 err.append(code)1128 executor = ThreadPoolExecutor(max_workers=4)1129 res = {1130 executor.submit(__saving_work,1131 __index_list.index[i_][0],1132 coll)1133 for i_ in range(len(__index_list))1134 } # multi index ./.1135 count = 01136 for i_ in concurrent.futures.as_completed(res):1137 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(1138 str(float(count / len(__index_list) * 100))[0:4] + '%'1139 )1140 intLogProgress = int(float(count / len(__index_list) * 10000.0))1141 QA_util_log_info(1142 'The {} of Total {}'.format(count,1143 len(__index_list)),1144 ui_log=ui_log1145 )1146 QA_util_log_info(1147 strLogProgress,1148 ui_log=ui_log,1149 ui_progress=ui_progress,1150 ui_progress_int_value=intLogProgress1151 )1152 count = count + 11153 if len(err) < 1:1154 QA_util_log_info('SUCCESS', ui_log=ui_log)1155 else:1156 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)1157 QA_util_log_info(err, ui_log=ui_log)1158def QA_SU_save_single_index_min(code : str, client=DATABASE, ui_log=None, ui_progress=None):1159 """save single index_min1160 Keyword Arguments:1161 client {[type]} -- [description] (default: {DATABASE})1162 """1163 #__index_list = QA_fetch_get_stock_list('index')1164 __index_list = [code]1165 coll = client.index_min1166 coll.create_index(1167 [1168 ('code',1169 pymongo.ASCENDING),1170 ('time_stamp',1171 pymongo.ASCENDING),1172 ('date_stamp',1173 pymongo.ASCENDING)1174 ]1175 )1176 err = []1177 def __saving_work(code, coll):1178 QA_util_log_info(1179 '##JOB05 Now Saving Index_MIN ==== {}'.format(str(code)),1180 ui_log=ui_log1181 )1182 try:1183 for type in ['1min', '5min', '15min', '30min', '60min']:1184 ref_ = coll.find({'code': str(code)[0:6], 'type': type})1185 end_time = str(now_time())[0:19]1186 if ref_.count() > 0:1187 start_time = ref_[ref_.count() - 1]['datetime']1188 QA_util_log_info(1189 '##JOB05.{} Now Saving {} from {} to {} =={} '.format(1190 ['1min',1191 '5min',1192 '15min',1193 '30min',1194 '60min'].index(type),1195 str(code),1196 start_time,1197 end_time,1198 type1199 ),1200 ui_log=ui_log1201 )1202 if start_time != end_time:1203 __data = QA_fetch_get_index_min(1204 str(code),1205 start_time,1206 end_time,1207 type1208 )1209 if len(__data) > 1:1210 coll.insert_many(1211 QA_util_to_json_from_pandas(__data[1::])1212 )1213 else:1214 start_time = '2015-01-01'1215 QA_util_log_info(1216 '##JOB05.{} Now Saving {} from {} to {} =={} '.format(1217 ['1min',1218 '5min',1219 '15min',1220 '30min',1221 '60min'].index(type),1222 str(code),1223 start_time,1224 end_time,1225 type1226 ),1227 ui_log=ui_log1228 )1229 if start_time != end_time:1230 __data = QA_fetch_get_index_min(1231 str(code),1232 start_time,1233 end_time,1234 type1235 )1236 if len(__data) > 1:1237 coll.insert_many(1238 QA_util_to_json_from_pandas(__data)1239 )1240 except:1241 err.append(code)1242 executor = ThreadPoolExecutor(max_workers=4)1243 res = {1244 executor.submit(__saving_work,1245 __index_list[i_],1246 coll)1247 for i_ in range(len(__index_list))1248 } # multi index ./.1249 count = 11250 for i_ in concurrent.futures.as_completed(res):1251 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(1252 str(float(count / len(__index_list) * 100))[0:4] + '%'1253 )1254 intLogProgress = int(float(count / len(__index_list) * 10000.0))1255 QA_util_log_info(1256 'The {} of Total {}'.format(count,1257 len(__index_list)),1258 ui_log=ui_log1259 )1260 QA_util_log_info(1261 strLogProgress,1262 ui_log=ui_log,1263 ui_progress=ui_progress,1264 ui_progress_int_value=intLogProgress1265 )1266 count = count + 11267 if len(err) < 1:1268 QA_util_log_info('SUCCESS', ui_log=ui_log)1269 else:1270 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)1271 QA_util_log_info(err, ui_log=ui_log)1272def QA_SU_save_single_etf_day(code : str, client=DATABASE, ui_log=None):1273 """save etf_day1274 Keyword Arguments:1275 code : single etf code1276 client {[type]} -- [description] (default: {DATABASE})1277 """1278 #__index_list = QA_fetch_get_stock_list('etf')1279 coll = client.index_day1280 coll.create_index(1281 [('code',1282 pymongo.ASCENDING),1283 ('date_stamp',1284 pymongo.ASCENDING)]1285 )1286 err = []1287 def __saving_work(code, coll):1288 try:1289 ref_ = coll.find({'code': str(code)[0:6]})1290 end_time = str(now_time())[0:10]1291 if ref_.count() > 0:1292 start_time = ref_[ref_.count() - 1]['date']1293 QA_util_log_info(1294 '##JOB06 Now Saving ETF_DAY==== \n Trying updating {} from {} to {}'1295 .format(code,1296 start_time,1297 end_time),1298 ui_log=ui_log1299 )1300 if start_time != end_time:1301 coll.insert_many(1302 QA_util_to_json_from_pandas(1303 QA_fetch_get_index_day(1304 str(code),1305 QA_util_get_next_day(start_time),1306 end_time1307 )1308 )1309 )1310 else:1311 start_time = '1990-01-01'1312 QA_util_log_info(1313 '##JOB06 Now Saving ETF_DAY==== \n Trying updating {} from {} to {}'1314 .format(code,1315 start_time,1316 end_time),1317 ui_log=ui_log1318 )1319 if start_time != end_time:1320 coll.insert_many(1321 QA_util_to_json_from_pandas(1322 QA_fetch_get_index_day(1323 str(code),1324 start_time,1325 end_time1326 )1327 )1328 )1329 except:1330 err.append(str(code))1331 __saving_work(code, coll)1332 if len(err) < 1:1333 QA_util_log_info('SUCCESS', ui_log=ui_log)1334 else:1335 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)1336 QA_util_log_info(err, ui_log=ui_log)1337def QA_SU_save_etf_day(client=DATABASE, ui_log=None, ui_progress=None):1338 """save etf_day1339 Keyword Arguments:1340 client {[type]} -- [description] (default: {DATABASE})1341 """1342 __index_list = QA_fetch_get_stock_list('etf')1343 coll = client.index_day1344 coll.create_index(1345 [('code',1346 pymongo.ASCENDING),1347 ('date_stamp',1348 pymongo.ASCENDING)]1349 )1350 err = []1351 def __saving_work(code, coll):1352 try:1353 ref_ = coll.find({'code': str(code)[0:6]})1354 end_time = str(now_time())[0:10]1355 if ref_.count() > 0:1356 start_time = ref_[ref_.count() - 1]['date']1357 QA_util_log_info(1358 '##JOB06 Now Saving ETF_DAY==== \n Trying updating {} from {} to {}'1359 .format(code,1360 start_time,1361 end_time),1362 ui_log=ui_log1363 )1364 if start_time != end_time:1365 coll.insert_many(1366 QA_util_to_json_from_pandas(1367 QA_fetch_get_index_day(1368 str(code),1369 QA_util_get_next_day(start_time),1370 end_time1371 )1372 )1373 )1374 else:1375 start_time = '1990-01-01'1376 QA_util_log_info(1377 '##JOB06 Now Saving ETF_DAY==== \n Trying updating {} from {} to {}'1378 .format(code,1379 start_time,1380 end_time),1381 ui_log=ui_log1382 )1383 if start_time != end_time:1384 coll.insert_many(1385 QA_util_to_json_from_pandas(1386 QA_fetch_get_index_day(1387 str(code),1388 start_time,1389 end_time1390 )1391 )1392 )1393 except:1394 err.append(str(code))1395 for i_ in range(len(__index_list)):1396 # __saving_work('000001')1397 QA_util_log_info(1398 'The {} of Total {}'.format(i_,1399 len(__index_list)),1400 ui_log=ui_log1401 )1402 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(1403 str(float(i_ / len(__index_list) * 100))[0:4] + '%'1404 )1405 intLogProgress = int(float(i_ / len(__index_list) * 10000.0))1406 QA_util_log_info(1407 strLogProgress,1408 ui_log=ui_log,1409 ui_progress=ui_progress,1410 ui_progress_int_value=intLogProgress1411 )1412 __saving_work(__index_list.index[i_][0], coll)1413 if len(err) < 1:1414 QA_util_log_info('SUCCESS', ui_log=ui_log)1415 else:1416 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)1417 QA_util_log_info(err, ui_log=ui_log)1418def QA_SU_save_etf_min(client=DATABASE, ui_log=None, ui_progress=None):1419 """save etf_min1420 Keyword Arguments:1421 client {[type]} -- [description] (default: {DATABASE})1422 """1423 __index_list = QA_fetch_get_stock_list('etf')1424 coll = client.index_min1425 coll.create_index(1426 [1427 ('code',1428 pymongo.ASCENDING),1429 ('time_stamp',1430 pymongo.ASCENDING),1431 ('date_stamp',1432 pymongo.ASCENDING)1433 ]1434 )1435 err = []1436 def __saving_work(code, coll):1437 QA_util_log_info(1438 '##JOB07 Now Saving ETF_MIN ==== {}'.format(str(code)),1439 ui_log=ui_log1440 )1441 try:1442 for type in ['1min', '5min', '15min', '30min', '60min']:1443 ref_ = coll.find({'code': str(code)[0:6], 'type': type})1444 end_time = str(now_time())[0:19]1445 if ref_.count() > 0:1446 start_time = ref_[ref_.count() - 1]['datetime']1447 QA_util_log_info(1448 '##JOB07.{} Now Saving {} from {} to {} =={} '.format(1449 ['1min',1450 '5min',1451 '15min',1452 '30min',1453 '60min'].index(type),1454 str(code),1455 start_time,1456 end_time,1457 type1458 ),1459 ui_log=ui_log1460 )1461 if start_time != end_time:1462 __data = QA_fetch_get_index_min(1463 str(code),1464 start_time,1465 end_time,1466 type1467 )1468 if len(__data) > 1:1469 coll.insert_many(1470 QA_util_to_json_from_pandas(__data[1::])1471 )1472 else:1473 start_time = '2015-01-01'1474 QA_util_log_info(1475 '##JOB07.{} Now Saving {} from {} to {} =={} '.format(1476 ['1min',1477 '5min',1478 '15min',1479 '30min',1480 '60min'].index(type),1481 str(code),1482 start_time,1483 end_time,1484 type1485 ),1486 ui_log=ui_log1487 )1488 if start_time != end_time:1489 __data = QA_fetch_get_index_min(1490 str(code),1491 start_time,1492 end_time,1493 type1494 )1495 if len(__data) > 1:1496 coll.insert_many(1497 QA_util_to_json_from_pandas(__data)1498 )1499 except:1500 err.append(code)1501 executor = ThreadPoolExecutor(max_workers=4)1502 res = {1503 executor.submit(__saving_work,1504 __index_list.index[i_][0],1505 coll)1506 for i_ in range(len(__index_list))1507 } # multi index ./.1508 count = 11509 for i_ in concurrent.futures.as_completed(res):1510 QA_util_log_info(1511 'The {} of Total {}'.format(count,1512 len(__index_list)),1513 ui_log=ui_log1514 )1515 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(1516 str(float(count / len(__index_list) * 100))[0:4] + '%'1517 )1518 intLogProgress = int(float(count / len(__index_list) * 10000.0))1519 QA_util_log_info(1520 strLogProgress,1521 ui_log=ui_log,1522 ui_progress=ui_progress,1523 ui_progress_int_value=intLogProgress1524 )1525 count = count + 11526 if len(err) < 1:1527 QA_util_log_info('SUCCESS', ui_log=ui_log)1528 else:1529 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)1530 QA_util_log_info(err, ui_log=ui_log)1531def QA_SU_save_single_etf_min(code : str, client=DATABASE, ui_log=None, ui_progress=None):1532 """save single etf_min1533 Keyword Arguments:1534 client {[type]} -- [description] (default: {DATABASE})1535 """1536 #__index_list = QA_fetch_get_stock_list('etf')1537 __index_list = [code]1538 coll = client.index_min1539 coll.create_index(1540 [1541 ('code',1542 pymongo.ASCENDING),1543 ('time_stamp',1544 pymongo.ASCENDING),1545 ('date_stamp',1546 pymongo.ASCENDING)1547 ]1548 )1549 err = []1550 def __saving_work(code, coll):1551 QA_util_log_info(1552 '##JOB07 Now Saving ETF_MIN ==== {}'.format(str(code)),1553 ui_log=ui_log1554 )1555 try:1556 for type in ['1min', '5min', '15min', '30min', '60min']:1557 ref_ = coll.find({'code': str(code)[0:6], 'type': type})1558 end_time = str(now_time())[0:19]1559 if ref_.count() > 0:1560 start_time = ref_[ref_.count() - 1]['datetime']1561 QA_util_log_info(1562 '##JOB07.{} Now Saving {} from {} to {} =={} '.format(1563 ['1min',1564 '5min',1565 '15min',1566 '30min',1567 '60min'].index(type),1568 str(code),1569 start_time,1570 end_time,1571 type1572 ),1573 ui_log=ui_log1574 )1575 if start_time != end_time:1576 __data = QA_fetch_get_index_min(1577 str(code),1578 start_time,1579 end_time,1580 type1581 )1582 if len(__data) > 1:1583 coll.insert_many(1584 QA_util_to_json_from_pandas(__data[1::])1585 )1586 else:1587 start_time = '2015-01-01'1588 QA_util_log_info(1589 '##JOB07.{} Now Saving {} from {} to {} =={} '.format(1590 ['1min',1591 '5min',1592 '15min',1593 '30min',1594 '60min'].index(type),1595 str(code),1596 start_time,1597 end_time,1598 type1599 ),1600 ui_log=ui_log1601 )1602 if start_time != end_time:1603 __data = QA_fetch_get_index_min(1604 str(code),1605 start_time,1606 end_time,1607 type1608 )1609 if len(__data) > 1:1610 coll.insert_many(1611 QA_util_to_json_from_pandas(__data)1612 )1613 except:1614 err.append(code)1615 executor = ThreadPoolExecutor(max_workers=4)1616 res = {1617 executor.submit(__saving_work,1618 __index_list[i_],1619 coll)1620 for i_ in range(len(__index_list))1621 } # multi index ./.1622 count = 11623 for i_ in concurrent.futures.as_completed(res):1624 QA_util_log_info(1625 'The {} of Total {}'.format(count,1626 len(__index_list)),1627 ui_log=ui_log1628 )1629 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(1630 str(float(count / len(__index_list) * 100))[0:4] + '%'1631 )1632 intLogProgress = int(float(count / len(__index_list) * 10000.0))1633 QA_util_log_info(1634 strLogProgress,1635 ui_log=ui_log,1636 ui_progress=ui_progress,1637 ui_progress_int_value=intLogProgress1638 )1639 count = count + 11640 if len(err) < 1:1641 QA_util_log_info('SUCCESS', ui_log=ui_log)1642 else:1643 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)1644 QA_util_log_info(err, ui_log=ui_log)1645def QA_SU_save_stock_list(client=DATABASE, ui_log=None, ui_progress=None):1646 """save stock_list1647 Keyword Arguments:1648 client {[type]} -- [description] (default: {DATABASE})1649 """1650 client.drop_collection('stock_list')1651 coll = client.stock_list1652 coll.create_index('code')1653 try:1654 # 🛠todo 这个应该是第一个任务 JOB01， 先更新股票列表！！1655 QA_util_log_info(1656 '##JOB08 Now Saving STOCK_LIST ====',1657 ui_log=ui_log,1658 ui_progress=ui_progress,1659 ui_progress_int_value=50001660 )1661 stock_list_from_tdx = QA_fetch_get_stock_list()1662 pandas_data = QA_util_to_json_from_pandas(stock_list_from_tdx)1663 coll.insert_many(pandas_data)1664 QA_util_log_info(1665 "完成股票列表获取",1666 ui_log=ui_log,1667 ui_progress=ui_progress,1668 ui_progress_int_value=100001669 )1670 except Exception as e:1671 QA_util_log_info(e, ui_log=ui_log)1672 print(" Error save_tdx.QA_SU_save_stock_list exception!")1673 pass1674def QA_SU_save_etf_list(client=DATABASE, ui_log=None, ui_progress=None):1675 """save etf_list1676 Keyword Arguments:1677 client {[type]} -- [description] (default: {DATABASE})1678 """1679 try:1680 QA_util_log_info(1681 '##JOB16 Now Saving ETF_LIST ====',1682 ui_log=ui_log,1683 ui_progress=ui_progress,1684 ui_progress_int_value=50001685 )1686 etf_list_from_tdx = QA_fetch_get_stock_list(type_="etf")1687 pandas_data = QA_util_to_json_from_pandas(etf_list_from_tdx)1688 if len(pandas_data) > 0:1689 # 获取到数据后才进行drop collection 操作1690 client.drop_collection('etf_list')1691 coll = client.etf_list1692 coll.create_index('code')1693 coll.insert_many(pandas_data)1694 QA_util_log_info(1695 "完成ETF列表获取",1696 ui_log=ui_log,1697 ui_progress=ui_progress,1698 ui_progress_int_value=100001699 )1700 except Exception as e:1701 QA_util_log_info(e, ui_log=ui_log)1702 print(" Error save_tdx.QA_SU_save_etf_list exception!")1703 pass1704def QA_SU_save_stock_block(client=DATABASE, ui_log=None, ui_progress=None):1705 """save stock_block1706 Keyword Arguments:1707 client {[type]} -- [description] (default: {DATABASE})1708 """1709 client.drop_collection('stock_block')1710 coll = client.stock_block1711 coll.create_index('code')1712 try:1713 QA_util_log_info(1714 '##JOB09 Now Saving STOCK_BlOCK ====',1715 ui_log=ui_log,1716 ui_progress=ui_progress,1717 ui_progress_int_value=50001718 )1719 coll.insert_many(1720 QA_util_to_json_from_pandas(QA_fetch_get_stock_block('tdx'))1721 )1722 QA_util_log_info(1723 'tdx Block ====',1724 ui_log=ui_log,1725 ui_progress=ui_progress,1726 ui_progress_int_value=50001727 )1728 # # 🛠todo fixhere here 获取同花顺板块， 还是调用tdx的1729 # coll.insert_many(1730 # QA_util_to_json_from_pandas(QA_fetch_get_stock_block('ths'))1731 # )1732 # QA_util_log_info(1733 # 'ths Block ====',1734 # ui_log=ui_log,1735 # ui_progress=ui_progress,1736 # ui_progress_int_value=80001737 # )1738 # tushare 的板块数据有中证500成分，增加获取中证500成分 ——阿财1739 coll.insert_many(1740 QA_util_to_json_from_pandas(QA_fetch_get_stock_block('tushare'))1741 )1742 QA_util_log_info(1743 'tushare Block ====',1744 ui_log=ui_log,1745 ui_progress=ui_progress,1746 ui_progress_int_value=90001747 )1748 QA_util_log_info(1749 '完成股票板块获取=',1750 ui_log=ui_log,1751 ui_progress=ui_progress,1752 ui_progress_int_value=100001753 )1754 # coll.insert_many(1755 # QA_util_to_json_from_pandas(QA_fetch_get_stock_block('QA'))1756 # )1757 # QA_util_log_info(1758 # 'QA_Select Block ====',1759 # ui_log=ui_log,1760 # ui_progress=ui_progress,1761 # ui_progress_int_value=80001762 # )1763 QA_util_log_info(1764 '完成股票板块获取=',1765 ui_log=ui_log,1766 ui_progress=ui_progress,1767 ui_progress_int_value=100001768 )1769 except Exception as e:1770 QA_util_log_info(e, ui_log=ui_log)1771 print(" Error save_tdx.QA_SU_save_stock_block exception!")1772 pass1773def QA_SU_save_stock_info(client=DATABASE, ui_log=None, ui_progress=None):1774 """save stock_info1775 Keyword Arguments:1776 client {[type]} -- [description] (default: {DATABASE})1777 """1778 client.drop_collection('stock_info')1779 stock_list = QA_fetch_get_stock_list().code.unique().tolist()1780 coll = client.stock_info1781 coll.create_index('code')1782 err = []1783 def __saving_work(code, coll):1784 QA_util_log_info(1785 '##JOB10 Now Saving STOCK INFO ==== {}'.format(str(code)),1786 ui_log=ui_log1787 )1788 try:1789 coll.insert_many(1790 QA_util_to_json_from_pandas(QA_fetch_get_stock_info(str(code)))1791 )1792 except:1793 err.append(str(code))1794 for i_ in range(len(stock_list)):1795 # __saving_work('000001')1796 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(1797 str(float(i_ / len(stock_list) * 100))[0:4] + '%'1798 )1799 intLogProgress = int(float(i_ / len(stock_list) * 10000.0))1800 QA_util_log_info('The {} of Total {}'.format(i_, len(stock_list)))1801 QA_util_log_info(1802 strLogProgress,1803 ui_log=ui_log,1804 ui_progress=ui_progress,1805 ui_progress_int_value=intLogProgress1806 )1807 __saving_work(stock_list[i_], coll)1808 if len(err) < 1:1809 QA_util_log_info('SUCCESS', ui_log=ui_log)1810 else:1811 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)1812 QA_util_log_info(err, ui_log=ui_log)1813def QA_SU_save_stock_transaction(1814 client=DATABASE,1815 ui_log=None,1816 ui_progress=None1817):1818 """save stock_transaction1819 Keyword Arguments:1820 client {[type]} -- [description] (default: {DATABASE})1821 """1822 stock_list = QA_fetch_get_stock_list().code.unique().tolist()1823 coll = client.stock_transaction1824 coll.create_index(1825 [1826 ('code',1827 pymongo.ASCENDING),1828 ('time_stamp',1829 pymongo.ASCENDING),1830 ('date_stamp',1831 pymongo.ASCENDING)1832 ]1833 )1834 err = []1835 def __saving_work(code):1836 QA_util_log_info(1837 '##JOB11 Now Saving STOCK_TRANSACTION ==== {}'.format(str(code)),1838 ui_log=ui_log1839 )1840 try:1841 coll.insert_many(1842 QA_util_to_json_from_pandas(1843 # 🛠todo str(stock_list[code]) 参数不对？1844 QA_fetch_get_stock_transaction(1845 str(code),1846 '2019-01-01',1847 str(now_time())[0:10]1848 )1849 )1850 )1851 except:1852 err.append(str(code))1853 for i_ in range(len(stock_list)):1854 # __saving_work('000001')1855 QA_util_log_info(1856 'The {} of Total {}'.format(i_,1857 len(stock_list)),1858 ui_log=ui_log1859 )1860 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(1861 str(float(i_ / len(stock_list) * 100))[0:4] + '%'1862 )1863 intLogProgress = int(float(i_ / len(stock_list) * 10000.0))1864 QA_util_log_info(1865 strLogProgress,1866 ui_log=ui_log,1867 ui_progress=ui_progress,1868 ui_progress_int_value=intLogProgress1869 )1870 __saving_work(stock_list[i_])1871 if len(err) < 1:1872 QA_util_log_info('SUCCESS', ui_log=ui_log)1873 else:1874 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)1875 QA_util_log_info(err, ui_log=ui_log)1876def QA_SU_save_index_transaction(1877 client=DATABASE,1878 ui_log=None,1879 ui_progress=None1880):1881 """save index_transaction1882 Keyword Arguments:1883 client {[type]} -- [description] (default: {DATABASE})1884 """1885 index_list = QA_fetch_get_index_list().code.unique().tolist()1886 coll = client.index_transaction1887 coll.create_index(1888 [1889 ('code',1890 pymongo.ASCENDING),1891 ('time_stamp',1892 pymongo.ASCENDING),1893 ('date_stamp',1894 pymongo.ASCENDING)1895 ]1896 )1897 err = []1898 def __saving_work(code):1899 QA_util_log_info(1900 '##JOB11 Now Saving INDEX_TRANSACTION ==== {}'.format(str(code)),1901 ui_log=ui_log1902 )1903 try:1904 coll.insert_many(1905 QA_util_to_json_from_pandas(1906 # 🛠todo str(stock_list[code]) 参数不对？1907 QA_fetch_get_index_transaction(1908 str(code),1909 '2019-01-01',1910 str(now_time())[0:10]1911 )1912 )1913 )1914 except:1915 err.append(str(code))1916 for i_ in range(len(index_list)):1917 # __saving_work('000001')1918 QA_util_log_info(1919 'The {} of Total {}'.format(i_,1920 len(index_list)),1921 ui_log=ui_log1922 )1923 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(1924 str(float(i_ / len(index_list) * 100))[0:4] + '%'1925 )1926 intLogProgress = int(float(i_ / len(index_list) * 10000.0))1927 QA_util_log_info(1928 strLogProgress,1929 ui_log=ui_log,1930 ui_progress=ui_progress,1931 ui_progress_int_value=intLogProgress1932 )1933 __saving_work(index_list[i_])1934 if len(err) < 1:1935 QA_util_log_info('SUCCESS', ui_log=ui_log)1936 else:1937 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)1938 QA_util_log_info(err, ui_log=ui_log)1939########################################################################################################1940def _save_option_commodity_ru_day(1941 client=DATABASE,1942 ui_log=None,1943 ui_progress=None1944):1945 ##################### ru 天然橡胶 ############################################################################1946 option_ru_contract_list = QA_fetch_get_commodity_option_RU_contract_time_to_market()1947 coll_option_commodity_ru_day = client.option_commodity_ru_day1948 coll_option_commodity_ru_day.create_index(1949 [("code",1950 pymongo.ASCENDING),1951 ("date_stamp",1952 pymongo.ASCENDING)]1953 )1954 err = []1955 def __saving_work(code, coll_option_commodity_ru_day):1956 try:1957 QA_util_log_info(1958 '##JOB12 Now Saving OPTION_DAY_COMMODITY_RU 天然橡胶 ==== {}'.format(1959 str(code)1960 ),1961 ui_log=ui_log1962 )1963 # 首选查找数据库 是否 有 这个代码的数据1964 ref = coll_option_commodity_ru_day.find({'code': str(code)[0:8]})1965 end_date = str(now_time())[0:10]1966 # 当前数据库已经包含了这个代码的数据， 继续增量更新1967 # 加入这个判断的原因是因为如果是刚上市的 数据库会没有数据 所以会有负索引问题出现1968 if ref.count() > 0:1969 # 接着上次获取的日期继续更新1970 start_date = ref[ref.count() - 1]['date']1971 QA_util_log_info(1972 ' 上次获取 期权ru 天然橡胶 日线数据的最后日期是 {}'.format(start_date),1973 ui_log=ui_log1974 )1975 QA_util_log_info(1976 'UPDATE_OPTION_RU_DAY \n 从上一次下载数据开始继续 Trying update {} from {} to {}'1977 .format(code,1978 start_date,1979 end_date),1980 ui_log=ui_log1981 )1982 if start_date != end_date:1983 start_date0 = QA_util_get_next_day(start_date)1984 df0 = QA_fetch_get_option_day(1985 code=code,1986 start_date=start_date0,1987 end_date=end_date,1988 frequence='day',1989 ip=None,1990 port=None1991 )1992 retCount = df0.iloc[:, 0].size1993 QA_util_log_info(1994 "日期从开始{}-结束{} , 合约代码{} , 返回了{}条记录 , 准备写入数据库".format(1995 start_date0,1996 end_date,1997 code,1998 retCount1999 ),2000 ui_log=ui_log2001 )2002 coll_option_commodity_ru_day.insert_many(2003 QA_util_to_json_from_pandas(df0)2004 )2005 else:2006 QA_util_log_info(2007 "^已经获取过这天的数据了^ {}".format(start_date),2008 ui_log=ui_log2009 )2010 else:2011 start_date = '1990-01-01'2012 QA_util_log_info(2013 'UPDATE_M_OPTION_DAY \n 从新开始下载数据 Trying update {} from {} to {}'2014 .format(code,2015 start_date,2016 end_date),2017 ui_log=ui_log2018 )2019 if start_date != end_date:2020 df0 = QA_fetch_get_option_day(2021 code=code,2022 start_date=start_date,2023 end_date=end_date,2024 frequence='day',2025 ip=None,2026 port=None2027 )2028 retCount = df0.iloc[:, 0].size2029 QA_util_log_info(2030 "日期从开始{}-结束{} , 合约代码{} , 获取了{}条记录 , 准备写入数据库^_^ ".format(2031 start_date,2032 end_date,2033 code,2034 retCount2035 ),2036 ui_log=ui_log2037 )2038 coll_option_commodity_ru_day.insert_many(2039 QA_util_to_json_from_pandas(df0)2040 )2041 else:2042 QA_util_log_info(2043 "*已经获取过这天的数据了* {}".format(start_date),2044 ui_log=ui_log2045 )2046 except Exception as error0:2047 print(error0)2048 err.append(str(code))2049 for item in range(len(option_ru_contract_list)):2050 QA_util_log_info(2051 'The {} of Total {}'.format(item,2052 len(option_ru_contract_list)),2053 ui_log=ui_log2054 )2055 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(2056 str(float(item / len(option_ru_contract_list) * 100))[0:4] + '%'2057 )2058 intLogProgress = int(2059 float(item / len(option_ru_contract_list) * 10000.0)2060 )2061 QA_util_log_info(2062 strLogProgress,2063 ui_log=ui_log,2064 ui_progress=ui_progress,2065 ui_progress_int_value=intLogProgress2066 )2067 __saving_work(2068 option_ru_contract_list[item].code,2069 coll_option_commodity_ru_day2070 )2071 if len(err) < 1:2072 QA_util_log_info('SUCCESS save option ru day ^_^ ', ui_log=ui_log)2073 else:2074 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)2075 QA_util_log_info(err, ui_log=ui_log)2076def _save_option_commodity_c_day(2077 client=DATABASE,2078 ui_log=None,2079 ui_progress=None,2080):2081 ##################### c 玉米 ############################################################################2082 option_c_contract_list = QA_fetch_get_commodity_option_C_contract_time_to_market()2083 coll_option_commodity_c_day = client.option_commodity_c_day2084 coll_option_commodity_c_day.create_index(2085 [("code",2086 pymongo.ASCENDING),2087 ("date_stamp",2088 pymongo.ASCENDING)]2089 )2090 err = []2091 def __saving_work(code, coll_option_commodity_c_day):2092 try:2093 QA_util_log_info(2094 '##JOB12 Now Saving OPTION_DAY_COMMODITY_C 玉米 ==== {}'.format(2095 str(code)2096 ),2097 ui_log=ui_log2098 )2099 # 首选查找数据库 是否 有 这个代码的数据2100 ref = coll_option_commodity_c_day.find({'code': str(code)[0:8]})2101 end_date = str(now_time())[0:10]2102 # 当前数据库已经包含了这个代码的数据， 继续增量更新2103 # 加入这个判断的原因是因为如果是刚上市的 数据库会没有数据 所以会有负索引问题出现2104 if ref.count() > 0:2105 # 接着上次获取的日期继续更新2106 start_date = ref[ref.count() - 1]['date']2107 QA_util_log_info(2108 ' 上次获取 玉米C 天然橡胶 日线数据的最后日期是 {}'.format(start_date),2109 ui_log=ui_log2110 )2111 QA_util_log_info(2112 'UPDATE_OPTION_C_DAY \n 从上一次下载数据开始继续 Trying update {} from {} to {}'2113 .format(code,2114 start_date,2115 end_date),2116 ui_log=ui_log2117 )2118 if start_date != end_date:2119 start_date0 = QA_util_get_next_day(start_date)2120 df0 = QA_fetch_get_option_day(2121 code=code,2122 start_date=start_date0,2123 end_date=end_date,2124 frequence='day',2125 ip=None,2126 port=None2127 )2128 retCount = df0.iloc[:, 0].size2129 QA_util_log_info(2130 "日期从开始{}-结束{} , 合约代码{} , 返回了{}条记录 , 准备写入数据库".format(2131 start_date0,2132 end_date,2133 code,2134 retCount2135 ),2136 ui_log=ui_log2137 )2138 coll_option_commodity_c_day.insert_many(2139 QA_util_to_json_from_pandas(df0)2140 )2141 else:2142 QA_util_log_info(2143 "^已经获取过这天的数据了^ {}".format(start_date),2144 ui_log=ui_log2145 )2146 else:2147 start_date = '1990-01-01'2148 QA_util_log_info(2149 'UPDATE_C_OPTION_DAY \n 从新开始下载数据 Trying update {} from {} to {}'2150 .format(code,2151 start_date,2152 end_date),2153 ui_log=ui_log2154 )2155 if start_date != end_date:2156 df0 = QA_fetch_get_option_day(2157 code=code,2158 start_date=start_date,2159 end_date=end_date,2160 frequence='day',2161 ip=None,2162 port=None2163 )2164 retCount = df0.iloc[:, 0].size2165 QA_util_log_info(2166 "日期从开始{}-结束{} , 合约代码{} , 获取了{}条记录 , 准备写入数据库^_^ ".format(2167 start_date,2168 end_date,2169 code,2170 retCount2171 ),2172 ui_log=ui_log2173 )2174 coll_option_commodity_c_day.insert_many(2175 QA_util_to_json_from_pandas(df0)2176 )2177 else:2178 QA_util_log_info(2179 "*已经获取过这天的数据了* {}".format(start_date),2180 ui_log=ui_log2181 )2182 except Exception as error0:2183 print(error0)2184 err.append(str(code))2185 for item in range(len(option_c_contract_list)):2186 QA_util_log_info(2187 'The {} of Total {}'.format(item,2188 len(option_c_contract_list)),2189 ui_log=ui_log2190 )2191 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(2192 str(float(item / len(option_c_contract_list) * 100))[0:4] + '%'2193 )2194 intLogProgress = int(2195 float(item / len(option_c_contract_list) * 10000.0)2196 )2197 QA_util_log_info(2198 strLogProgress,2199 ui_log=ui_log,2200 ui_progress=ui_progress,2201 ui_progress_int_value=intLogProgress2202 )2203 __saving_work(2204 option_c_contract_list[item].code,2205 coll_option_commodity_c_day2206 )2207 if len(err) < 1:2208 QA_util_log_info('SUCCESS save option ru day ^_^ ', ui_log=ui_log)2209 else:2210 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)2211 QA_util_log_info(err, ui_log=ui_log)2212def _save_option_commodity_cf_day(2213 client=DATABASE,2214 ui_log=None,2215 ui_progress=None2216):2217 ##################### cf 棉花 ############################################################################2218 option_cf_contract_list = QA_fetch_get_commodity_option_CF_contract_time_to_market()2219 coll_option_commodity_cf_day = client.option_commodity_cf_day2220 coll_option_commodity_cf_day.create_index(2221 [("code",2222 pymongo.ASCENDING),2223 ("date_stamp",2224 pymongo.ASCENDING)]2225 )2226 err = []2227 def __saving_work(code, coll_option_commodity_cf_day):2228 try:2229 QA_util_log_info(2230 '##JOB12 Now Saving OPTION_DAY_COMMODITY_CF 棉花 ==== {}'.format(2231 str(code)2232 ),2233 ui_log=ui_log2234 )2235 # 首选查找数据库 是否 有 这个代码的数据2236 ref = coll_option_commodity_cf_day.find({'code': str(code)[0:8]})2237 end_date = str(now_time())[0:10]2238 # 当前数据库已经包含了这个代码的数据， 继续增量更新2239 # 加入这个判断的原因是因为如果是刚上市的 数据库会没有数据 所以会有负索引问题出现2240 if ref.count() > 0:2241 # 接着上次获取的日期继续更新2242 start_date = ref[ref.count() - 1]['date']2243 QA_util_log_info(2244 ' 上次获取 期权ru 天然橡胶 日线数据的最后日期是 {}'.format(start_date),2245 ui_log=ui_log2246 )2247 QA_util_log_info(2248 'UPDATE_OPTION_CF_DAY \n 从上一次下载数据开始继续 Trying update {} from {} to {}'2249 .format(code,2250 start_date,2251 end_date),2252 ui_log=ui_log2253 )2254 if start_date != end_date:2255 start_date0 = QA_util_get_next_day(start_date)2256 df0 = QA_fetch_get_option_day(2257 code=code,2258 start_date=start_date0,2259 end_date=end_date,2260 frequence='day',2261 ip=None,2262 port=None2263 )2264 retCount = df0.iloc[:, 0].size2265 QA_util_log_info(2266 "日期从开始{}-结束{} , 合约代码{} , 返回了{}条记录 , 准备写入数据库".format(2267 start_date0,2268 end_date,2269 code,2270 retCount2271 ),2272 ui_log=ui_log2273 )2274 coll_option_commodity_cf_day.insert_many(2275 QA_util_to_json_from_pandas(df0)2276 )2277 else:2278 QA_util_log_info(2279 "^已经获取过这天的数据了^ {}".format(start_date),2280 ui_log=ui_log2281 )2282 else:2283 start_date = '1990-01-01'2284 QA_util_log_info(2285 'UPDATE_M_OPTION_DAY \n 从新开始下载数据 Trying update {} from {} to {}'2286 .format(code,2287 start_date,2288 end_date),2289 ui_log=ui_log2290 )2291 if start_date != end_date:2292 df0 = QA_fetch_get_option_day(2293 code=code,2294 start_date=start_date,2295 end_date=end_date,2296 frequence='day',2297 ip=None,2298 port=None2299 )2300 retCount = df0.iloc[:, 0].size2301 QA_util_log_info(2302 "日期从开始{}-结束{} , 合约代码{} , 获取了{}条记录 , 准备写入数据库^_^ ".format(2303 start_date,2304 end_date,2305 code,2306 retCount2307 ),2308 ui_log=ui_log2309 )2310 coll_option_commodity_cf_day.insert_many(2311 QA_util_to_json_from_pandas(df0)2312 )2313 else:2314 QA_util_log_info(2315 "*已经获取过这天的数据了* {}".format(start_date),2316 ui_log=ui_log2317 )2318 except Exception as error0:2319 print(error0)2320 err.append(str(code))2321 for item in range(len(option_cf_contract_list)):2322 QA_util_log_info(2323 'The {} of Total {}'.format(item,2324 len(option_cf_contract_list)),2325 ui_log=ui_log2326 )2327 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(2328 str(float(item / len(option_cf_contract_list) * 100))[0:4] + '%'2329 )2330 intLogProgress = int(2331 float(item / len(option_cf_contract_list) * 10000.0)2332 )2333 QA_util_log_info(2334 strLogProgress,2335 ui_log=ui_log,2336 ui_progress=ui_progress,2337 ui_progress_int_value=intLogProgress2338 )2339 __saving_work(2340 option_cf_contract_list[item].code,2341 coll_option_commodity_cf_day2342 )2343 if len(err) < 1:2344 QA_util_log_info('SUCCESS save option ru day ^_^ ', ui_log=ui_log)2345 else:2346 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)2347 QA_util_log_info(err, ui_log=ui_log)2348def _save_option_commodity_sr_day(2349 client=DATABASE,2350 ui_log=None,2351 ui_progress=None2352):2353 ##################### sr 白糖 ############################################################################2354 option_sr_contract_list = QA_fetch_get_commodity_option_SR_contract_time_to_market(2355 )2356 coll_option_commodity_sr_day = client.option_commodity_sr_day2357 coll_option_commodity_sr_day.create_index(2358 [("code",2359 pymongo.ASCENDING),2360 ("date_stamp",2361 pymongo.ASCENDING)]2362 )2363 err = []2364 def __saving_work(code, coll_option_commodity_sr_day):2365 try:2366 QA_util_log_info(2367 '##JOB12 Now Saving OPTION_DAY_COMMODITY_SR 白糖 ==== {}'.format(2368 str(code)2369 ),2370 ui_log=ui_log2371 )2372 # 首选查找数据库 是否 有 这个代码的数据2373 ref = coll_option_commodity_sr_day.find({'code': str(code)[0:8]})2374 end_date = str(now_time())[0:10]2375 # 当前数据库已经包含了这个代码的数据， 继续增量更新2376 # 加入这个判断的原因是因为如果是刚上市的 数据库会没有数据 所以会有负索引问题出现2377 if ref.count() > 0:2378 # 接着上次获取的日期继续更新2379 start_date = ref[ref.count() - 1]['date']2380 QA_util_log_info(2381 ' 上次获取期权sr白糖日线数据的最后日期是 {}'.format(start_date),2382 ui_log=ui_log2383 )2384 QA_util_log_info(2385 'UPDATE_OPTION_M_DAY \n 从上一次下载数据开始继续 Trying update {} from {} to {}'2386 .format(code,2387 start_date,2388 end_date),2389 ui_log=ui_log2390 )2391 if start_date != end_date:2392 start_date0 = QA_util_get_next_day(start_date)2393 df0 = QA_fetch_get_option_day(2394 code=code,2395 start_date=start_date0,2396 end_date=end_date,2397 frequence='day',2398 ip=None,2399 port=None2400 )2401 retCount = df0.iloc[:, 0].size2402 QA_util_log_info(2403 "日期从开始{}-结束{} , 合约代码{} , 返回了{}条记录 , 准备写入数据库".format(2404 start_date0,2405 end_date,2406 code,2407 retCount2408 ),2409 ui_log=ui_log2410 )2411 coll_option_commodity_sr_day.insert_many(2412 QA_util_to_json_from_pandas(df0)2413 )2414 else:2415 QA_util_log_info(2416 "^已经获取过这天的数据了^ {}".format(start_date),2417 ui_log=ui_log2418 )2419 else:2420 start_date = '1990-01-01'2421 QA_util_log_info(2422 'UPDATE_M_OPTION_DAY \n 从新开始下载数据 Trying update {} from {} to {}'2423 .format(code,2424 start_date,2425 end_date),2426 ui_log=ui_log2427 )2428 if start_date != end_date:2429 df0 = QA_fetch_get_option_day(2430 code=code,2431 start_date=start_date,2432 end_date=end_date,2433 frequence='day',2434 ip=None,2435 port=None2436 )2437 retCount = df0.iloc[:, 0].size2438 QA_util_log_info(2439 "日期从开始{}-结束{} , 合约代码{} , 获取了{}条记录 , 准备写入数据库^_^ ".format(2440 start_date,2441 end_date,2442 code,2443 retCount2444 ),2445 ui_log=ui_log2446 )2447 coll_option_commodity_sr_day.insert_many(2448 QA_util_to_json_from_pandas(df0)2449 )2450 else:2451 QA_util_log_info(2452 "*已经获取过这天的数据了* {}".format(start_date),2453 ui_log=ui_log2454 )2455 except Exception as error0:2456 print(error0)2457 err.append(str(code))2458 for item in range(len(option_sr_contract_list)):2459 QA_util_log_info(2460 'The {} of Total {}'.format(item,2461 len(option_sr_contract_list)),2462 ui_log=ui_log2463 )2464 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(2465 str(float(item / len(option_sr_contract_list) * 100))[0:4] + '%'2466 )2467 intLogProgress = int(2468 float(item / len(option_sr_contract_list) * 10000.0)2469 )2470 QA_util_log_info(2471 strLogProgress,2472 ui_log=ui_log,2473 ui_progress=ui_progress,2474 ui_progress_int_value=intLogProgress2475 )2476 __saving_work(2477 option_sr_contract_list[item].code,2478 coll_option_commodity_sr_day2479 )2480 if len(err) < 1:2481 QA_util_log_info('SUCCESS save option sr day ^_^ ', ui_log=ui_log)2482 else:2483 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)2484 QA_util_log_info(err, ui_log=ui_log)2485def _save_option_commodity_m_day(2486 client=DATABASE,2487 ui_log=None,2488 ui_progress=None2489):2490 ##################### M 豆粕 ############################################################################2491 option_m_contract_list = QA_fetch_get_commodity_option_M_contract_time_to_market(2492 )2493 coll_option_commodity_m_day = client.option_commodity_m_day2494 coll_option_commodity_m_day.create_index(2495 [("code",2496 pymongo.ASCENDING),2497 ("date_stamp",2498 pymongo.ASCENDING)]2499 )2500 err = []2501 def __saving_work(code, coll_option_commodity_m_day):2502 try:2503 QA_util_log_info(2504 '##JOB12 Now Saving OPTION_DAY_COMMODITY_M 豆粕 ==== {}'.format(2505 str(code)2506 ),2507 ui_log=ui_log2508 )2509 # 首选查找数据库 是否 有 这个代码的数据2510 # M XXXXXX 编码格式2511 ref = coll_option_commodity_m_day.find({'code': str(code)[0:8]})2512 end_date = str(now_time())[0:10]2513 # 当前数据库已经包含了这个代码的数据， 继续增量更新2514 # 加入这个判断的原因是因为如果是刚上市的 数据库会没有数据 所以会有负索引问题出现2515 if ref.count() > 0:2516 # 接着上次获取的日期继续更新2517 start_date = ref[ref.count() - 1]['date']2518 QA_util_log_info(2519 ' 上次获取期权M豆粕日线数据的最后日期是 {}'.format(start_date),2520 ui_log=ui_log2521 )2522 QA_util_log_info(2523 'UPDATE_OPTION_M_DAY \n 从上一次下载数据开始继续 Trying update {} from {} to {}'2524 .format(code,2525 start_date,2526 end_date),2527 ui_log=ui_log2528 )2529 if start_date != end_date:2530 start_date0 = QA_util_get_next_day(start_date)2531 df0 = QA_fetch_get_option_day(2532 code=code,2533 start_date=start_date0,2534 end_date=end_date,2535 frequence='day',2536 ip=None,2537 port=None2538 )2539 retCount = df0.iloc[:, 0].size2540 QA_util_log_info(2541 "日期从开始{}-结束{} , 合约代码{} , 返回了{}条记录 , 准备写入数据库".format(2542 start_date0,2543 end_date,2544 code,2545 retCount2546 ),2547 ui_log=ui_log2548 )2549 coll_option_commodity_m_day.insert_many(2550 QA_util_to_json_from_pandas(df0)2551 )2552 else:2553 QA_util_log_info(2554 "^已经获取过这天的数据了^ {}".format(start_date),2555 ui_log=ui_log2556 )2557 else:2558 start_date = '1990-01-01'2559 QA_util_log_info(2560 'UPDATE_M_OPTION_DAY \n 从新开始下载数据 Trying update {} from {} to {}'2561 .format(code,2562 start_date,2563 end_date),2564 ui_log=ui_log2565 )2566 if start_date != end_date:2567 df0 = QA_fetch_get_option_day(2568 code=code,2569 start_date=start_date,2570 end_date=end_date,2571 frequence='day',2572 ip=None,2573 port=None2574 )2575 retCount = df0.iloc[:, 0].size2576 QA_util_log_info(2577 "日期从开始{}-结束{} , 合约代码{} , 获取了{}条记录 , 准备写入数据库^_^ ".format(2578 start_date,2579 end_date,2580 code,2581 retCount2582 ),2583 ui_log=ui_log2584 )2585 coll_option_commodity_m_day.insert_many(2586 QA_util_to_json_from_pandas(df0)2587 )2588 else:2589 QA_util_log_info(2590 "*已经获取过这天的数据了* {}".format(start_date),2591 ui_log=ui_log2592 )2593 except Exception as error0:2594 print(error0)2595 err.append(str(code))2596 for item in range(len(option_m_contract_list)):2597 QA_util_log_info(2598 'The {} of Total {}'.format(item,2599 len(option_m_contract_list)),2600 ui_log=ui_log2601 )2602 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(2603 str(float(item / len(option_m_contract_list) * 100))[0:4] + '%'2604 )2605 intLogProgress = int(2606 float(item / len(option_m_contract_list) * 10000.0)2607 )2608 QA_util_log_info(2609 strLogProgress,2610 ui_log=ui_log,2611 ui_progress=ui_progress,2612 ui_progress_int_value=intLogProgress2613 )2614 __saving_work(2615 option_m_contract_list[item].code,2616 coll_option_commodity_m_day2617 )2618 if len(err) < 1:2619 QA_util_log_info('SUCCESS save option m day ^_^ ', ui_log=ui_log)2620 else:2621 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)2622 QA_util_log_info(err, ui_log=ui_log)2623def _save_option_commodity_al_day(client=DATABASE,ui_log=None,ui_progress=None):2624 ##################### Al 铝 ############################################################################2625 option_al_contract_list = QA_fetch_get_commodity_option_AL_contract_time_to_market()2626 coll_option_commodity_al_day = client.option_commodity_al_day2627 coll_option_commodity_al_day.create_index(2628 [("code",2629 pymongo.ASCENDING),2630 ("date_stamp",2631 pymongo.ASCENDING)]2632 )2633 err = []2634 def __saving_work(code, coll_option_commodity_al_day):2635 try:2636 QA_util_log_info(2637 '##JOB13 Now Saving OPTION_DAY_COMMODITY_AL 铝 ==== {}'.format(2638 str(code)2639 ),2640 ui_log=ui_log2641 )2642 # 首选查找数据库 是否 有 这个代码的数据2643 # 期权代码 从 10000001 开始编码 100012282644 ref = coll_option_commodity_al_day.find({'code': str(code)[0:8]})2645 end_date = str(now_time())[0:10]2646 # 当前数据库已经包含了这个代码的数据， 继续增量更新2647 # 加入这个判断的原因是因为如果是刚上市的 数据库会没有数据 所以会有负索引问题出现2648 if ref.count() > 0:2649 # 接着上次获取的日期继续更新2650 start_date = ref[ref.count() - 1]['date']2651 QA_util_log_info(2652 ' 上次获取期权AU日线数据的最后日期是 {}'.format(start_date),2653 ui_log=ui_log2654 )2655 QA_util_log_info(2656 'UPDATE_OPTION_AU_DAY \n 从上一次下载数据开始继续 Trying update {} from {} to {}'2657 .format(code,2658 start_date,2659 end_date),2660 ui_log=ui_log2661 )2662 if start_date != end_date:2663 start_date0 = QA_util_get_next_day(start_date)2664 df0 = QA_fetch_get_option_day(2665 code=code,2666 start_date=start_date0,2667 end_date=end_date,2668 frequence='day',2669 ip=None,2670 port=None2671 )2672 retCount = df0.iloc[:, 0].size2673 QA_util_log_info(2674 "日期从开始{}-结束{} , 合约代码{} , 返回了{}条记录 , 准备写入数据库".format(2675 start_date0,2676 end_date,2677 code,2678 retCount2679 ),2680 ui_log=ui_log2681 )2682 coll_option_commodity_al_day.insert_many(2683 QA_util_to_json_from_pandas(df0)2684 )2685 else:2686 QA_util_log_info(2687 "^已经获取过这天的数据了^ {}".format(start_date),2688 ui_log=ui_log2689 )2690 else:2691 start_date = '1990-01-01'2692 QA_util_log_info(2693 'UPDATE_AU_OPTION_DAY \n 从新开始下载数据 Trying update {} from {} to {}'2694 .format(code,2695 start_date,2696 end_date),2697 ui_log=ui_log2698 )2699 if start_date != end_date:2700 df0 = QA_fetch_get_option_day(2701 code=code,2702 start_date=start_date,2703 end_date=end_date,2704 frequence='day',2705 ip=None,2706 port=None2707 )2708 retCount = df0.iloc[:, 0].size2709 QA_util_log_info(2710 "日期从开始{}-结束{} , 合约代码{} , 获取了{}条记录 , 准备写入数据库^_^ ".format(2711 start_date,2712 end_date,2713 code,2714 retCount2715 ),2716 ui_log=ui_log2717 )2718 coll_option_commodity_al_day.insert_many(2719 QA_util_to_json_from_pandas(df0)2720 )2721 else:2722 QA_util_log_info(2723 "*已经获取过这天的数据了* {}".format(start_date),2724 ui_log=ui_log2725 )2726 except Exception as error0:2727 print(error0)2728 err.append(str(code))2729 for item in range(len(option_al_contract_list)):2730 QA_util_log_info(2731 'The {} of Total {}'.format(item, len(option_al_contract_list)),2732 ui_log=ui_log2733 )2734 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(2735 str(float(item / len(option_al_contract_list) * 100))[0:4] + '%'2736 )2737 intLogProgress = int(2738 float(item / len(option_al_contract_list) * 10000.0)2739 )2740 QA_util_log_info(2741 strLogProgress,2742 ui_log=ui_log,2743 ui_progress=ui_progress,2744 ui_progress_int_value=intLogProgress2745 )2746 __saving_work(2747 option_al_contract_list[item].code,2748 coll_option_commodity_al_day2749 )2750 if len(err) < 1:2751 QA_util_log_info('SUCCESS save option au day ^_^ ', ui_log=ui_log)2752 else:2753 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)2754 QA_util_log_info(err, ui_log=ui_log)2755def _save_option_commodity_cu_day(2756 client=DATABASE,2757 ui_log=None,2758 ui_progress=None2759):2760 ##################### CU 铜 ############################################################################2761 option_cu_contract_list = QA_fetch_get_commodity_option_CU_contract_time_to_market(2762 )2763 coll_option_commodity_cu_day = client.option_commodity_cu_day2764 coll_option_commodity_cu_day.create_index(2765 [("code",2766 pymongo.ASCENDING),2767 ("date_stamp",2768 pymongo.ASCENDING)]2769 )2770 err = []2771 def __saving_work(code, coll_option_commodity_cu_day):2772 try:2773 QA_util_log_info(2774 '##JOB12 Now Saving OPTION_DAY_COMMODITY_CU 铜 ==== {}'.format(2775 str(code)2776 ),2777 ui_log=ui_log2778 )2779 # 首选查找数据库 是否 有 这个代码的数据2780 # 期权代码 从 10000001 开始编码 100012282781 ref = coll_option_commodity_cu_day.find({'code': str(code)[0:8]})2782 end_date = str(now_time())[0:10]2783 # 当前数据库已经包含了这个代码的数据， 继续增量更新2784 # 加入这个判断的原因是因为如果是刚上市的 数据库会没有数据 所以会有负索引问题出现2785 if ref.count() > 0:2786 # 接着上次获取的日期继续更新2787 start_date = ref[ref.count() - 1]['date']2788 QA_util_log_info(2789 ' 上次获取期权CU日线数据的最后日期是 {}'.format(start_date),2790 ui_log=ui_log2791 )2792 QA_util_log_info(2793 'UPDATE_OPTION_CU_DAY \n 从上一次下载数据开始继续 Trying update {} from {} to {}'2794 .format(code,2795 start_date,2796 end_date),2797 ui_log=ui_log2798 )2799 if start_date != end_date:2800 start_date0 = QA_util_get_next_day(start_date)2801 df0 = QA_fetch_get_option_day(2802 code=code,2803 start_date=start_date0,2804 end_date=end_date,2805 frequence='day',2806 ip=None,2807 port=None2808 )2809 retCount = df0.iloc[:, 0].size2810 QA_util_log_info(2811 "日期从开始{}-结束{} , 合约代码{} , 返回了{}条记录 , 准备写入数据库".format(2812 start_date0,2813 end_date,2814 code,2815 retCount2816 ),2817 ui_log=ui_log2818 )2819 coll_option_commodity_cu_day.insert_many(2820 QA_util_to_json_from_pandas(df0)2821 )2822 else:2823 QA_util_log_info(2824 "^已经获取过这天的数据了^ {}".format(start_date),2825 ui_log=ui_log2826 )2827 else:2828 start_date = '1990-01-01'2829 QA_util_log_info(2830 'UPDATE_CU_OPTION_DAY \n 从新开始下载数据 Trying update {} from {} to {}'2831 .format(code,2832 start_date,2833 end_date),2834 ui_log=ui_log2835 )2836 if start_date != end_date:2837 df0 = QA_fetch_get_option_day(2838 code=code,2839 start_date=start_date,2840 end_date=end_date,2841 frequence='day',2842 ip=None,2843 port=None2844 )2845 retCount = df0.iloc[:, 0].size2846 QA_util_log_info(2847 "日期从开始{}-结束{} , 合约代码{} , 获取了{}条记录 , 准备写入数据库^_^ ".format(2848 start_date,2849 end_date,2850 code,2851 retCount2852 ),2853 ui_log=ui_log2854 )2855 coll_option_commodity_cu_day.insert_many(2856 QA_util_to_json_from_pandas(df0)2857 )2858 else:2859 QA_util_log_info(2860 "*已经获取过这天的数据了* {}".format(start_date),2861 ui_log=ui_log2862 )2863 except Exception as error0:2864 print(error0)2865 err.append(str(code))2866 for item in range(len(option_cu_contract_list)):2867 QA_util_log_info(2868 'The {} of Total {}'.format(item,2869 len(option_cu_contract_list)),2870 ui_log=ui_log2871 )2872 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(2873 str(float(item / len(option_cu_contract_list) * 100))[0:4] + '%'2874 )2875 intLogProgress = int(2876 float(item / len(option_cu_contract_list) * 10000.0)2877 )2878 QA_util_log_info(2879 strLogProgress,2880 ui_log=ui_log,2881 ui_progress=ui_progress,2882 ui_progress_int_value=intLogProgress2883 )2884 __saving_work(2885 option_cu_contract_list[item].code,2886 coll_option_commodity_cu_day2887 )2888 if len(err) < 1:2889 QA_util_log_info('SUCCESS save option cu day ^_^ ', ui_log=ui_log)2890 else:2891 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)2892 QA_util_log_info(err, ui_log=ui_log)2893def _save_option_commodity_au_day(client=DATABASE,ui_log=None,ui_progress=None):2894 ##################### Au 金 ############################################################################2895 option_au_contract_list = QA_fetch_get_commodity_option_AU_contract_time_to_market()2896 coll_option_commodity_au_day = client.option_commodity_au_day2897 coll_option_commodity_au_day.create_index(2898 [("code",2899 pymongo.ASCENDING),2900 ("date_stamp",2901 pymongo.ASCENDING)]2902 )2903 err = []2904 def __saving_work(code, coll_option_commodity_au_day):2905 try:2906 QA_util_log_info(2907 '##JOB13 Now Saving OPTION_DAY_COMMODITY_AU 金 ==== {}'.format(2908 str(code)2909 ),2910 ui_log=ui_log2911 )2912 # 首选查找数据库 是否 有 这个代码的数据2913 # 期权代码 从 10000001 开始编码 100012282914 ref = coll_option_commodity_au_day.find({'code': str(code)[0:8]})2915 end_date = str(now_time())[0:10]2916 # 当前数据库已经包含了这个代码的数据， 继续增量更新2917 # 加入这个判断的原因是因为如果是刚上市的 数据库会没有数据 所以会有负索引问题出现2918 if ref.count() > 0:2919 # 接着上次获取的日期继续更新2920 start_date = ref[ref.count() - 1]['date']2921 QA_util_log_info(2922 ' 上次获取期权AU日线数据的最后日期是 {}'.format(start_date),2923 ui_log=ui_log2924 )2925 QA_util_log_info(2926 'UPDATE_OPTION_AU_DAY \n 从上一次下载数据开始继续 Trying update {} from {} to {}'2927 .format(code,2928 start_date,2929 end_date),2930 ui_log=ui_log2931 )2932 if start_date != end_date:2933 start_date0 = QA_util_get_next_day(start_date)2934 df0 = QA_fetch_get_option_day(2935 code=code,2936 start_date=start_date0,2937 end_date=end_date,2938 frequence='day',2939 ip=None,2940 port=None2941 )2942 retCount = df0.iloc[:, 0].size2943 QA_util_log_info(2944 "日期从开始{}-结束{} , 合约代码{} , 返回了{}条记录 , 准备写入数据库".format(2945 start_date0,2946 end_date,2947 code,2948 retCount2949 ),2950 ui_log=ui_log2951 )2952 coll_option_commodity_au_day.insert_many(2953 QA_util_to_json_from_pandas(df0)2954 )2955 else:2956 QA_util_log_info(2957 "^已经获取过这天的数据了^ {}".format(start_date),2958 ui_log=ui_log2959 )2960 else:2961 start_date = '1990-01-01'2962 QA_util_log_info(2963 'UPDATE_AU_OPTION_DAY \n 从新开始下载数据 Trying update {} from {} to {}'2964 .format(code,2965 start_date,2966 end_date),2967 ui_log=ui_log2968 )2969 if start_date != end_date:2970 df0 = QA_fetch_get_option_day(2971 code=code,2972 start_date=start_date,2973 end_date=end_date,2974 frequence='day',2975 ip=None,2976 port=None2977 )2978 retCount = df0.iloc[:, 0].size2979 QA_util_log_info(2980 "日期从开始{}-结束{} , 合约代码{} , 获取了{}条记录 , 准备写入数据库^_^ ".format(2981 start_date,2982 end_date,2983 code,2984 retCount2985 ),2986 ui_log=ui_log2987 )2988 coll_option_commodity_au_day.insert_many(2989 QA_util_to_json_from_pandas(df0)2990 )2991 else:2992 QA_util_log_info(2993 "*已经获取过这天的数据了* {}".format(start_date),2994 ui_log=ui_log2995 )2996 except Exception as error0:2997 print(error0)2998 err.append(str(code))2999 for item in range(len(option_au_contract_list)):3000 QA_util_log_info(3001 'The {} of Total {}'.format(item,3002 len(option_au_contract_list)),3003 ui_log=ui_log3004 )3005 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(3006 str(float(item / len(option_au_contract_list) * 100))[0:4] + '%'3007 )3008 intLogProgress = int(3009 float(item / len(option_au_contract_list) * 10000.0)3010 )3011 QA_util_log_info(3012 strLogProgress,3013 ui_log=ui_log,3014 ui_progress=ui_progress,3015 ui_progress_int_value=intLogProgress3016 )3017 __saving_work(3018 option_au_contract_list[item].code,3019 coll_option_commodity_au_day3020 )3021 if len(err) < 1:3022 QA_util_log_info('SUCCESS save option au day ^_^ ', ui_log=ui_log)3023 else:3024 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)3025 QA_util_log_info(err, ui_log=ui_log)3026def QA_SU_save_option_commodity_day(3027 client=DATABASE,3028 ui_log=None,3029 ui_progress=None3030):3031 '''3032 :param client:3033 :return:3034 '''3035 _save_option_commodity_cu_day(3036 client=client,3037 ui_log=ui_log,3038 ui_progress=ui_progress,3039 )3040 _save_option_commodity_m_day(3041 client=client,3042 ui_log=ui_log,3043 ui_progress=ui_progress3044 )3045 _save_option_commodity_sr_day(3046 client=client,3047 ui_log=ui_log,3048 ui_progress=ui_progress3049 )3050 _save_option_commodity_ru_day(3051 client=client,3052 ui_log=ui_log,3053 ui_progress=ui_progress3054 )3055 _save_option_commodity_cf_day(3056 client=client,3057 ui_log=ui_log,3058 ui_progress=ui_progress3059 )3060 _save_option_commodity_c_day(3061 client=client,3062 ui_log=ui_log,3063 ui_progress=ui_progress3064 )3065 _save_option_commodity_au_day(3066 client=client,3067 ui_log=ui_log,3068 ui_progress=ui_progress3069 )3070 _save_option_commodity_al_day(3071 client=client,3072 ui_log=ui_log,3073 ui_progress=ui_progress3074 )3075'''3076期权分钟线3077todo: 代码需要重构 ， 把重复的代码合并在一起3078'''3079def _save_option_commodity_ru_min(3080 client=DATABASE,3081 ui_log=None,3082 ui_progress=None3083):3084 '''3085 :param client:3086 :param ui_log:3087 :param ui_progress:3088 :return:3089 '''3090 '''3091 :param client:3092 :return:3093 '''3094 option_contract_list = QA_fetch_get_commodity_option_C_contract_time_to_market()3095 coll_option_min = client.option_commodity_ru_min3096 coll_option_min.create_index(3097 [("code",3098 pymongo.ASCENDING),3099 ("date_stamp",3100 pymongo.ASCENDING)]3101 )3102 err = []3103 # 索引 code3104 err = []3105 def __saving_work(code, coll):3106 QA_util_log_info(3107 '##JOB13 Now Saving Option RU 棉花 MIN ==== {}'.format(str(code)),3108 ui_log=ui_log3109 )3110 try:3111 for type in ['1min', '5min', '15min', '30min', '60min']:3112 ref_ = coll.find({'code': str(code)[0:8], 'type': type})3113 end_time = str(now_time())[0:19]3114 if ref_.count() > 0:3115 start_time = ref_[ref_.count() - 1]['datetime']3116 QA_util_log_info(3117 '##JOB13.{} Now Saving Option RU 天然橡胶 {} from {} to {} =={} '3118 .format(3119 ['1min',3120 '5min',3121 '15min',3122 '30min',3123 '60min'].index(type),3124 str(code),3125 start_time,3126 end_time,3127 type3128 ),3129 ui_log=ui_log3130 )3131 if start_time != end_time:3132 __data = QA_fetch_get_future_min(3133 str(code),3134 start_time,3135 end_time,3136 type3137 )3138 if len(__data) > 1:3139 QA_util_log_info(3140 " 写入 新增历史合约记录数 {} ".format(len(__data))3141 )3142 coll.insert_many(3143 QA_util_to_json_from_pandas(__data[1::])3144 )3145 else:3146 start_time = '2015-01-01'3147 QA_util_log_info(3148 '##JOB13.{} Now Option RU 天然橡胶 {} from {} to {} =={} '3149 .format(3150 ['1min',3151 '5min',3152 '15min',3153 '30min',3154 '60min'].index(type),3155 str(code),3156 start_time,3157 end_time,3158 type3159 ),3160 ui_log=ui_log3161 )3162 if start_time != end_time:3163 __data = QA_fetch_get_future_min(3164 str(code),3165 start_time,3166 end_time,3167 type3168 )3169 if len(__data) > 1:3170 QA_util_log_info(3171 " 写入 新增合约记录数 {} ".format(len(__data))3172 )3173 coll.insert_many(3174 QA_util_to_json_from_pandas(__data)3175 )3176 except:3177 err.append(code)3178 executor = ThreadPoolExecutor(max_workers=4)3179 res = {3180 executor.submit(3181 __saving_work,3182 option_contract_list[i_]["code"],3183 coll_option_min3184 )3185 for i_ in range(len(option_contract_list))3186 } # multi index ./.3187 count = 03188 for i_ in concurrent.futures.as_completed(res):3189 QA_util_log_info(3190 'The {} of Total {}'.format(count,3191 len(option_contract_list)),3192 ui_log=ui_log3193 )3194 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(3195 str(float(count / len(option_contract_list) * 100))[0:4] + '%'3196 )3197 intLogProgress = int(float(count / len(option_contract_list) * 10000.0))3198 QA_util_log_info(3199 strLogProgress,3200 ui_log=ui_log,3201 ui_progress=ui_progress,3202 ui_progress_int_value=intLogProgress3203 )3204 count = count + 13205 if len(err) < 1:3206 QA_util_log_info('SUCCESS', ui_log=ui_log)3207 else:3208 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)3209 QA_util_log_info(err, ui_log=ui_log)3210 pass3211def _save_option_commodity_c_min(3212 client=DATABASE,3213 ui_log=None,3214 ui_progress=None3215):3216 '''3217 :param client:3218 :param ui_log:3219 :param ui_progress:3220 :return:3221 '''3222 '''3223 :param client:3224 :return:3225 '''3226 option_contract_list = QA_fetch_get_commodity_option_C_contract_time_to_market()3227 coll_option_min = client.option_commodity_c_min3228 coll_option_min.create_index(3229 [("code",3230 pymongo.ASCENDING),3231 ("date_stamp",3232 pymongo.ASCENDING)]3233 )3234 err = []3235 # 索引 code3236 err = []3237 def __saving_work(code, coll):3238 QA_util_log_info(3239 '##JOB13 Now Saving Option C 玉米 MIN ==== {}'.format(str(code)),3240 ui_log=ui_log3241 )3242 try:3243 for type in ['1min', '5min', '15min', '30min', '60min']:3244 ref_ = coll.find({'code': str(code)[0:8], 'type': type})3245 end_time = str(now_time())[0:19]3246 if ref_.count() > 0:3247 start_time = ref_[ref_.count() - 1]['datetime']3248 QA_util_log_info(3249 '##JOB13.{} Now Saving Option C 玉米 {} from {} to {} =={} '3250 .format(3251 ['1min',3252 '5min',3253 '15min',3254 '30min',3255 '60min'].index(type),3256 str(code),3257 start_time,3258 end_time,3259 type3260 ),3261 ui_log=ui_log3262 )3263 if start_time != end_time:3264 __data = QA_fetch_get_future_min(3265 str(code),3266 start_time,3267 end_time,3268 type3269 )3270 if len(__data) > 1:3271 QA_util_log_info(3272 " 写入 新增历史合约记录数 {} ".format(len(__data))3273 )3274 coll.insert_many(3275 QA_util_to_json_from_pandas(__data[1::])3276 )3277 else:3278 start_time = '2015-01-01'3279 QA_util_log_info(3280 '##JOB13.{} Now Option C 玉米 {} from {} to {} =={} '3281 .format(3282 ['1min',3283 '5min',3284 '15min',3285 '30min',3286 '60min'].index(type),3287 str(code),3288 start_time,3289 end_time,3290 type3291 ),3292 ui_log=ui_log3293 )3294 if start_time != end_time:3295 __data = QA_fetch_get_future_min(3296 str(code),3297 start_time,3298 end_time,3299 type3300 )3301 if len(__data) > 1:3302 QA_util_log_info(3303 " 写入 新增合约记录数 {} ".format(len(__data))3304 )3305 coll.insert_many(3306 QA_util_to_json_from_pandas(__data)3307 )3308 except:3309 err.append(code)3310 executor = ThreadPoolExecutor(max_workers=4)3311 res = {3312 executor.submit(3313 __saving_work,3314 option_contract_list[i_]["code"],3315 coll_option_min3316 )3317 for i_ in range(len(option_contract_list))3318 } # multi index ./.3319 count = 03320 for i_ in concurrent.futures.as_completed(res):3321 QA_util_log_info(3322 'The {} of Total {}'.format(count,3323 len(option_contract_list)),3324 ui_log=ui_log3325 )3326 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(3327 str(float(count / len(option_contract_list) * 100))[0:4] + '%'3328 )3329 intLogProgress = int(float(count / len(option_contract_list) * 10000.0))3330 QA_util_log_info(3331 strLogProgress,3332 ui_log=ui_log,3333 ui_progress=ui_progress,3334 ui_progress_int_value=intLogProgress3335 )3336 count = count + 13337 if len(err) < 1:3338 QA_util_log_info('SUCCESS', ui_log=ui_log)3339 else:3340 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)3341 QA_util_log_info(err, ui_log=ui_log)3342 pass3343def _save_option_commodity_cf_min(3344 client=DATABASE,3345 ui_log=None,3346 ui_progress=None3347):3348 '''3349 :param client:3350 :param ui_log:3351 :param ui_progress:3352 :return:3353 '''3354 '''3355 :param client:3356 :return:3357 '''3358 option_contract_list = QA_fetch_get_commodity_option_CF_contract_time_to_market()3359 coll_option_min = client.option_commodity_cf_min3360 coll_option_min.create_index(3361 [("code",3362 pymongo.ASCENDING),3363 ("date_stamp",3364 pymongo.ASCENDING)]3365 )3366 err = []3367 # 索引 code3368 err = []3369 def __saving_work(code, coll):3370 QA_util_log_info(3371 '##JOB13 Now Saving Option CF 棉花 MIN ==== {}'.format(str(code)),3372 ui_log=ui_log3373 )3374 try:3375 for type in ['1min', '5min', '15min', '30min', '60min']:3376 ref_ = coll.find({'code': str(code)[0:8], 'type': type})3377 end_time = str(now_time())[0:19]3378 if ref_.count() > 0:3379 start_time = ref_[ref_.count() - 1]['datetime']3380 QA_util_log_info(3381 '##JOB13.{} Now Saving Option CF 棉花 {} from {} to {} =={} '3382 .format(3383 ['1min',3384 '5min',3385 '15min',3386 '30min',3387 '60min'].index(type),3388 str(code),3389 start_time,3390 end_time,3391 type3392 ),3393 ui_log=ui_log3394 )3395 if start_time != end_time:3396 __data = QA_fetch_get_future_min(3397 str(code),3398 start_time,3399 end_time,3400 type3401 )3402 if len(__data) > 1:3403 QA_util_log_info(3404 " 写入 新增历史合约记录数 {} ".format(len(__data))3405 )3406 coll.insert_many(3407 QA_util_to_json_from_pandas(__data[1::])3408 )3409 else:3410 start_time = '2015-01-01'3411 QA_util_log_info(3412 '##JOB13.{} Now Option CF 棉花 {} from {} to {} =={} '3413 .format(3414 ['1min',3415 '5min',3416 '15min',3417 '30min',3418 '60min'].index(type),3419 str(code),3420 start_time,3421 end_time,3422 type3423 ),3424 ui_log=ui_log3425 )3426 if start_time != end_time:3427 __data = QA_fetch_get_future_min(3428 str(code),3429 start_time,3430 end_time,3431 type3432 )3433 if len(__data) > 1:3434 QA_util_log_info(3435 " 写入 新增合约记录数 {} ".format(len(__data))3436 )3437 coll.insert_many(3438 QA_util_to_json_from_pandas(__data)3439 )3440 except:3441 err.append(code)3442 executor = ThreadPoolExecutor(max_workers=4)3443 res = {3444 executor.submit(3445 __saving_work,3446 option_contract_list[i_]["code"],3447 coll_option_min3448 )3449 for i_ in range(len(option_contract_list))3450 } # multi index ./.3451 count = 03452 for i_ in concurrent.futures.as_completed(res):3453 QA_util_log_info(3454 'The {} of Total {}'.format(count,3455 len(option_contract_list)),3456 ui_log=ui_log3457 )3458 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(3459 str(float(count / len(option_contract_list) * 100))[0:4] + '%'3460 )3461 intLogProgress = int(float(count / len(option_contract_list) * 10000.0))3462 QA_util_log_info(3463 strLogProgress,3464 ui_log=ui_log,3465 ui_progress=ui_progress,3466 ui_progress_int_value=intLogProgress3467 )3468 count = count + 13469 if len(err) < 1:3470 QA_util_log_info('SUCCESS', ui_log=ui_log)3471 else:3472 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)3473 QA_util_log_info(err, ui_log=ui_log)3474 pass3475def _save_option_commodity_ru_min(3476 client=DATABASE,3477 ui_log=None,3478 ui_progress=None3479):3480 '''3481 :param client:3482 :param ui_log:3483 :param ui_progress:3484 :return:3485 '''3486 '''3487 :param client:3488 :return:3489 '''3490 option_contract_list = QA_fetch_get_commodity_option_RU_contract_time_to_market(3491 )3492 coll_option_min = client.option_commodity_ru_min3493 coll_option_min.create_index(3494 [("code",3495 pymongo.ASCENDING),3496 ("date_stamp",3497 pymongo.ASCENDING)]3498 )3499 err = []3500 # 索引 code3501 err = []3502 def __saving_work(code, coll):3503 QA_util_log_info(3504 '##JOB13 Now Saving Option RU 天然橡胶 MIN ==== {}'.format(str(code)),3505 ui_log=ui_log3506 )3507 try:3508 for type in ['1min', '5min', '15min', '30min', '60min']:3509 ref_ = coll.find({'code': str(code)[0:8], 'type': type})3510 end_time = str(now_time())[0:19]3511 if ref_.count() > 0:3512 start_time = ref_[ref_.count() - 1]['datetime']3513 QA_util_log_info(3514 '##JOB13.{} Now Saving Option RU 天然橡胶 {} from {} to {} =={} '3515 .format(3516 ['1min',3517 '5min',3518 '15min',3519 '30min',3520 '60min'].index(type),3521 str(code),3522 start_time,3523 end_time,3524 type3525 ),3526 ui_log=ui_log3527 )3528 if start_time != end_time:3529 __data = QA_fetch_get_future_min(3530 str(code),3531 start_time,3532 end_time,3533 type3534 )3535 if len(__data) > 1:3536 QA_util_log_info(3537 " 写入 新增历史合约记录数 {} ".format(len(__data))3538 )3539 coll.insert_many(3540 QA_util_to_json_from_pandas(__data[1::])3541 )3542 else:3543 start_time = '2015-01-01'3544 QA_util_log_info(3545 '##JOB13.{} Now Option RU 天然橡胶 {} from {} to {} =={} '3546 .format(3547 ['1min',3548 '5min',3549 '15min',3550 '30min',3551 '60min'].index(type),3552 str(code),3553 start_time,3554 end_time,3555 type3556 ),3557 ui_log=ui_log3558 )3559 if start_time != end_time:3560 __data = QA_fetch_get_future_min(3561 str(code),3562 start_time,3563 end_time,3564 type3565 )3566 if len(__data) > 1:3567 QA_util_log_info(3568 " 写入 新增合约记录数 {} ".format(len(__data))3569 )3570 coll.insert_many(3571 QA_util_to_json_from_pandas(__data)3572 )3573 except:3574 err.append(code)3575 executor = ThreadPoolExecutor(max_workers=4)3576 res = {3577 executor.submit(3578 __saving_work,3579 option_contract_list[i_]["code"],3580 coll_option_min3581 )3582 for i_ in range(len(option_contract_list))3583 } # multi index ./.3584 count = 03585 for i_ in concurrent.futures.as_completed(res):3586 QA_util_log_info(3587 'The {} of Total {}'.format(count,3588 len(option_contract_list)),3589 ui_log=ui_log3590 )3591 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(3592 str(float(count / len(option_contract_list) * 100))[0:4] + '%'3593 )3594 intLogProgress = int(float(count / len(option_contract_list) * 10000.0))3595 QA_util_log_info(3596 strLogProgress,3597 ui_log=ui_log,3598 ui_progress=ui_progress,3599 ui_progress_int_value=intLogProgress3600 )3601 count = count + 13602 if len(err) < 1:3603 QA_util_log_info('SUCCESS', ui_log=ui_log)3604 else:3605 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)3606 QA_util_log_info(err, ui_log=ui_log)3607 pass3608def _save_option_commodity_cu_min(3609 client=DATABASE,3610 ui_log=None,3611 ui_progress=None3612):3613 '''3614 :param client:3615 :param ui_log:3616 :param ui_progress:3617 :return:3618 '''3619 '''3620 :param client:3621 :return:3622 '''3623 option_contract_list = QA_fetch_get_commodity_option_CU_contract_time_to_market(3624 )3625 coll_option_min = client.option_commodity_cu_min3626 coll_option_min.create_index(3627 [("code",3628 pymongo.ASCENDING),3629 ("date_stamp",3630 pymongo.ASCENDING)]3631 )3632 err = []3633 # 索引 code3634 err = []3635 def __saving_work(code, coll):3636 QA_util_log_info(3637 '##JOB13 Now Saving Option CU 铜 MIN ==== {}'.format(str(code)),3638 ui_log=ui_log3639 )3640 try:3641 for type in ['1min', '5min', '15min', '30min', '60min']:3642 ref_ = coll.find({'code': str(code)[0:8], 'type': type})3643 end_time = str(now_time())[0:19]3644 if ref_.count() > 0:3645 start_time = ref_[ref_.count() - 1]['datetime']3646 QA_util_log_info(3647 '##JOB13.{} Now Saving Option CU 铜 {} from {} to {} =={} '3648 .format(3649 ['1min',3650 '5min',3651 '15min',3652 '30min',3653 '60min'].index(type),3654 str(code),3655 start_time,3656 end_time,3657 type3658 ),3659 ui_log=ui_log3660 )3661 if start_time != end_time:3662 __data = QA_fetch_get_future_min(3663 str(code),3664 start_time,3665 end_time,3666 type3667 )3668 if len(__data) > 1:3669 QA_util_log_info(3670 " 写入 新增历史合约记录数 {} ".format(len(__data))3671 )3672 coll.insert_many(3673 QA_util_to_json_from_pandas(__data[1::])3674 )3675 else:3676 start_time = '2015-01-01'3677 QA_util_log_info(3678 '##JOB13.{} Now Option CU 铜 {} from {} to {} =={} '3679 .format(3680 ['1min',3681 '5min',3682 '15min',3683 '30min',3684 '60min'].index(type),3685 str(code),3686 start_time,3687 end_time,3688 type3689 ),3690 ui_log=ui_log3691 )3692 if start_time != end_time:3693 __data = QA_fetch_get_future_min(3694 str(code),3695 start_time,3696 end_time,3697 type3698 )3699 if len(__data) > 1:3700 QA_util_log_info(3701 " 写入 新增合约记录数 {} ".format(len(__data))3702 )3703 coll.insert_many(3704 QA_util_to_json_from_pandas(__data)3705 )3706 except:3707 err.append(code)3708 executor = ThreadPoolExecutor(max_workers=4)3709 res = {3710 executor.submit(3711 __saving_work,3712 option_contract_list[i_]["code"],3713 coll_option_min3714 )3715 for i_ in range(len(option_contract_list))3716 } # multi index ./.3717 count = 03718 for i_ in concurrent.futures.as_completed(res):3719 QA_util_log_info(3720 'The {} of Total {}'.format(count,3721 len(option_contract_list)),3722 ui_log=ui_log3723 )3724 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(3725 str(float(count / len(option_contract_list) * 100))[0:4] + '%'3726 )3727 intLogProgress = int(float(count / len(option_contract_list) * 10000.0))3728 QA_util_log_info(3729 strLogProgress,3730 ui_log=ui_log,3731 ui_progress=ui_progress,3732 ui_progress_int_value=intLogProgress3733 )3734 count = count + 13735 if len(err) < 1:3736 QA_util_log_info('SUCCESS', ui_log=ui_log)3737 else:3738 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)3739 QA_util_log_info(err, ui_log=ui_log)3740 pass3741def _save_option_commodity_au_min(3742 client=DATABASE,3743 ui_log=None,3744 ui_progress=None3745):3746 '''3747 :param client:3748 :param ui_log:3749 :param ui_progress:3750 :return:3751 '''3752 '''3753 :param client:3754 :return:3755 '''3756 option_contract_list = QA_fetch_get_commodity_option_AU_contract_time_to_market(3757 )3758 coll_option_min = client.option_commodity_au_min3759 coll_option_min.create_index(3760 [("code",3761 pymongo.ASCENDING),3762 ("date_stamp",3763 pymongo.ASCENDING)]3764 )3765 err = []3766 # 索引 code3767 err = []3768 def __saving_work(code, coll):3769 QA_util_log_info(3770 '##JOB13 Now Saving Option AU 金 MIN ==== {}'.format(str(code)),3771 ui_log=ui_log3772 )3773 try:3774 for type in ['1min', '5min', '15min', '30min', '60min']:3775 ref_ = coll.find({'code': str(code)[0:8], 'type': type})3776 end_time = str(now_time())[0:19]3777 if ref_.count() > 0:3778 start_time = ref_[ref_.count() - 1]['datetime']3779 QA_util_log_info(3780 '##JOB13.{} Now Saving Option AU 金 {} from {} to {} =={} '3781 .format(3782 ['1min',3783 '5min',3784 '15min',3785 '30min',3786 '60min'].index(type),3787 str(code),3788 start_time,3789 end_time,3790 type3791 ),3792 ui_log=ui_log3793 )3794 if start_time != end_time:3795 __data = QA_fetch_get_future_min(3796 str(code),3797 start_time,3798 end_time,3799 type3800 )3801 if len(__data) > 1:3802 QA_util_log_info(3803 " 写入 新增历史合约记录数 {} ".format(len(__data))3804 )3805 coll.insert_many(3806 QA_util_to_json_from_pandas(__data[1::])3807 )3808 else:3809 start_time = '2015-01-01'3810 QA_util_log_info(3811 '##JOB13.{} Now Option AU 金 {} from {} to {} =={} '3812 .format(3813 ['1min',3814 '5min',3815 '15min',3816 '30min',3817 '60min'].index(type),3818 str(code),3819 start_time,3820 end_time,3821 type3822 ),3823 ui_log=ui_log3824 )3825 if start_time != end_time:3826 __data = QA_fetch_get_future_min(3827 str(code),3828 start_time,3829 end_time,3830 type3831 )3832 if len(__data) > 1:3833 QA_util_log_info(3834 " 写入 新增合约记录数 {} ".format(len(__data))3835 )3836 coll.insert_many(3837 QA_util_to_json_from_pandas(__data)3838 )3839 except:3840 err.append(code)3841 executor = ThreadPoolExecutor(max_workers=4)3842 res = {3843 executor.submit(3844 __saving_work,3845 option_contract_list[i_]["code"],3846 coll_option_min3847 )3848 for i_ in range(len(option_contract_list))3849 } # multi index ./.3850 count = 03851 for i_ in concurrent.futures.as_completed(res):3852 QA_util_log_info(3853 'The {} of Total {}'.format(count,3854 len(option_contract_list)),3855 ui_log=ui_log3856 )3857 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(3858 str(float(count / len(option_contract_list) * 100))[0:4] + '%'3859 )3860 intLogProgress = int(float(count / len(option_contract_list) * 10000.0))3861 QA_util_log_info(3862 strLogProgress,3863 ui_log=ui_log,3864 ui_progress=ui_progress,3865 ui_progress_int_value=intLogProgress3866 )3867 count = count + 13868 if len(err) < 1:3869 QA_util_log_info('SUCCESS', ui_log=ui_log)3870 else:3871 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)3872 QA_util_log_info(err, ui_log=ui_log)3873 pass3874def _save_option_commodity_al_min(3875 client=DATABASE,3876 ui_log=None,3877 ui_progress=None3878):3879 '''3880 :param client:3881 :param ui_log:3882 :param ui_progress:3883 :return:3884 '''3885 '''3886 :param client:3887 :return:3888 '''3889 option_contract_list = QA_fetch_get_commodity_option_AL_contract_time_to_market()3890 coll_option_min = client.option_commodity_al_min3891 coll_option_min.create_index(3892 [("code",3893 pymongo.ASCENDING),3894 ("date_stamp",3895 pymongo.ASCENDING)]3896 )3897 err = []3898 # 索引 code3899 err = []3900 def __saving_work(code, coll):3901 QA_util_log_info(3902 '##JOB20 Now Saving Option AL 铝 MIN ==== {}'.format(str(code)),3903 ui_log=ui_log3904 )3905 try:3906 for type in ['1min', '5min', '15min', '30min', '60min']:3907 ref_ = coll.find({'code': str(code)[0:8], 'type': type})3908 end_time = str(now_time())[0:19]3909 if ref_.count() > 0:3910 start_time = ref_[ref_.count() - 1]['datetime']3911 QA_util_log_info(3912 '##JOB20.{} Now Saving Option AL 铝 {} from {} to {} =={} '3913 .format(3914 ['1min',3915 '5min',3916 '15min',3917 '30min',3918 '60min'].index(type),3919 str(code),3920 start_time,3921 end_time,3922 type3923 ),3924 ui_log=ui_log3925 )3926 if start_time != end_time:3927 __data = QA_fetch_get_future_min(3928 str(code),3929 start_time,3930 end_time,3931 type3932 )3933 if len(__data) > 1:3934 QA_util_log_info(3935 " 写入 新增历史合约记录数 {} ".format(len(__data))3936 )3937 coll.insert_many(3938 QA_util_to_json_from_pandas(__data[1::])3939 )3940 else:3941 start_time = '2015-01-01'3942 QA_util_log_info(3943 '##JOB20.{} Now Option AL 铝 {} from {} to {} =={} '3944 .format(3945 ['1min',3946 '5min',3947 '15min',3948 '30min',3949 '60min'].index(type),3950 str(code),3951 start_time,3952 end_time,3953 type3954 ),3955 ui_log=ui_log3956 )3957 if start_time != end_time:3958 __data = QA_fetch_get_future_min(3959 str(code),3960 start_time,3961 end_time,3962 type3963 )3964 if len(__data) > 1:3965 QA_util_log_info(3966 " 写入 新增合约记录数 {} ".format(len(__data))3967 )3968 coll.insert_many(3969 QA_util_to_json_from_pandas(__data)3970 )3971 except:3972 err.append(code)3973 executor = ThreadPoolExecutor(max_workers=4)3974 res = {3975 executor.submit(3976 __saving_work,3977 option_contract_list[i_]["code"],3978 coll_option_min3979 )3980 for i_ in range(len(option_contract_list))3981 } # multi index ./.3982 count = 03983 for i_ in concurrent.futures.as_completed(res):3984 QA_util_log_info(3985 'The {} of Total {}'.format(count,3986 len(option_contract_list)),3987 ui_log=ui_log3988 )3989 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(3990 str(float(count / len(option_contract_list) * 100))[0:4] + '%'3991 )3992 intLogProgress = int(float(count / len(option_contract_list) * 10000.0))3993 QA_util_log_info(3994 strLogProgress,3995 ui_log=ui_log,3996 ui_progress=ui_progress,3997 ui_progress_int_value=intLogProgress3998 )3999 count = count + 14000 if len(err) < 1:4001 QA_util_log_info('SUCCESS', ui_log=ui_log)4002 else:4003 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)4004 QA_util_log_info(err, ui_log=ui_log)4005 pass4006def _save_option_commodity_sr_min(4007 client=DATABASE,4008 ui_log=None,4009 ui_progress=None4010):4011 '''4012 :param client:4013 :param ui_log:4014 :param ui_progress:4015 :return:4016 '''4017 '''4018 :param client:4019 :return:4020 '''4021 option_contract_list = QA_fetch_get_commodity_option_SR_contract_time_to_market(4022 )4023 coll_option_min = client.option_commodity_sr_min4024 coll_option_min.create_index(4025 [("code",4026 pymongo.ASCENDING),4027 ("date_stamp",4028 pymongo.ASCENDING)]4029 )4030 err = []4031 # 索引 code4032 err = []4033 def __saving_work(code, coll):4034 QA_util_log_info(4035 '##JOB13 Now Saving Option SR 白糖 ==== {}'.format(str(code)),4036 ui_log=ui_log4037 )4038 try:4039 for type in ['1min', '5min', '15min', '30min', '60min']:4040 ref_ = coll.find({'code': str(code)[0:8], 'type': type})4041 end_time = str(now_time())[0:19]4042 if ref_.count() > 0:4043 start_time = ref_[ref_.count() - 1]['datetime']4044 QA_util_log_info(4045 '##JOB13.{} Now Saving Option SR 白糖 {} from {} to {} =={} '4046 .format(4047 ['1min',4048 '5min',4049 '15min',4050 '30min',4051 '60min'].index(type),4052 str(code),4053 start_time,4054 end_time,4055 type4056 ),4057 ui_log=ui_log4058 )4059 if start_time != end_time:4060 __data = QA_fetch_get_future_min(4061 str(code),4062 start_time,4063 end_time,4064 type4065 )4066 if len(__data) > 1:4067 QA_util_log_info(4068 " 写入 新增历史合约记录数 {} ".format(len(__data))4069 )4070 coll.insert_many(4071 QA_util_to_json_from_pandas(__data[1::])4072 )4073 else:4074 start_time = '2015-01-01'4075 QA_util_log_info(4076 '##JOB13.{} Now Option SR 白糖 {} from {} to {} =={} '4077 .format(4078 ['1min',4079 '5min',4080 '15min',4081 '30min',4082 '60min'].index(type),4083 str(code),4084 start_time,4085 end_time,4086 type4087 ),4088 ui_log=ui_log4089 )4090 if start_time != end_time:4091 __data = QA_fetch_get_future_min(4092 str(code),4093 start_time,4094 end_time,4095 type4096 )4097 if len(__data) > 1:4098 QA_util_log_info(4099 " 写入 新增合约记录数 {} ".format(len(__data))4100 )4101 coll.insert_many(4102 QA_util_to_json_from_pandas(__data)4103 )4104 except:4105 err.append(code)4106 executor = ThreadPoolExecutor(max_workers=4)4107 res = {4108 executor.submit(4109 __saving_work,4110 option_contract_list[i_]["code"],4111 coll_option_min4112 )4113 for i_ in range(len(option_contract_list))4114 } # multi index ./.4115 count = 04116 for i_ in concurrent.futures.as_completed(res):4117 QA_util_log_info(4118 'The {} of Total {}'.format(count,4119 len(option_contract_list)),4120 ui_log=ui_log4121 )4122 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(4123 str(float(count / len(option_contract_list) * 100))[0:4] + '%'4124 )4125 intLogProgress = int(float(count / len(option_contract_list) * 10000.0))4126 QA_util_log_info(4127 strLogProgress,4128 ui_log=ui_log,4129 ui_progress=ui_progress,4130 ui_progress_int_value=intLogProgress4131 )4132 count = count + 14133 if len(err) < 1:4134 QA_util_log_info('SUCCESS', ui_log=ui_log)4135 else:4136 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)4137 QA_util_log_info(err, ui_log=ui_log)4138 pass4139def _save_option_commodity_m_min(4140 client=DATABASE,4141 ui_log=None,4142 ui_progress=None4143):4144 '''4145 :param client:4146 :param ui_log:4147 :param ui_progress:4148 :return:4149 '''4150 option_contract_list = QA_fetch_get_commodity_option_M_contract_time_to_market(4151 )4152 coll_option_min = client.option_commodity_m_min4153 coll_option_min.create_index(4154 [("code",4155 pymongo.ASCENDING),4156 ("date_stamp",4157 pymongo.ASCENDING)]4158 )4159 err = []4160 # 索引 code4161 err = []4162 def __saving_work(code, coll):4163 QA_util_log_info(4164 '##JOB13 Now Saving Option M 豆粕 ==== {}'.format(str(code)),4165 ui_log=ui_log4166 )4167 try:4168 for type in ['1min', '5min', '15min', '30min', '60min']:4169 ref_ = coll.find({'code': str(code)[0:8], 'type': type})4170 end_time = str(now_time())[0:19]4171 if ref_.count() > 0:4172 start_time = ref_[ref_.count() - 1]['datetime']4173 QA_util_log_info(4174 '##JOB13.{} Now Saving Option M 豆粕 {} from {} to {} =={} '4175 .format(4176 ['1min',4177 '5min',4178 '15min',4179 '30min',4180 '60min'].index(type),4181 str(code),4182 start_time,4183 end_time,4184 type4185 ),4186 ui_log=ui_log4187 )4188 if start_time != end_time:4189 __data = QA_fetch_get_future_min(4190 str(code),4191 start_time,4192 end_time,4193 type4194 )4195 if len(__data) > 1:4196 QA_util_log_info(4197 " 写入 新增历史合约记录数 {} ".format(len(__data))4198 )4199 coll.insert_many(4200 QA_util_to_json_from_pandas(__data[1::])4201 )4202 else:4203 start_time = '2015-01-01'4204 QA_util_log_info(4205 '##JOB13.{} Now Option M 豆粕 {} from {} to {} =={} '4206 .format(4207 ['1min',4208 '5min',4209 '15min',4210 '30min',4211 '60min'].index(type),4212 str(code),4213 start_time,4214 end_time,4215 type4216 ),4217 ui_log=ui_log4218 )4219 if start_time != end_time:4220 __data = QA_fetch_get_future_min(4221 str(code),4222 start_time,4223 end_time,4224 type4225 )4226 if len(__data) > 1:4227 QA_util_log_info(4228 " 写入 新增合约记录数 {} ".format(len(__data))4229 )4230 coll.insert_many(4231 QA_util_to_json_from_pandas(__data)4232 )4233 except:4234 err.append(code)4235 executor = ThreadPoolExecutor(max_workers=4)4236 res = {4237 executor.submit(4238 __saving_work,4239 option_contract_list[i_]["code"],4240 coll_option_min4241 )4242 for i_ in range(len(option_contract_list))4243 } # multi index ./.4244 count = 04245 for i_ in concurrent.futures.as_completed(res):4246 QA_util_log_info(4247 'The {} of Total {}'.format(count,4248 len(option_contract_list)),4249 ui_log=ui_log4250 )4251 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(4252 str(float(count / len(option_contract_list) * 100))[0:4] + '%'4253 )4254 intLogProgress = int(float(count / len(option_contract_list) * 10000.0))4255 QA_util_log_info(4256 strLogProgress,4257 ui_log=ui_log,4258 ui_progress=ui_progress,4259 ui_progress_int_value=intLogProgress4260 )4261 count = count + 14262 if len(err) < 1:4263 QA_util_log_info('SUCCESS', ui_log=ui_log)4264 else:4265 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)4266 QA_util_log_info(err, ui_log=ui_log)4267 pass4268def QA_SU_save_option_commodity_min(4269 client=DATABASE,4270 ui_log=None,4271 ui_progress=None4272):4273 '''4274 :param client:4275 :return:4276 '''4277 # 测试中发现， 一起回去，容易出现错误，每次获取一个品种后 ，更换服务ip继续获取 ？4278 _save_option_commodity_cu_min(4279 client=client,4280 ui_log=ui_log,4281 ui_progress=ui_progress4282 )4283 _save_option_commodity_sr_min(4284 client=client,4285 ui_log=ui_log,4286 ui_progress=ui_progress4287 )4288 _save_option_commodity_m_min(4289 client=client,4290 ui_log=ui_log,4291 ui_progress=ui_progress4292 )4293 _save_option_commodity_ru_min(4294 client=client,4295 ui_log=ui_log,4296 ui_progress=ui_progress4297 )4298 _save_option_commodity_cf_min(4299 client=client,4300 ui_log=ui_log,4301 ui_progress=ui_progress4302 )4303 _save_option_commodity_c_min(4304 client=client,4305 ui_log=ui_log,4306 ui_progress=ui_progress4307 )4308 _save_option_commodity_au_min(4309 client=client,4310 ui_log=ui_log,4311 ui_progress=ui_progress4312 )4313 _save_option_commodity_al_min(4314 client=client,4315 ui_log=ui_log,4316 ui_progress=ui_progress4317 )4318def QA_SU_save_option_50etf_min(client=DATABASE, ui_log=None, ui_progress=None):4319 '''4320 :param client:4321 :return:4322 '''4323 option_contract_list = QA_fetch_get_option_50etf_contract_time_to_market()4324 coll_option_min = client.option_day_min4325 coll_option_min.create_index(4326 [("code",4327 pymongo.ASCENDING),4328 ("date_stamp",4329 pymongo.ASCENDING)]4330 )4331 err = []4332 # 索引 code4333 err = []4334 def __saving_work(code, coll):4335 QA_util_log_info(4336 '##JOB13 Now Saving Option 50ETF MIN ==== {}'.format(str(code)),4337 ui_log=ui_log4338 )4339 try:4340 for type in ['1min', '5min', '15min', '30min', '60min']:4341 ref_ = coll.find({'code': str(code)[0:8], 'type': type})4342 end_time = str(now_time())[0:19]4343 if ref_.count() > 0:4344 start_time = ref_[ref_.count() - 1]['datetime']4345 QA_util_log_info(4346 '##JOB13.{} Now Saving Option 50ETF {} from {} to {} =={} '4347 .format(4348 ['1min',4349 '5min',4350 '15min',4351 '30min',4352 '60min'].index(type),4353 str(code),4354 start_time,4355 end_time,4356 type4357 ),4358 ui_log=ui_log4359 )4360 if start_time != end_time:4361 __data = QA_fetch_get_future_min(4362 str(code),4363 start_time,4364 end_time,4365 type4366 )4367 if len(__data) > 1:4368 QA_util_log_info(4369 " 写入 新增历史合约记录数 {} ".format(len(__data))4370 )4371 coll.insert_many(4372 QA_util_to_json_from_pandas(__data[1::])4373 )4374 else:4375 start_time = '2015-01-01'4376 QA_util_log_info(4377 '##JOB13.{} Now Option 50ETF {} from {} to {} =={} '4378 .format(4379 ['1min',4380 '5min',4381 '15min',4382 '30min',4383 '60min'].index(type),4384 str(code),4385 start_time,4386 end_time,4387 type4388 ),4389 ui_log=ui_log4390 )4391 if start_time != end_time:4392 __data = QA_fetch_get_future_min(4393 str(code),4394 start_time,4395 end_time,4396 type4397 )4398 if len(__data) > 1:4399 QA_util_log_info(4400 " 写入 新增合约记录数 {} ".format(len(__data))4401 )4402 coll.insert_many(4403 QA_util_to_json_from_pandas(__data)4404 )4405 except:4406 err.append(code)4407 executor = ThreadPoolExecutor(max_workers=4)4408 res = {4409 executor.submit(4410 __saving_work,4411 option_contract_list[i_]["code"],4412 coll_option_min4413 )4414 for i_ in range(len(option_contract_list))4415 } # multi index ./.4416 count = 04417 for i_ in concurrent.futures.as_completed(res):4418 QA_util_log_info(4419 'The {} of Total {}'.format(count,4420 len(option_contract_list)),4421 ui_log=ui_log4422 )4423 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(4424 str(float(count / len(option_contract_list) * 100))[0:4] + '%'4425 )4426 intLogProgress = int(float(count / len(option_contract_list) * 10000.0))4427 QA_util_log_info(4428 strLogProgress,4429 ui_log=ui_log,4430 ui_progress=ui_progress,4431 ui_progress_int_value=intLogProgress4432 )4433 count = count + 14434 if len(err) < 1:4435 QA_util_log_info('SUCCESS', ui_log=ui_log)4436 else:4437 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)4438 QA_util_log_info(err, ui_log=ui_log)4439def QA_SU_save_option_50etf_day(client=DATABASE, ui_log=None, ui_progress=None):4440 '''4441 :param client:4442 :return:4443 '''4444 option_contract_list = QA_fetch_get_option_50etf_contract_time_to_market()4445 coll_option_day = client.option_day4446 coll_option_day.create_index(4447 [("code",4448 pymongo.ASCENDING),4449 ("date_stamp",4450 pymongo.ASCENDING)]4451 )4452 err = []4453 # 索引 code4454 def __saving_work(code, coll_option_day):4455 try:4456 QA_util_log_info(4457 '##JOB12 Now Saving OPTION_DAY==== {}'.format(str(code)),4458 ui_log=ui_log4459 )4460 # 首选查找数据库 是否 有 这个代码的数据4461 # 期权代码 从 10000001 开始编码 100012284462 ref = coll_option_day.find({'code': str(code)[0:8]})4463 end_date = str(now_time())[0:10]4464 # 当前数据库已经包含了这个代码的数据， 继续增量更新4465 # 加入这个判断的原因是因为如果是刚上市的 数据库会没有数据 所以会有负索引问题出现4466 if ref.count() > 0:4467 # 接着上次获取的日期继续更新4468 start_date = ref[ref.count() - 1]['date']4469 QA_util_log_info(4470 ' 上次获取期权日线数据的最后日期是 {}'.format(start_date),4471 ui_log=ui_log4472 )4473 QA_util_log_info(4474 'UPDATE_OPTION_DAY \n 从上一次下载数据开始继续 Trying update {} from {} to {}'4475 .format(code,4476 start_date,4477 end_date),4478 ui_log=ui_log4479 )4480 if start_date != end_date:4481 start_date0 = QA_util_get_next_day(start_date)4482 df0 = QA_fetch_get_option_day(4483 code=code,4484 start_date=start_date0,4485 end_date=end_date,4486 frequence='day',4487 ip=None,4488 port=None4489 )4490 retCount = df0.iloc[:, 0].size4491 QA_util_log_info(4492 "日期从开始{}-结束{} , 合约代码{} , 返回了{}条记录 , 准备写入数据库".format(4493 start_date0,4494 end_date,4495 code,4496 retCount4497 ),4498 ui_log=ui_log4499 )4500 coll_option_day.insert_many(4501 QA_util_to_json_from_pandas(df0)4502 )4503 else:4504 QA_util_log_info(4505 "^已经获取过这天的数据了^ {}".format(start_date),4506 ui_log=ui_log4507 )4508 else:4509 start_date = '1990-01-01'4510 QA_util_log_info(4511 'UPDATE_OPTION_DAY \n 从新开始下载数据 Trying update {} from {} to {}'4512 .format(code,4513 start_date,4514 end_date),4515 ui_log=ui_log4516 )4517 if start_date != end_date:4518 df0 = QA_fetch_get_option_day(4519 code=code,4520 start_date=start_date,4521 end_date=end_date,4522 frequence='day',4523 ip=None,4524 port=None4525 )4526 retCount = df0.iloc[:, 0].size4527 QA_util_log_info(4528 "日期从开始{}-结束{} , 合约代码{} , 获取了{}条记录 , 准备写入数据库^_^ ".format(4529 start_date,4530 end_date,4531 code,4532 retCount4533 ),4534 ui_log=ui_log4535 )4536 coll_option_day.insert_many(4537 QA_util_to_json_from_pandas(df0)4538 )4539 else:4540 QA_util_log_info(4541 "*已经获取过这天的数据了* {}".format(start_date),4542 ui_log=ui_log4543 )4544 except Exception as error0:4545 print(error0)4546 err.append(str(code))4547 for item in range(len(option_contract_list)):4548 QA_util_log_info(4549 'The {} of Total {}'.format(item,4550 len(option_contract_list)),4551 ui_log=ui_log4552 )4553 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(4554 str(float(item / len(option_contract_list) * 100))[0:4] + '%'4555 )4556 intLogProgress = int(float(item / len(option_contract_list) * 10000.0))4557 QA_util_log_info(4558 strLogProgress,4559 ui_log=ui_log,4560 ui_progress=ui_progress,4561 ui_progress_int_value=intLogProgress4562 )4563 __saving_work(option_contract_list[item].code, coll_option_day)4564 if len(err) < 1:4565 QA_util_log_info('SUCCESS save option day ^_^ ', ui_log=ui_log)4566 else:4567 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)4568 QA_util_log_info(err, ui_log=ui_log)4569def QA_SU_save_option_300etf_min(client=DATABASE, ui_log=None, ui_progress=None):4570 '''4571 :param client:4572 :return:4573 '''4574 option_contract_list = QA_fetch_get_option_300etf_contract_time_to_market()4575 coll_option_min = client.option_day_min4576 coll_option_min.create_index(4577 [("code",4578 pymongo.ASCENDING),4579 ("date_stamp",4580 pymongo.ASCENDING)]4581 )4582 err = []4583 # 索引 code4584 err = []4585 def __saving_work(code, coll):4586 QA_util_log_info(4587 '##JOB13 Now Saving Option shanghai sse 300 ETF MIN ==== {}'.format(str(code)),4588 ui_log=ui_log4589 )4590 try:4591 for type in ['1min', '5min', '15min', '30min', '60min']:4592 ref_ = coll.find({'code': str(code)[0:8], 'type': type})4593 end_time = str(now_time())[0:19]4594 if ref_.count() > 0:4595 start_time = ref_[ref_.count() - 1]['datetime']4596 QA_util_log_info(4597 '##JOB13.{} Now Saving Option shanghai 300ETF {} from {} to {} =={} '4598 .format(4599 ['1min',4600 '5min',4601 '15min',4602 '30min',4603 '60min'].index(type),4604 str(code),4605 start_time,4606 end_time,4607 type4608 ),4609 ui_log=ui_log4610 )4611 if start_time != end_time:4612 __data = QA_fetch_get_future_min(4613 str(code),4614 start_time,4615 end_time,4616 type4617 )4618 if len(__data) > 1:4619 QA_util_log_info(4620 " 写入 新增历史合约记录数 {} ".format(len(__data))4621 )4622 coll.insert_many(4623 QA_util_to_json_from_pandas(__data[1::])4624 )4625 else:4626 start_time = '2015-01-01'4627 QA_util_log_info(4628 '##JOB13.{} Now Option shanghai sse 300ETF {} from {} to {} =={} '4629 .format(4630 ['1min',4631 '5min',4632 '15min',4633 '30min',4634 '60min'].index(type),4635 str(code),4636 start_time,4637 end_time,4638 type4639 ),4640 ui_log=ui_log4641 )4642 if start_time != end_time:4643 __data = QA_fetch_get_future_min(4644 str(code),4645 start_time,4646 end_time,4647 type4648 )4649 if len(__data) > 1:4650 QA_util_log_info(4651 " 写入 新增合约记录数 {} ".format(len(__data))4652 )4653 coll.insert_many(4654 QA_util_to_json_from_pandas(__data)4655 )4656 except:4657 err.append(code)4658 executor = ThreadPoolExecutor(max_workers=4)4659 res = {4660 executor.submit(4661 __saving_work,4662 option_contract_list[i_]["code"],4663 coll_option_min4664 )4665 for i_ in range(len(option_contract_list))4666 } # multi index ./.4667 count = 04668 for i_ in concurrent.futures.as_completed(res):4669 QA_util_log_info(4670 'The {} of Total {}'.format(count,4671 len(option_contract_list)),4672 ui_log=ui_log4673 )4674 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(4675 str(float(count / len(option_contract_list) * 100))[0:4] + '%'4676 )4677 intLogProgress = int(float(count / len(option_contract_list) * 10000.0))4678 QA_util_log_info(4679 strLogProgress,4680 ui_log=ui_log,4681 ui_progress=ui_progress,4682 ui_progress_int_value=intLogProgress4683 )4684 count = count + 14685 if len(err) < 1:4686 QA_util_log_info('SUCCESS', ui_log=ui_log)4687 else:4688 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)4689 QA_util_log_info(err, ui_log=ui_log)4690def QA_SU_save_option_300etf_day(client=DATABASE, ui_log=None, ui_progress=None):4691 '''4692 :param client:4693 :return:4694 '''4695 option_contract_list = QA_fetch_get_option_300etf_contract_time_to_market()4696 coll_option_day = client.option_day4697 coll_option_day.create_index(4698 [("code",4699 pymongo.ASCENDING),4700 ("date_stamp",4701 pymongo.ASCENDING)]4702 )4703 err = []4704 # 索引 code4705 def __saving_work(code, coll_option_day):4706 try:4707 QA_util_log_info(4708 '##JOB12 Now Saving shanghai sse 300 etf OPTION_DAY==== {}'.format(str(code)),4709 ui_log=ui_log4710 )4711 # 首选查找数据库 是否 有 这个代码的数据4712 # 期权代码 从 10000001 开始编码 100012284713 ref = coll_option_day.find({'code': str(code)[0:8]})4714 end_date = str(now_time())[0:10]4715 # 当前数据库已经包含了这个代码的数据， 继续增量更新4716 # 加入这个判断的原因是因为如果是刚上市的 数据库会没有数据 所以会有负索引问题出现4717 if ref.count() > 0:4718 # 接着上次获取的日期继续更新4719 start_date = ref[ref.count() - 1]['date']4720 QA_util_log_info(4721 ' 上次获取期权日线数据的最后日期是 {}'.format(start_date),4722 ui_log=ui_log4723 )4724 QA_util_log_info(4725 'UPDATE_OPTION_DAY shanghai sse 300 etf \n 从上一次下载数据开始继续 Trying update {} from {} to {}'4726 .format(code,4727 start_date,4728 end_date),4729 ui_log=ui_log4730 )4731 if start_date != end_date:4732 start_date0 = QA_util_get_next_day(start_date)4733 df0 = QA_fetch_get_option_day(4734 code=code,4735 start_date=start_date0,4736 end_date=end_date,4737 frequence='day',4738 ip=None,4739 port=None4740 )4741 retCount = df0.iloc[:, 0].size4742 QA_util_log_info(4743 "日期从开始{}-结束{} , 合约代码{} , 返回了{}条记录 , 准备写入数据库".format(4744 start_date0,4745 end_date,4746 code,4747 retCount4748 ),4749 ui_log=ui_log4750 )4751 coll_option_day.insert_many(4752 QA_util_to_json_from_pandas(df0)4753 )4754 else:4755 QA_util_log_info(4756 "^已经获取过这天的数据了^ {}".format(start_date),4757 ui_log=ui_log4758 )4759 else:4760 start_date = '1990-01-01'4761 QA_util_log_info(4762 'UPDATE_OPTION_DAY shanghai sse 300 etf \n 从新开始下载数据 Trying update {} from {} to {}'4763 .format(code,4764 start_date,4765 end_date),4766 ui_log=ui_log4767 )4768 if start_date != end_date:4769 df0 = QA_fetch_get_option_day(4770 code=code,4771 start_date=start_date,4772 end_date=end_date,4773 frequence='day',4774 ip=None,4775 port=None4776 )4777 retCount = df0.iloc[:, 0].size4778 QA_util_log_info(4779 "日期从开始{}-结束{} , 合约代码{} , 获取了{}条记录 , 准备写入数据库^_^ ".format(4780 start_date,4781 end_date,4782 code,4783 retCount4784 ),4785 ui_log=ui_log4786 )4787 coll_option_day.insert_many(4788 QA_util_to_json_from_pandas(df0)4789 )4790 else:4791 QA_util_log_info(4792 "*已经获取过这天的数据了* {}".format(start_date),4793 ui_log=ui_log4794 )4795 except Exception as error0:4796 print(error0)4797 err.append(str(code))4798 for item in range(len(option_contract_list)):4799 QA_util_log_info(4800 'The {} of Total {}'.format(item,4801 len(option_contract_list)),4802 ui_log=ui_log4803 )4804 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(4805 str(float(item / len(option_contract_list) * 100))[0:4] + '%'4806 )4807 intLogProgress = int(float(item / len(option_contract_list) * 10000.0))4808 QA_util_log_info(4809 strLogProgress,4810 ui_log=ui_log,4811 ui_progress=ui_progress,4812 ui_progress_int_value=intLogProgress4813 )4814 __saving_work(option_contract_list[item].code, coll_option_day)4815 if len(err) < 1:4816 QA_util_log_info('SUCCESS save option day ^_^ ', ui_log=ui_log)4817 else:4818 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)4819 QA_util_log_info(err, ui_log=ui_log)4820def QA_SU_save_option_contract_list(4821 client=DATABASE,4822 ui_log=None,4823 ui_progress=None):4824 rows_of_option = QA_fetch_get_option_all_contract_time_to_market()4825 # rows_cu = QA_fetch_get_commodity_option_CU_contract_time_to_market()4826 # rows_m = QA_fetch_get_commodity_option_M_contract_time_to_market()4827 # rows_sr = QA_fetch_get_commodity_option_SR_contract_time_to_market()4828 # rows_cf = QA_fetch_get_commodity_option_CF_contract_time_to_market()4829 # rows_ru = QA_fetch_get_commodity_option_RU_contract_time_to_market()4830 # rows_c = QA_fetch_get_commodity_option_C_contract_time_to_market()4831 try:4832 # 🛠todo 这个应该是第一个任务 JOB01， 先更新股票列表！！4833 QA_util_log_info(4834 '##JOB15 Now Saving OPTION_CONTRACT_LIST ====',4835 ui_log=ui_log,4836 ui_progress=ui_progress,4837 ui_progress_int_value=50004838 )4839 coll = client.option_contract_list4840 coll.create_index([('desc', pymongo.ASCENDING)], unique=True)4841 # todo fixhere4842 # from_items is deprecated. Please use DataFrame.from_dict(dict(items), ...) instead. DataFrame.from_dict4843 try:4844 df = pd.DataFrame.from_items([(s.desc, s) for s in rows_of_option])4845 df = (df.T)4846 js = QA_util_to_json_from_pandas(df)4847 # result0 = coll.insert_many(js)4848 for a_js_row in js:4849 # print(a_js_row)4850 akey = a_js_row['desc']4851 id0 = coll.find_one({'desc': akey})4852 if id0 is None:4853 coll.insert(a_js_row)4854 # print(result0)4855 except pymongo.errors.BulkWriteError as e:4856 # https://ask.helplib.com/python/post_127405304857 panic = filter(4858 lambda x: x['code'] != 11000,4859 e.details['writeErrors']4860 )4861 # if len(panic) > 0:4862 # print4863 # "really panic"4864 QA_util_log_info(4865 "完成合约列表更新",4866 ui_log=ui_log,4867 ui_progress=ui_progress,4868 ui_progress_int_value=100004869 )4870 except Exception as e:4871 QA_util_log_info(e, ui_log=ui_log)4872 print(" Error save_tdx.QA_SU_save_option_contract_list exception!")4873def QA_SU_save_option_day_all(client=DATABASE,ui_log=None,ui_progress=None):4874 option_contract_list = QA_fetch_get_option_all_contract_time_to_market()4875 coll_option_day = client.option_day_all4876 coll_option_day.create_index(4877 [("code",4878 pymongo.ASCENDING),4879 ("date_stamp",4880 pymongo.ASCENDING)]4881 )4882 err = []4883 # 索引 code4884 def __saving_work(code, coll_option_day):4885 try:4886 QA_util_log_info(4887 '##JOB12 Now Saving OPTION_DAY ALL ==== {}'.format(str(code)),4888 ui_log=ui_log4889 )4890 # 首选查找数据库 是否 有 这个代码的数据4891 # 期权代码 从 10000001 开始编码 100012284892 ref = coll_option_day.find({'code': str(code)[0:8]})4893 end_date = str(now_time())[0:10]4894 # 当前数据库已经包含了这个代码的数据， 继续增量更新4895 # 加入这个判断的原因是因为如果是刚上市的 数据库会没有数据 所以会有负索引问题出现4896 if ref.count() > 0:4897 # 接着上次获取的日期继续更新4898 start_date = ref[ref.count() - 1]['date']4899 QA_util_log_info(4900 ' 上次获取期权日线数据的最后日期是 {}'.format(start_date),4901 ui_log=ui_log4902 )4903 QA_util_log_info(4904 'UPDATE_OPTION_DAY \n 从上一次下载数据开始继续 Trying update {} from {} to {}'4905 .format(code,4906 start_date,4907 end_date),4908 ui_log=ui_log4909 )4910 if start_date != end_date:4911 start_date0 = QA_util_get_next_day(start_date)4912 df0 = QA_fetch_get_option_day(4913 code=code,4914 start_date=start_date0,4915 end_date=end_date,4916 frequence='day',4917 ip=None,4918 port=None4919 )4920 retCount = df0.iloc[:, 0].size4921 QA_util_log_info(4922 "日期从开始{}-结束{} , 合约代码{} , 返回了{}条记录 , 准备写入数据库".format(4923 start_date0,4924 end_date,4925 code,4926 retCount4927 ),4928 ui_log=ui_log4929 )4930 coll_option_day.insert_many(4931 QA_util_to_json_from_pandas(df0)4932 )4933 else:4934 QA_util_log_info(4935 "^已经获取过这天的数据了^ {}".format(start_date),4936 ui_log=ui_log4937 )4938 else:4939 start_date = '1990-01-01'4940 QA_util_log_info(4941 'UPDATE_OPTION_DAY \n 从新开始下载数据 Trying update {} from {} to {}'4942 .format(code,4943 start_date,4944 end_date),4945 ui_log=ui_log4946 )4947 if start_date != end_date:4948 df0 = QA_fetch_get_option_day(4949 code=code,4950 start_date=start_date,4951 end_date=end_date,4952 frequence='day',4953 ip=None,4954 port=None4955 )4956 retCount = df0.iloc[:, 0].size4957 QA_util_log_info(4958 "日期从开始{}-结束{} , 合约代码{} , 获取了{}条记录 , 准备写入数据库^_^ ".format(4959 start_date,4960 end_date,4961 code,4962 retCount4963 ),4964 ui_log=ui_log4965 )4966 coll_option_day.insert_many(4967 QA_util_to_json_from_pandas(df0)4968 )4969 else:4970 QA_util_log_info(4971 "*已经获取过这天的数据了* {}".format(start_date),4972 ui_log=ui_log4973 )4974 except Exception as error0:4975 print(error0)4976 err.append(str(code))4977 for item in range(len(option_contract_list)):4978 QA_util_log_info(4979 'The {} of Total {}'.format(item,4980 len(option_contract_list)),4981 ui_log=ui_log4982 )4983 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(4984 str(float(item / len(option_contract_list) * 100))[0:4] + '%'4985 )4986 intLogProgress = int(float(item / len(option_contract_list) * 10000.0))4987 QA_util_log_info(4988 strLogProgress,4989 ui_log=ui_log,4990 ui_progress=ui_progress,4991 ui_progress_int_value=intLogProgress4992 )4993 __saving_work(option_contract_list[item].code, coll_option_day)4994 if len(err) < 1:4995 QA_util_log_info('SUCCESS save option day all contract ^_^ ', ui_log=ui_log)4996 else:4997 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)4998 QA_util_log_info(err, ui_log=ui_log)4999def QA_SU_save_option_min_all(client=DATABASE, ui_log=None, ui_progress=None):5000 '''5001 :param client:5002 :return:5003 '''5004 option_contract_list = QA_fetch_get_option_all_contract_time_to_market()5005 coll_option_min = client.option_min_all5006 coll_option_min.create_index(5007 [("code",5008 pymongo.ASCENDING),5009 ("date_stamp",5010 pymongo.ASCENDING)]5011 )5012 err = []5013 # 索引 code5014 err = []5015 def __saving_work(code, coll):5016 QA_util_log_info(5017 '##JOB15 Now Saving Option ALL MIN ==== {}'.format(str(code)),5018 ui_log=ui_log5019 )5020 try:5021 for type in ['1min', '5min', '15min', '30min', '60min']:5022 ref_ = coll.find({'code': str(code)[0:8], 'type': type})5023 end_time = str(now_time())[0:19]5024 if ref_.count() > 0:5025 start_time = ref_[ref_.count() - 1]['datetime']5026 QA_util_log_info(5027 '##JOB99.{} Now Saving Option ALL MIN {} from {} to {} =={} '5028 .format(5029 ['1min',5030 '5min',5031 '15min',5032 '30min',5033 '60min'].index(type),5034 str(code),5035 start_time,5036 end_time,5037 type5038 ),5039 ui_log=ui_log5040 )5041 if start_time != end_time:5042 __data = QA_fetch_get_future_min(5043 str(code),5044 start_time,5045 end_time,5046 type5047 )5048 if len(__data) > 1:5049 QA_util_log_info(5050 " 写入 新增历史合约记录数 {} ".format(len(__data))5051 )5052 coll.insert_many(5053 QA_util_to_json_from_pandas(__data[1::])5054 )5055 else:5056 start_time = '2015-01-01'5057 QA_util_log_info(5058 '##JOB15.{} Now Option ALL MIN {} from {} to {} =={} '5059 .format(5060 ['1min',5061 '5min',5062 '15min',5063 '30min',5064 '60min'].index(type),5065 str(code),5066 start_time,5067 end_time,5068 type5069 ),5070 ui_log=ui_log5071 )5072 if start_time != end_time:5073 __data = QA_fetch_get_future_min(5074 str(code),5075 start_time,5076 end_time,5077 type5078 )5079 if len(__data) > 1:5080 QA_util_log_info(5081 " 写入 新增合约记录数 {} ".format(len(__data))5082 )5083 coll.insert_many(5084 QA_util_to_json_from_pandas(__data)5085 )5086 except:5087 err.append(code)5088 executor = ThreadPoolExecutor(max_workers=4)5089 res = {5090 executor.submit(5091 __saving_work,5092 option_contract_list[i_]["code"],5093 coll_option_min5094 )5095 for i_ in range(len(option_contract_list))5096 } # multi index ./.5097 count = 05098 for i_ in concurrent.futures.as_completed(res):5099 QA_util_log_info(5100 'The {} of Total {}'.format(count,5101 len(option_contract_list)),5102 ui_log=ui_log5103 )5104 strLogProgress = 'DOWNLOAD PROGRESS {} '.format(5105 str(float(count / len(option_contract_list) * 100))[0:4] + '%'5106 )5107 intLogProgress = int(float(count / len(option_contract_list) * 10000.0))5108 QA_util_log_info(5109 strLogProgress,5110 ui_log=ui_log,5111 ui_progress=ui_progress,5112 ui_progress_int_value=intLogProgress5113 )5114 count = count + 15115 if len(err) < 1:5116 QA_util_log_info('SUCCESS', ui_log=ui_log)5117 else:5118 QA_util_log_info(' ERROR CODE \n ', ui_log=ui_log)5119 QA_util_log_info(err, ui_log=ui_log)5120#######################################################################################5121def QA_SU_save_future_list(client=DATABASE, ui_log=None, ui_progress=None):5122 future_list = QA_fetch_get_future_list()5123 coll_future_list = client.future_list5124 coll_future_list.create_index("code", unique=True)5125 try:5126 coll_future_list.insert_many(5127 QA_util_to_json_from_pandas(future_list),5128 ordered=False5129 )5130 except:5131 pass5132def QA_SU_save_index_list(client=DATABASE, ui_log=None, ui_progress=None):5133 index_list = QA_fetch_get_index_list()5134 coll_index_list = client.index_list5135 coll_index_list.create_index("code", unique=True)5136 try:5137 coll_index_list.insert_many(5138 QA_util_to_json_from_pandas(index_list),5139 ordered=False5140 )5141 except:5142 pass5143def QA_SU_save_single_future_day(code : str, client=DATABASE, ui_log=None, ui_progress=None):5144 '''5145 save single_future_day5146 保存单个期货数据日线数据5147 :param client:5148 :param ui_log: 给GUI qt 界面使用5149 :param ui_progress: 给GUI qt 界面使用5150 :param ui_progress_int_value: 给GUI qt 界面使用5151 :return:5152 '''5153 coll_future_day = client.future_day5154 coll_future_day.create_index(5155 [("code",5156 pymongo.ASCENDING),5157 ("date_stamp",5158 pymongo.ASCENDING)]5159 )5160 err = []5161 def __saving_work(code, coll_future_day):5162 try:5163 QA_util_log_info(5164 '##JOB12 Now Saving Future_DAY==== {}'.format(str(code)),5165 ui_log5166 )5167 # 首选查找数据库 是否 有 这个代码的数据5168 ref = coll_future_day.find({'code': str(code)[0:4]})5169 end_date = str(now_time())[0:10]5170 # 当前数据库已经包含了这个代码的数据， 继续增量更新5171 # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现5172 if ref.count() > 0:5173 # 接着上次获取的日期继续更新5174 start_date = ref[ref.count() - 1]['date']5175 QA_util_log_info(5176 'UPDATE_Future_DAY \n Trying updating {} from {} to {}'5177 .format(code,5178 start_date,5179 end_date),5180 ui_log5181 )5182 if start_date != end_date:5183 coll_future_day.insert_many(5184 QA_util_to_json_from_pandas(5185 QA_fetch_get_future_day(5186 str(code),5187 QA_util_get_next_day(start_date),5188 end_date5189 )5190 )5191 )5192 # 当前数据库中没有这个代码的股票数据， 从1990-01-01 开始下载所有的数据5193 else:5194 start_date = '2001-01-01'5195 QA_util_log_info(5196 'UPDATE_Future_DAY \n Trying updating {} from {} to {}'5197 .format(code,5198 start_date,5199 end_date),5200 ui_log5201 )5202 if start_date != end_date:5203 coll_future_day.insert_many(5204 QA_util_to_json_from_pandas(5205 QA_fetch_get_future_day(5206 str(code),5207 start_date,5208 end_date5209 )5210 )5211 )5212 except Exception as error0:5213 print(error0)5214 err.append(str(code))5215 __saving_work(code, coll_future_day)5216 if len(err) < 1:5217 QA_util_log_info('SUCCESS save future day ^_^', ui_log)5218 else:5219 QA_util_log_info(' ERROR CODE \n ', ui_log)5220 QA_util_log_info(err, ui_log)5221def QA_SU_save_future_day(client=DATABASE, ui_log=None, ui_progress=None):5222 '''5223 save future_day5224 保存日线数据5225 :param client:5226 :param ui_log: 给GUI qt 界面使用5227 :param ui_progress: 给GUI qt 界面使用5228 :param ui_progress_int_value: 给GUI qt 界面使用5229 :return:5230 '''5231 future_list = [5232 item for item in QA_fetch_get_future_list().code.unique().tolist()5233 if str(item)[-2:] in ['L8',5234 'L9']5235 ]5236 coll_future_day = client.future_day5237 coll_future_day.create_index(5238 [("code",5239 pymongo.ASCENDING),5240 ("date_stamp",5241 pymongo.ASCENDING)]5242 )5243 err = []5244 def __saving_work(code, coll_future_day):5245 try:5246 QA_util_log_info(5247 '##JOB12 Now Saving Future_DAY==== {}'.format(str(code)),5248 ui_log5249 )5250 # 首选查找数据库 是否 有 这个代码的数据5251 ref = coll_future_day.find({'code': str(code)[0:4]})5252 end_date = str(now_time())[0:10]5253 # 当前数据库已经包含了这个代码的数据， 继续增量更新5254 # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现5255 if ref.count() > 0:5256 # 接着上次获取的日期继续更新5257 start_date = ref[ref.count() - 1]['date']5258 QA_util_log_info(5259 'UPDATE_Future_DAY \n Trying updating {} from {} to {}'5260 .format(code,5261 start_date,5262 end_date),5263 ui_log5264 )5265 if start_date != end_date:5266 coll_future_day.insert_many(5267 QA_util_to_json_from_pandas(5268 QA_fetch_get_future_day(5269 str(code),5270 QA_util_get_next_day(start_date),5271 end_date5272 )5273 )5274 )5275 # 当前数据库中没有这个代码的股票数据， 从1990-01-01 开始下载所有的数据5276 else:5277 start_date = '2001-01-01'5278 QA_util_log_info(5279 'UPDATE_Future_DAY \n Trying updating {} from {} to {}'5280 .format(code,5281 start_date,5282 end_date),5283 ui_log5284 )5285 if start_date != end_date:5286 coll_future_day.insert_many(5287 QA_util_to_json_from_pandas(5288 QA_fetch_get_future_day(5289 str(code),5290 start_date,5291 end_date5292 )5293 )5294 )5295 except Exception as error0:5296 print(error0)5297 err.append(str(code))5298 for item in range(len(future_list)):5299 QA_util_log_info('The {} of Total {}'.format(item, len(future_list)))5300 strProgressToLog = 'DOWNLOAD PROGRESS {} {}'.format(5301 str(float(item / len(future_list) * 100))[0:4] + '%',5302 ui_log5303 )5304 intProgressToLog = int(float(item / len(future_list) * 100))5305 QA_util_log_info(5306 strProgressToLog,5307 ui_log=ui_log,5308 ui_progress=ui_progress,5309 ui_progress_int_value=intProgressToLog5310 )5311 __saving_work(future_list[item], coll_future_day)5312 if len(err) < 1:5313 QA_util_log_info('SUCCESS save future day ^_^', ui_log)5314 else:5315 QA_util_log_info(' ERROR CODE \n ', ui_log)5316 QA_util_log_info(err, ui_log)5317def QA_SU_save_future_day_all(client=DATABASE, ui_log=None, ui_progress=None):5318 '''5319 save future_day_all5320 保存日线数据(全部, 包含单月合约)5321 :param client:5322 :param ui_log: 给GUI qt 界面使用5323 :param ui_progress: 给GUI qt 界面使用5324 :param ui_progress_int_value: 给GUI qt 界面使用5325 :return:5326 '''5327 future_list = QA_fetch_get_future_list().code.unique().tolist()5328 coll_future_day = client.future_day5329 coll_future_day.create_index(5330 [("code",5331 pymongo.ASCENDING),5332 ("date_stamp",5333 pymongo.ASCENDING)]5334 )5335 err = []5336 def __saving_work(code, coll_future_day):5337 try:5338 QA_util_log_info(5339 '##JOB12 Now Saving Future_DAY==== {}'.format(str(code)),5340 ui_log5341 )5342 # 首选查找数据库 是否 有 这个代码的数据5343 ref = coll_future_day.find({'code': str(code)[0:6]})5344 end_date = str(now_time())[0:10]5345 # 当前数据库已经包含了这个代码的数据， 继续增量更新5346 # 加入这个判断的原因是因为如果股票是刚上市的 数据库会没有数据 所以会有负索引问题出现5347 if ref.count() > 0:5348 # 接着上次获取的日期继续更新5349 start_date = ref[ref.count() - 1]['date']5350 QA_util_log_info(5351 'UPDATE_Future_DAY \n Trying updating {} from {} to {}'5352 .format(code,5353 start_date,5354 end_date),5355 ui_log5356 )5357 if start_date != end_date:5358 coll_future_day.insert_many(5359 QA_util_to_json_from_pandas(5360 QA_fetch_get_future_day(5361 str(code),5362 QA_util_get_next_day(start_date),5363 end_date5364 )5365 )5366 )5367 # 当前数据库中没有这个代码的股票数据， 从1990-01-01 开始下载所有的数据5368 else:5369 start_date = '2001-01-01'5370 QA_util_log_info(5371 'UPDATE_Future_DAY \n Trying updating {} from {} to {}'5372 .format(code,5373 start_date,5374 end_date),5375 ui_log5376 )5377 if start_date != end_date:5378 coll_future_day.insert_many(5379 QA_util_to_json_from_pandas(5380 QA_fetch_get_future_day(5381 str(code),5382 start_date,5383 end_date5384 )5385 )5386 )5387 except Exception as error0:5388 print(error0)5389 err.append(str(code))5390 for item in range(len(future_list)):5391 QA_util_log_info('The {} of Total {}'.format(item, len(future_list)))5392 strProgressToLog = 'DOWNLOAD PROGRESS {} {}'.format(5393 str(float(item / len(future_list) * 100))[0:4] + '%',5394 ui_log5395 )5396 intProgressToLog = int(float(item / len(future_list) * 100))5397 QA_util_log_info(5398 strProgressToLog,5399 ui_log=ui_log,5400 ui_progress=ui_progress,5401 ui_progress_int_value=intProgressToLog5402 )5403 __saving_work(future_list[item], coll_future_day)5404 if len(err) < 1:5405 QA_util_log_info('SUCCESS save future day ^_^', ui_log)5406 else:5407 QA_util_log_info(' ERROR CODE \n ', ui_log)5408 QA_util_log_info(err, ui_log)5409def QA_SU_save_single_future_min(code : str, client=DATABASE, ui_log=None, ui_progress=None):5410 """save single_future_min5411 Keyword Arguments:5412 client {[type]} -- [description] (default: {DATABASE})5413 """5414 coll = client.future_min5415 coll.create_index(