DigitalMgnt
/
tfse-etl-api-v0.2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'wsc' into 'master' 

Wsc

See merge request root/etl_tfse!19
This commit is contained in:
FECR-IBD 2022-05-04 19:27:52 +00:00
parent 070bcadce7 25e02afd5f
commit 4f88426357
11 changed files with 169 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
CompanyData/CompanyDataImpl.py
View File

@ -9,7 +9,7 @@ from Utils.ErrorUtil import LogConditionCheckFailed
from CompanyData.CompanyDataObj import CompanyData, BasicInfo, ShareHolder, MainMember, ComprehensiveCreditAnalysis, \
FinancialElementsAnalysis, BusinessElementsAnalysis, FinancialIndex, BalanceSheet, ProfitSheet, AppendixDataSheet, \
EsgRatingAnalysis, FinancialIndexs
from common.utils import sub_dict, df_iterrows
from Utils.CommonUtil import sub_dict, df_iterrows
class CompanyDataImpl(CompanyData):
@ -28,10 +28,8 @@ class CompanyDataImpl(CompanyData):
return True if res.status_code == 200 else False
def prepare_company_init_data(self):
""""""
def prepare_business_data():
""""""
data = self.db_tyc.find_single_column(
"公司背景",
"基本信息",
@ -56,17 +54,9 @@ class CompanyDataImpl(CompanyData):
self.basic_info = basic_info
self.db_tfse.upsert_single_data(
"企业数据",
"企业数据_更新汇总",
{"企业ID": self.cid},
self.dict_to_save()
)
def prepare_share_holders():
""""""
try:
self.share_holder = list()
self.share_holders = list()
data = self.db_tyc.find_single_column(
"公司背景",
"企业股东",
@ -106,10 +96,10 @@ class CompanyDataImpl(CompanyData):
share_holder.payment_method = None
share_holder.payment_time = None
self.share_holder.append(share_holder)
self.share_holders.append(share_holder)
except TypeError:
self.share_holder = list()
self.share_holders = list()
data = self.db_tyc.find_single_column(
"公司背景",
"十大股东",
@ -133,17 +123,9 @@ class CompanyDataImpl(CompanyData):
share_holder.paid_amount = None
share_holder.payment_method = None
share_holder.payment_time = None
self.share_holder.append(share_holder)
self.db_tfse.upsert_single_data(
"企业数据",
"企业数据_更新汇总",
{"企业ID": self.cid},
self.dict_to_save()
)
self.share_holders.append(share_holder)
def prepare_main_members():
""""""
self.main_members = list()
@ -165,13 +147,6 @@ class CompanyDataImpl(CompanyData):
main_member.job_title = result['typeJoin']
self.main_members.append(main_member)
self.db_tfse.upsert_single_data(
"企业数据",
"企业数据_更新汇总",
{"企业ID": self.cid},
self.dict_to_save()
)
def prepare_industry_l1():
"""一级行业数据"""
db = MongoHelper("tfse_v0.21")
@ -192,11 +167,39 @@ class CompanyDataImpl(CompanyData):
"远东_一级行业"
)
db.upsert_single_data(
self.industry_l1 = fecr_industry_l1
def save_to_basic_business_info():
self.db_tfse.upsert_single_data(
"企业数据",
"基本工商信息",
{"企业ID": self.cid},
{
"企业ID": self.cid,
"企业名称": self.name,
"更新日期": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
"工商信息": self.basic_info.dict_keys_toggle(),
"股东信息": [item.dict_keys_toggle() for item in self.share_holders],
"主要成员": [item.dict_keys_toggle() for item in self.main_members]
}
)
def save_to_update_summary():
self.update_time = {
"工商信息": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
"财务信息": None,
"综信评价": None,
"ESG评价": None,
}
self.db_tfse.upsert_single_data(
"企业数据",
"企业数据_更新汇总",
{"企业ID": self.cid},
{"一级行业": fecr_industry_l1}
self.dict_keys_toggle()
)
def __main__():
@ -204,6 +207,8 @@ class CompanyDataImpl(CompanyData):
prepare_share_holders()
prepare_main_members()
prepare_industry_l1()
save_to_basic_business_info()
save_to_update_summary()
__main__()

2
CompanyData/CompanyDataObj.py
View File

@ -164,7 +164,7 @@ class CompanyData(SpecObject):
industry_l1 = ValidateAttr(field='industry_l1', type=str)
industry_l2 = ValidateAttr(field='industry_l2', type=str)
basic_info = ValidateAttr(field='basic_info', type=BasicInfo)
share_holder = ValidateAttr(field='share_holders', instance_list=ShareHolder)
share_holders = ValidateAttr(field='share_holders', instance_list=ShareHolder)
main_members = ValidateAttr(field="main_members", instance_list=MainMember)
balance_sheet = ValidateAttr(field='balance_sheet', type=BalanceSheet)
profit_sheet = ValidateAttr(field='profit_sheet', type=ProfitSheet)

23
CompanyData/CompanyDataRoutes.py
View File

@ -3,7 +3,7 @@ from flask import Blueprint, request
from CompanyData.CompanyDataImpl import CompanyDataImpl, CreditAnalysisImpl, FinancialAnalysisImpl, \
BusinessAnalysisImpl, EsgAnalysisImpl
from Utils.ErrorUtil import ReturnConditionCheckFailed
from common.utils import verify_token
from Utils.CommonUtil import verify_token
from CompanyData.job_scripts import update_process_scripts
company_route = Blueprint('company', __name__)
@ -15,26 +15,19 @@ def basic_info():
"""
初始化企业数据
"""
try:
req = request.json
company_data = CompanyDataImpl()
company_data.cid = req['cid']
company_data.name = req['company_name']
company_data.drag_data_from_tyc_api()
# company_data.drag_data_from_tyc_api()
company_data.prepare_company_init_data()
return {"info": "企业数据初始化成功"}, 200
# try:
# req = request.json
# company_data = CompanyDataImpl()
# company_data.cid = req['cid']
# company_data.name = req['company_name']
# company_data.drag_data_from_tyc_api()
# company_data.prepare_company_init_data()
# return {"info": "企业数据初始化成功"}, 200
# except ReturnConditionCheckFailed as e:
# e.log_error()
# return {"info": e.failed_info}, e.status_code
# except KeyError:
# return {"info": "参数错误"}, 400
except ReturnConditionCheckFailed as e:
e.log_error()
return {"info": e.failed_info}, e.status_code
except KeyError:
return {"info": "参数错误"}, 400
@company_route.route('/company_index_etl', methods=['POST'])

2
CompanyData/job_scripts.py
View File

@ -5,7 +5,7 @@ import requests
import pandas as pd
from common.utils import read_json_file, sub_dict, df_iterrows
from Utils.CommonUtil import read_json_file, sub_dict, df_iterrows
from CompanyData.db import FIND_DATA_IN_TYC, UPSERT_SINGLE_DATA, UPDATE_SINGLE_DATA, FIND_SINGLE_DATA_BY_REQ, FIND_ALL_DATA_BY_REQ
from RatingData.scripts.risk_detail import associate_risk_detail, change_log_detail

2
RatingData/EsgReportEtl.py
View File

@ -1,4 +1,4 @@
from common.utils import read_json_file
from Utils.CommonUtil import read_json_file
from RatingData.scripts.common import industry_processing
from RatingData.scripts.esg.environment import *
from RatingData.scripts.esg.governance import *

2
RatingData/routes.py
View File

@ -1,5 +1,5 @@
from flask import Blueprint, request
from common.utils import verify_token
from Utils.CommonUtil import verify_token
from RatingData.EsgReportEtl import EsgCleanModel
from RatingData.ReportEtl import CleanModel
from RatingData.scripts.common import get_tfse_data, get_esg_data

2
RatingData/scripts/report/abstract.py
View File

@ -1,4 +1,4 @@
from common.utils import read_json_file
from Utils.CommonUtil import read_json_file
from RatingData.scripts.common import *

0
common/utils.py → Utils/CommonUtil.py
View File

60
Utils/ObjUtil.py
View File

@ -5,29 +5,69 @@ class SpecObject(object):
fields_map = {}
def dict_to_show(self, **kwargs):
"""显示对象"""
def dict_to_save(self, **kwargs):
"""存储对象"""
def dict_keys_toggle(self, **kwargs):
"""字典键值切换"""
_dict_ = dict()
default_types = ['str', 'int', 'float', 'dict', 'bool', 'tuple']
for key in self.__dict__.keys():
if type(self.__dict__[key]).__name__ in ['str', 'int', 'float', 'dict', 'bool', 'tuple']:
if type(self.__dict__[key]).__name__ in default_types:
_dict_[self.fields_map[key]] = self.__dict__[key]
elif type(self.__dict__[key]).__name__ == 'list':
if len(self.__dict__[key]) == 0:
_dict_[self.fields_map[key]] = self.__dict__[key]
elif type(self.__dict__[key][0]).__name__ in ['str', 'int', 'float', 'dict', 'bool', 'tuple']:
elif type(self.__dict__[key][0]).__name__ in default_types:
_dict_[self.fields_map[key]] = self.__dict__[key]
else:
_dict_[self.fields_map[key]] = [item.dict_to_save() for item in self.__dict__[key]]
_dict_[self.fields_map[key]] = [item.dict_keys_toggle() for item in self.__dict__[key]]
elif self.__dict__[key] is None:
_dict_[self.fields_map[key]] = self.__dict__[key]
else:
_dict_[self.fields_map[key]] = self.__dict__[key].dict_to_save()
_dict_[self.fields_map[key]] = self.__dict__[key].dict_keys_toggle()
if 'columns' in kwargs:
_dict_ = {key: _dict_[key] for key in kwargs['columns']}
return _dict_
def dict_to_show(self, **kwargs):
"""字典显示格式"""
return self.dict_keys_toggle(**kwargs)
def dict_to_save(self, **kwargs):
"""字典存储格式"""
return self.dict_keys_toggle(**kwargs)
def dict_to_return(self, **kwargs):
"""字典返回格式"""
@staticmethod
def dict_to_set(**kwargs):
"""实例设值"""
instance = kwargs['instance']()
fields_map = dict([v, k] for k, v in instance.fields_map.items())
for field in list(kwargs['data'].keys()):
instance.__setattr__(fields_map[field], kwargs['data'][field])
return instance
@staticmethod
def instance_list_to_set(**kwargs):
"""实例数组设值"""
list_ = list()
for item in list(kwargs['data']):
instance = kwargs['instance']()
fields_map = dict([v, k] for k, v in instance.fields_map.items())
for field in list(item.keys()):
instance.__setattr__(fields_map[field], item[field])
list_.append(instance)
return list_
@staticmethod
def get_attr(_dict_, _key_, **kwargs):
"""获取字典属性值"""
value = kwargs['default'] if kwargs.__contains__('default') else None
try:
value = _dict_[_key_]
except KeyError:
pass
return value

63
Utils/ValidateUtil.py
View File

@ -4,10 +4,11 @@ from Utils.ErrorUtil import ReturnConditionCheckFailed
class Validate(object):
"""常用格式检查"""
@staticmethod
def email(param):
""""""
"""邮箱格式"""
regex = "^.+\\@(\\[?)[a-zA-Z0-9\\-\\.]+\\.([a-zA-Z]{2,3}|[0-9]{1,3})(\\]?)$"
case = (len(param) > 7) and (re.match(regex, param) is not None)
result = True if case else False
@ -15,11 +16,18 @@ class Validate(object):
@staticmethod
def password(param):
""""""
"""密码格式"""
regex = "^(?![A-Za-z0-9]+$)(?![a-z0-9\\W]+$)(?![A-Za-z\\W]+$)(?![A-Z0-9\\W]+$)^.{8,}$"
case = (len(param) >= 8) and (re.match(regex, param) is not None)
return True if case else False
@staticmethod
def telephone(param):
"""手机号格式"""
regex = "(^(13[0-9]|14[01456879]|15[0-3,5-9]|16[2567]|17[0-8]|18[0-9]|19[0-3,5-9])d{8}$)"
case = re.match(regex, param) is not None
return True if case else False
@staticmethod
def date_format(param):
"""yyyy-mm-dd"""
@ -36,79 +44,82 @@ class Validate(object):
@staticmethod
def image(param):
""""""
"""图片格式"""
mimetype = param.mimetype
return True if mimetype in ['image/jpeg', 'image/png'] else False
class ValidateAttr(object):
"""对象属性值检查"""
FIELD_ERROR_INFO = '字段【{}】异常'
FILED_MAP_ERROR = '字段未映射完整'
def __init__(self, **kwargs):
""""""
self.kwargs = kwargs
self.error_info = kwargs['error_info'] if 'error_info' in kwargs else ('{}异常'.format(kwargs['mark']) if 'mark' in kwargs else None)
self.error_code = kwargs['error_code'] if 'error_code' in kwargs else 200
self.code = kwargs['error_code'] if 'error_code' in kwargs else 200
if 'error_info' in kwargs:
self.info = kwargs['error_info']
elif 'mark' in kwargs:
self.info = self.FIELD_ERROR_INFO.format(kwargs['mark'])
else:
self.info = None
def __get__(self, instance, owner):
if self.kwargs['field'] in instance.__dict__:
return instance.__dict__[self.kwargs['field']]
def __set__(self, instance, value):
""""""
try:
self.error_info = '{}异常'.format(instance.fields_map[self.kwargs['field']])
if not self.info:
self.info = self.FIELD_ERROR_INFO.format(instance.fields_map[self.kwargs['field']])
except AttributeError:
pass
def not_default():
"""无默认值时属性设值检查"""
raise ReturnConditionCheckFailed(self.FILED_MAP_ERROR, 200)
if 'type' in self.kwargs:
"""检查实例的属性类型"""
if type(self.kwargs['type']) is list:
if type(value) not in self.kwargs['type']:
raise ReturnConditionCheckFailed(self.error_info, self.error_code)
raise ReturnConditionCheckFailed(self.info, self.code)
else:
if not isinstance(value, self.kwargs['type']):
raise ReturnConditionCheckFailed(self.error_info, self.error_code)
if value is not None and 'default' not in self.kwargs:
raise ReturnConditionCheckFailed(self.info, self.code)
if 'length' in self.kwargs:
"""检查实例的属性值长度(一般是str类型)"""
if len(value) != self.kwargs['length']:
raise ReturnConditionCheckFailed(self.error_info, self.error_code)
raise ReturnConditionCheckFailed(self.info, self.code)
if 'in_list' in self.kwargs:
"""检查实例属性是否包含于列表中(属性有列表和非列表两种情况)"""
if type(value) is not list:
if value not in self.kwargs['in_list']:
raise ReturnConditionCheckFailed(self.error_info, self.error_code)
raise ReturnConditionCheckFailed(self.info, self.code)
else:
for item in value:
if item not in self.kwargs['in_list']:
raise ReturnConditionCheckFailed(self.error_info, self.error_code)
raise ReturnConditionCheckFailed(self.info, self.code)
if 'instance_list' in self.kwargs:
"""检查实例列表"""
if type(value) is not list:
raise ReturnConditionCheckFailed(self.error_info, self.error_code)
raise ReturnConditionCheckFailed(self.info, self.code)
else:
for item in value:
if not isinstance(item, self.kwargs['instance_list']):
raise ReturnConditionCheckFailed(self.error_info, self.error_code)
raise ReturnConditionCheckFailed(self.info, self.code)
if 'func' in self.kwargs:
"""属性检查函数"""
if not list(map(self.kwargs['func'], [value]))[0]:
raise ReturnConditionCheckFailed(self.error_info, self.error_code)
instance.__dict__[self.kwargs['field']] = value
raise ReturnConditionCheckFailed(self.info, self.code)
if 'default' in self.kwargs:
"""实例属性默认值"""
if value is None:
value = self.kwargs['default']
instance.__dict__[self.kwargs['field']] = value
else:
not_default()
else:
not_default()

0
common/__init__.py
View File