mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-10-31 04:45:23 +08:00
48 lines
1.4 KiB
Python
48 lines
1.4 KiB
Python
import re
|
|
import logging
|
|
|
|
from .sites import model
|
|
|
|
|
|
def process(url: str, brief_len: int, **kwargs) -> str:
|
|
"""处理网页内容"""
|
|
|
|
brief_len = model.clamp_brief_len(brief_len)
|
|
|
|
adapter_cls: model.SiteAdapterBase = model.SiteAdapterBase
|
|
|
|
found = False
|
|
for adapter in model.__site_adapters__:
|
|
for regexp in adapter['regexp']:
|
|
if re.match(regexp, url):
|
|
if not adapter['cls'].feed(url):
|
|
# 匹中了这个适配器，但适配器不接受这个链接
|
|
# 则直接检查下一个适配器
|
|
break
|
|
adapter_cls = adapter['cls']
|
|
found = True
|
|
break
|
|
if found:
|
|
break
|
|
|
|
logging.debug("site adapter: {}".format(adapter_cls))
|
|
|
|
processed: dict = adapter_cls.process(url, brief_len, **kwargs)
|
|
|
|
logging.debug("site adapter ret: {}".format(processed))
|
|
|
|
# 处理成纯文本
|
|
|
|
text: str = ""
|
|
|
|
if processed.get('status', 200) == 200:
|
|
if 'content' not in processed:
|
|
text = processed.get('message', "nothing found")
|
|
else:
|
|
text += ('title: '+processed['content']['title']+"\n") if 'title' in processed['content'] else ''
|
|
text += '\n'.join(processed['content'].get('briefs', []))
|
|
else:
|
|
raise Exception(processed.get('message', "error"))
|
|
|
|
return text
|