xinyin025/LangBot
1
0
Fork 0
mirror of https://github.com/langbot-app/LangBot.git synced 2026-06-27 07:54:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: 提交部分测试文件

Browse Source
This commit is contained in:
RockChinQ
2023-07-31 16:24:39 +08:00
parent 749d0219fb
commit 2b9612e933
3 changed files with 223 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/bs_test/bs_test.py
+42
View File
@@ -0,0 +1,42 @@
import requests
from bs4 import BeautifulSoup
import os
import random
import sys
user_agents = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Version/14.1.2 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Version/14.1 Safari/537.36',
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0',
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:88.0) Gecko/20100101 Firefox/88.0'
]
r = requests.get(
sys.argv[1],
headers={
"User-Agent": random.choice(user_agents)
}
)
soup = BeautifulSoup(r.text, 'html.parser')
# print(soup.get_text())
raw = soup.get_text()
import re
# strip每一行
# raw = '\n'.join([line.strip() for line in raw.split('\n')])
# # 删除所有空行或只有空格的行
# raw = re.sub(r'\n\s*\n', '\n', raw)
print(raw)