|
|
@@ -11,7 +11,7 @@ from random import Random
|
|
|
|
|
|
import requests
|
|
|
|
|
|
-from tools import redis_client
|
|
|
+from tools import redis_client, loggerKit
|
|
|
|
|
|
|
|
|
# 随机生成token
|
|
|
@@ -170,8 +170,9 @@ def save_proxy_ip(page, page_size):
|
|
|
i = i - 1
|
|
|
|
|
|
|
|
|
-def save_all_proxy_ip(page, page_size, total_page):
|
|
|
- for t in range(0, total_page):
|
|
|
+def save_all_proxy_ip(page_size, total_page):
|
|
|
+ for t in range(1, total_page):
|
|
|
+ loggerKit.info(f't:{t}')
|
|
|
response = requests.get(
|
|
|
f"https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page={t}&page_size={page_size}",
|
|
|
headers={"Authorization": "Token 0pl6i8wbtukg4sdnf3b61wwzhi0h95pze47w5h3p"}
|
|
|
@@ -179,8 +180,7 @@ def save_all_proxy_ip(page, page_size, total_page):
|
|
|
|
|
|
if response.status_code == 200:
|
|
|
data = json.loads(response.text)
|
|
|
- # total = page * page_size
|
|
|
- print(f"代理总数:{data['count']}, 下一页:{data['next']}, 上一页:{data['previous']}")
|
|
|
+ loggerKit.info(f"代理总数:{data['count']}, 下一页:{data['next']}, 上一页:{data['previous']}")
|
|
|
i = t * page_size - 1
|
|
|
for r_proxy in data['results']:
|
|
|
if r_proxy['valid']:
|
|
|
@@ -285,4 +285,4 @@ if __name__ == "__main__":
|
|
|
# external_ip = get_external_ip()
|
|
|
# print(f"本机网络出口 IP 地址:", external_ip)
|
|
|
# IP代理池一次性存储到redis
|
|
|
- save_all_proxy_ip(None, 100, 9)
|
|
|
+ save_all_proxy_ip(100, 9)
|
derrickx