Prechádzať zdrojové kódy

IP被封问题处理

xubo 1 rok pred
rodič
commit
394a5821e4

BIN
__pycache__/app.cpython-311.pyc


BIN
__pycache__/litter_helper.cpython-311.pyc


+ 19 - 7
app.py

@@ -15,7 +15,7 @@ import requests
 
 from tools import loggerKit, redis_client, file_writer
 from tools.utils import get_random_browser, get_external_ip, get_random_proxy_at_redis, save_all_proxy_ip, \
-    save_all_proxy_ip_v2
+    save_all_proxy_ip_v2, get_proxy_from_file
 
 # 创建定时器对象
 scheduler = sched.scheduler(time.time, time.sleep)
@@ -41,7 +41,7 @@ def save_account_info(account, passwd, target_url, single_proxy):
             # single_proxy = 'rrehqcjf:33f90umk8x32@185.199.228.220:7300'
             proxies = {
                 'http': f'http://{single_proxy}',
-                # 'https': f'https://{single_proxy}'
+                'https': f'https://{single_proxy}'
             }
 
             loggerKit.info(f'proxies: {proxies}')
@@ -82,7 +82,7 @@ def save_account_info(account, passwd, target_url, single_proxy):
                 # loggerKit.info(f'login info: {content}')
                 if 'token' not in content:
                     loggerKit.info(
-                        f'login account:{account}, 本机出口IP:{get_external_ip()}、代理IP:{single_proxy} 被封!!!')
+                        f'login account:{account}, 登录失败,正在重试!!!')
 
                     # # 生成 2 到 6 分之间的随机等待时间
                     # wait_time = random.uniform(120, 180)
@@ -168,7 +168,7 @@ def get_ticket(account, passwd, target_url, single_proxy):
                 # loggerKit.info(f'login info: {content}')
                 if 'token' not in content:
                     loggerKit.info(
-                        f'login account:{account}, 本机出口IP:{get_external_ip()}、代理IP:{single_proxy} 被封!!!')
+                        f'login account:{account}, 登录失败,正在重试!!!')
 
                     # # 生成 2 到 6 分之间的随机等待时间
                     # wait_time = random.uniform(120, 180)
@@ -421,7 +421,7 @@ def get_ticket_order(account, passwd, target_url, single_proxy):
                 # loggerKit.info(f'login info: {content}')
                 if 'token' not in content:
                     loggerKit.info(
-                        f'login account:{account}, 本机出口IP:{get_external_ip()}、代理IP:{single_proxy} 被封!!!')
+                        f'login account:{account}, 登录失败,正在重试!!!')
 
                     # # 生成 2 到 6 分之间的随机等待时间
                     # wait_time = random.uniform(120, 180)
@@ -657,7 +657,13 @@ def process_running_manual(max_threads, pocket_url):
 
         threads = []
         for account in account_list:
-            single_proxy, random_index = get_random_proxy_at_redis(600)
+            proxy_generator = get_proxy_from_file("proxy_list.txt")
+            single_proxy = None
+            while True:
+                single_proxy = next(proxy_generator)
+                print(f"当前账号: {account}, 当前代理:{single_proxy}")
+                break
+
             thread = threading.Thread(target=process_account,
                                       args=(pocket_url, account.strip(), single_proxy))
             threads.append(thread)
@@ -734,7 +740,13 @@ def run_threading_order(max_threads, account_path, url):
 
             threads = []
             for account in account_list:
-                single_proxy, random_index = get_random_proxy_at_redis(100)
+                proxy_generator = get_proxy_from_file("proxy_list.txt")
+                single_proxy = None
+                while True:
+                    single_proxy = next(proxy_generator)
+                    print(f"当前账号: {account}, 当前代理:{single_proxy}")
+                    break
+
                 thread = threading.Thread(target=get_ticket_order,
                                           args=(account, "panyue666", url, single_proxy))
                 threads.append(thread)

+ 0 - 0
logs/spider-terminal_2024-07-01.log


BIN
tools/__pycache__/__init__.cpython-311.pyc


BIN
tools/__pycache__/document_util.cpython-311.pyc


BIN
tools/__pycache__/logger_util.cpython-311.pyc


BIN
tools/__pycache__/redis_pool.cpython-311.pyc


BIN
tools/__pycache__/redis_util.cpython-311.pyc


BIN
tools/__pycache__/utils.cpython-311.pyc


+ 22 - 0
tools/utils.py

@@ -253,6 +253,28 @@ def get_random_proxy_at_redis(total):
     return single_proxy, random_index
 
 
+def get_proxy_from_file(file_path):
+    """
+    从文件中逐行读取代理,循环读取。
+
+    Args:
+        file_path: 代理文件路径
+
+    Returns:
+        一个代理字符串
+    """
+    with open(file_path, 'r') as f:
+        proxies = f.readlines()
+        # 去除换行符
+        proxies = [proxy.strip() for proxy in proxies]
+        # 使用循环逐行读取
+        for i, proxy in enumerate(proxies):
+            yield proxy
+            # 如果读到最后一行,则从第一行循环读取
+            if i == len(proxies) - 1:
+                i = -1
+
+
 def process_account(account_number):
     print(f"Processing account {account_number}")