代码优化

livepocket/302_full.py

@@ -1,4 +1,5 @@
 import datetime
+import hashlib
 import json
 import multiprocessing
 import random
@@ -8,6 +9,7 @@ import sys
 import threading
 import time
 import traceback
+from hashlib import md5
 from urllib import parse
 
 import requests
@@ -29,10 +31,16 @@ def poc_2(account, passwd, target_url, log_name, single_proxy, random_index):
         # 开始业务逻辑
         lock = threading.Lock()
         with lock:
+            # event_cname
+            # url = 'https://t.livepocket.jp/e/lxyyc'
+            sub_str = target_url.replace("https://t.livepocket.jp/", "")
+            strs = sub_str.split('/')
+            cname = strs[len(strs) - 1]
+            # print(cname)
 
             loggerKit.info(f'account: {account}, passwd: {passwd}')
             writer = FileWriter(log_name)
-            writer.write_to_file(f'------{account}, {passwd}------ \n')
+            # writer.write_to_file(f'------{account}, {passwd}------ \n')
             user_agent = get_random_browser()
             # single_proxy = 'rrehqcjf:33f90umk8x32@185.199.228.220:7300'
             proxies = {
@@ -41,57 +49,68 @@ def poc_2(account, passwd, target_url, log_name, single_proxy, random_index):
             }
 
             loggerKit.info(f'proxies: {proxies}')
-            writer.write_to_file(f'account: {account}, passwd: {passwd}, proxies:{proxies} \n')
+            # writer.write_to_file(f'account: {account}, passwd: {passwd}, proxies:{proxies} \n')
 
             # session
-            """
-            模拟登录
-            """
-            # 定义请求地址
-            stamp = str(int(time.time()))
-            login_url = f"https://t.livepocket.jp/api/sessions/create?mytimestamp={stamp}"
-
-            # 定义请求header
-            headers = {'Content-Type': 'application/x-www-form-urlencoded;',
-                       'Referer': 'https://t.livepocket.jp/login?acroot=header-new_p_u_nl',
-                       'Sec-Ch-Ua': 'Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99',
-                       'User-Agent': user_agent,
-                       'path': f'/api/sessions/create?mytimestamp={stamp}',
-                       'Origin': 'https://t.livepocket.jp',
-                       'Sec-Ch-Ua-Mobile': '?0',
-                       'Sec-Fetch-Mode': 'cors',
-                       'Sec-Fetch-Site': 'same-origin',
-                       'Priority': 'u=1, i'
-                       }
-
-            # 通过字典方式定义请求body
-            form_data = {"login": account,
-                         "password": passwd,
-                         "auto_login": "on",
-                         "login_password": f"{account}&{passwd}"
-                         }
-
-            data = parse.urlencode(form_data)
-            session = requests.session()
-            content = session.post(url=login_url, headers=headers, data=data, proxies=proxies).text
-            # loggerKit.info(f'login info: {content}')
-            if 'token' not in content:
-                loggerKit.info(f'login account:{account}, 本机出口IP:{get_external_ip()}、代理IP:{single_proxy} 被封!!!')
-                # 将被封的IP数据从redis中去掉
-                key = f'proxy_{str(random_index)}'
-                loggerKit.info(f'key:{key}')
-                redis_client.delete(key=key)
-                return
-            cookie = session.cookies
-            response_cookie = cookie.get_dict()
-            login_resp = json.loads(content)
-            token = login_resp['result']['token']
-            login_session = response_cookie['ci_session']
-            php_session = response_cookie['PHPSESSID']
+            if redis_client.get(f'token_{cname}') is None:
+
+                """
+                模拟登录
+                """
+                # 定义请求地址
+                stamp = str(int(time.time()))
+                login_url = f"https://t.livepocket.jp/api/sessions/create?mytimestamp={stamp}"
+
+                # 定义请求header
+                headers = {'Content-Type': 'application/x-www-form-urlencoded;',
+                           'Referer': 'https://t.livepocket.jp/login?acroot=header-new_p_u_nl',
+                           'Sec-Ch-Ua': 'Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99',
+                           'User-Agent': user_agent,
+                           'path': f'/api/sessions/create?mytimestamp={stamp}',
+                           'Origin': 'https://t.livepocket.jp',
+                           'Sec-Ch-Ua-Mobile': '?0',
+                           'Sec-Fetch-Mode': 'cors',
+                           'Sec-Fetch-Site': 'same-origin',
+                           'Priority': 'u=1, i'
+                           }
+
+                # 通过字典方式定义请求body
+                form_data = {"login": account,
+                             "password": passwd,
+                             "auto_login": "on",
+                             "login_password": f"{account}&{passwd}"
+                             }
+
+                data = parse.urlencode(form_data)
+                session = requests.session()
+                content = session.post(url=login_url, headers=headers, data=data, proxies=proxies).text
+                # loggerKit.info(f'login info: {content}')
+                if 'token' not in content:
+                    loggerKit.info(f'login account:{account}, 本机出口IP:{get_external_ip()}、代理IP:{single_proxy} 被封!!!')
+                    # 将被封的IP数据从redis中去掉
+                    key = f'proxy_{str(random_index)}'
+                    loggerKit.info(f'key:{key}')
+                    redis_client.delete(key=key)
+                    return
+
+                cookie = session.cookies
+                response_cookie = cookie.get_dict()
+                login_resp = json.loads(content)
+                token = login_resp['result']['token']
+                login_session = response_cookie['ci_session']
+                php_session = response_cookie['PHPSESSID']
+                loggerKit.info(f'login account:{account}, 代理IP:{single_proxy}, token: {token} 登录成功!!!')
+                redis_client.set(f'token_{cname}', token)
+                redis_client.set(f'php_session_{cname}', php_session)
+                redis_client.set(f'ci_session_{cname}', login_session)
+            else:
+                token = redis_client.get(f'token_{cname}')
+                login_session = redis_client.get(f'ci_session_{cname}')
+                php_session = redis_client.get(f'php_session_{cname}')
 
             # event_id
             # 从redis中查询，查询不到再去请求
-            event_id = redis_client.get(f'event_id_{target_url}')
+            event_id = redis_client.get(f'event_id_{cname}')
             if event_id == 0 or event_id is None:
                 response = requests.get(target_url, proxies=proxies)
                 if response.status_code == 200:
@@ -102,7 +121,7 @@ def poc_2(account, passwd, target_url, log_name, single_proxy, random_index):
                         if end_index != -1:
                             event_id = html_string[
                                        start_index + len('https://t.livepocket.jp/purchase/verify?event_id='):end_index]
-                            redis_client.set(f'event_id_{target_url}', event_id)
+                            redis_client.set(f'event_id_{cname}', event_id)
                             loggerKit.info(f'event_id: {event_id}')
                         else:
                             loggerKit.warning('End index not found.')
@@ -113,7 +132,7 @@ def poc_2(account, passwd, target_url, log_name, single_proxy, random_index):
 
             # ticket_id
             # 从redis中查询，不存在则请求
-            ticket_id = redis_client.get(f'ticket_id_{target_url}')
+            ticket_id = redis_client.get(f'ticket_id_{cname}')
             if ticket_id == 0 or ticket_id is None:
                 response = requests.get(target_url, proxies=proxies)
                 if response.status_code == 200:
@@ -121,22 +140,11 @@ def poc_2(account, passwd, target_url, log_name, single_proxy, random_index):
                     match = re.search(r'id="js_order_limited_(\d+)"', html_string)
                     if match:
                         ticket_id = match.group(1)
-                        redis_client.set(f'ticket_id_{target_url}', ticket_id)
+                        redis_client.set(f'ticket_id_{cname}', ticket_id)
                         loggerKit.info(f'ticket_id: {ticket_id}')
                     else:
                         loggerKit.warning("未找到匹配的数值")
 
-            # event_cname
-            # url = 'https://t.livepocket.jp/e/lxyyc'
-            # 从redis中查询，不存在则处理
-            cname = redis_client.get(f'cname_{target_url}')
-            if cname is None:
-                sub_str = target_url.replace("https://t.livepocket.jp/", "")
-                strs = sub_str.split('/')
-                cname = strs[len(strs) - 1]
-                redis_client.set(f'cname_{target_url}', cname)
-                # print(cname)
-
             # redirect_url1
             net_url = 'https://t.livepocket.jp/purchase?type=new'
             result = requests.Session()
@@ -201,7 +209,7 @@ def poc_2(account, passwd, target_url, log_name, single_proxy, random_index):
 
             # https://t.livepocket.jp/purchase/confirm?id=309887&reserved_session_id=794986&otoken=mMyZTgzMmQ4OWQ5ZjY5NWFlNGE2ZmJlMWM5Mm
             # drawing_tickets
-            drawing_url = 'https://t.livepocket.jp/api/drawing_tickets/entry?mytimestamp=' + str(int(time.time()))
+            # drawing_url = 'https://t.livepocket.jp/api/drawing_tickets/entry?mytimestamp=' + str(int(time.time()))
 
             region_string = redirect_url3.split('?')[1]
             params = region_string.split('&')
@@ -229,7 +237,7 @@ def poc_2(account, passwd, target_url, log_name, single_proxy, random_index):
                 'reserve_ticket[0][amount]': 1,
                 'payment_type': 'credit'
             }
-            drawing_response = result.post(drawing_url, headers=drawing_headers, cookies=drawing_cookies,
+            drawing_response = result.post(f'https://t.livepocket.jp/api/drawing_tickets/entry?mytimestamp={str(int(time.time()))}', headers=drawing_headers, cookies=drawing_cookies,
                                            data=drawing_form_data, allow_redirects=False, proxies=proxies)
             loggerKit.info('drawing_response.text: {0}', drawing_response.text)
             drawing_data = json.loads(drawing_response.text)
@@ -240,13 +248,12 @@ def poc_2(account, passwd, target_url, log_name, single_proxy, random_index):
                 loggerKit.info('account: {0}, order_id: {1}, onetime_token_name: {2}, onetime_token_value: {3}',
                                account, order_id, onetime_token_name, onetime_token_value)
 
-                writer.write_to_file(
-                    f'account:{account}, order_id:{order_id}, onetime_token_name:{onetime_token_name}, onetime_token_value:{onetime_token_value} \n')
+                # writer.write_to_file(f'account:{account}, order_id:{order_id}, onetime_token_name:{onetime_token_name}, onetime_token_value:{onetime_token_value} \n')
 
 
 def process_account(account, log_name, single_proxy, random_index):
     try:
-        run_poc(account.strip(), 'panyue666', 'https://t.livepocket.jp/e/pua9z', log_name, single_proxy, random_index)
+        run_poc(account.strip(), 'panyue666', 'https://t.livepocket.jp/e/c1rbq', log_name, single_proxy, random_index)
     except Exception as ex:
         loggerKit.error(f'error: {traceback.format_exc()}, account:{account}')
 
@@ -310,4 +317,6 @@ if __name__ == '__main__':
     # 新增下面一行代码即可打包多进程
     multiprocessing.freeze_support()
 
-    run_daily_job(18, 48)
+    # run_daily_job(8, 52)
+
+    looper_job()

tools/logger_util.py

@@ -53,7 +53,7 @@ class Logger:
         # 项目日志目录
         project_log_dir = os.path.join(project_path, 'logs')
         # 日志文件名
-        project_log_filename = 'rpa-terminal_{}.log'.format(datetime.date.today())
+        project_log_filename = 'spider-terminal_{}.log'.format(datetime.date.today())
         # 日志文件路径
         project_log_path = os.path.join(project_log_dir, project_log_filename)
         # 返回日志路径

tools/logs/rpa-terminal_2024-05-11.log

@@ -1,20 +0,0 @@
-{"text": "2024-05-11 07:14:44 INFO From utils.get_external_ip : 公网出口IP地址：\n", "record": {"elapsed": {"repr": "0:00:00.531623", "seconds": 0.531623}, "exception": null, "extra": {}, "file": {"name": "utils.py", "path": "/Volumes/work/project/awesome-python/tools/utils.py"}, "function": "get_external_ip", "level": {"icon": "ℹ️", "name": "INFO", "no": 20}, "line": 275, "message": "公网出口IP地址：", "module": "utils", "name": "__main__", "process": {"id": 18439, "name": "MainProcess"}, "thread": {"id": 140704326104832, "name": "MainThread"}, "time": {"repr": "2024-05-11 07:14:44.775411+08:00", "timestamp": 1715382884.775411}}}
-{"text": "2024-05-11 07:15:13 INFO From utils.get_external_ip : 公网出口IP地址：\n", "record": {"elapsed": {"repr": "0:00:00.405496", "seconds": 0.405496}, "exception": null, "extra": {}, "file": {"name": "utils.py", "path": "/Volumes/work/project/awesome-python/tools/utils.py"}, "function": "get_external_ip", "level": {"icon": "ℹ️", "name": "INFO", "no": 20}, "line": 275, "message": "公网出口IP地址：", "module": "utils", "name": "__main__", "process": {"id": 18462, "name": "MainProcess"}, "thread": {"id": 140704326104832, "name": "MainThread"}, "time": {"repr": "2024-05-11 07:15:13.968268+08:00", "timestamp": 1715382913.968268}}}
-{"text": "2024-05-11 07:15:31 INFO From utils.get_external_ip : 公网出口IP地址：203.184.131.116\n", "record": {"elapsed": {"repr": "0:00:00.389999", "seconds": 0.389999}, "exception": null, "extra": {}, "file": {"name": "utils.py", "path": "/Volumes/work/project/awesome-python/tools/utils.py"}, "function": "get_external_ip", "level": {"icon": "ℹ️", "name": "INFO", "no": 20}, "line": 275, "message": "公网出口IP地址：203.184.131.116", "module": "utils", "name": "__main__", "process": {"id": 18473, "name": "MainProcess"}, "thread": {"id": 140704326104832, "name": "MainThread"}, "time": {"repr": "2024-05-11 07:15:31.853945+08:00", "timestamp": 1715382931.853945}}}
-{"text": "2024-05-11 10:21:44 INFO From utils.save_all_proxy_ip : t:1\n", "record": {"elapsed": {"repr": "0:00:00.079265", "seconds": 0.079265}, "exception": null, "extra": {}, "file": {"name": "utils.py", "path": "/Users/mac/Documents/code/awesome-python/tools/utils.py"}, "function": "save_all_proxy_ip", "level": {"icon": "\u2139\ufe0f", "name": "INFO", "no": 20}, "line": 175, "message": "t:1", "module": "utils", "name": "__main__", "process": {"id": 10538, "name": "MainProcess"}, "thread": {"id": 140704738600704, "name": "MainThread"}, "time": {"repr": "2024-05-11 10:21:44.722805+08:00", "timestamp": 1715394104.722805}}}
-{"text": "2024-05-11 10:23:16 INFO From utils.save_all_proxy_ip : t:1\n", "record": {"elapsed": {"repr": "0:00:00.092009", "seconds": 0.092009}, "exception": null, "extra": {}, "file": {"name": "utils.py", "path": "/Users/mac/Documents/code/awesome-python/tools/utils.py"}, "function": "save_all_proxy_ip", "level": {"icon": "\u2139\ufe0f", "name": "INFO", "no": 20}, "line": 175, "message": "t:1", "module": "utils", "name": "__main__", "process": {"id": 11224, "name": "MainProcess"}, "thread": {"id": 140704738600704, "name": "MainThread"}, "time": {"repr": "2024-05-11 10:23:16.545531+08:00", "timestamp": 1715394196.545531}}}
-{"text": "2024-05-11 10:23:19 INFO From utils.save_all_proxy_ip : \u4ee3\u7406\u603b\u6570:1000, \u4e0b\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=2&page_size=100, \u4e0a\u4e00\u9875:None\n", "record": {"elapsed": {"repr": "0:00:02.691374", "seconds": 2.691374}, "exception": null, "extra": {}, "file": {"name": "utils.py", "path": "/Users/mac/Documents/code/awesome-python/tools/utils.py"}, "function": "save_all_proxy_ip", "level": {"icon": "\u2139\ufe0f", "name": "INFO", "no": 20}, "line": 183, "message": "\u4ee3\u7406\u603b\u6570:1000, \u4e0b\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=2&page_size=100, \u4e0a\u4e00\u9875:None", "module": "utils", "name": "__main__", "process": {"id": 11224, "name": "MainProcess"}, "thread": {"id": 140704738600704, "name": "MainThread"}, "time": {"repr": "2024-05-11 10:23:19.144896+08:00", "timestamp": 1715394199.144896}}}
-{"text": "2024-05-11 10:23:19 INFO From utils.save_all_proxy_ip : t:2\n", "record": {"elapsed": {"repr": "0:00:02.730438", "seconds": 2.730438}, "exception": null, "extra": {}, "file": {"name": "utils.py", "path": "/Users/mac/Documents/code/awesome-python/tools/utils.py"}, "function": "save_all_proxy_ip", "level": {"icon": "\u2139\ufe0f", "name": "INFO", "no": 20}, "line": 175, "message": "t:2", "module": "utils", "name": "__main__", "process": {"id": 11224, "name": "MainProcess"}, "thread": {"id": 140704738600704, "name": "MainThread"}, "time": {"repr": "2024-05-11 10:23:19.183960+08:00", "timestamp": 1715394199.18396}}}
-{"text": "2024-05-11 10:23:21 INFO From utils.save_all_proxy_ip : \u4ee3\u7406\u603b\u6570:1000, \u4e0b\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=3&page_size=100, \u4e0a\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=1&page_size=100\n", "record": {"elapsed": {"repr": "0:00:05.295095", "seconds": 5.295095}, "exception": null, "extra": {}, "file": {"name": "utils.py", "path": "/Users/mac/Documents/code/awesome-python/tools/utils.py"}, "function": "save_all_proxy_ip", "level": {"icon": "\u2139\ufe0f", "name": "INFO", "no": 20}, "line": 183, "message": "\u4ee3\u7406\u603b\u6570:1000, \u4e0b\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=3&page_size=100, \u4e0a\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=1&page_size=100", "module": "utils", "name": "__main__", "process": {"id": 11224, "name": "MainProcess"}, "thread": {"id": 140704738600704, "name": "MainThread"}, "time": {"repr": "2024-05-11 10:23:21.748617+08:00", "timestamp": 1715394201.748617}}}
-{"text": "2024-05-11 10:23:21 INFO From utils.save_all_proxy_ip : t:3\n", "record": {"elapsed": {"repr": "0:00:05.336567", "seconds": 5.336567}, "exception": null, "extra": {}, "file": {"name": "utils.py", "path": "/Users/mac/Documents/code/awesome-python/tools/utils.py"}, "function": "save_all_proxy_ip", "level": {"icon": "\u2139\ufe0f", "name": "INFO", "no": 20}, "line": 175, "message": "t:3", "module": "utils", "name": "__main__", "process": {"id": 11224, "name": "MainProcess"}, "thread": {"id": 140704738600704, "name": "MainThread"}, "time": {"repr": "2024-05-11 10:23:21.790089+08:00", "timestamp": 1715394201.790089}}}
-{"text": "2024-05-11 10:23:29 INFO From utils.save_all_proxy_ip : \u4ee3\u7406\u603b\u6570:1000, \u4e0b\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=4&page_size=100, \u4e0a\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=2&page_size=100\n", "record": {"elapsed": {"repr": "0:00:12.632495", "seconds": 12.632495}, "exception": null, "extra": {}, "file": {"name": "utils.py", "path": "/Users/mac/Documents/code/awesome-python/tools/utils.py"}, "function": "save_all_proxy_ip", "level": {"icon": "\u2139\ufe0f", "name": "INFO", "no": 20}, "line": 183, "message": "\u4ee3\u7406\u603b\u6570:1000, \u4e0b\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=4&page_size=100, \u4e0a\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=2&page_size=100", "module": "utils", "name": "__main__", "process": {"id": 11224, "name": "MainProcess"}, "thread": {"id": 140704738600704, "name": "MainThread"}, "time": {"repr": "2024-05-11 10:23:29.086017+08:00", "timestamp": 1715394209.086017}}}
-{"text": "2024-05-11 10:23:29 INFO From utils.save_all_proxy_ip : t:4\n", "record": {"elapsed": {"repr": "0:00:12.671311", "seconds": 12.671311}, "exception": null, "extra": {}, "file": {"name": "utils.py", "path": "/Users/mac/Documents/code/awesome-python/tools/utils.py"}, "function": "save_all_proxy_ip", "level": {"icon": "\u2139\ufe0f", "name": "INFO", "no": 20}, "line": 175, "message": "t:4", "module": "utils", "name": "__main__", "process": {"id": 11224, "name": "MainProcess"}, "thread": {"id": 140704738600704, "name": "MainThread"}, "time": {"repr": "2024-05-11 10:23:29.124833+08:00", "timestamp": 1715394209.124833}}}
-{"text": "2024-05-11 10:23:31 INFO From utils.save_all_proxy_ip : \u4ee3\u7406\u603b\u6570:1000, \u4e0b\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=5&page_size=100, \u4e0a\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=3&page_size=100\n", "record": {"elapsed": {"repr": "0:00:15.459800", "seconds": 15.4598}, "exception": null, "extra": {}, "file": {"name": "utils.py", "path": "/Users/mac/Documents/code/awesome-python/tools/utils.py"}, "function": "save_all_proxy_ip", "level": {"icon": "\u2139\ufe0f", "name": "INFO", "no": 20}, "line": 183, "message": "\u4ee3\u7406\u603b\u6570:1000, \u4e0b\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=5&page_size=100, \u4e0a\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=3&page_size=100", "module": "utils", "name": "__main__", "process": {"id": 11224, "name": "MainProcess"}, "thread": {"id": 140704738600704, "name": "MainThread"}, "time": {"repr": "2024-05-11 10:23:31.913322+08:00", "timestamp": 1715394211.913322}}}
-{"text": "2024-05-11 10:23:31 INFO From utils.save_all_proxy_ip : t:5\n", "record": {"elapsed": {"repr": "0:00:15.483191", "seconds": 15.483191}, "exception": null, "extra": {}, "file": {"name": "utils.py", "path": "/Users/mac/Documents/code/awesome-python/tools/utils.py"}, "function": "save_all_proxy_ip", "level": {"icon": "\u2139\ufe0f", "name": "INFO", "no": 20}, "line": 175, "message": "t:5", "module": "utils", "name": "__main__", "process": {"id": 11224, "name": "MainProcess"}, "thread": {"id": 140704738600704, "name": "MainThread"}, "time": {"repr": "2024-05-11 10:23:31.936713+08:00", "timestamp": 1715394211.936713}}}
-{"text": "2024-05-11 10:23:34 INFO From utils.save_all_proxy_ip : \u4ee3\u7406\u603b\u6570:1000, \u4e0b\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=6&page_size=100, \u4e0a\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=4&page_size=100\n", "record": {"elapsed": {"repr": "0:00:17.650404", "seconds": 17.650404}, "exception": null, "extra": {}, "file": {"name": "utils.py", "path": "/Users/mac/Documents/code/awesome-python/tools/utils.py"}, "function": "save_all_proxy_ip", "level": {"icon": "\u2139\ufe0f", "name": "INFO", "no": 20}, "line": 183, "message": "\u4ee3\u7406\u603b\u6570:1000, \u4e0b\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=6&page_size=100, \u4e0a\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=4&page_size=100", "module": "utils", "name": "__main__", "process": {"id": 11224, "name": "MainProcess"}, "thread": {"id": 140704738600704, "name": "MainThread"}, "time": {"repr": "2024-05-11 10:23:34.103926+08:00", "timestamp": 1715394214.103926}}}
-{"text": "2024-05-11 10:23:34 INFO From utils.save_all_proxy_ip : t:6\n", "record": {"elapsed": {"repr": "0:00:17.683347", "seconds": 17.683347}, "exception": null, "extra": {}, "file": {"name": "utils.py", "path": "/Users/mac/Documents/code/awesome-python/tools/utils.py"}, "function": "save_all_proxy_ip", "level": {"icon": "\u2139\ufe0f", "name": "INFO", "no": 20}, "line": 175, "message": "t:6", "module": "utils", "name": "__main__", "process": {"id": 11224, "name": "MainProcess"}, "thread": {"id": 140704738600704, "name": "MainThread"}, "time": {"repr": "2024-05-11 10:23:34.136869+08:00", "timestamp": 1715394214.136869}}}
-{"text": "2024-05-11 10:23:36 INFO From utils.save_all_proxy_ip : \u4ee3\u7406\u603b\u6570:1000, \u4e0b\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=7&page_size=100, \u4e0a\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=5&page_size=100\n", "record": {"elapsed": {"repr": "0:00:20.104617", "seconds": 20.104617}, "exception": null, "extra": {}, "file": {"name": "utils.py", "path": "/Users/mac/Documents/code/awesome-python/tools/utils.py"}, "function": "save_all_proxy_ip", "level": {"icon": "\u2139\ufe0f", "name": "INFO", "no": 20}, "line": 183, "message": "\u4ee3\u7406\u603b\u6570:1000, \u4e0b\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=7&page_size=100, \u4e0a\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=5&page_size=100", "module": "utils", "name": "__main__", "process": {"id": 11224, "name": "MainProcess"}, "thread": {"id": 140704738600704, "name": "MainThread"}, "time": {"repr": "2024-05-11 10:23:36.558139+08:00", "timestamp": 1715394216.558139}}}
-{"text": "2024-05-11 10:23:36 INFO From utils.save_all_proxy_ip : t:7\n", "record": {"elapsed": {"repr": "0:00:20.133808", "seconds": 20.133808}, "exception": null, "extra": {}, "file": {"name": "utils.py", "path": "/Users/mac/Documents/code/awesome-python/tools/utils.py"}, "function": "save_all_proxy_ip", "level": {"icon": "\u2139\ufe0f", "name": "INFO", "no": 20}, "line": 175, "message": "t:7", "module": "utils", "name": "__main__", "process": {"id": 11224, "name": "MainProcess"}, "thread": {"id": 140704738600704, "name": "MainThread"}, "time": {"repr": "2024-05-11 10:23:36.587330+08:00", "timestamp": 1715394216.58733}}}
-{"text": "2024-05-11 10:23:38 INFO From utils.save_all_proxy_ip : \u4ee3\u7406\u603b\u6570:1000, \u4e0b\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=8&page_size=100, \u4e0a\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=6&page_size=100\n", "record": {"elapsed": {"repr": "0:00:22.367462", "seconds": 22.367462}, "exception": null, "extra": {}, "file": {"name": "utils.py", "path": "/Users/mac/Documents/code/awesome-python/tools/utils.py"}, "function": "save_all_proxy_ip", "level": {"icon": "\u2139\ufe0f", "name": "INFO", "no": 20}, "line": 183, "message": "\u4ee3\u7406\u603b\u6570:1000, \u4e0b\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=8&page_size=100, \u4e0a\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=6&page_size=100", "module": "utils", "name": "__main__", "process": {"id": 11224, "name": "MainProcess"}, "thread": {"id": 140704738600704, "name": "MainThread"}, "time": {"repr": "2024-05-11 10:23:38.820984+08:00", "timestamp": 1715394218.820984}}}
-{"text": "2024-05-11 10:23:38 INFO From utils.save_all_proxy_ip : t:8\n", "record": {"elapsed": {"repr": "0:00:22.390455", "seconds": 22.390455}, "exception": null, "extra": {}, "file": {"name": "utils.py", "path": "/Users/mac/Documents/code/awesome-python/tools/utils.py"}, "function": "save_all_proxy_ip", "level": {"icon": "\u2139\ufe0f", "name": "INFO", "no": 20}, "line": 175, "message": "t:8", "module": "utils", "name": "__main__", "process": {"id": 11224, "name": "MainProcess"}, "thread": {"id": 140704738600704, "name": "MainThread"}, "time": {"repr": "2024-05-11 10:23:38.843977+08:00", "timestamp": 1715394218.843977}}}
-{"text": "2024-05-11 10:23:41 INFO From utils.save_all_proxy_ip : \u4ee3\u7406\u603b\u6570:1000, \u4e0b\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=9&page_size=100, \u4e0a\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=7&page_size=100\n", "record": {"elapsed": {"repr": "0:00:24.825158", "seconds": 24.825158}, "exception": null, "extra": {}, "file": {"name": "utils.py", "path": "/Users/mac/Documents/code/awesome-python/tools/utils.py"}, "function": "save_all_proxy_ip", "level": {"icon": "\u2139\ufe0f", "name": "INFO", "no": 20}, "line": 183, "message": "\u4ee3\u7406\u603b\u6570:1000, \u4e0b\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=9&page_size=100, \u4e0a\u4e00\u9875:https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page=7&page_size=100", "module": "utils", "name": "__main__", "process": {"id": 11224, "name": "MainProcess"}, "thread": {"id": 140704738600704, "name": "MainThread"}, "time": {"repr": "2024-05-11 10:23:41.278680+08:00", "timestamp": 1715394221.27868}}}

tools/utils.py

@@ -311,6 +311,6 @@ if __name__ == "__main__":
     # external_ip = get_external_ip()
     # print(f"本机网络出口 IP 地址:", external_ip)
     # IP代理池一次性存储到redis
-    # save_all_proxy_ip(100, 9)
+    save_all_proxy_ip(100, 9)
     # get_proxy_poc()
-    redis_del_poc()
+    # redis_del_poc()