import datetime import json import os import random import sched import threading import time import urllib import urllib.request from random import Random import requests from tools import loggerKit, redis_client # 随机生成token def get_token(): length_r = 32 token = '' chars = '01' length = len(chars) - 1 random = Random() for i in range(length_r): token += chars[random.randint(0, length)] return token # 随机获取user_agent def get_random_browser(): user_agent = [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/82.0.4085.133 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0) Gecko/20100101 Firefox/86.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:85.0) Gecko/20100101 Firefox/85.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:81.0) Gecko/20100101 Firefox/81.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:80.0) Gecko/20100101 Firefox/80.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:79.0) Gecko/20100101 Firefox/79.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) Gecko/20100101 Firefox/78.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:75.0) Gecko/20100101 Firefox/75.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:74.0) Gecko/20100101 Firefox/74.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:73.0) Gecko/20100101 Firefox/73.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:72.0) Gecko/20100101 Firefox/72.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:70.0) Gecko/20100101 Firefox/70.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:68.0) Gecko/20100101 Firefox/68.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36 Edg/88.0.705.81", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.66", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36 Edg/86.0.622.69", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36 Edg/85.0.564.63", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.125 Safari/537.36 Edg/84.0.522.63", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36 Edg/83.0.478.45", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/82.0.4086.0 Safari/537.36 Edg/82.0.459.1", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36 Edg/81.0.416.72", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36 Edg/80.0.361.62", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36 Edg/79.0.309.71", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36 Edg/78.0.276.19", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36 Edg/77.0.235.27", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36 Edg/76.0.182.6", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36 Edg/75.0.139.20", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36 Edg/74.1.96.24", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36 Edg/73.0.139.7", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36 Edg/72.0.130.14", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36 Edg/71.0.194.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edg/70.0.3538.102", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Safari/605.1.15", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.2 Safari/605.1.15", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.2 Safari/605.1.15", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/10.1.2 Safari/605.1.15", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/9.1.2 Safari/605.1.15", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/8.1.2 Safari/605.1.15", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/7.1.2 Safari/605.1.15", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/6.1.2 Safari/605.1.15", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/5.1.2 Safari/605.1.15", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/4.1.2 Safari/605.1.15", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/3.1.2 Safari/605.1.15", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/2.1.2 Safari/605.1.15", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/1.1.2 Safari/605.1.15", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/0.1.2 Safari/605.1.15", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/99.1.2 Safari/605.1.15", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/98.1.2 Safari/605.1.15", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/97.1.2 Safari/605.1.15", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/96.1.2 Safari/605.1.15", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/95.1.2 Safari/605.1.15", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" ] return random.choice(user_agent) def get_proxy_ip(page, page_size): response = requests.get( f"https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page={page}&page_size={page_size}", # headers={"Authorization": "Token pmdz7f3wdrwhwp88zwxmgtvn9dsusk0557vd16sp"} headers={"Authorization": "Token 0pl6i8wbtukg4sdnf3b61wwzhi0h95pze47w5h3p"} ) if response.status_code == 200: content = json.loads(response.text) # print(content["results"]) random_proxy = random.choice(content["results"]) print(random_proxy) if random_proxy['valid']: username = random_proxy['username'] password = random_proxy['password'] proxy_address_ip = random_proxy['proxy_address'] port = random_proxy['port'] # print(proxy_address_ip, port) proxy = f'{username}:{password}@{proxy_address_ip}:{port}' return proxy else: return None, None """ 每天预先半小时存储好(早上8点半定时执行) """ def save_proxy_ip(page, page_size): response = requests.get( f"https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page={page}&page_size={page_size}", headers={"Authorization": "Token 0pl6i8wbtukg4sdnf3b61wwzhi0h95pze47w5h3p"} ) if response.status_code == 200: data = json.loads(response.text) # print(content['results']) # total = page * page_size print(f"代理总数:{data['count']}, 下一页:{data['next']}, 上一页:{data['previous']}") i = page * page_size - 1 for r_proxy in data['results']: if r_proxy['valid']: username = r_proxy['username'] password = r_proxy['password'] proxy_address_ip = r_proxy['proxy_address'] port = r_proxy['port'] proxy = f'{username}:{password}@{proxy_address_ip}:{port}' # 连接Redis并存储代理IP redis_client.set(f'proxy_{str(i)}', proxy) i = i - 1 def save_all_proxy_ip(page_size, total_page): for t in range(1, total_page): loggerKit.info(f't:{t}') url = f"https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page={t}&page_size={page_size}" headers = { "Authorization": "Token 0pl6i8wbtukg4sdnf3b61wwzhi0h95pze47w5h3p" } # 添加country查询参数来获取新加坡的IP parameters = { "country": "SG" } response = requests.get(url, headers=headers, params=parameters) if response.status_code == 200: data = json.loads(response.text) loggerKit.info(f"代理总数:{data['count']}, 下一页:{data['next']}, 上一页:{data['previous']}") i = t * page_size - 1 for r_proxy in data['results']: if r_proxy['valid']: username = r_proxy['username'] password = r_proxy['password'] proxy_address_ip = r_proxy['proxy_address'] port = r_proxy['port'] proxy = f'{username}:{password}@{proxy_address_ip}:{port}' # 连接Redis并存储代理IP redis_client.set(f'proxy_{str(i)}', proxy) i = i - 1 """ 读取指定proxy_list文件 """ def save_all_proxy_ip_v2(): i = 0 with open('proxy_list.txt', 'r') as proxies: lines = [line.strip() for line in proxies] for single_proxy in lines: print(f'proxy:{single_proxy}') # 存入redis redis_client.set(f'proxy_{str(i)}', single_proxy, 24 * 60 * 60) i = i + 1 """ 读取指定proxy_list文件 """ def save_all_proxy_ip_v3(proxy_file_path): i = 0 with open(proxy_file_path, 'r') as proxies: lines = [line.strip() for line in proxies] for single_proxy in lines: print(f'proxy:{single_proxy}') # 存入redis redis_client.set(f'proxy_{str(i)}', single_proxy, 24 * 60 * 60) i = i + 1 def get_proxy_poc(): response = requests.get( "https://proxy.webshare.io/api/v2/proxy/list/?page=1&page_size=10", headers={"Authorization": "Token 0pl6i8wbtukg4sdnf3b61wwzhi0h95pze47w5h3p"} ) if response.status_code == 200: data = json.loads(response.text) print(f"{json.loads(data)}") def get_random_proxy_at_redis(total): random_index = random.randint(0, total - 1) # 连接Redis并获取代理IP single_proxy = redis_client.get(f'proxy_{str(random_index)}') return single_proxy, random_index def process_account(account_number): print(f"Processing account {account_number}") def task_running(num_account): threads = [] for account_number in range(num_account): thread = threading.Thread(target=process_account, args=(account_number,)) threads.append(thread) thread.start() for thread in threads: thread.join() class Singleton(type): _instances = {} def __call__(cls, *args, **kwargs): if cls not in cls._instances: cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) return cls._instances[cls] class FileWriter(metaclass=Singleton): def __init__(self, file_path): if not isinstance(file_path, str): raise TypeError("file_path should be a string") self.file_path = file_path if not os.path.exists(self.file_path): with open(self.file_path, 'a') as f: print(f'File {self.file_path} created successfully!') def write_to_file(self, content): try: with open(self.file_path, 'a') as file: file.write(content) except Exception as e: print(f"An error occurred: {e}") def run_daily_task(hour, minute, page_size, total): # 获取当前日期和时间 now = datetime.datetime.now() # 计算下次运行时间 run_time = datetime.datetime(now.year, now.month, now.day, hour, minute) if run_time < now: # 如果运行时间小于当前时间，则在明天的相应时间运行任务 run_time = run_time + datetime.timedelta(days=1) # 计算等待时间 wait_time = (run_time - now).total_seconds() # 创建定时器对象 scheduler = sched.scheduler(time.time, time.sleep) # 添加定时任务 scheduler.enter(wait_time, 1, save_all_proxy_ip, (page_size, total)) # 启动定时器 scheduler.run() def get_external_ip(): # 获取公网出口IP地址 url = "https://api.ipify.org/?format=json" # 定义查询 API 的 URL try: response = urllib.request.urlopen(url) # 向查询 API 发送请求并获取响应 data = response.read() # 读取响应中的数据（字节流） data = data.decode("utf-8") # 将响应数据从字节流转换为字符串 data = json.loads(data) # 将响应数据解析为 JSON 格式 public_ip = data["ip"] # 从 JSON 数据中提取公网IP地址 loggerKit.info(f"公网出口IP地址：{public_ip}") return public_ip except Exception as ex: err_msg = "无法获取公网出口IP地址" # 查询失败时输出提示信息 loggerKit.error(f'err_msg: {err_msg}') def redis_del_poc(): redis_client.delete('proxy_370')