awesome-python/tools/utils.py

import datetime
import json
import os
import random
import sched
import threading
import time
import urllib
import urllib.request
from random import Random

import requests

from tools import loggerKit, redis_client


# 随机生成token
def get_token():
    length_r = 32
    token = ''
    chars = '01'
    length = len(chars) - 1
    random = Random()
    for i in range(length_r):
        token += chars[random.randint(0, length)]
    return token


# 随机获取user_agent
def get_random_browser():
    user_agent = [
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/82.0.4085.133 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0) Gecko/20100101 Firefox/86.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:85.0) Gecko/20100101 Firefox/85.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:81.0) Gecko/20100101 Firefox/81.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:80.0) Gecko/20100101 Firefox/80.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:79.0) Gecko/20100101 Firefox/79.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) Gecko/20100101 Firefox/78.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:75.0) Gecko/20100101 Firefox/75.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:74.0) Gecko/20100101 Firefox/74.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:73.0) Gecko/20100101 Firefox/73.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:72.0) Gecko/20100101 Firefox/72.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:70.0) Gecko/20100101 Firefox/70.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:68.0) Gecko/20100101 Firefox/68.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36 Edg/88.0.705.81",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.66",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36 Edg/86.0.622.69",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36 Edg/85.0.564.63",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.125 Safari/537.36 Edg/84.0.522.63",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36 Edg/83.0.478.45",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/82.0.4086.0 Safari/537.36 Edg/82.0.459.1",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36 Edg/81.0.416.72",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36 Edg/80.0.361.62",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36 Edg/79.0.309.71",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36 Edg/78.0.276.19",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36 Edg/77.0.235.27",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36 Edg/76.0.182.6",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36 Edg/75.0.139.20",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36 Edg/74.1.96.24",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36 Edg/73.0.139.7",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36 Edg/72.0.130.14",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36 Edg/71.0.194.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edg/70.0.3538.102",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/10.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/9.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/8.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/7.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/6.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/5.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/4.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/3.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/2.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/1.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/0.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/99.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/98.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/97.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/96.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/95.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
    ]

    return random.choice(user_agent)


def get_proxy_ip(page, page_size):
    response = requests.get(
        f"https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page={page}&page_size={page_size}",
        # headers={"Authorization": "Token pmdz7f3wdrwhwp88zwxmgtvn9dsusk0557vd16sp"}
        headers={"Authorization": "Token 0pl6i8wbtukg4sdnf3b61wwzhi0h95pze47w5h3p"}
    )

    if response.status_code == 200:
        content = json.loads(response.text)
        # print(content["results"])
        random_proxy = random.choice(content["results"])
        print(random_proxy)
        if random_proxy['valid']:
            username = random_proxy['username']
            password = random_proxy['password']
            proxy_address_ip = random_proxy['proxy_address']
            port = random_proxy['port']
            # print(proxy_address_ip, port)
            proxy = f'{username}:{password}@{proxy_address_ip}:{port}'

            return proxy

    else:
        return None, None


"""
每天预先半小时存储好(早上8点半定时执行)
"""


def save_proxy_ip(page, page_size):
    response = requests.get(
        f"https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page={page}&page_size={page_size}",
        headers={"Authorization": "Token 0pl6i8wbtukg4sdnf3b61wwzhi0h95pze47w5h3p"}
    )

    if response.status_code == 200:
        data = json.loads(response.text)
        # print(content['results'])
        # total = page * page_size
        print(f"代理总数:{data['count']}, 下一页:{data['next']}, 上一页:{data['previous']}")
        i = page * page_size - 1
        for r_proxy in data['results']:
            if r_proxy['valid']:
                username = r_proxy['username']
                password = r_proxy['password']
                proxy_address_ip = r_proxy['proxy_address']
                port = r_proxy['port']
                proxy = f'{username}:{password}@{proxy_address_ip}:{port}'

                # 连接Redis并存储代理IP
                redis_client.set(f'proxy_{str(i)}', proxy)
                i = i - 1


def save_all_proxy_ip(page_size, total_page):
    for t in range(1, total_page):
        loggerKit.info(f't:{t}')
        response = requests.get(
            f"https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page={t}&page_size={page_size}",
            headers={"Authorization": "Token 0pl6i8wbtukg4sdnf3b61wwzhi0h95pze47w5h3p"}
        )

        if response.status_code == 200:
            data = json.loads(response.text)
            loggerKit.info(f"代理总数:{data['count']}, 下一页:{data['next']}, 上一页:{data['previous']}")
            i = t * page_size - 1
            for r_proxy in data['results']:
                if r_proxy['valid']:
                    username = r_proxy['username']
                    password = r_proxy['password']
                    proxy_address_ip = r_proxy['proxy_address']
                    port = r_proxy['port']
                    proxy = f'{username}:{password}@{proxy_address_ip}:{port}'

                    # 连接Redis并存储代理IP
                    redis_client.set(f'proxy_{str(i)}', proxy)
                    i = i - 1


def get_proxy_poc():
    response = requests.get(
        "https://proxy.webshare.io/api/v2/proxy/list/?page=1&page_size=10",
        headers={"Authorization": "Token 0pl6i8wbtukg4sdnf3b61wwzhi0h95pze47w5h3p"}
    )

    if response.status_code == 200:
        data = json.loads(response.text)
        print(f"{json.loads(data)}")


def get_random_proxy_at_redis(total):
    random_index = random.randint(0, total - 1)
    # 连接Redis并获取代理IP
    single_proxy = redis_client.get(f'proxy_{str(random_index)}')

    return single_proxy, random_index


def process_account(account_number):
    print(f"Processing account {account_number}")


def task_running(num_account):
    threads = []
    for account_number in range(num_account):
        thread = threading.Thread(target=process_account, args=(account_number,))
        threads.append(thread)
        thread.start()

    for thread in threads:
        thread.join()


class Singleton(type):
    _instances = {}

    def __call__(cls, *args, **kwargs):
        if cls not in cls._instances:
            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
        return cls._instances[cls]


class FileWriter(metaclass=Singleton):

    def __init__(self, file_path):
        if not isinstance(file_path, str):
            raise TypeError("file_path should be a string")
        self.file_path = file_path
        if not os.path.exists(self.file_path):
            with open(self.file_path, 'a') as f:
                print(f'File {self.file_path} created successfully!')

    def write_to_file(self, content):
        try:
            with open(self.file_path, 'a') as file:
                file.write(content)
        except Exception as e:
            print(f"An error occurred: {e}")


def run_daily_task(hour, minute, page, page_size):
    # 获取当前日期和时间
    now = datetime.datetime.now()
    # 计算下次运行时间
    run_time = datetime.datetime(now.year, now.month, now.day, hour, minute)
    if run_time < now:
        # 如果运行时间小于当前时间，则在明天的相应时间运行任务
        run_time = run_time + datetime.timedelta(days=1)
    # 计算等待时间
    wait_time = (run_time - now).total_seconds()
    # 创建定时器对象
    scheduler = sched.scheduler(time.time, time.sleep)
    # 添加定时任务
    scheduler.enter(wait_time, 1, save_proxy_ip, (page, page_size))
    # 启动定时器
    scheduler.run()


def get_external_ip():
    # 获取公网出口IP地址
    url = "https://api.ipify.org/?format=json"  # 定义查询 API 的 URL
    try:
        response = urllib.request.urlopen(url)  # 向查询 API 发送请求并获取响应
        data = response.read()  # 读取响应中的数据（字节流）
        data = data.decode("utf-8")  # 将响应数据从字节流转换为字符串
        data = json.loads(data)  # 将响应数据解析为 JSON 格式
        public_ip = data["ip"]  # 从 JSON 数据中提取公网IP地址
        loggerKit.info(f"公网出口IP地址：{public_ip}")
        return public_ip
    except Exception as ex:
        err_msg = "无法获取公网出口IP地址"  # 查询失败时输出提示信息
        loggerKit.error(f'err_msg: {err_msg}')


def redis_del_poc():
    redis_client.delete('proxy_370')


if __name__ == "__main__":
    # 代理池
    # print(get_proxy_ip(2, 100))
    # 多线程
    # num = 5
    # task_running(num)
    # 写文件
    # file_writer = FileWriter('output.txt')
    # file_writer.write_to_file('Hello, World!')
    # 代理池存储到redis
    # save_proxy_ip(9, 100)
    # 每天早上8点30开始执行
    # run_daily_task(8, 30, 1, 100)
    # 打印本机网络出口IP
    # external_ip = get_external_ip()
    # print(f"本机网络出口 IP 地址:", external_ip)
    # IP代理池一次性存储到redis
    # save_all_proxy_ip(100, 9)
    # get_proxy_poc()
    redis_del_poc()