awesome-python/tools/utils.py

import datetime
import json
import os
import random
import sched
import threading
import time
import urllib
import urllib.request
from random import Random

import requests
import yaml

from tools import loggerKit, redis_client


# 随机生成token
def get_token():
    length_r = 32
    token = ''
    chars = '01'
    length = len(chars) - 1
    random = Random()
    for i in range(length_r):
        token += chars[random.randint(0, length)]
    return token


# 随机获取user_agent
def get_random_browser():
    user_agent = [
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/82.0.4085.133 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0) Gecko/20100101 Firefox/86.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:85.0) Gecko/20100101 Firefox/85.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:81.0) Gecko/20100101 Firefox/81.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:80.0) Gecko/20100101 Firefox/80.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:79.0) Gecko/20100101 Firefox/79.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) Gecko/20100101 Firefox/78.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:75.0) Gecko/20100101 Firefox/75.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:74.0) Gecko/20100101 Firefox/74.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:73.0) Gecko/20100101 Firefox/73.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:72.0) Gecko/20100101 Firefox/72.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:70.0) Gecko/20100101 Firefox/70.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:68.0) Gecko/20100101 Firefox/68.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36 Edg/88.0.705.81",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36 Edg/87.0.664.66",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36 Edg/86.0.622.69",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36 Edg/85.0.564.63",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.125 Safari/537.36 Edg/84.0.522.63",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36 Edg/83.0.478.45",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/82.0.4086.0 Safari/537.36 Edg/82.0.459.1",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36 Edg/81.0.416.72",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36 Edg/80.0.361.62",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36 Edg/79.0.309.71",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36 Edg/78.0.276.19",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36 Edg/77.0.235.27",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36 Edg/76.0.182.6",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36 Edg/75.0.139.20",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36 Edg/74.1.96.24",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36 Edg/73.0.139.7",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36 Edg/72.0.130.14",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36 Edg/71.0.194.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edg/70.0.3538.102",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/10.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/9.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/8.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/7.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/6.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/5.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/4.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/3.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/2.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/1.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/0.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/99.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/98.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/97.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/96.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/95.1.2 Safari/605.1.15",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
    ]

    return random.choice(user_agent)


def get_proxy_ip(page, page_size):
    response = requests.get(
        f"https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page={page}&page_size={page_size}",
        # headers={"Authorization": "Token pmdz7f3wdrwhwp88zwxmgtvn9dsusk0557vd16sp"}
        headers={"Authorization": "Token 0pl6i8wbtukg4sdnf3b61wwzhi0h95pze47w5h3p"}
    )

    if response.status_code == 200:
        content = json.loads(response.text)
        # print(content["results"])
        random_proxy = random.choice(content["results"])
        print(random_proxy)
        if random_proxy['valid']:
            username = random_proxy['username']
            password = random_proxy['password']
            proxy_address_ip = random_proxy['proxy_address']
            port = random_proxy['port']
            # print(proxy_address_ip, port)
            proxy = f'{username}:{password}@{proxy_address_ip}:{port}'

            return proxy

    else:
        return None, None


"""
每天预先半小时存储好(早上8点半定时执行)
"""


def save_proxy_ip(page, page_size):
    response = requests.get(
        f"https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page={page}&page_size={page_size}",
        headers={"Authorization": "Token 0pl6i8wbtukg4sdnf3b61wwzhi0h95pze47w5h3p"}
    )

    if response.status_code == 200:
        data = json.loads(response.text)
        # print(content['results'])
        # total = page * page_size
        print(f"代理总数:{data['count']}, 下一页:{data['next']}, 上一页:{data['previous']}")
        i = page * page_size - 1
        for r_proxy in data['results']:
            if r_proxy['valid']:
                username = r_proxy['username']
                password = r_proxy['password']
                proxy_address_ip = r_proxy['proxy_address']
                port = r_proxy['port']
                proxy = f'{username}:{password}@{proxy_address_ip}:{port}'

                # 连接Redis并存储代理IP
                redis_client.set(f'proxy_{str(i)}', proxy)
                i = i - 1


def save_all_proxy_ip(page_size, total_page):
    for t in range(1, total_page):
        loggerKit.info(f't:{t}')

        url = f"https://proxy.webshare.io/api/v2/proxy/list/?mode=direct&page={t}&page_size={page_size}"
        headers = {
            "Authorization": "Token 0pl6i8wbtukg4sdnf3b61wwzhi0h95pze47w5h3p"
        }

        # 添加country查询参数来获取新加坡的IP
        parameters = {
            "country": "SG"
        }
        response = requests.get(url, headers=headers, params=parameters)
        if response.status_code == 200:
            data = json.loads(response.text)
            loggerKit.info(f"代理总数:{data['count']}, 下一页:{data['next']}, 上一页:{data['previous']}")
            i = t * page_size - 1
            for r_proxy in data['results']:
                if r_proxy['valid']:
                    username = r_proxy['username']
                    password = r_proxy['password']
                    proxy_address_ip = r_proxy['proxy_address']
                    port = r_proxy['port']
                    proxy = f'{username}:{password}@{proxy_address_ip}:{port}'

                    # 连接Redis并存储代理IP
                    redis_client.set(f'proxy_{str(i)}', proxy)
                    i = i - 1


"""
读取指定proxy_list文件
"""


def save_all_proxy_ip_v2():
    i = 0
    with open('proxy_list.txt', 'r') as proxies:
        lines = [line.strip() for line in proxies]
        for single_proxy in lines:
            print(f'proxy:{single_proxy}')
            # 存入redis
            redis_client.set(f'proxy_{str(i)}', single_proxy, 24 * 60 * 60)
            i = i + 1


"""
读取指定proxy_list文件
"""


def save_all_proxy_ip_v3(proxy_file_path):
    i = 0
    with open(proxy_file_path, 'r') as proxies:
        lines = [line.strip() for line in proxies]
        for single_proxy in lines:
            print(f'proxy:{single_proxy}')
            # 存入redis
            redis_client.set(f'proxy_{str(i)}', single_proxy, 24 * 60 * 60)
            i = i + 1


def get_proxy_poc():
    response = requests.get(
        "https://proxy.webshare.io/api/v2/proxy/list/?page=1&page_size=10",
        headers={"Authorization": "Token 0pl6i8wbtukg4sdnf3b61wwzhi0h95pze47w5h3p"}
    )

    if response.status_code == 200:
        data = json.loads(response.text)
        print(f"{json.loads(data)}")


def get_random_proxy_at_redis(total):
    random_index = random.randint(0, total - 1)
    # 连接Redis并获取代理IP
    single_proxy = redis_client.get(f'proxy_{str(random_index)}')

    return single_proxy, random_index


def get_proxy_from_file(file_path):
    """
    从文件中逐行读取代理，循环读取。

    Args:
        file_path: 代理文件路径

    Returns:
        一个代理字符串
    """
    with open(file_path, 'r') as f:
        proxies = f.readlines()
        # 去除换行符
        proxies = [proxy.strip() for proxy in proxies]
        # 使用循环逐行读取
        for i, proxy in enumerate(proxies):
            yield proxy
            # 如果读到最后一行，则从第一行循环读取
            if i == len(proxies) - 1:
                i = -1


def process_account(account_number):
    print(f"Processing account {account_number}")


def task_running(num_account):
    threads = []
    for account_number in range(num_account):
        thread = threading.Thread(target=process_account, args=(account_number,))
        threads.append(thread)
        thread.start()

    for thread in threads:
        thread.join()


class Singleton(type):
    _instances = {}

    def __call__(cls, *args, **kwargs):
        if cls not in cls._instances:
            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
        return cls._instances[cls]


class FileWriter(metaclass=Singleton):

    def __init__(self, file_path):
        if not isinstance(file_path, str):
            raise TypeError("file_path should be a string")
        self.file_path = file_path
        if not os.path.exists(self.file_path):
            with open(self.file_path, 'a') as f:
                print(f'File {self.file_path} created successfully!')

    def write_to_file(self, content):
        try:
            with open(self.file_path, 'a') as file:
                file.write(content)
        except Exception as e:
            print(f"An error occurred: {e}")


def run_daily_task(hour, minute, page_size, total):
    # 获取当前日期和时间
    now = datetime.datetime.now()
    # 计算下次运行时间
    run_time = datetime.datetime(now.year, now.month, now.day, hour, minute)
    if run_time < now:
        # 如果运行时间小于当前时间，则在明天的相应时间运行任务
        run_time = run_time + datetime.timedelta(days=1)
    # 计算等待时间
    wait_time = (run_time - now).total_seconds()
    # 创建定时器对象
    scheduler = sched.scheduler(time.time, time.sleep)
    # 添加定时任务
    scheduler.enter(wait_time, 1, save_all_proxy_ip, (page_size, total))
    # 启动定时器
    scheduler.run()


def get_external_ip():
    # 获取公网出口IP地址
    url = "https://api.ipify.org/?format=json"  # 定义查询 API 的 URL
    try:
        response = urllib.request.urlopen(url)  # 向查询 API 发送请求并获取响应
        data = response.read()  # 读取响应中的数据（字节流）
        data = data.decode("utf-8")  # 将响应数据从字节流转换为字符串
        data = json.loads(data)  # 将响应数据解析为 JSON 格式
        public_ip = data["ip"]  # 从 JSON 数据中提取公网IP地址
        loggerKit.info(f"公网出口IP地址：{public_ip}")
        return public_ip
    except Exception as ex:
        err_msg = "无法获取公网出口IP地址"  # 查询失败时输出提示信息
        loggerKit.error(f'err_msg: {err_msg}')


def redis_del_poc():
    redis_client.delete('proxy_370')