Cx4x/ptb
1
0
Fork 0
Code Issues Pull Requests Actions2 Packages Projects Releases Wiki Activity

feat:speed and delay

Browse Source
This commit is contained in:
guorong.zheng 2024-12-10 17:11:14 +08:00
parent 822e47be9b
commit b55fcb87fc
8 changed files with 118 additions and 82 deletions

7
config/config.ini

@ -9,12 +9,13 @@ online_search_page_num = 1
urls_limit = 10
open_keep_all = False
open_sort = True
sort_timeout = 5
sort_timeout = 10
open_ffmpeg = True
open_filter_resolution = True
min_resolution = 1920x1080
response_time_weight = 0.5
resolution_weight = 0.5
delay_weight = 0.25
speed_weight = 0.5
resolution_weight = 0.25
recent_days = 30
ipv_type = 全部
ipv_type_prefer = 自动

5
docs/config.md

@ -17,8 +17,9 @@
| open_m3u_result | True | 开启转换生成 m3u 文件类型结果链接，支持显示频道图标 |
| open_filter_resolution | True | 开启分辨率过滤，低于最小分辨率（min_resolution）的接口将会被过滤 |
| min_resolution | 1920x1080 | 接口最小分辨率，需要开启 open_filter_resolution 才能生效 |
| response_time_weight | 0.5 | 响应时间权重值（所有权重值总和应为 1） |
| resolution_weight | 0.5 | 分辨率权重值 （所有权重值总和应为 1） |
| speed_weight | 0.5 | 速率权重值（所有权重值总和应为 1） |
| delay_weight | 0.25 | 响应时间权重值（所有权重值总和应为 1） |
| resolution_weight | 0.25 | 分辨率权重值 （所有权重值总和应为 1） |
| recent_days | 30 | 获取最近时间范围内更新的接口（单位天），适当减小可避免出现匹配问题 |
| ipv_type | 全部 | 生成结果中接口的协议类型，可选值：ipv4、ipv6、全部、all |
| ipv_type_prefer | 自动 | 接口协议类型偏好，优先将该类型的接口排在结果前面，可选值：IPv4、IPv6、自动、auto |

5
docs/config_en.md

@ -17,8 +17,9 @@
| open_m3u_result | True | Enable the conversion to generate m3u file type result links, supporting the display of channel icons |
| open_filter_resolution | True | Enable resolution filtering, interfaces with resolution lower than the minimum resolution (min_resolution) will be filtered |
| min_resolution | 1920x1080 | Minimum interface resolution, requires enabling open_filter_resolution to take effect |
| response_time_weight | 0.5 | Response time weight value (the sum of all weight values should be 1) |
| resolution_weight | 0.5 | Resolution weight value (the sum of all weight values should be 1) |
| speed_weight | 0.5 | Speed weight value (the sum of all weight values should be 1) |
| delay_weight | 0.25 | Response time weight value (the sum of all weight values should be 1) |
| resolution_weight | 0.25 | Resolution weight value (the sum of all weight values should be 1) |
| recent_days | 30 | Retrieve interfaces updated within a recent time range (in days), reducing appropriately can avoid matching issues |
| ipv_type | all | The protocol type of interface in the generated result, optional values: ipv4, ipv6, all |
| ipv_type_prefer | auto | Interface protocol type preference, prioritize interfaces of this type in the results, optional values: IPv4, IPv6, auto |

35
tkinter_ui/default.py

@ -1,9 +1,10 @@
import tkinter as tk
from utils.config import config
from tkinter import ttk
from tkinter import scrolledtext
from tkinter import filedialog
import os
import tkinter as tk
from tkinter import filedialog
from tkinter import scrolledtext
from tkinter import ttk
from utils.config import config
class DefaultUI:
@ -292,20 +293,20 @@ class DefaultUI:
frame_default_sort_params_column2 = tk.Frame(frame_default_sort_params)
frame_default_sort_params_column2.pack(side=tk.RIGHT, fill=tk.Y)
self.response_time_weight_label = tk.Label(
self.delay_weight_label = tk.Label(
frame_default_sort_params_column1, text="响应时间权重:", width=12
)
self.response_time_weight_label.pack(side=tk.LEFT, padx=4, pady=8)
self.response_time_weight_scale = tk.Scale(
self.delay_weight_label.pack(side=tk.LEFT, padx=4, pady=8)
self.delay_weight_scale = tk.Scale(
frame_default_sort_params_column1,
from_=0,
to=1,
orient=tk.HORIZONTAL,
resolution=0.1,
command=self.update_response_time_weight,
command=self.update_delay_weight,
)
self.response_time_weight_scale.pack(side=tk.LEFT, padx=4, pady=8)
self.response_time_weight_scale.set(config.response_time_weight)
self.delay_weight_scale.pack(side=tk.LEFT, padx=4, pady=8)
self.delay_weight_scale.set(config.delay_weight)
self.resolution_weight_label = tk.Label(
frame_default_sort_params_column2, text="分辨率权重:", width=12
@ -458,19 +459,19 @@ class DefaultUI:
def update_urls_limit(self, event):
config.set("Settings", "urls_limit", self.urls_limit_entry.get())
def update_response_time_weight(self, event):
weight1 = self.response_time_weight_scale.get()
def update_delay_weight(self, event):
weight1 = self.delay_weight_scale.get()
weight2 = 1 - weight1
self.resolution_weight_scale.set(weight2)
config.set("Settings", "response_time_weight", str(weight1))
config.set("Settings", "delay_weight", str(weight1))
config.set("Settings", "resolution_weight", str(weight2))
def update_resolution_weight(self, event):
weight1 = self.resolution_weight_scale.get()
weight2 = 1 - weight1
self.response_time_weight_scale.set(weight2)
self.delay_weight_scale.set(weight2)
config.set("Settings", "resolution_weight", str(weight1))
config.set("Settings", "response_time_weight", str(weight2))
config.set("Settings", "delay_weight", str(weight2))
def update_open_update_time(self):
config.set("Settings", "open_update_time", str(self.open_update_time_var.get()))
@ -518,7 +519,7 @@ class DefaultUI:
"open_filter_resolution_checkbutton",
"min_resolution_entry",
"urls_limit_entry",
"response_time_weight_scale",
"delay_weight_scale",
"resolution_weight_scale",
"open_update_time_checkbutton",
"open_url_info_checkbutton",

4
tkinter_ui/tkinter_ui.py

@ -1,5 +1,5 @@
import sys
import os
import sys
sys.path.append(os.path.dirname(sys.path[0]))
import tkinter as tk
@ -55,7 +55,7 @@ class TkinterUI:
"open_sort": self.default_ui.open_sort_var.get(),
"open_filter_resolution": self.default_ui.open_filter_resolution_var.get(),
"min_resolution": self.default_ui.min_resolution_entry.get(),
"response_time_weight": self.default_ui.response_time_weight_scale.get(),
"delay_weight": self.default_ui.delay_weight_scale.get(),
"resolution_weight": self.default_ui.resolution_weight_scale.get(),
"ipv_type": self.default_ui.ipv_type_combo.get(),
"url_keywords_blacklist": self.default_ui.url_keywords_blacklist_text.get(

10
updates/proxy/request.py

@ -1,12 +1,14 @@
from asyncio import Semaphore
from concurrent.futures import ThreadPoolExecutor
from tqdm import tqdm
from tqdm.asyncio import tqdm_asyncio
from utils.config import config
from utils.speed import get_speed_requests
from concurrent.futures import ThreadPoolExecutor
from driver.utils import get_soup_driver
from requests_custom.utils import get_soup_requests, close_session
from utils.config import config
from utils.retry import retry_func
from utils.speed import get_delay_requests
def get_proxy_list(page_count=1):
@ -71,7 +73,7 @@ async def get_proxy_list_with_test(base_url, proxy_list):
async def get_speed_task(url, timeout, proxy):
async with semaphore:
return await get_speed_requests(url, timeout=timeout, proxy=proxy)
return await get_delay_requests(url, timeout=timeout, proxy=proxy)
response_times = await tqdm_asyncio.gather(
*(get_speed_task(base_url, timeout=30, proxy=url) for url in proxy_list),

16
utils/config.py

@ -1,8 +1,8 @@
import os
import configparser
import os
import re
import shutil
import sys
import re
def resource_path(relative_path, persistent=False):
@ -63,7 +63,7 @@ class ConfigManager:
@property
def open_ipv6(self):
return (
"ipv6" in self.ipv_type or "all" in self.ipv_type or "全部" in self.ipv_type
"ipv6" in self.ipv_type or "all" in self.ipv_type or "全部" in self.ipv_type
)
@property
@ -297,8 +297,12 @@ class ConfigManager:
]
@property
def response_time_weight(self):
return self.config.getfloat("Settings", "response_time_weight", fallback=0.5)
def delay_weight(self):
return self.config.getfloat("Settings", "delay_weight", fallback=0.5)
@property
def speed_weight(self):
return self.config.getfloat("Settings", "speed_weight", fallback=0.5)
@property
def resolution_weight(self):
@ -370,7 +374,7 @@ class ConfigManager:
for src_file in files_to_copy:
dest_path = os.path.join(dest_folder, os.path.basename(src_file))
if os.path.abspath(src_file) == os.path.abspath(
dest_path
dest_path
) or os.path.exists(dest_path):
continue
shutil.copy(src_file, dest_folder)

118
utils/speed.py

@ -15,18 +15,22 @@ from utils.tools import is_ipv6, remove_cache_info, get_resolution_value, get_lo
logger = get_logger(constants.log_path)
async def get_speed_with_download(url, timeout=config.sort_timeout):
async def get_speed_with_download(url: str, timeout: int = config.sort_timeout) -> dict[str, float | None]:
"""
Get the speed of the url with a total timeout
"""
start_time = time()
total_size = 0
total_time = 0
info = {'speed': None, 'delay': None}
try:
async with ClientSession(
connector=TCPConnector(ssl=False), trust_env=True
) as session:
async with session.get(url, timeout=timeout) as response:
if response.status == 404:
return info
info['delay'] = int(round((time() - start_time) * 1000))
async for chunk in response.content.iter_any():
if chunk:
total_size += len(chunk)
@ -35,25 +39,33 @@ async def get_speed_with_download(url, timeout=config.sort_timeout):
finally:
end_time = time()
total_time += end_time - start_time
average_speed = (total_size / total_time if total_time > 0 else 0) / 1024
return average_speed
info['speed'] = (total_size / total_time if total_time > 0 else 0) / 1024 / 1024
return info
async def get_speed_m3u8(url, timeout=config.sort_timeout):
async def get_speed_m3u8(url: str, timeout: int = config.sort_timeout) -> dict[str, float | None]:
"""
Get the speed of the m3u8 url with a total timeout
"""
url = quote(url, safe=':/?$&=@')
m3u8_obj = m3u8.load(url)
speed_list = []
start_time = time()
for segment in m3u8_obj.segments:
if time() - start_time > timeout:
break
ts_url = segment.absolute_uri
speed = await get_speed_with_download(ts_url, timeout)
speed_list.append(speed)
return sum(speed_list) / len(speed_list) if speed_list else 0
info = {'speed': None, 'delay': None}
try:
url = quote(url, safe=':/?$&=@')
m3u8_obj = m3u8.load(url, timeout=2)
speed_list = []
start_time = time()
for segment in m3u8_obj.segments:
if time() - start_time > timeout:
break
ts_url = segment.absolute_uri
download_info = await get_speed_with_download(ts_url, timeout)
speed_list.append(download_info['speed'])
if info['delay'] is None and download_info['delay'] is not None:
info['delay'] = download_info['delay']
info['speed'] = sum(speed_list) / len(speed_list) if speed_list else 0
except:
pass
finally:
return info
def get_info_yt_dlp(url, timeout=config.sort_timeout):
@ -72,9 +84,9 @@ def get_info_yt_dlp(url, timeout=config.sort_timeout):
return ydl.sanitize_info(ydl.extract_info(url, download=False))
async def get_speed_yt_dlp(url, timeout=config.sort_timeout):
async def get_delay_yt_dlp(url, timeout=config.sort_timeout):
"""
Get the speed of the url by yt_dlp
Get the delay of the url by yt_dlp
"""
try:
start_time = time()
@ -92,9 +104,9 @@ async def get_speed_yt_dlp(url, timeout=config.sort_timeout):
return float("inf"), None
async def get_speed_requests(url, timeout=config.sort_timeout, proxy=None):
async def get_delay_requests(url, timeout=config.sort_timeout, proxy=None):
"""
Get the speed of the url by requests
Get the delay of the url by requests
"""
async with ClientSession(
connector=TCPConnector(ssl=False), trust_env=True
@ -176,9 +188,9 @@ def get_video_info(video_info):
return frame_size, resolution
async def check_stream_speed(url_info):
async def check_stream_delay(url_info):
"""
Check the stream speed
Check the stream delay
"""
try:
url = url_info[0]
@ -195,13 +207,14 @@ async def check_stream_speed(url_info):
return float("inf")
speed_cache = {}
cache = {}
async def get_speed(url, ipv6_proxy=None, callback=None):
"""
Get the speed (response time and resolution) of the url
"""
data = {'speed': None, 'delay': None, 'resolution': None}
try:
cache_key = None
url_is_ipv6 = is_ipv6(url)
@ -210,19 +223,20 @@ async def get_speed(url, ipv6_proxy=None, callback=None):
matcher = re.search(r"cache:(.*)", cache_info)
if matcher:
cache_key = matcher.group(1)
if cache_key in speed_cache:
return speed_cache[cache_key][0]
if cache_key in cache:
return cache[cache_key][0]
if ipv6_proxy and url_is_ipv6:
speed = (0, None)
# elif '.m3u8' in url:
# speed = await get_speed_m3u8(url)
data['speed'] = float("inf")
data['delay'] = float("-inf")
elif '.m3u8' in url:
data.update(await get_speed_m3u8(url))
else:
speed = await get_speed_yt_dlp(url)
if cache_key and cache_key not in speed_cache:
speed_cache[cache_key] = speed
return speed
data.update(await get_speed_with_download(url))
if cache_key and cache_key not in cache:
cache[cache_key] = data
return data
except:
return float("inf"), None
return data
finally:
if callback:
callback()
@ -239,32 +253,44 @@ def sort_urls_by_speed_and_resolution(name, data, logger=None):
continue
cache_key_match = re.search(r"cache:(.*)", url.partition("$")[2])
cache_key = cache_key_match.group(1) if cache_key_match else None
if cache_key and cache_key in speed_cache:
cache = speed_cache[cache_key]
if cache:
response_time, cache_resolution = cache
if cache_key and cache_key in cache:
cache_item = cache[cache_key]
if cache_item:
speed, delay, cache_resolution = cache_item['speed'], cache_item['delay'], cache_item['resolution']
resolution = cache_resolution or resolution
if response_time != float("inf"):
if speed is not None:
url = remove_cache_info(url)
try:
if logger:
logger.info(
f"Name: {name}, URL: {url}, Date: {date}, Resolution: {resolution}, Response Time: {response_time} ms"
f"Name: {name}, URL: {url}, Date: {date}, Delay: {delay} ms, Speed: {speed:.2f} M/s, Resolution: {resolution}"
)
except Exception as e:
print(e)
filter_data.append((url, date, resolution, origin))
filter_data.append(
{
"url": url,
"date": date,
"delay": delay,
"speed": speed,
"resolution": resolution,
"origin": origin
}
)
def combined_key(item):
_, _, resolution, origin = item
speed, delay, resolution, origin = item["speed"], item["delay"], item["resolution"], item["origin"]
if origin == "important":
return -float("inf")
return float("inf")
else:
resolution_value = get_resolution_value(resolution) if resolution else 0
return (
config.response_time_weight * response_time
- config.resolution_weight * resolution_value
config.speed_weight * (speed * 1024 if speed is not None else float("-inf"))
- config.delay_weight * (delay if delay is not None else float("inf"))
+ config.resolution_weight * (get_resolution_value(resolution) if resolution else 0)
)
filter_data.sort(key=combined_key)
return filter_data
filter_data.sort(key=combined_key, reverse=True)
return [
(item["url"], item["date"], item["resolution"], item["origin"])
for item in filter_data
]