feat:sort_duplicate_limit
This commit is contained in:
guorong.zheng
parent
c1a66e8578
commit
5e07d8e982
@ -154,6 +154,7 @@ https://cdn.jsdelivr.net/gh/Guovin/iptv-api@gd/source.json
|
||||
| recent_days | 获取最近时间范围内更新的接口（单位天），适当减小可避免出现匹配问题 | 30 |
|
||||
| request_timeout | 查询请求超时时长，单位秒(s)，用于控制查询接口文本链接的超时时长以及重试时长，调整此值能优化更新时间 | 10 |
|
||||
| sort_timeout | 单个接口测速超时时长，单位秒(s)；数值越大测速所属时间越长，能提高获取接口数量，但质量会有所下降；数值越小测速所需时间越短，能获取低延时的接口，质量较好；调整此值能优化更新时间 | 10 |
|
||||
| sort_duplicate_limit | 相同域名接口允许重复执行次数，用于控制执行测速、获取分辨率时的重复次数，数值越大结果越准确，但耗时会成倍增加 | 2 |
|
||||
| source_file | 模板文件路径 | config/demo.txt |
|
||||
| subscribe_num | 结果中偏好的订阅源接口数量 | 10 |
|
||||
| time_zone | 时区，可用于控制更新时间显示的时区，可选值：Asia/Shanghai 或其它时区编码 | Asia/Shanghai |
|
||||
|
||||
@ -155,6 +155,7 @@ https://cdn.jsdelivr.net/gh/Guovin/iptv-api@gd/source.json
|
||||
| recent_days | Retrieve interfaces updated within a recent time range (in days), reducing appropriately can avoid matching issues | 30 |
|
||||
| request_timeout | Query request timeout duration, in seconds (s), used to control the timeout and retry duration for querying interface text links. Adjusting this value can optimize update time. | 10 |
|
||||
| sort_timeout | The timeout duration for speed testing of a single interface, in seconds (s). A larger value means a longer testing period, which can increase the number of interfaces obtained but may decrease their quality. A smaller value means a shorter testing time, which can obtain low-latency interfaces with better quality. Adjusting this value can optimize the update time. | 10 |
|
||||
| sort_duplicate_limit | Number of allowed repetitions for the same domain interface, used to control the number of repetitions when performing speed tests and obtaining resolutions. The larger the value, the more accurate the results, but the time consumption will increase exponentially | 2 |
|
||||
| source_file | Template file path | config/demo.txt |
|
||||
| subscribe_num | The number of preferred subscribe source interfaces in the results | 10 |
|
||||
| time_zone | Time zone, can be used to control the time zone displayed by the update time, optional values: Asia/Shanghai or other time zone codes | Asia/Shanghai |
|
||||
|
||||
@ -95,6 +95,8 @@ recent_days = 30
|
||||
request_timeout = 10
|
||||
# 单个接口测速超时时长，单位秒(s)；数值越大测速所属时间越长，能提高获取接口数量，但质量会有所下降；数值越小测速所需时间越短，能获取低延时的接口，质量较好；调整此值能优化更新时间 | Single interface speed measurement timeout duration, unit seconds (s); The larger the value, the longer the speed measurement time, which can increase the number of interfaces obtained, but the quality will decrease; The smaller the value, the shorter the speed measurement time, which can obtain interfaces with low latency and better quality; Adjusting this value can optimize the update time
|
||||
sort_timeout = 10
|
||||
# 相同域名接口允许重复执行次数，用于控制执行测速、获取分辨率时的重复次数，数值越大结果越准确，但耗时会成倍增加 | Number of allowed repetitions for the same domain interface, used to control the number of repetitions when performing speed tests and obtaining resolutions. The larger the value, the more accurate the results, but the time consumption will increase exponentially
|
||||
sort_duplicate_limit = 2
|
||||
# 模板文件路径， 默认值: config/demo.txt | Template file path, Default value: config/demo.txt
|
||||
source_file = config/demo.txt
|
||||
# 结果中偏好的订阅源接口数量 | Preferred number of subscription source interfaces in the result
|
||||
|
||||
@ -47,6 +47,7 @@
|
||||
| recent_days | 获取最近时间范围内更新的接口（单位天），适当减小可避免出现匹配问题 | 30 |
|
||||
| request_timeout | 查询请求超时时长，单位秒(s)，用于控制查询接口文本链接的超时时长以及重试时长，调整此值能优化更新时间 | 10 |
|
||||
| sort_timeout | 单个接口测速超时时长，单位秒(s)；数值越大测速所属时间越长，能提高获取接口数量，但质量会有所下降；数值越小测速所需时间越短，能获取低延时的接口，质量较好；调整此值能优化更新时间 | 10 |
|
||||
| sort_duplicate_limit | 相同域名接口允许重复执行次数，用于控制执行测速、获取分辨率时的重复次数，数值越大结果越准确，但耗时会成倍增加 | 2 |
|
||||
| source_file | 模板文件路径 | config/demo.txt |
|
||||
| subscribe_num | 结果中偏好的订阅源接口数量 | 10 |
|
||||
| time_zone | 时区，可用于控制更新时间显示的时区，可选值：Asia/Shanghai 或其它时区编码 | Asia/Shanghai |
|
||||
|
||||
@ -47,6 +47,7 @@
|
||||
| recent_days | Retrieve interfaces updated within a recent time range (in days), reducing appropriately can avoid matching issues | 30 |
|
||||
| request_timeout | Query request timeout duration, in seconds (s), used to control the timeout and retry duration for querying interface text links. Adjusting this value can optimize update time. | 10 |
|
||||
| sort_timeout | The timeout duration for speed testing of a single interface, in seconds (s). A larger value means a longer testing period, which can increase the number of interfaces obtained but may decrease their quality. A smaller value means a shorter testing time, which can obtain low-latency interfaces with better quality. Adjusting this value can optimize the update time. | 10 |
|
||||
| sort_duplicate_limit | Number of allowed repetitions for the same domain interface, used to control the number of repetitions when performing speed tests and obtaining resolutions. The larger the value, the more accurate the results, but the time consumption will increase exponentially | 2 |
|
||||
| source_file | Template file path | config/demo.txt |
|
||||
| subscribe_num | The number of preferred subscribe source interfaces in the results | 10 |
|
||||
| time_zone | Time zone, can be used to control the time zone displayed by the update time, optional values: Asia/Shanghai or other time zone codes | Asia/Shanghai |
|
||||
|
||||
2
main.py
2
main.py
@ -97,7 +97,7 @@ class UpdateSource:
|
||||
def get_urls_len(self, filter=False):
|
||||
data = copy.deepcopy(self.channel_data)
|
||||
if filter:
|
||||
process_nested_dict(data, seen=set(), flag=r"cache:(.*)", force_str="!")
|
||||
process_nested_dict(data, seen={}, flag=r"cache:(.*)", force_str="!")
|
||||
processed_urls = set(
|
||||
url_info[0]
|
||||
for channel_obj in data.values()
|
||||
|
||||
@ -124,10 +124,17 @@ class SpeedUI:
|
||||
|
||||
frame_default_sort_params = tk.Frame(root)
|
||||
frame_default_sort_params.pack(fill=tk.X)
|
||||
frame_default_sort_params_column1 = tk.Frame(frame_default_sort_params)
|
||||
frame_default_sort_params_column1.pack(side=tk.LEFT, fill=tk.Y)
|
||||
frame_default_sort_params_column2 = tk.Frame(frame_default_sort_params)
|
||||
frame_default_sort_params_column2.pack(side=tk.RIGHT, fill=tk.Y)
|
||||
|
||||
self.sort_duplicate_limit_label = tk.Label(
|
||||
frame_default_sort_params, text="重复执行:", width=12
|
||||
)
|
||||
self.sort_duplicate_limit_label.pack(side=tk.LEFT, padx=4, pady=8)
|
||||
self.sort_duplicate_limit_entry = tk.Entry(
|
||||
frame_default_sort_params, width=10
|
||||
)
|
||||
self.sort_duplicate_limit_entry.pack(side=tk.LEFT, padx=4, pady=8)
|
||||
self.sort_duplicate_limit_entry.insert(0, config.sort_duplicate_limit)
|
||||
self.sort_duplicate_limit_entry.bind("<KeyRelease>", self.update_sort_duplicate_limit)
|
||||
|
||||
def update_open_sort(self):
|
||||
config.set("Settings", "open_sort", str(self.open_sort_var.get()))
|
||||
@ -155,6 +162,9 @@ class SpeedUI:
|
||||
def update_min_resolution(self, event):
|
||||
config.set("Settings", "min_resolution", self.min_resolution_entry.get())
|
||||
|
||||
def update_sort_duplicate_limit(self, event):
|
||||
config.set("Settings", "sort_duplicate_limit", self.sort_duplicate_limit_entry.get())
|
||||
|
||||
def change_entry_state(self, state):
|
||||
for entry in [
|
||||
"open_sort_checkbutton",
|
||||
@ -162,6 +172,7 @@ class SpeedUI:
|
||||
"open_filter_speed_checkbutton",
|
||||
"min_speed_entry",
|
||||
"open_filter_resolution_checkbutton",
|
||||
"min_resolution_entry"
|
||||
"min_resolution_entry",
|
||||
"sort_duplicate_limit_entry"
|
||||
]:
|
||||
getattr(self, entry).config(state=state)
|
||||
|
||||
@ -605,7 +605,7 @@ async def process_sort_channel_list(data, ipv6=False, callback=None):
|
||||
get_resolution = open_filter_resolution and check_ffmpeg_installed_status()
|
||||
sort_timeout = config.sort_timeout
|
||||
need_sort_data = copy.deepcopy(data)
|
||||
process_nested_dict(need_sort_data, seen=set(), flag=r"cache:(.*)", force_str="!")
|
||||
process_nested_dict(need_sort_data, seen={}, flag=r"cache:(.*)", force_str="!")
|
||||
result = {}
|
||||
semaphore = asyncio.Semaphore(10)
|
||||
|
||||
|
||||
@ -329,6 +329,10 @@ class ConfigManager:
|
||||
def local_num(self):
|
||||
return self.config.getint("Settings", "local_num", fallback=10)
|
||||
|
||||
@property
|
||||
def sort_duplicate_limit(self):
|
||||
return self.config.getint("Settings", "sort_duplicate_limit", fallback=2)
|
||||
|
||||
def load(self):
|
||||
"""
|
||||
Load the config
|
||||
|
||||
@ -145,15 +145,15 @@ async def get_delay_requests(url, timeout=config.sort_timeout, proxy=None):
|
||||
try:
|
||||
async with session.get(url, timeout=timeout, proxy=proxy) as response:
|
||||
if response.status == 404:
|
||||
return float("inf")
|
||||
return -1
|
||||
content = await response.read()
|
||||
if content:
|
||||
end = time()
|
||||
else:
|
||||
return float("inf")
|
||||
return -1
|
||||
except Exception as e:
|
||||
return float("inf")
|
||||
return int(round((end - start) * 1000)) if end else float("inf")
|
||||
return -1
|
||||
return int(round((end - start) * 1000)) if end else -1
|
||||
|
||||
|
||||
def check_ffmpeg_installed_status():
|
||||
@ -238,7 +238,7 @@ def get_video_info(video_info):
|
||||
"""
|
||||
Get the video info
|
||||
"""
|
||||
frame_size = float("inf")
|
||||
frame_size = -1
|
||||
resolution = None
|
||||
if video_info is not None:
|
||||
info_data = video_info.replace(" ", "")
|
||||
@ -259,15 +259,15 @@ async def check_stream_delay(url_info):
|
||||
url = url_info[0]
|
||||
video_info = await ffmpeg_url(url)
|
||||
if video_info is None:
|
||||
return float("inf")
|
||||
return -1
|
||||
frame, resolution = get_video_info(video_info)
|
||||
if frame is None or frame == float("inf"):
|
||||
return float("inf")
|
||||
if frame is None or frame == -1:
|
||||
return -1
|
||||
url_info[2] = resolution
|
||||
return url_info, frame
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return float("inf")
|
||||
return -1
|
||||
|
||||
|
||||
cache = {}
|
||||
@ -287,11 +287,9 @@ async def get_speed(url, ipv6_proxy=None, filter_resolution=config.open_filter_r
|
||||
matcher = re.search(r"cache:(.*)", cache_info)
|
||||
if matcher:
|
||||
cache_key = matcher.group(1)
|
||||
if cache_key in cache:
|
||||
return cache[cache_key][0]
|
||||
if ipv6_proxy and url_is_ipv6:
|
||||
data['speed'] = float("inf")
|
||||
data['delay'] = float("-inf")
|
||||
data['delay'] = 0
|
||||
data['resolution'] = "1920x1080"
|
||||
elif re.match(constants.rtmp_url_pattern, url) is not None:
|
||||
start_time = time()
|
||||
@ -300,8 +298,8 @@ async def get_speed(url, ipv6_proxy=None, filter_resolution=config.open_filter_r
|
||||
data['speed'] = float("inf") if data['resolution'] is not None else 0
|
||||
else:
|
||||
data.update(await get_speed_m3u8(url, filter_resolution, timeout))
|
||||
if cache_key and cache_key not in cache:
|
||||
cache[cache_key] = data
|
||||
if cache_key:
|
||||
cache.setdefault(cache_key, []).append(data)
|
||||
return data
|
||||
except:
|
||||
return data
|
||||
@ -343,24 +341,24 @@ def sort_urls(name, data, supply=config.open_supply, filter_speed=config.open_fi
|
||||
cache_key_match = re.search(r"cache:(.*)", url.partition("$")[2])
|
||||
cache_key = cache_key_match.group(1) if cache_key_match else None
|
||||
if cache_key and cache_key in cache:
|
||||
cache_item = cache[cache_key]
|
||||
if cache_item:
|
||||
speed, delay, cache_resolution = cache_item['speed'] or 0, cache_item['delay'] or -1, cache_item[
|
||||
'resolution']
|
||||
resolution = cache_resolution or resolution
|
||||
cache_list = cache[cache_key]
|
||||
if cache_list:
|
||||
avg_speed = sum(item['speed'] or 0 for item in cache_list) / len(cache_list)
|
||||
avg_delay = max(int(sum(item['delay'] or -1 for item in cache_list) / len(cache_list)), -1)
|
||||
resolution = max((item['resolution'] for item in cache_list), key=get_resolution_value) or resolution
|
||||
try:
|
||||
if logger:
|
||||
logger.info(
|
||||
f"Name: {name}, URL: {result["url"]}, Date: {date}, Delay: {delay} ms, Speed: {speed:.2f} M/s, Resolution: {resolution}"
|
||||
f"Name: {name}, URL: {result["url"]}, Date: {date}, Delay: {avg_delay} ms, Speed: {avg_speed:.2f} M/s, Resolution: {resolution}"
|
||||
)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
if (not supply and filter_speed and speed < min_speed) or (
|
||||
if (not supply and filter_speed and avg_speed < min_speed) or (
|
||||
not supply and filter_resolution and get_resolution_value(resolution) < min_resolution) or (
|
||||
supply and delay == -1):
|
||||
supply and avg_delay < 0):
|
||||
continue
|
||||
result["delay"] = delay
|
||||
result["speed"] = speed
|
||||
result["delay"] = avg_delay
|
||||
result["speed"] = avg_speed
|
||||
result["resolution"] = resolution
|
||||
filter_data.append(result)
|
||||
filter_data.sort(key=sort_urls_key, reverse=True)
|
||||
|
||||
@ -429,8 +429,9 @@ def remove_duplicates_from_tuple_list(tuple_list, seen, flag=None, force_str=Non
|
||||
matcher = re.search(flag, item_first)
|
||||
if matcher:
|
||||
part = matcher.group(1)
|
||||
if part not in seen:
|
||||
seen.add(part)
|
||||
seen_num = seen.get(part, 0)
|
||||
if (seen_num < config.sort_duplicate_limit) or (seen_num == 0 and config.sort_duplicate_limit == 0):
|
||||
seen[part] = seen_num + 1
|
||||
unique_list.append(item)
|
||||
return unique_list
|
||||
|
||||
@ -483,7 +484,7 @@ def remove_cache_info(string):
|
||||
"""
|
||||
Remove the cache info from the string
|
||||
"""
|
||||
return re.sub(r"[.*]?\$?cache:.*", "", string)
|
||||
return re.sub(r"[.*]?\$?-?cache:.*", "", string)
|
||||
|
||||
|
||||
def resource_path(relative_path, persistent=False):
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user