feat:region channel
This commit is contained in:
guorong.zheng
parent
e40b5183b4
commit
63423b76e1
@ -27,3 +27,4 @@ extend_base_urls = [
|
||||
"https://m3u.ibert.me/txt/o_cn.txt",
|
||||
"https://m3u.ibert.me/txt/j_iptv.txt",
|
||||
]
|
||||
region_list = ["广东"]
|
||||
|
||||
25
fofa_map.py
Normal file
25
fofa_map.py
Normal file
@ -0,0 +1,25 @@
|
||||
region_url = {
|
||||
"广东": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iR3Vhbmdkb25nIg%3D%3D",
|
||||
"北京": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iQmVpamluZyI%3D",
|
||||
"湖南": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iSHVuYW4i",
|
||||
"湖北": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iSHViZWki",
|
||||
"浙江": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iWmhlamlhbmci",
|
||||
"上海": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iU2hhbmdoYWki",
|
||||
"天津": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iVGlhbmppbiI%3D",
|
||||
"江苏": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iSmlhbmdzdSI%3D",
|
||||
"山东": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iU2hhbmRvbmci",
|
||||
"河南": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iSGVuYW4i",
|
||||
"河北": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iSGViZWki",
|
||||
"山西": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iU2hhbnhpIg%3D%3D",
|
||||
"陕西": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iU2hhYW54aSI%3D",
|
||||
"安徽": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iQW5odWki",
|
||||
"重庆": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iQ2hvbmdxaW5nIg%3D%3D",
|
||||
"福建": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iRnVqaWFuIg%3D%3D",
|
||||
"江西": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iSmlhbmd4aSI%3D",
|
||||
"辽宁": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iTGlhb25pbmci",
|
||||
"黑龙江": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iSGVpbG9uZ2ppYW5nIg%3D%3D",
|
||||
"吉林": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0i5ZCJ5p6XIg%3D%3D",
|
||||
"四川": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iU2ljaHVhbiI%3D",
|
||||
"云南": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iWXVubmFuIg%3D%3D",
|
||||
"香港": "https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iSEsi",
|
||||
}
|
||||
24
main.py
24
main.py
@ -21,13 +21,16 @@ from utils import (
|
||||
useAccessibleUrl,
|
||||
getChannelsByExtendBaseUrls,
|
||||
checkUrlByPatterns,
|
||||
getFOFAUrlsFromRegionList,
|
||||
getChannelsByFOFA,
|
||||
mergeObjects,
|
||||
)
|
||||
import logging
|
||||
from logging.handlers import RotatingFileHandler
|
||||
import os
|
||||
from tqdm import tqdm
|
||||
import re
|
||||
import fofa_map
|
||||
import time
|
||||
|
||||
handler = RotatingFileHandler("result_new.log", encoding="utf-8")
|
||||
@ -69,14 +72,17 @@ class UpdateSource:
|
||||
]
|
||||
extendResults = await getChannelsByExtendBaseUrls(channelNames)
|
||||
print(f"Getting channels by FOFA...")
|
||||
self.driver.get(
|
||||
"https://fofa.info/result?qbase64=ImlwdHYvbGl2ZS96aF9jbi5qcyIgJiYgY291bnRyeT0iQ04iICYmIHJlZ2lvbj0iR3Vhbmdkb25nIg%3D%3D"
|
||||
)
|
||||
time.sleep(10)
|
||||
fofa_source = re.sub(
|
||||
r"<!--.*?-->", "", self.driver.page_source, flags=re.DOTALL
|
||||
)
|
||||
fofa_channels = getChannelsByFOFA(fofa_source)
|
||||
fofa_urls = getFOFAUrlsFromRegionList()
|
||||
fofa_results = {}
|
||||
for url in fofa_urls:
|
||||
if url:
|
||||
self.driver.get(url)
|
||||
time.sleep(10)
|
||||
fofa_source = re.sub(
|
||||
r"<!--.*?-->", "", self.driver.page_source, flags=re.DOTALL
|
||||
)
|
||||
fofa_channels = getChannelsByFOFA(fofa_source)
|
||||
fofa_results = mergeObjects(fofa_results, fofa_channels)
|
||||
total_channels = len(channelNames)
|
||||
pbar = tqdm(total=total_channels)
|
||||
pageUrl = await useAccessibleUrl()
|
||||
@ -92,7 +98,7 @@ class UpdateSource:
|
||||
for url, date, resolution in extendResults.get(name, []):
|
||||
if url and checkUrlByPatterns(url):
|
||||
infoList.append((url, None, resolution))
|
||||
for url in fofa_channels.get(name, []):
|
||||
for url in fofa_results.get(name, []):
|
||||
if url and checkUrlByPatterns(url):
|
||||
infoList.append((url, None, None))
|
||||
if pageUrl:
|
||||
|
||||
86
utils.py
86
utils.py
@ -14,6 +14,7 @@ from urllib.parse import urlparse
|
||||
import requests
|
||||
import re
|
||||
from bs4 import NavigableString
|
||||
import fofa_map
|
||||
|
||||
|
||||
def getChannelItems():
|
||||
@ -175,12 +176,6 @@ def checkNameMatch(name, result_name):
|
||||
result_name,
|
||||
re.IGNORECASE,
|
||||
):
|
||||
print(
|
||||
"Name test match:",
|
||||
name.lower(),
|
||||
result_name.lower(),
|
||||
name.lower() == result_name.lower(),
|
||||
)
|
||||
return name.lower() == result_name.lower()
|
||||
else:
|
||||
return True
|
||||
@ -389,6 +384,19 @@ async def useAccessibleUrl():
|
||||
return baseUrl2
|
||||
|
||||
|
||||
def getFOFAUrlsFromRegionList():
|
||||
"""
|
||||
Get the FOFA url from region
|
||||
"""
|
||||
region_list = getattr(config, "region_list", [])
|
||||
urls = []
|
||||
for region in region_list:
|
||||
region_url = getattr(fofa_map, "region_url")
|
||||
if region in region_url:
|
||||
urls.append(region_url[region])
|
||||
return urls
|
||||
|
||||
|
||||
def getChannelsByFOFA(source):
|
||||
"""
|
||||
Get the channel by FOFA
|
||||
@ -398,26 +406,50 @@ def getChannelsByFOFA(source):
|
||||
for url in urls:
|
||||
try:
|
||||
response = requests.get(url + "/iptv/live/1000.json?key=txiptv", timeout=2)
|
||||
json_data = response.json()
|
||||
if json_data["code"] == 0:
|
||||
try:
|
||||
for item in json_data["data"]:
|
||||
if isinstance(item, dict):
|
||||
item_name = item.get("name")
|
||||
item_url = item.get("url")
|
||||
if item_name and item_url:
|
||||
total_url = url + item_url
|
||||
if item_name not in channels:
|
||||
channels[item_name] = [total_url]
|
||||
else:
|
||||
channels[item_name].append(total_url)
|
||||
except Exception as e:
|
||||
print(f"Error on fofa: {e}")
|
||||
continue
|
||||
except (
|
||||
requests.exceptions.ConnectTimeout,
|
||||
requests.exceptions.ReadTimeout,
|
||||
):
|
||||
print(f"Connection to {url} timed out.")
|
||||
try:
|
||||
json_data = response.json()
|
||||
if json_data["code"] == 0:
|
||||
try:
|
||||
for item in json_data["data"]:
|
||||
if isinstance(item, dict):
|
||||
item_name = item.get("name")
|
||||
item_url = item.get("url")
|
||||
if item_name and item_url:
|
||||
total_url = url + item_url
|
||||
if item_name not in channels:
|
||||
channels[item_name] = [total_url]
|
||||
else:
|
||||
channels[item_name].append(total_url)
|
||||
except Exception as e:
|
||||
# print(f"Error on fofa: {e}")
|
||||
continue
|
||||
except Exception as e:
|
||||
# print(f"{url}: {e}")
|
||||
continue
|
||||
except Exception as e:
|
||||
# print(f"{url}: {e}")
|
||||
continue
|
||||
return channels
|
||||
|
||||
|
||||
def mergeObjects(*objects):
|
||||
"""
|
||||
Merge objects
|
||||
"""
|
||||
merged_dict = {}
|
||||
for obj in objects:
|
||||
if not isinstance(obj, dict):
|
||||
raise TypeError("All input objects must be dictionaries")
|
||||
for key, value in obj.items():
|
||||
if key not in merged_dict:
|
||||
merged_dict[key] = set()
|
||||
if isinstance(value, set):
|
||||
merged_dict[key].update(value)
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
merged_dict[key].add(item)
|
||||
else:
|
||||
merged_dict[key].add(value)
|
||||
for key, value in merged_dict.items():
|
||||
merged_dict[key] = list(value)
|
||||
return merged_dict
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user