Cx4x/ptb
1
0
Fork 0
Code Issues Pull Requests Actions2 Packages Projects Releases Wiki Activity

feat:fofa

Browse Source
This commit is contained in:
guorong.zheng 2024-08-19 17:30:09 +08:00
parent b77368529f
commit fabaec6137
5 changed files with 42 additions and 34 deletions

5
config/config.ini

@ -29,4 +29,7 @@ open_proxy = False
open_driver = False
open_hotel = False
hotel_region_list = 广东
hotel_region_list = 广东
open_fofa = False
fofa_region_list = all

46
main.py

@ -16,6 +16,7 @@ from utils.speed import is_ffmpeg_installed
from updates.subscribe import get_channels_by_subscribe_urls
from updates.multicast import get_channels_by_multicast
from updates.hotel import get_channels_by_hotel
from updates.fofa import get_channels_by_fofa
from updates.online_search import get_channels_by_online_search
import os
from tqdm import tqdm
@ -45,6 +46,7 @@ class UpdateSource:
self.subscribe_result = {}
self.multicast_result = {}
self.hotel_result = {}
self.fofa_result = {}
self.online_search_result = {}
self.channel_data = {}
self.pbar = None
@ -53,30 +55,25 @@ class UpdateSource:
self.sort_n = 0
async def visit_page(self, channel_names=None):
if config.getboolean("Settings", "open_subscribe"):
subscribe_task = asyncio.create_task(
get_channels_by_subscribe_urls(callback=self.update_progress)
)
self.tasks.append(subscribe_task)
self.subscribe_result = await subscribe_task
if config.getboolean("Settings", "open_multicast"):
multicast_task = asyncio.create_task(
get_channels_by_multicast(channel_names, self.update_progress)
)
self.tasks.append(multicast_task)
self.multicast_result = await multicast_task
if config.getboolean("Settings", "open_hotel"):
hotel_task = asyncio.create_task(
get_channels_by_hotel(channel_names, self.update_progress)
)
self.tasks.append(hotel_task)
self.hotel_result = await hotel_task
if config.getboolean("Settings", "open_online_search"):
online_search_task = asyncio.create_task(
get_channels_by_online_search(channel_names, self.update_progress)
)
self.tasks.append(online_search_task)
self.online_search_result = await online_search_task
tasks_config = [
("open_subscribe", get_channels_by_subscribe_urls, "subscribe_result"),
("open_multicast", get_channels_by_multicast, "multicast_result"),
("open_hotel", get_channels_by_hotel, "hotel_result"),
("open_fofa", get_channels_by_fofa, "fofa_result"),
(
"open_online_search",
get_channels_by_online_search,
"online_search_result",
),
]
for setting, task_func, result_attr in tasks_config:
if config.getboolean("Settings", setting):
task = asyncio.create_task(
task_func(channel_names, self.update_progress)
)
self.tasks.append(task)
setattr(self, result_attr, await task)
def pbar_update(self, name="", n=0):
if not n:
@ -106,6 +103,7 @@ class UpdateSource:
self.subscribe_result,
self.multicast_result,
self.hotel_result,
self.fofa_result,
self.online_search_result,
)
if config.getboolean("Settings", "open_sort"):

19
updates/fofa/request.py

@ -19,7 +19,7 @@ def get_fofa_urls_from_region_list():
"""
Get the FOFA url from region
"""
region_list = config.get("Settings", "region_list").split(",")
region_list = config.get("Settings", "fofa_region_list").split(",")
urls = []
region_url = getattr(fofa_map, "region_url")
if "all" in region_list:
@ -31,16 +31,16 @@ def get_fofa_urls_from_region_list():
return urls
async def get_channels_by_fofa(callback):
async def get_channels_by_fofa(names, callback):
"""
Get the channel by FOFA
"""
fofa_urls = get_fofa_urls_from_region_list()
fofa_urls_len = len(fofa_urls)
pbar = tqdm_asyncio(total=fofa_urls_len, desc="Processing multicast")
pbar = tqdm_asyncio(total=fofa_urls_len, desc="Processing fofa")
start_time = time()
fofa_results = {}
callback(f"正在获取组播源更新, 共{fofa_urls_len}个地区", 0)
callback(f"正在获取Fofa源更新, 共{fofa_urls_len}个地区", 0)
proxy = None
open_proxy = config.getboolean("Settings", "open_proxy")
open_driver = config.getboolean("Settings", "open_driver")
@ -83,7 +83,7 @@ async def get_channels_by_fofa(callback):
pbar.update()
remain = fofa_urls_len - pbar.n
callback(
f"正在获取组播源更新, 剩余{remain}个地区待获取, 预计剩余时间: {get_pbar_remaining(n=pbar.n, total=pbar.total, start_time=start_time)}",
f"正在获取Fofa源更新, 剩余{remain}个地区待获取, 预计剩余时间: {get_pbar_remaining(n=pbar.n, total=pbar.total, start_time=start_time)}",
int((pbar.n / fofa_urls_len) * 100),
)
return results
@ -95,10 +95,13 @@ async def get_channels_by_fofa(callback):
]
for future in futures:
fofa_results = merge_objects(fofa_results, future.result())
channels = {}
for name in names:
channels[name] = fofa_results.get(format_channel_name(name), [])
if not open_driver:
close_session()
pbar.close()
return fofa_results
return channels
def process_fofa_json_url(url):
@ -124,9 +127,9 @@ def process_fofa_json_url(url):
if item_name and item_url:
total_url = url + item_url
if item_name not in channels:
channels[item_name] = [total_url]
channels[item_name] = [(total_url, None, None)]
else:
channels[item_name].append(total_url)
channels[item_name].append((total_url, None, None))
except Exception as e:
# print(f"Error on fofa: {e}")
pass

2
updates/hotel/request.py

@ -6,7 +6,7 @@ from utils.channel import (
format_channel_name,
)
from utils.tools import get_pbar_remaining, get_soup
from utils.config import config, resource_path
from utils.config import config
from updates.proxy import get_proxy, get_proxy_next
from time import time, sleep
from driver.setup import setup_driver

4
utils/channel.py

@ -477,6 +477,7 @@ def append_all_method_data(
subscribe_result=None,
multicast_result=None,
hotel_result=None,
fofa_result=None,
online_search_result=None,
):
"""
@ -488,6 +489,7 @@ def append_all_method_data(
("subscribe", subscribe_result),
("multicast", multicast_result),
("hotel", hotel_result),
("fofa", fofa_result),
("online_search", online_search_result),
]:
if config.getboolean("Settings", f"open_{method}"):
@ -526,6 +528,7 @@ def append_all_method_data_keep_all(
subscribe_result=None,
multicast_result=None,
hotel_result=None,
fofa_result=None,
online_search_result=None,
):
"""
@ -536,6 +539,7 @@ def append_all_method_data_keep_all(
("subscribe", subscribe_result),
("multicast", multicast_result),
("hotel", hotel_result),
("fofa", fofa_result),
("online_search", online_search_result),
]:
if result and config.getboolean("Settings", f"open_{result_name}"):