Govin
GitHub
commit
fff0b55735
13
Dockerfile
Normal file
13
Dockerfile
Normal file
@ -0,0 +1,13 @@
|
||||
FROM python:3.8-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY . /app
|
||||
|
||||
# RUN pip install --trusted-host pypi.python.org Flask
|
||||
|
||||
EXPOSE 80
|
||||
|
||||
ENV NAME World
|
||||
|
||||
CMD ["python", "./main.py"]
|
||||
1
Pipfile
1
Pipfile
@ -20,6 +20,7 @@ tqdm = "*"
|
||||
async-timeout = "*"
|
||||
pyinstaller = "*"
|
||||
aiohttp = "*"
|
||||
flask = "*"
|
||||
|
||||
[requires]
|
||||
python_version = "3.8"
|
||||
|
||||
117
Pipfile.lock
generated
117
Pipfile.lock
generated
@ -1,7 +1,7 @@
|
||||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "4bfb309bc7d7f25a5cb114e48b398480e2ff489ff884e3d4c07d4b726d58875a"
|
||||
"sha256": "3e131626f5e67fd5e09513344aee61f34c1f56f1e96f1cf523a66b82b7146cf6"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
@ -139,6 +139,14 @@
|
||||
"markers": "python_full_version >= '3.6.0'",
|
||||
"version": "==4.12.3"
|
||||
},
|
||||
"blinker": {
|
||||
"hashes": [
|
||||
"sha256:1779309f71bf239144b9399d06ae925637cf6634cf6bd131104184531bf67c01",
|
||||
"sha256:8f77b09d3bf7c795e969e9486f39c2c5e9c39d4ee07424be2bc594ece9642d83"
|
||||
],
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==1.8.2"
|
||||
},
|
||||
"bs4": {
|
||||
"hashes": [
|
||||
"sha256:a48685c58f50fe127722417bae83fe6badf500d54b55f7e39ffe43b798653925",
|
||||
@ -309,6 +317,14 @@
|
||||
"markers": "python_full_version >= '3.7.0'",
|
||||
"version": "==3.3.2"
|
||||
},
|
||||
"click": {
|
||||
"hashes": [
|
||||
"sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28",
|
||||
"sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==8.1.7"
|
||||
},
|
||||
"colorama": {
|
||||
"hashes": [
|
||||
"sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44",
|
||||
@ -334,6 +350,15 @@
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==6.0.11"
|
||||
},
|
||||
"flask": {
|
||||
"hashes": [
|
||||
"sha256:34e815dfaa43340d1d15a5c3a02b8476004037eb4840b34910c6e21679d288f3",
|
||||
"sha256:ceb27b0af3823ea2737928a4d99d125a06175b8512c445cbd9a9ce200ef76842"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==3.0.3"
|
||||
},
|
||||
"frozenlist": {
|
||||
"hashes": [
|
||||
"sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7",
|
||||
@ -441,6 +466,88 @@
|
||||
"markers": "python_version < '3.10'",
|
||||
"version": "==8.0.0"
|
||||
},
|
||||
"itsdangerous": {
|
||||
"hashes": [
|
||||
"sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef",
|
||||
"sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173"
|
||||
],
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==2.2.0"
|
||||
},
|
||||
"jinja2": {
|
||||
"hashes": [
|
||||
"sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369",
|
||||
"sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==3.1.4"
|
||||
},
|
||||
"markupsafe": {
|
||||
"hashes": [
|
||||
"sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf",
|
||||
"sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff",
|
||||
"sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f",
|
||||
"sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3",
|
||||
"sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532",
|
||||
"sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f",
|
||||
"sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617",
|
||||
"sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df",
|
||||
"sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4",
|
||||
"sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906",
|
||||
"sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f",
|
||||
"sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4",
|
||||
"sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8",
|
||||
"sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371",
|
||||
"sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2",
|
||||
"sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465",
|
||||
"sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52",
|
||||
"sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6",
|
||||
"sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169",
|
||||
"sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad",
|
||||
"sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2",
|
||||
"sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0",
|
||||
"sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029",
|
||||
"sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f",
|
||||
"sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a",
|
||||
"sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced",
|
||||
"sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5",
|
||||
"sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c",
|
||||
"sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf",
|
||||
"sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9",
|
||||
"sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb",
|
||||
"sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad",
|
||||
"sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3",
|
||||
"sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1",
|
||||
"sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46",
|
||||
"sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc",
|
||||
"sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a",
|
||||
"sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee",
|
||||
"sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900",
|
||||
"sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5",
|
||||
"sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea",
|
||||
"sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f",
|
||||
"sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5",
|
||||
"sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e",
|
||||
"sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a",
|
||||
"sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f",
|
||||
"sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50",
|
||||
"sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a",
|
||||
"sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b",
|
||||
"sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4",
|
||||
"sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff",
|
||||
"sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2",
|
||||
"sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46",
|
||||
"sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b",
|
||||
"sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf",
|
||||
"sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5",
|
||||
"sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5",
|
||||
"sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab",
|
||||
"sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd",
|
||||
"sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==2.1.5"
|
||||
},
|
||||
"multidict": {
|
||||
"hashes": [
|
||||
"sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556",
|
||||
@ -735,6 +842,14 @@
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==1.8.0"
|
||||
},
|
||||
"werkzeug": {
|
||||
"hashes": [
|
||||
"sha256:097e5bfda9f0aba8da6b8545146def481d06aa7d3266e7448e2cccf67dd8bd18",
|
||||
"sha256:fc9645dc43e03e4d630d23143a04a7f947a9a3b5727cd535fdfe155a17cc48c8"
|
||||
],
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==3.0.3"
|
||||
},
|
||||
"wsproto": {
|
||||
"hashes": [
|
||||
"sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065",
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
open_update = True
|
||||
source_file = "demo.txt"
|
||||
final_file = "result.txt"
|
||||
favorite_list = [
|
||||
|
||||
22
main.py
22
main.py
@ -20,6 +20,9 @@ import os
|
||||
from tqdm import tqdm
|
||||
from tqdm.asyncio import tqdm_asyncio
|
||||
from time import time
|
||||
from flask import Flask, render_template_string
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
config_path = resource_path("user_config.py")
|
||||
default_config_path = resource_path("config.py")
|
||||
@ -42,6 +45,13 @@ class UpdateSource:
|
||||
self.total = 0
|
||||
self.start_time = None
|
||||
|
||||
@app.route("/")
|
||||
def show_result():
|
||||
user_final_file = getattr(config, "final_file", "result.txt")
|
||||
with open(user_final_file, "r", encoding="utf-8") as file:
|
||||
content = file.read()
|
||||
return render_template_string("<pre>{{ content }}</pre>", content=content)
|
||||
|
||||
def check_info_data(self, cate, name):
|
||||
if self.channel_data.get(cate) is None:
|
||||
self.channel_data[cate] = {}
|
||||
@ -246,7 +256,11 @@ class UpdateSource:
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
update_source = UpdateSource()
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
loop.run_until_complete(update_source.start())
|
||||
if config.open_update:
|
||||
update_source = UpdateSource()
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
loop.run_until_complete(update_source.start())
|
||||
github_actions = os.environ.get("GITHUB_ACTIONS")
|
||||
if not github_actions:
|
||||
app.run(host="0.0.0.0", port=80)
|
||||
|
||||
136
utils.py
136
utils.py
@ -18,6 +18,7 @@ from tqdm.asyncio import tqdm_asyncio
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
from selenium.common.exceptions import TimeoutException
|
||||
from selenium_stealth import stealth
|
||||
import concurrent.futures
|
||||
import sys
|
||||
@ -34,6 +35,7 @@ def retry_func(func, retries=max_retries + 1, name=""):
|
||||
"""
|
||||
for i in range(retries):
|
||||
try:
|
||||
sleep(3)
|
||||
return func()
|
||||
except Exception as e:
|
||||
count = retries - 1
|
||||
@ -45,6 +47,34 @@ def retry_func(func, retries=max_retries + 1, name=""):
|
||||
continue
|
||||
|
||||
|
||||
def locate_element_with_retry(driver, locator, timeout=timeout, retries=max_retries):
|
||||
"""
|
||||
Locate the element with retry
|
||||
"""
|
||||
wait = WebDriverWait(driver, timeout)
|
||||
for _ in range(retries):
|
||||
try:
|
||||
return wait.until(EC.presence_of_element_located(locator))
|
||||
except TimeoutException:
|
||||
driver.refresh()
|
||||
return None
|
||||
|
||||
|
||||
def find_clickable_element_with_retry(
|
||||
driver, locator, timeout=timeout, retries=max_retries
|
||||
):
|
||||
"""
|
||||
Find the clickable element with retry
|
||||
"""
|
||||
wait = WebDriverWait(driver, timeout)
|
||||
for _ in range(retries):
|
||||
try:
|
||||
return wait.until(EC.element_to_be_clickable(locator))
|
||||
except TimeoutException:
|
||||
driver.refresh()
|
||||
return None
|
||||
|
||||
|
||||
def resource_path(relative_path, persistent=False):
|
||||
"""
|
||||
Get the resource path
|
||||
@ -128,8 +158,7 @@ def get_proxy_list(page_count=1):
|
||||
]
|
||||
proxy_list = []
|
||||
driver = setup_driver()
|
||||
pbar = tqdm_asyncio(total=page_count)
|
||||
pbar.set_description(f"Getting proxy list, {page_count} pages remaining")
|
||||
pbar = tqdm_asyncio(total=page_count, desc="Getting proxy list")
|
||||
for page_index in range(1, page_count + 1):
|
||||
for pattern in url_pattern:
|
||||
url = pattern.format(page_index)
|
||||
@ -151,9 +180,6 @@ def get_proxy_list(page_count=1):
|
||||
proxy = f"http://{ip}:{port}"
|
||||
proxy_list.append(proxy)
|
||||
pbar.update()
|
||||
pbar.set_description(
|
||||
f"Getting proxy list, {page_count - page_index} pages remaining"
|
||||
)
|
||||
pbar.close()
|
||||
return proxy_list
|
||||
|
||||
@ -362,69 +388,81 @@ async def get_channels_by_online_search(names, callback):
|
||||
|
||||
def process_channel_by_online_search(name, proxy=None):
|
||||
driver = setup_driver(proxy)
|
||||
wait = WebDriverWait(driver, timeout)
|
||||
info_list = []
|
||||
try:
|
||||
retry_func(lambda: driver.get(pageUrl), name=f"online search:{name}")
|
||||
search_box = retry_func(
|
||||
lambda: wait.until(
|
||||
EC.presence_of_element_located((By.XPATH, '//input[@type="text"]'))
|
||||
)
|
||||
search_box = locate_element_with_retry(
|
||||
driver, (By.XPATH, '//input[@type="text"]')
|
||||
)
|
||||
if not search_box:
|
||||
return
|
||||
search_box.clear()
|
||||
search_box.send_keys(name)
|
||||
submit_button = retry_func(
|
||||
lambda: wait.until(
|
||||
EC.element_to_be_clickable((By.XPATH, '//input[@type="submit"]'))
|
||||
)
|
||||
submit_button = find_clickable_element_with_retry(
|
||||
driver, (By.XPATH, '//input[@type="submit"]')
|
||||
)
|
||||
if not submit_button:
|
||||
return
|
||||
sleep(3)
|
||||
driver.execute_script("arguments[0].click();", submit_button)
|
||||
isFavorite = name in config.favorite_list
|
||||
pageNum = (
|
||||
config.favorite_page_num if isFavorite else config.default_page_num
|
||||
)
|
||||
retry_limit = 3
|
||||
for page in range(1, pageNum + 1):
|
||||
try:
|
||||
if page > 1:
|
||||
sleep(1)
|
||||
page_link = retry_func(
|
||||
lambda: wait.until(
|
||||
EC.element_to_be_clickable(
|
||||
(
|
||||
By.XPATH,
|
||||
f'//a[contains(@href, "={page}") and contains(@href, "{name}")]',
|
||||
)
|
||||
)
|
||||
retries = 0
|
||||
while retries < retry_limit:
|
||||
try:
|
||||
if page > 1:
|
||||
page_link = find_clickable_element_with_retry(
|
||||
driver,
|
||||
(
|
||||
By.XPATH,
|
||||
f'//a[contains(@href, "={page}") and contains(@href, "{name}")]',
|
||||
),
|
||||
)
|
||||
if not page_link:
|
||||
break
|
||||
sleep(3)
|
||||
driver.execute_script("arguments[0].click();", page_link)
|
||||
sleep(3)
|
||||
source = re.sub(
|
||||
r"<!--.*?-->",
|
||||
"",
|
||||
driver.page_source,
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
if not page_link:
|
||||
soup = BeautifulSoup(source, "html.parser")
|
||||
if soup:
|
||||
results = get_results_from_soup(soup, name)
|
||||
print(name, "page:", page, "results num:", len(results))
|
||||
if len(results) == 0 and retries < retry_limit - 1:
|
||||
print(
|
||||
f"{name}:No results found, refreshing page and retrying..."
|
||||
)
|
||||
driver.refresh()
|
||||
retries += 1
|
||||
continue
|
||||
for result in results:
|
||||
url, date, resolution = result
|
||||
if url and check_url_by_patterns(url):
|
||||
info_list.append((url, date, resolution))
|
||||
break
|
||||
else:
|
||||
print(
|
||||
f"{name}:No results found, refreshing page and retrying..."
|
||||
)
|
||||
driver.refresh()
|
||||
retries += 1
|
||||
continue
|
||||
driver.execute_script("arguments[0].click();", page_link)
|
||||
source = re.sub(
|
||||
r"<!--.*?-->",
|
||||
"",
|
||||
driver.page_source,
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
soup = BeautifulSoup(source, "html.parser")
|
||||
if soup:
|
||||
results = get_results_from_soup(soup, name)
|
||||
print(name, "page:", page, "results num:", len(results))
|
||||
for result in results:
|
||||
url, date, resolution = result
|
||||
if url and check_url_by_patterns(url):
|
||||
info_list.append((url, date, resolution))
|
||||
else:
|
||||
print(f"No results found for {name}")
|
||||
except Exception as e:
|
||||
print(f"Error on page {page}: {e}")
|
||||
continue
|
||||
except Exception as e:
|
||||
print(f"{name}:Error on page {page}: {e}")
|
||||
break
|
||||
if retries == retry_limit:
|
||||
print(f"{name}:Reached retry limit, moving to next page")
|
||||
except Exception as e:
|
||||
print(f"Error on search: {e}")
|
||||
print(f"{name}:Error on search: {e}")
|
||||
pass
|
||||
finally:
|
||||
channels[format_channel_name(name)] = info_list
|
||||
@ -451,9 +489,7 @@ async def get_channels_by_online_search(names, callback):
|
||||
loop = asyncio.get_running_loop()
|
||||
name = await names_queue.get()
|
||||
proxy = (
|
||||
proxy_list_test[proxy_index]
|
||||
if config.open_proxy and proxy_list_test
|
||||
else None
|
||||
proxy_list_test[0] if config.open_proxy and proxy_list_test else None
|
||||
)
|
||||
if config.open_proxy and proxy_list_test:
|
||||
proxy_index = (proxy_index + 1) % len(proxy_list_test)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user