dr_py/txt/ts/tools.ts
2023-05-09 18:29:45 +08:00

588 lines
19 KiB
TypeScript

import axios from 'axios';
import axiosRetry from 'axios-retry';
import { XMLParser } from 'fast-xml-parser';
import * as cheerio from 'cheerio';
import { Parser as M3u8Parser } from 'm3u8-parser';
import _ from 'lodash';
import { sites } from '@/lib/dexie';
const iconv = require('iconv-lite');
const dns = require('dns');
const net = require('net');
axiosRetry(axios, {
retries: 3,
retryDelay: (retryCount) => {
return retryCount * 1000;
}
});
// 初始化对象xml转json https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/docs/v4/1.GettingStarted.md
const options = { // XML 转 JSON 配置
trimValues: true,
textNodeName: '_t',
ignoreAttributes: false,
attributeNamePrefix: '_',
parseAttributeValue: true
}
const parser = new XMLParser(options);
Object.fromEntries = function fromEntries (iterable) {
return [...iterable].reduce((obj, [key, val]) => {
obj[key] = val;
return obj;
}, {});
};
const buildUrl = function (url,params_str){
const u = new URL(url);
const p = new URLSearchParams(params_str);
const api = u.origin + u.pathname;
let params = Object.fromEntries(u.searchParams.entries());
let params_obj = Object.fromEntries(p.entries());
Object.assign(params,params_obj);
let plist = [];
for(let key in params){
plist.push(key+'='+params[key]);
}
return api + '?' + plist.join('&')
};
// 资源爬虫
const zy = {
/**
* 获取资源分类 和 所有资源的总数, 分页等信息
* @param {*} key 资源网 key
* @returns
*/
async class (key) {
try {
const site = await sites.find({key:key});
const url = site.api;
const res = await axios.get(url);
const json = res.data;
const jsondata = json?.rss === undefined ? json : json.rss;
if (!jsondata?.class || !jsondata?.list) return null;
return {
class: jsondata.class,
page: jsondata.page,
pagecount: jsondata.pagecount,
pagesize: parseInt(jsondata.limit),
recordcount: jsondata.total
};
} catch (err) {
throw err;
}
},
/**
* 获取资源列表
* @param {*} key 资源网 key
* @param {number} [pg=1] 翻页 page
* @param {*} t 分类 type
* @returns
*/
async list(key, pg = 1, t) {
try {
const site = await sites.find({key:key});
const url = t ? buildUrl(site.api,`?ac=videolist&t=${t}&pg=${pg}`) : buildUrl(site.api,`?ac=videolist&pg=${pg}`);
const res = await axios.get(url);
const json = res.data;
const jsondata = json.rss || json;
const videoList = jsondata.list || [];
return videoList;
} catch (err) {
throw err;
}
},
/**
* 获取资源热榜列表
* @param {*} key 资源网 key
* @param {number} [pg=1] 翻页 page
* @param {*} t 分类 type
* @param {*} h 时间 time
* @returns
*/
// https://y.ioszxc123.me/api/v1/Vod/hot?limit=10&order=1&os=2&page=1&type=2
async hot(key, h) {
try {
const site = await sites.find({key:key});
const url = buildUrl(site.api,`?ac=hot&h=${h}`);
const res = await axios.get(url);
const json = res.data;
const jsondata = json.rss || json;
const videoList = jsondata.list || [];
const data = [];
for (let i = 0; i < 10; i++) {
const item = videoList[i]
if ( i in [0, 1, 2, 3 ]) {
const pic = await this.detail(key, item.vod_id);
item['vod_pic'] = pic.vod_pic
}
data.push(item);
}
return data;
} catch (err) {
throw err;
}
},
/**
* 获取总资源数, 以及页数
* @param {*} key 资源网
* @param {*} t 分类 type
* @returns page object
*/
async page (key, t) {
try {
const site = await sites.find({key:key});
let url = buildUrl(site.api,`?ac=videolist`);
if (t) url += `&t=${t}`;
const res = await axios.get(url);
// 某些源站不含页码时获取到的数据parser无法解析
const data = res.data.match(/<list [^>]*>/)[0] + '</list>';
const json = parser.parse(data);
const { _page, _pagecount, _pagesize, _recordcount } = json.rss?.list || {};
const pg = {
page: _page,
pagecount: _pagecount,
pagesize: _pagesize,
recordcount: _recordcount
};
// const jsondata = json.rss === undefined ? json : json.rss
// const pg = {
// page: jsondata.list._page,
// pagecount: jsondata.list._pagecount,
// pagesize: jsondata.list._pagesize,
// recordcount: jsondata.list._recordcount
// }
return pg;
} catch (err) {
throw err;
}
},
/**
* 搜索资源
* @param {*} key 资源网 key
* @param {*} wd 搜索关键字
* @returns
*/
async search(key, wd) {
try {
const site = await sites.find({key:key});
const url = buildUrl(site.api,`?wd=${encodeURIComponent(wd)}`);
const res = await axios.get(url, { timeout: 3000 });
const json = res.data;
const jsondata = json?.rss ?? json;
if (json && jsondata.total > 0) {
let videoList = jsondata.list;
if (Array.isArray(videoList)) {
return videoList;
}
}
} catch (err) {
throw err;
}
},
/**
* 搜索资源详情
* @param {*} key 资源网 key
* @param {*} wd 搜索关键字
* @returns
*/
async searchFirstDetail(key, wd) {
try {
const site = await sites.find({key:key});
const url = buildUrl(site.api,`?wd=${encodeURI(wd)}`)
const res = await axios.get(url, { timeout: 3000 })
const json = res.data
const jsondata = json?.rss === undefined ? json : json.rss
if (jsondata || jsondata?.list) {
let videoList = jsondata.list
if (Object.prototype.toString.call(videoList) === '[object Object]') videoList = [].concat(videoList)
if (videoList?.length) {
const detailRes = await this.detail(key, videoList[0].vod_id)
return detailRes
} else return null
} else return null
} catch (err) {
throw err;
}
},
/**
* 获取资源详情
* @param {*} key 资源网 key
* @param {*} id 资源唯一标识符 id
* @returns
*/
async detail(key, id) {
try {
const site = await sites.find({key:key});
const url = buildUrl(site.api,`?ac=videolist&ids=${id}`);
const res = await axios.get(url);
const json = res.data;
const jsondata = json?.rss ?? json;
const videoList = jsondata?.list?.[0];
if (!videoList) return;
// Parse video
// 播放源
const playFrom = videoList.vod_play_from;
const playSource = playFrom.split('$').filter(Boolean);
// 剧集
const playUrl = videoList.vod_play_url;
const playUrlDiffPlaySource = playUrl.split('$$$'); // 分离不同播放源
const playEpisodes = playUrlDiffPlaySource.map((item) => {
return item.replace(/\$+/g, '$').split('#').filter(e => {
const isHttp = e.startsWith('http');
const hasHttp = e.split('$')[1]?.startsWith('http');
return Boolean(e && (isHttp || hasHttp));
});
});
const fullList = Object.fromEntries(playSource.map((key, index) => [key, playEpisodes[index]]));
videoList.fullList = fullList;
return videoList;
} catch (err) {
throw err;
}
},
/**
* 检查资源
* @param {*} key 资源网 key
* @returns boolean
*/
async check (key) {
try {
const cls = await this.class(key)
if (cls) return true
else return false
} catch (err) {
console.log(err)
return false
}
},
/**
* 检查直播源
* @param {*} channel 直播频道 url
* @returns boolean
*/
async checkChannel(url) {
try {
const res = await axios.get(url);
const manifest = res.data;
const parser = new M3u8Parser();
parser.push(manifest);
parser.end();
const parsedManifest = parser.manifest;
if (parsedManifest.segments.length > 0) {
return true;
}
// 兼容性处理 抓包多次请求规则 #EXT-X-STREAM-INF 带文件路径的相对路径
const responseURL = res.request.responseURL
const { uri } = parsedManifest.playlists[0]
let newUrl
if (res.data.indexOf("encoder") > 0) {
// request1: http://1.204.169.243/live.aishang.ctlcdn.com/00000110240389_1/playlist.m3u8?CONTENTID=00000110240389_1&AUTHINFO=FABqh274XDn8fkurD5614t%2B1RvYajgx%2Ba3PxUJe1SMO4OjrtFitM6ZQbSJEFffaD35hOAhZdTXOrK0W8QvBRom%2BXaXZYzB%2FQfYjeYzGgKhP%2Fdo%2BXpr4quVxlkA%2BubKvbU1XwJFRgrbX%2BnTs60JauQUrav8kLj%2FPH8LxkDFpzvkq75UfeY%2FVNDZygRZLw4j%2BXtwhj%2FIuXf1hJAU0X%2BheT7g%3D%3D&USERTOKEN=eHKuwve%2F35NVIR5qsO5XsuB0O2BhR0KR
// #EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=8000000,CODECS="avc,mp21" encoder/0/playlist.m3u8?CONTENTID=00000110240127_1&AUTHINFO=FABqh274XDn8fkurD5614t%2B1RvYajgx%2Ba3PxUJe1SMO4OjrtFitM6ZQbSJEFffaD35hOAhZdTXOrK0W8QvBRom%2BXaXZYzB%2FQfYjeYzGgKhP%2Fdo%2BXpr4quVxlkA%2BubKvbU1XwJFRgrbX%2BnTs60JauQUrav8kLj%2FPH8LxkDFpzvkq75UfeY%2FVNDZygRZLw4j%2BXtwhj%2FIuXf1hJAU0X%2BheT7g%3D%3D&USERTOKEN=eHKuwve%2F35NVIR5qsO5XsuB0O2BhR0KR
// request2: http://1.204.169.243/live.aishang.ctlcdn.com/00000110240303_1/encoder/0/playlist.m3u8?CONTENTID=00000110240303_1&AUTHINFO=FABqh274XDn8fkurD5614t%2B1RvYajgx%2Ba3PxUJe1SMO4OjrtFitM6ZQbSJEFffaD35hOAhZdTXOrK0W8QvBRom%2BXaXZYzB%2FQfYjeYzGgKhP%2Fdo%2BXpr4quVxlkA%2BubKvbU1XwJFRgrbX%2BnTs60JauQUrav8kLj%2FPH8LxkDFpzvkq75UfeY%2FVNDZygRZLw4j%2BXtwhj%2FIuXf1hJAU0X%2BheT7g%3D%3D&USERTOKEN=eHKuwve%2F35NVIR5qsO5XsuB0O2BhR0KR
const index = responseURL.lastIndexOf("\/");
const urlLastParam= responseURL.substring(0, index+1);
newUrl = urlLastParam + uri;
return this.checkChannel(newUrl);
} else if (uri.indexOf("http") === 0|| uri.indexOf("//") === 0) {
// request1: http://[2409:8087:3869:8021:1001::e5]:6610/PLTV/88888888/224/3221225491/2/index.m3u8?IASHttpSessionId=OTT8798520230127055253191816
// #EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=8468480 http://[2409:8087:3869:8021:1001::e5]:6610/PLTV/88888888/224/3221225491/2/1000.m3u8?IASHttpSessionId=OTT8798520230127055253191816&zte_bandwidth=1000&bandwidth=8468480&ispcode=888&timeformat=local&channel=3221225491&m3u8_level=2&ztecid=3221225491
// request2: http://[2409:8087:3869:8021:1001::e5]:6610/PLTV/88888888/224/3221225491/2/1000.m3u8?IASHttpSessionId=OTT8867820230127053805215983&zte_bandwidth=1000&bandwidth=8467456&ispcode=888&timeformat=local&channel=3221225491&m3u8_level=2&ztecid=3221225491
newUrl = uri
return this.checkChannel(newUrl);
} else if (/^\/[^\/]/.test(uri) || (/^[^\/]/.test(uri) && uri.indexOf("http") === 0)) {
// request1: http://baidu.live.cqccn.com/__cl/cg:live/__c/hxjc_4K/__op/default/__f//index.m3u8
// #EXT-X-STREAM-INF:BANDWIDTH=15435519,AVERAGE-BANDWIDTH=15435519,RESOLUTION=3840x2160,CODECS="hvc1.1.6.L150.b0,mp4a.40.2",AUDIO="audio_mp4a.40.2_48000",CLOSED-CAPTIONS=NONE,FRAME-RATE=25 1/v15M/index.m3u8
// request2: http://baidu.live.cqccn.com/__cl/cg:live/__c/hxjc_4K/__op/default/__f//1/v15M/index.m3u8
const index = responseURL.lastIndexOf("\/");
const urlLastParam= responseURL.substring(0, index+1);
newUrl = urlLastParam + uri;
return this.checkChannel(newUrl);
}
return false;
} catch (err) {
throw err;
}
},
/**
* 提取ck/dp播放器m3u8
* @param {*} parserFilmUrl film url
* @returns boolean
*/
async parserFilmUrl(url) {
const urlDomain = url.match(/(\w+):\/\/([^\:|\/]+)(\:\d*)?(\/)/)[0];
try {
const response = await axios.get(url);
let urlPlay;
// 全局提取完整地址
const urlGlobal = response.data.match(/(https?:\/\/[^\s]+\.m3u8)/);
if (urlGlobal) {
urlPlay = urlGlobal[0];
return urlPlay;
}
// 局部提取地址 提取参数拼接域名
const urlParm = response.data.match(/\/(.*?)(\.m3u8)/);
if (urlParm) urlPlay = urlDomain + urlParm[0];
return urlPlay;
} catch (err) {
throw err;
}
},
/**
* 获取电子节目单
* @param {*} url epg阶段单api
* @param {*} tvg_name 节目名称
* @param {*} date 日期 2023-01-31
* @returns 电子节目单列表
*/
async iptvEpg(url, tvg_name, date) {
try {
const res = await axios.get(url, {
params: {
ch: tvg_name,
date: date
}
});
const epgData = res.data.epg_data;
return epgData;
} catch (err) {
throw err;
}
},
/**
* 判断 m3u8 文件是否为直播流
* @param {*} url m3u8地址
* @returns 是否是直播流
*/
async isLiveM3U8(url) {
try {
const res = await axios.get(url);
const m3u8Content = res.data;
// 从m3u8文件中解析媒体段(MEDIA-SEQUENCE)的值
const mediaSequenceMatch = m3u8Content.match(/#EXT-X-MEDIA-SEQUENCE:(\d+)/);
const mediaSequence = mediaSequenceMatch ? parseInt(mediaSequenceMatch[1]) : null;
// 判断是直播还是点播
const isLiveStream = mediaSequence === null || mediaSequence === 0;
return !isLiveStream;
} catch (err) {
throw err;
}
},
/**
* 获取豆瓣页面链接
* @param {*} id 视频唯一标识
* @param {*} name 视频名称
* @param {*} year 视频年份
* @returns 豆瓣页面链接,如果没有搜到该视频,返回搜索页面链接
*/
async doubanLink(id, name, year) {
const nameToSearch = encodeURI(name.trim())
const doubanSearchLink = id && parseInt(id) !== 0 ? `https://movie.douban.com/subject/${id}` : `https://www.douban.com/search?cat=1002&q=${nameToSearch}`
try {
const res = await axios.get(doubanSearchLink)
const $ = cheerio.load(res.data)
let link = ''
$('div.result').each(function () {
const linkInDouban = $(this).find('div>div>h3>a').first()
const nameInDouban = linkInDouban.text().replace(/\s/g, '')
const subjectCast = $(this).find('span.subject-cast').text()
if (nameToSearch === encodeURI(nameInDouban) && subjectCast && subjectCast.includes(year)) {
link = linkInDouban.attr('href')
return
}
})
return link || doubanSearchLink
} catch (err) {
throw err
}
},
/**
* 获取豆瓣评分
* @param {*} id 视频唯一标识
* @param {*} name 视频名称
* @param {*} year 视频年份
* @returns 豆瓣评分
*/
async doubanRate(id, name, year) {
try {
const link = await this.doubanLink(id, name, year);
if (link.includes('https://www.douban.com/search')) {
return '暂无评分';
} else {
const response = await axios.get(link);
const parsedHtml = cheerio.load(response.data);
const rating = parsedHtml('body').find('#interest_sectl').first().find('strong').first().text().replace(/\s/g, '');
return rating || '暂无评分';
// const rating = parsedHtml('body').find('#interest_sectl').first().find('strong').first();
// if (rating.text()) {
// return rating.text().replace(/\s/g, '');
// } else {
// return '暂无评分';
// }
}
} catch (err) {
throw err;
}
},
/**
* 获取豆瓣相关视频推荐列表
* @param {*} id 视频唯一标识
* @param {*} name 视频名称
* @param {*} year 视频年份
* @returns 豆瓣相关视频推荐列表
*/
async doubanRecommendations(id, name, year) {
try {
const link = await this.doubanLink(id, name, year);
if (link.includes('https://www.douban.com/search')) {
return [];
} else {
const response = await axios.get(link);
const $ = cheerio.load(response.data);
const recommendations = $('div.recommendations-bd').find('div>dl>dd>a').map((index, element) => $(element).text()).get();
return recommendations;
}
} catch (err) {
throw err;
}
},
/**
* 获取豆瓣热点视频列表
* @param {*} type 类型
* @param {*} tag 标签
* @param {*} limit 显示条数
* @param {*} start 跳过
* @returns 豆瓣热点视频推荐列表
*/
async doubanHot(type, tag, limit = 20, start = 0) {
const doubanHotLink = `https://movie.douban.com/j/search_subjects?type=${type}&tag=${encodeURI(tag)}&page_limit=${limit}&page_start=${start}`;
try {
const { data: { subjects } } = await axios.get(doubanHotLink);
return subjects.map(item => ({
vod_id: item.id,
vod_name: item.title,
vod_remarks: item.episodes_info,
vod_pic: item.cover,
}));
} catch (err) {
throw err;
}
},
/**
* 获取酷云热点视频列表
* @param {*} date 日期2023-5-3
* @param {*} type 类型 1.电影 2.剧集 3.综艺
* @param {*} plat 平台 1.腾讯视频 2.爱奇艺 3.优酷 4.芒果
* @returns 酷云热点视频推荐列表
*/
async kuyunHot( date, type, plat) {
const kuyunHotLink = `https://eye.kuyun.com/api/netplat/ranking?date=${date}&type=${type}&plat=${plat}`;
try {
const { data: { data: { list } } } = await axios.get(kuyunHotLink);
return list.map(item => ({
vod_id: item.ca_id,
vod_name: item.name,
vod_hot: item.num,
}));
} catch (err) {
throw err;
}
},
/**
* 获取解析url链接的标题
* @param {*} url 需要解析的地址
* @returns 解析标题
*/
async getAnalysizeTitle (url) {
try {
const res = await axios.get(url, { responseType: 'arraybuffer' });
let html = '';
if (url.includes('sohu')) {
html = iconv.decode(Buffer.from(res.data), 'gb2312');
} else {
html = iconv.decode(Buffer.from(res.data), 'utf-8');
}
const $ = cheerio.load(html);
return $("title").text();
} catch (err) {
throw err;
}
},
/**
* 获取配置文件
* @param {*} url 需要获取的地址
* @returns 配置文件
*/
async getConfig(url) {
try {
const res = await axios.get(url);
return res.data || false;
} catch (err) {
throw err;
}
},
/**
* 判断是否支持ipv6
* @returns ture/false
*/
async checkSupportIpv6() {
try {
const res = await axios.get('https://6.ipw.cn');
const ip = res.data;
const isIpv6 = /([0-9a-z]*:{1,4}){1,7}[0-9a-z]{1,4}/i.test(ip);
return isIpv6;
} catch (err) {
throw err;
}
},
/**
* 判断url是否为ipv6
* @returns ture/false
*/
async checkUrlIpv6(url) {
let hostname = new URL(url).hostname;
const ipv6Regex = /^\[([\da-fA-F:]+)\]$/; // 匹配 IPv6 地址
const match = ipv6Regex.exec(hostname);
if(match){
// console.log(match[1])
hostname = match[1];
}
const ipType = net.isIP(hostname);
if (ipType === 4) {
// console.log(`1.ipv4:${hostname}`)
return 'IPv4';
} else if (ipType === 6) {
// console.log(`1.ipv6:${hostname}`)
return 'IPv6';
} else {
try {
const addresses = await dns.promises.resolve(hostname);
const ipType = net.isIP(addresses[0]);
if (ipType === 4) {
// console.log(`2.ipv4:${addresses[0]}`)
return 'IPv4';
} else if (ipType === 6) {
// console.log(`2.ipv6:${addresses[0]}`)
return 'IPv6';
} else {
return 'Unknown';
}
} catch (err) {
console.log(url,hostname)
throw err;
}
}
}
}
export default zy