import axios from 'axios'; import axiosRetry from 'axios-retry'; import { XMLParser } from 'fast-xml-parser'; import * as cheerio from 'cheerio'; import { Parser as M3u8Parser } from 'm3u8-parser'; import _ from 'lodash'; import { sites } from '@/lib/dexie'; const iconv = require('iconv-lite'); const dns = require('dns'); const net = require('net'); axiosRetry(axios, { retries: 3, retryDelay: (retryCount) => { return retryCount * 1000; } }); // 初始化对象xml转json https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/docs/v4/1.GettingStarted.md const options = { // XML 转 JSON 配置 trimValues: true, textNodeName: '_t', ignoreAttributes: false, attributeNamePrefix: '_', parseAttributeValue: true } const parser = new XMLParser(options); Object.fromEntries = function fromEntries (iterable) { return [...iterable].reduce((obj, [key, val]) => { obj[key] = val; return obj; }, {}); }; const buildUrl = function (url,params_str){ const u = new URL(url); const p = new URLSearchParams(params_str); const api = u.origin + u.pathname; let params = Object.fromEntries(u.searchParams.entries()); let params_obj = Object.fromEntries(p.entries()); Object.assign(params,params_obj); let plist = []; for(let key in params){ plist.push(key+'='+params[key]); } return api + '?' + plist.join('&') }; // 资源爬虫 const zy = { /** * 获取资源分类 和 所有资源的总数, 分页等信息 * @param {*} key 资源网 key * @returns */ async class (key) { try { const site = await sites.find({key:key}); const url = site.api; const res = await axios.get(url); const json = res.data; const jsondata = json?.rss === undefined ? json : json.rss; if (!jsondata?.class || !jsondata?.list) return null; return { class: jsondata.class, page: jsondata.page, pagecount: jsondata.pagecount, pagesize: parseInt(jsondata.limit), recordcount: jsondata.total }; } catch (err) { throw err; } }, /** * 获取资源列表 * @param {*} key 资源网 key * @param {number} [pg=1] 翻页 page * @param {*} t 分类 type * @returns */ async list(key, pg = 1, t) { try { const site = await sites.find({key:key}); const url = t ? buildUrl(site.api,`?ac=videolist&t=${t}&pg=${pg}`) : buildUrl(site.api,`?ac=videolist&pg=${pg}`); const res = await axios.get(url); const json = res.data; const jsondata = json.rss || json; const videoList = jsondata.list || []; return videoList; } catch (err) { throw err; } }, /** * 获取资源热榜列表 * @param {*} key 资源网 key * @param {number} [pg=1] 翻页 page * @param {*} t 分类 type * @param {*} h 时间 time * @returns */ // https://y.ioszxc123.me/api/v1/Vod/hot?limit=10&order=1&os=2&page=1&type=2 async hot(key, h) { try { const site = await sites.find({key:key}); const url = buildUrl(site.api,`?ac=hot&h=${h}`); const res = await axios.get(url); const json = res.data; const jsondata = json.rss || json; const videoList = jsondata.list || []; const data = []; for (let i = 0; i < 10; i++) { const item = videoList[i] if ( i in [0, 1, 2, 3 ]) { const pic = await this.detail(key, item.vod_id); item['vod_pic'] = pic.vod_pic } data.push(item); } return data; } catch (err) { throw err; } }, /** * 获取总资源数, 以及页数 * @param {*} key 资源网 * @param {*} t 分类 type * @returns page object */ async page (key, t) { try { const site = await sites.find({key:key}); let url = buildUrl(site.api,`?ac=videolist`); if (t) url += `&t=${t}`; const res = await axios.get(url); // 某些源站不含页码时获取到的数据parser无法解析 const data = res.data.match(/]*>/)[0] + ''; const json = parser.parse(data); const { _page, _pagecount, _pagesize, _recordcount } = json.rss?.list || {}; const pg = { page: _page, pagecount: _pagecount, pagesize: _pagesize, recordcount: _recordcount }; // const jsondata = json.rss === undefined ? json : json.rss // const pg = { // page: jsondata.list._page, // pagecount: jsondata.list._pagecount, // pagesize: jsondata.list._pagesize, // recordcount: jsondata.list._recordcount // } return pg; } catch (err) { throw err; } }, /** * 搜索资源 * @param {*} key 资源网 key * @param {*} wd 搜索关键字 * @returns */ async search(key, wd) { try { const site = await sites.find({key:key}); const url = buildUrl(site.api,`?wd=${encodeURIComponent(wd)}`); const res = await axios.get(url, { timeout: 3000 }); const json = res.data; const jsondata = json?.rss ?? json; if (json && jsondata.total > 0) { let videoList = jsondata.list; if (Array.isArray(videoList)) { return videoList; } } } catch (err) { throw err; } }, /** * 搜索资源详情 * @param {*} key 资源网 key * @param {*} wd 搜索关键字 * @returns */ async searchFirstDetail(key, wd) { try { const site = await sites.find({key:key}); const url = buildUrl(site.api,`?wd=${encodeURI(wd)}`) const res = await axios.get(url, { timeout: 3000 }) const json = res.data const jsondata = json?.rss === undefined ? json : json.rss if (jsondata || jsondata?.list) { let videoList = jsondata.list if (Object.prototype.toString.call(videoList) === '[object Object]') videoList = [].concat(videoList) if (videoList?.length) { const detailRes = await this.detail(key, videoList[0].vod_id) return detailRes } else return null } else return null } catch (err) { throw err; } }, /** * 获取资源详情 * @param {*} key 资源网 key * @param {*} id 资源唯一标识符 id * @returns */ async detail(key, id) { try { const site = await sites.find({key:key}); const url = buildUrl(site.api,`?ac=videolist&ids=${id}`); const res = await axios.get(url); const json = res.data; const jsondata = json?.rss ?? json; const videoList = jsondata?.list?.[0]; if (!videoList) return; // Parse video // 播放源 const playFrom = videoList.vod_play_from; const playSource = playFrom.split('$').filter(Boolean); // 剧集 const playUrl = videoList.vod_play_url; const playUrlDiffPlaySource = playUrl.split('$$$'); // 分离不同播放源 const playEpisodes = playUrlDiffPlaySource.map((item) => { return item.replace(/\$+/g, '$').split('#').filter(e => { const isHttp = e.startsWith('http'); const hasHttp = e.split('$')[1]?.startsWith('http'); return Boolean(e && (isHttp || hasHttp)); }); }); const fullList = Object.fromEntries(playSource.map((key, index) => [key, playEpisodes[index]])); videoList.fullList = fullList; return videoList; } catch (err) { throw err; } }, /** * 检查资源 * @param {*} key 资源网 key * @returns boolean */ async check (key) { try { const cls = await this.class(key) if (cls) return true else return false } catch (err) { console.log(err) return false } }, /** * 检查直播源 * @param {*} channel 直播频道 url * @returns boolean */ async checkChannel(url) { try { const res = await axios.get(url); const manifest = res.data; const parser = new M3u8Parser(); parser.push(manifest); parser.end(); const parsedManifest = parser.manifest; if (parsedManifest.segments.length > 0) { return true; } // 兼容性处理 抓包多次请求规则 #EXT-X-STREAM-INF 带文件路径的相对路径 const responseURL = res.request.responseURL const { uri } = parsedManifest.playlists[0] let newUrl if (res.data.indexOf("encoder") > 0) { // request1: http://1.204.169.243/live.aishang.ctlcdn.com/00000110240389_1/playlist.m3u8?CONTENTID=00000110240389_1&AUTHINFO=FABqh274XDn8fkurD5614t%2B1RvYajgx%2Ba3PxUJe1SMO4OjrtFitM6ZQbSJEFffaD35hOAhZdTXOrK0W8QvBRom%2BXaXZYzB%2FQfYjeYzGgKhP%2Fdo%2BXpr4quVxlkA%2BubKvbU1XwJFRgrbX%2BnTs60JauQUrav8kLj%2FPH8LxkDFpzvkq75UfeY%2FVNDZygRZLw4j%2BXtwhj%2FIuXf1hJAU0X%2BheT7g%3D%3D&USERTOKEN=eHKuwve%2F35NVIR5qsO5XsuB0O2BhR0KR // #EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=8000000,CODECS="avc,mp21" encoder/0/playlist.m3u8?CONTENTID=00000110240127_1&AUTHINFO=FABqh274XDn8fkurD5614t%2B1RvYajgx%2Ba3PxUJe1SMO4OjrtFitM6ZQbSJEFffaD35hOAhZdTXOrK0W8QvBRom%2BXaXZYzB%2FQfYjeYzGgKhP%2Fdo%2BXpr4quVxlkA%2BubKvbU1XwJFRgrbX%2BnTs60JauQUrav8kLj%2FPH8LxkDFpzvkq75UfeY%2FVNDZygRZLw4j%2BXtwhj%2FIuXf1hJAU0X%2BheT7g%3D%3D&USERTOKEN=eHKuwve%2F35NVIR5qsO5XsuB0O2BhR0KR // request2: http://1.204.169.243/live.aishang.ctlcdn.com/00000110240303_1/encoder/0/playlist.m3u8?CONTENTID=00000110240303_1&AUTHINFO=FABqh274XDn8fkurD5614t%2B1RvYajgx%2Ba3PxUJe1SMO4OjrtFitM6ZQbSJEFffaD35hOAhZdTXOrK0W8QvBRom%2BXaXZYzB%2FQfYjeYzGgKhP%2Fdo%2BXpr4quVxlkA%2BubKvbU1XwJFRgrbX%2BnTs60JauQUrav8kLj%2FPH8LxkDFpzvkq75UfeY%2FVNDZygRZLw4j%2BXtwhj%2FIuXf1hJAU0X%2BheT7g%3D%3D&USERTOKEN=eHKuwve%2F35NVIR5qsO5XsuB0O2BhR0KR const index = responseURL.lastIndexOf("\/"); const urlLastParam= responseURL.substring(0, index+1); newUrl = urlLastParam + uri; return this.checkChannel(newUrl); } else if (uri.indexOf("http") === 0|| uri.indexOf("//") === 0) { // request1: http://[2409:8087:3869:8021:1001::e5]:6610/PLTV/88888888/224/3221225491/2/index.m3u8?IASHttpSessionId=OTT8798520230127055253191816 // #EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=8468480 http://[2409:8087:3869:8021:1001::e5]:6610/PLTV/88888888/224/3221225491/2/1000.m3u8?IASHttpSessionId=OTT8798520230127055253191816&zte_bandwidth=1000&bandwidth=8468480&ispcode=888&timeformat=local&channel=3221225491&m3u8_level=2&ztecid=3221225491 // request2: http://[2409:8087:3869:8021:1001::e5]:6610/PLTV/88888888/224/3221225491/2/1000.m3u8?IASHttpSessionId=OTT8867820230127053805215983&zte_bandwidth=1000&bandwidth=8467456&ispcode=888&timeformat=local&channel=3221225491&m3u8_level=2&ztecid=3221225491 newUrl = uri return this.checkChannel(newUrl); } else if (/^\/[^\/]/.test(uri) || (/^[^\/]/.test(uri) && uri.indexOf("http") === 0)) { // request1: http://baidu.live.cqccn.com/__cl/cg:live/__c/hxjc_4K/__op/default/__f//index.m3u8 // #EXT-X-STREAM-INF:BANDWIDTH=15435519,AVERAGE-BANDWIDTH=15435519,RESOLUTION=3840x2160,CODECS="hvc1.1.6.L150.b0,mp4a.40.2",AUDIO="audio_mp4a.40.2_48000",CLOSED-CAPTIONS=NONE,FRAME-RATE=25 1/v15M/index.m3u8 // request2: http://baidu.live.cqccn.com/__cl/cg:live/__c/hxjc_4K/__op/default/__f//1/v15M/index.m3u8 const index = responseURL.lastIndexOf("\/"); const urlLastParam= responseURL.substring(0, index+1); newUrl = urlLastParam + uri; return this.checkChannel(newUrl); } return false; } catch (err) { throw err; } }, /** * 提取ck/dp播放器m3u8 * @param {*} parserFilmUrl film url * @returns boolean */ async parserFilmUrl(url) { const urlDomain = url.match(/(\w+):\/\/([^\:|\/]+)(\:\d*)?(\/)/)[0]; try { const response = await axios.get(url); let urlPlay; // 全局提取完整地址 const urlGlobal = response.data.match(/(https?:\/\/[^\s]+\.m3u8)/); if (urlGlobal) { urlPlay = urlGlobal[0]; return urlPlay; } // 局部提取地址 提取参数拼接域名 const urlParm = response.data.match(/\/(.*?)(\.m3u8)/); if (urlParm) urlPlay = urlDomain + urlParm[0]; return urlPlay; } catch (err) { throw err; } }, /** * 获取电子节目单 * @param {*} url epg阶段单api * @param {*} tvg_name 节目名称 * @param {*} date 日期 2023-01-31 * @returns 电子节目单列表 */ async iptvEpg(url, tvg_name, date) { try { const res = await axios.get(url, { params: { ch: tvg_name, date: date } }); const epgData = res.data.epg_data; return epgData; } catch (err) { throw err; } }, /** * 判断 m3u8 文件是否为直播流 * @param {*} url m3u8地址 * @returns 是否是直播流 */ async isLiveM3U8(url) { try { const res = await axios.get(url); const m3u8Content = res.data; // 从m3u8文件中解析媒体段(MEDIA-SEQUENCE)的值 const mediaSequenceMatch = m3u8Content.match(/#EXT-X-MEDIA-SEQUENCE:(\d+)/); const mediaSequence = mediaSequenceMatch ? parseInt(mediaSequenceMatch[1]) : null; // 判断是直播还是点播 const isLiveStream = mediaSequence === null || mediaSequence === 0; return !isLiveStream; } catch (err) { throw err; } }, /** * 获取豆瓣页面链接 * @param {*} id 视频唯一标识 * @param {*} name 视频名称 * @param {*} year 视频年份 * @returns 豆瓣页面链接,如果没有搜到该视频,返回搜索页面链接 */ async doubanLink(id, name, year) { const nameToSearch = encodeURI(name.trim()) const doubanSearchLink = id && parseInt(id) !== 0 ? `https://movie.douban.com/subject/${id}` : `https://www.douban.com/search?cat=1002&q=${nameToSearch}` try { const res = await axios.get(doubanSearchLink) const $ = cheerio.load(res.data) let link = '' $('div.result').each(function () { const linkInDouban = $(this).find('div>div>h3>a').first() const nameInDouban = linkInDouban.text().replace(/\s/g, '') const subjectCast = $(this).find('span.subject-cast').text() if (nameToSearch === encodeURI(nameInDouban) && subjectCast && subjectCast.includes(year)) { link = linkInDouban.attr('href') return } }) return link || doubanSearchLink } catch (err) { throw err } }, /** * 获取豆瓣评分 * @param {*} id 视频唯一标识 * @param {*} name 视频名称 * @param {*} year 视频年份 * @returns 豆瓣评分 */ async doubanRate(id, name, year) { try { const link = await this.doubanLink(id, name, year); if (link.includes('https://www.douban.com/search')) { return '暂无评分'; } else { const response = await axios.get(link); const parsedHtml = cheerio.load(response.data); const rating = parsedHtml('body').find('#interest_sectl').first().find('strong').first().text().replace(/\s/g, ''); return rating || '暂无评分'; // const rating = parsedHtml('body').find('#interest_sectl').first().find('strong').first(); // if (rating.text()) { // return rating.text().replace(/\s/g, ''); // } else { // return '暂无评分'; // } } } catch (err) { throw err; } }, /** * 获取豆瓣相关视频推荐列表 * @param {*} id 视频唯一标识 * @param {*} name 视频名称 * @param {*} year 视频年份 * @returns 豆瓣相关视频推荐列表 */ async doubanRecommendations(id, name, year) { try { const link = await this.doubanLink(id, name, year); if (link.includes('https://www.douban.com/search')) { return []; } else { const response = await axios.get(link); const $ = cheerio.load(response.data); const recommendations = $('div.recommendations-bd').find('div>dl>dd>a').map((index, element) => $(element).text()).get(); return recommendations; } } catch (err) { throw err; } }, /** * 获取豆瓣热点视频列表 * @param {*} type 类型 * @param {*} tag 标签 * @param {*} limit 显示条数 * @param {*} start 跳过 * @returns 豆瓣热点视频推荐列表 */ async doubanHot(type, tag, limit = 20, start = 0) { const doubanHotLink = `https://movie.douban.com/j/search_subjects?type=${type}&tag=${encodeURI(tag)}&page_limit=${limit}&page_start=${start}`; try { const { data: { subjects } } = await axios.get(doubanHotLink); return subjects.map(item => ({ vod_id: item.id, vod_name: item.title, vod_remarks: item.episodes_info, vod_pic: item.cover, })); } catch (err) { throw err; } }, /** * 获取酷云热点视频列表 * @param {*} date 日期2023-5-3 * @param {*} type 类型 1.电影 2.剧集 3.综艺 * @param {*} plat 平台 1.腾讯视频 2.爱奇艺 3.优酷 4.芒果 * @returns 酷云热点视频推荐列表 */ async kuyunHot( date, type, plat) { const kuyunHotLink = `https://eye.kuyun.com/api/netplat/ranking?date=${date}&type=${type}&plat=${plat}`; try { const { data: { data: { list } } } = await axios.get(kuyunHotLink); return list.map(item => ({ vod_id: item.ca_id, vod_name: item.name, vod_hot: item.num, })); } catch (err) { throw err; } }, /** * 获取解析url链接的标题 * @param {*} url 需要解析的地址 * @returns 解析标题 */ async getAnalysizeTitle (url) { try { const res = await axios.get(url, { responseType: 'arraybuffer' }); let html = ''; if (url.includes('sohu')) { html = iconv.decode(Buffer.from(res.data), 'gb2312'); } else { html = iconv.decode(Buffer.from(res.data), 'utf-8'); } const $ = cheerio.load(html); return $("title").text(); } catch (err) { throw err; } }, /** * 获取配置文件 * @param {*} url 需要获取的地址 * @returns 配置文件 */ async getConfig(url) { try { const res = await axios.get(url); return res.data || false; } catch (err) { throw err; } }, /** * 判断是否支持ipv6 * @returns ture/false */ async checkSupportIpv6() { try { const res = await axios.get('https://6.ipw.cn'); const ip = res.data; const isIpv6 = /([0-9a-z]*:{1,4}){1,7}[0-9a-z]{1,4}/i.test(ip); return isIpv6; } catch (err) { throw err; } }, /** * 判断url是否为ipv6 * @returns ture/false */ async checkUrlIpv6(url) { let hostname = new URL(url).hostname; const ipv6Regex = /^\[([\da-fA-F:]+)\]$/; // 匹配 IPv6 地址 const match = ipv6Regex.exec(hostname); if(match){ // console.log(match[1]) hostname = match[1]; } const ipType = net.isIP(hostname); if (ipType === 4) { // console.log(`1.ipv4:${hostname}`) return 'IPv4'; } else if (ipType === 6) { // console.log(`1.ipv6:${hostname}`) return 'IPv6'; } else { try { const addresses = await dns.promises.resolve(hostname); const ipType = net.isIP(addresses[0]); if (ipType === 4) { // console.log(`2.ipv4:${addresses[0]}`) return 'IPv4'; } else if (ipType === 6) { // console.log(`2.ipv6:${addresses[0]}`) return 'IPv6'; } else { return 'Unknown'; } } catch (err) { console.log(url,hostname) throw err; } } } } export default zy