20 lines
818 B
JavaScript
20 lines
818 B
JavaScript
var URL = "http://dat.marsxplr.com/";
|
|
|
|
var child_process = require("child_process");
|
|
var fs = require("fs");
|
|
var parser = new (require("fast-xml-parser")).XMLParser();
|
|
|
|
var urls = [];
|
|
|
|
(function getPage(p, marker) {
|
|
child_process.execSync(`wget -nv ${URL}${marker ? `?marker=${marker}` : ''} -O ListBucketResult-${p}.xml`);
|
|
var xml = fs.readFileSync(`ListBucketResult-${p}.xml`, 'utf8');
|
|
var jso = parser.parse(xml);
|
|
urls.push(...jso.ListBucketResult.Contents.map(x => URL + x.Key).filter(x => !x.endsWith("/")));
|
|
if (jso.ListBucketResult.IsTruncated) getPage(++p, jso.ListBucketResult.Contents[jso.ListBucketResult.Contents.length-1].Key);
|
|
})(1);
|
|
|
|
fs.writeFileSync("urllist.txt", urls.join("\n"));
|
|
|
|
child_process.spawnSync(`wget`, ["-nv", "-x", "-N", "-i", "urllist.txt"], {stdio:"inherit"});
|