var URL = "http://dat.marsxplr.com/";

var child_process = require("child_process");
var fs = require("fs");
var parser = new (require("fast-xml-parser")).XMLParser();

var urls = [];

(function getPage(p, marker) {
	child_process.execSync(`wget -nv ${URL}${marker ? `?marker=${marker}` : ''} -O ListBucketResult-${p}.xml`);
	var xml = fs.readFileSync(`ListBucketResult-${p}.xml`, 'utf8');
	var jso = parser.parse(xml);
	urls.push(...jso.ListBucketResult.Contents.map(x => URL + x.Key).filter(x => !x.endsWith("/")));
	if (jso.ListBucketResult.IsTruncated) getPage(++p, jso.ListBucketResult.Contents[jso.ListBucketResult.Contents.length-1].Key);
})(1);

fs.writeFileSync("urllist.txt", urls.join("\n"));

child_process.spawnSync(`wget`, ["-nv", "-x", "-N", "-i", "urllist.txt"], {stdio:"inherit"});