I use Google Chrome 59.0.3071.115 in headless mode.
{ requestId: '1766.55',
frameId: '1766.1',
loaderId: '1766.2',
documentURL: 'https://www.golem.de/',
request:
{ url: 'https://s3-eu-central-1.amazonaws.com/prod.iqdcontroller.iqdigital/cdn_golem/live/iqadcontroller.js.gz',
method: 'GET',
headers:
{ Referer: 'https://www.golem.de/',
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36',
Intervention: '<https://www.chromestatus.com/feature/5718547946799104>; level="warning"' },
mixedContentType: 'none',
initialPriority: 'High',
referrerPolicy: 'no-referrer-when-downgrade' },
timestamp: 298.426415,
wallTime: 1501804201.99757,
initiator: { type: 'script', stack: { callFrames: [Array] } },
type: 'Other' }
{ requestId: '1766.55',
frameId: '1766.1',
loaderId: '1766.2',
timestamp: 298.539254,
type: 'Script',
response:
{ url: 'https://s3-eu-central-1.amazonaws.com/prod.iqdcontroller.iqdigital/cdn_golem/live/iqadcontroller.js.gz',
status: 200,
statusText: 'OK',
headers:
{ Date: 'Thu, 03 Aug 2017 23:50:04 GMT',
'Content-Encoding': 'gzip',
'Last-Modified': 'Thu, 29 Jun 2017 14:26:42 GMT',
Server: 'AmazonS3',
'x-amz-request-id': '5626885826712CCD',
ETag: '"afc86a3e20c32075c18daae3d167edc6"',
'Content-Type': 'text/javascript',
'Accept-Ranges': 'bytes',
'Content-Length': '40808',
'x-amz-id-2': 'aMjbzUHW1W74IHo9sjk2oMfy46xtppN3JvmE9+89fu8+ty8nvs80KeNuFV/BXkguB1BSyGfBme8=' },
mimeType: 'text/javascript',
connectionReused: true,
connectionId: 66,
remoteIPAddress: '52.219.73.0',
remotePort: 443,
fromDiskCache: false,
fromServiceWorker: false,
encodedDataLength: 388,
timing: ...,
protocol: 'http/1.1',
securityState: 'secure',
securityDetails: ... } }
{ scriptId: '37',
url: 'https://s3-eu-central-1.amazonaws.com/prod.iqdcontroller.iqdigital/cdn_golem/live/iqadcontroller.js.gz',
startLine: 0,
startColumn: 0,
endLine: 382,
endColumn: 0,
executionContextId: 3,
hash: '0FD2E9009E2C1EB731F0B111C86652591FBAA8C5',
executionContextAuxData: { isDefault: true, frameId: '1766.1' },
isLiveEdit: false,
sourceMapURL: '',
hasSourceURL: false,
isModule: false,
length: 172643 }
{ body: '', base64Encoded: false }
async function storeContent(content) {
if (content === null) return null;
let id = sha256(content);
let path = __dirname+"/../storage/contents/"+id.substr(0, 2)+"/"+id.substr(2, 2)+"/"+id.substr(4);
let exists = await fileExists(path);
if (!exists) {
await createDirIfNotExists(dirname(dirname(path)));
await createDirIfNotExists(dirname(path));
await writeGzipped(path, content);
}
return { id };
}
async getRequestBody(requestId) {
try {
let body = await this.protocol.Network.getResponseBody({ requestId });
if (body.base64Encoded)
return Buffer.from(body.body, "base64").toString("utf-8");
else
return body.body;
} catch (e) {
return null;
}
}
async getScriptSource(scriptId) {
try {
let source = await this.protocol.Debugger.getScriptSource({ scriptId });
return source.scriptSource;
} catch (e) {
return null;
}
}
// ...
Network.responseReceived(params => {
// ...
console.log(params);
contentPromises.push((async () => {
let body = await getRequestBody(params.requestId);
response.body = await storeContent(body);
});
});
Debugger.scriptParsed(script => {
// ...
console.log(script);
contentPromises.push((async () => {
let source = await getScriptSource(script.scriptId);
if (source !== null) {
let { id } = await storeContent(source);
// ...
}
});
});
// ...
await Promise.all(contentPromises);