Updating NPM dependencies

node_modules/jsdom/lib/api.js

@@ -1,9 +1,8 @@
 "use strict";
 const path = require("path");
-const fs = require("pn/fs");
+const fs = require("fs").promises;
 const vm = require("vm");
 const toughCookie = require("tough-cookie");
-const request = require("request-promise-native");
 const sniffHTMLEncoding = require("html-encoding-sniffer");
 const whatwgURL = require("whatwg-url");
 const whatwgEncoding = require("whatwg-encoding");
@@ -11,17 +10,11 @@ const { URL } = require("whatwg-url");
 const MIMEType = require("whatwg-mimetype");
 const idlUtils = require("./jsdom/living/generated/utils.js");
 const VirtualConsole = require("./jsdom/virtual-console.js");
-const Window = require("./jsdom/browser/Window.js");
-const { domToHtml } = require("./jsdom/browser/domtohtml.js");
-const { applyDocumentFeatures } = require("./jsdom/browser/documentfeatures.js");
-const { wrapCookieJarForRequest } = require("./jsdom/browser/resource-loader.js");
-const { version: packageVersion } = require("../package.json");
-
-const DEFAULT_USER_AGENT = `Mozilla/5.0 (${process.platform}) AppleWebKit/537.36 (KHTML, like Gecko) ` +
-                           `jsdom/${packageVersion}`;
-
-// This symbol allows us to smuggle a non-public option through to the JSDOM constructor, for use by JSDOM.fromURL.
-const transportLayerEncodingLabelHiddenOption = Symbol("transportLayerEncodingLabel");
+const { createWindow } = require("./jsdom/browser/Window.js");
+const { parseIntoDocument } = require("./jsdom/browser/parser");
+const { fragmentSerialization } = require("./jsdom/living/domparsing/serialization.js");
+const ResourceLoader = require("./jsdom/browser/resources/resource-loader.js");
+const NoOpResourceLoader = require("./jsdom/browser/resources/no-op-resource-loader.js");
 
 class CookieJar extends toughCookie.CookieJar {
   constructor(store, options) {
@@ -35,42 +28,25 @@ let sharedFragmentDocument = null;
 
 class JSDOM {
   constructor(input, options = {}) {
-    const { html, encoding } = normalizeHTML(input, options[transportLayerEncodingLabelHiddenOption]);
-    options = transformOptions(options, encoding);
+    const mimeType = new MIMEType(options.contentType === undefined ? "text/html" : options.contentType);
+    const { html, encoding } = normalizeHTML(input, mimeType);
 
-    this[window] = new Window(options.windowOptions);
+    options = transformOptions(options, encoding, mimeType);
 
-    // TODO NEWAPI: the whole "features" infrastructure is horrible and should be re-built. When we switch to newapi
-    // wholesale, or perhaps before, we should re-do it. For now, just adapt the new, nice, public API into the old,
-    // ugly, internal API.
-    const features = {
-      FetchExternalResources: [],
-      SkipExternalResources: false
-    };
-
-    if (options.resources === "usable") {
-      features.FetchExternalResources = ["link", "img", "frame", "iframe"];
-      if (options.windowOptions.runScripts === "dangerously") {
-        features.FetchExternalResources.push("script");
-      }
-
-      // Note that "img" will be ignored by the code in HTMLImageElement-impl.js if canvas is not installed.
-      // TODO NEWAPI: clean that up and centralize the logic here.
-    }
+    this[window] = createWindow(options.windowOptions);
 
     const documentImpl = idlUtils.implForWrapper(this[window]._document);
-    applyDocumentFeatures(documentImpl, features);
 
     options.beforeParse(this[window]._globalProxy);
 
-    // TODO NEWAPI: this is still pretty hacky. It's also different than jsdom.jsdom. Does it work? Can it be better?
-    documentImpl._htmlToDom.appendToDocument(html, documentImpl);
+    parseIntoDocument(html, documentImpl);
+
     documentImpl.close();
   }
 
   get window() {
-    // It's important to grab the global proxy, instead of just the result of `new Window(...)`, since otherwise things
-    // like `window.eval` don't exist.
+    // It's important to grab the global proxy, instead of just the result of `createWindow(...)`, since otherwise
+    // things like `window.eval` don't exist.
     return this[window]._globalProxy;
   }
 
@@ -84,24 +60,24 @@ class JSDOM {
   }
 
   serialize() {
-    return domToHtml([idlUtils.implForWrapper(this[window]._document)]);
+    return fragmentSerialization(idlUtils.implForWrapper(this[window]._document), { requireWellFormed: false });
   }
 
   nodeLocation(node) {
-    if (!idlUtils.implForWrapper(this[window]._document)._parseOptions.locationInfo) {
+    if (!idlUtils.implForWrapper(this[window]._document)._parseOptions.sourceCodeLocationInfo) {
       throw new Error("Location information was not saved for this jsdom. Use includeNodeLocations during creation.");
     }
 
-    return idlUtils.implForWrapper(node).__location;
+    return idlUtils.implForWrapper(node).sourceCodeLocation;
   }
 
-  runVMScript(script) {
+  getInternalVMContext() {
     if (!vm.isContext(this[window])) {
       throw new TypeError("This jsdom was not configured to allow script running. " +
         "Use the runScripts option during creation.");
     }
 
-    return script.runInContext(this[window]);
+    return this[window];
   }
 
   reconfigure(settings) {
@@ -118,11 +94,11 @@ class JSDOM {
       }
 
       document._URL = url;
-      document.origin = whatwgURL.serializeURLOrigin(document._URL);
+      document._origin = whatwgURL.serializeURLOrigin(document._URL);
     }
   }
 
-  static fragment(string) {
+  static fragment(string = "") {
     if (!sharedFragmentDocument) {
       sharedFragmentDocument = (new JSDOM()).window.document;
     }
@@ -134,50 +110,45 @@ class JSDOM {
 
   static fromURL(url, options = {}) {
     return Promise.resolve().then(() => {
+      // Remove the hash while sending this through the research loader fetch().
+      // It gets added back a few lines down when constructing the JSDOM object.
       const parsedURL = new URL(url);
+      const originalHash = parsedURL.hash;
+      parsedURL.hash = "";
       url = parsedURL.href;
+
       options = normalizeFromURLOptions(options);
 
-      const requestOptions = {
-        resolveWithFullResponse: true,
-        encoding: null, // i.e., give me the raw Buffer
-        gzip: true,
-        headers: {
-          "User-Agent": options.userAgent,
-          Referer: options.referrer,
-          Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-          "Accept-Language": "en"
-        },
-        jar: wrapCookieJarForRequest(options.cookieJar)
-      };
+      const resourceLoader = resourcesToResourceLoader(options.resources);
+      const resourceLoaderForInitialRequest = resourceLoader.constructor === NoOpResourceLoader ?
+        new ResourceLoader() :
+        resourceLoader;
 
-      return request(url, requestOptions).then(res => {
-        let transportLayerEncodingLabel;
-        if ("content-type" in res.headers) {
-          const mimeType = new MIMEType(res.headers["content-type"]);
-          transportLayerEncodingLabel = mimeType.parameters.get("charset");
-        }
+      const req = resourceLoaderForInitialRequest.fetch(url, {
+        accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        cookieJar: options.cookieJar,
+        referrer: options.referrer
+      });
+
+      return req.then(body => {
+        const res = req.response;
 
         options = Object.assign(options, {
-          url: res.request.href + parsedURL.hash,
+          url: req.href + originalHash,
           contentType: res.headers["content-type"],
-          referrer: res.request.getHeader("referer"),
-          [transportLayerEncodingLabelHiddenOption]: transportLayerEncodingLabel
+          referrer: req.getHeader("referer")
         });
 
-        return new JSDOM(res.body, options);
+        return new JSDOM(body, options);
       });
     });
   }
 
-  static fromFile(filename, options = {}) {
-    return Promise.resolve().then(() => {
-      options = normalizeFromFileOptions(filename, options);
+  static async fromFile(filename, options = {}) {
+    options = normalizeFromFileOptions(filename, options);
+    const buffer = await fs.readFile(filename);
 
-      return fs.readFile(filename).then(buffer => {
-        return new JSDOM(buffer, options);
-      });
-    });
+    return new JSDOM(buffer, options);
   }
 }
 
@@ -193,9 +164,6 @@ function normalizeFromURLOptions(options) {
   // Normalization of options which must be done before the rest of the fromURL code can use them, because they are
   // given to request()
   const normalized = Object.assign({}, options);
-  if (options.userAgent === undefined) {
-    normalized.userAgent = DEFAULT_USER_AGENT;
-  }
 
   if (options.referrer !== undefined) {
     normalized.referrer = (new URL(options.referrer)).href;
@@ -216,7 +184,7 @@ function normalizeFromFileOptions(filename, options) {
 
   if (normalized.contentType === undefined) {
     const extname = path.extname(filename);
-    if (extname === ".xhtml" || extname === ".xml") {
+    if (extname === ".xhtml" || extname === ".xht" || extname === ".xml") {
       normalized.contentType = "application/xhtml+xml";
     }
   }
@@ -228,7 +196,7 @@ function normalizeFromFileOptions(filename, options) {
   return normalized;
 }
 
-function transformOptions(options, encoding) {
+function transformOptions(options, encoding, mimeType) {
   const transformed = {
     windowOptions: {
       // Defaults
@@ -236,34 +204,33 @@ function transformOptions(options, encoding) {
       referrer: "",
       contentType: "text/html",
       parsingMode: "html",
-      userAgent: DEFAULT_USER_AGENT,
-      parseOptions: { locationInfo: false },
+      parseOptions: {
+        sourceCodeLocationInfo: false,
+        scriptingEnabled: false
+      },
       runScripts: undefined,
       encoding,
       pretendToBeVisual: false,
       storageQuota: 5000000,
 
       // Defaults filled in later
+      resourceLoader: undefined,
       virtualConsole: undefined,
       cookieJar: undefined
     },
 
     // Defaults
-    resources: undefined,
     beforeParse() { }
   };
 
-  if (options.contentType !== undefined) {
-    const mimeType = new MIMEType(options.contentType);
-
-    if (!mimeType.isHTML() && !mimeType.isXML()) {
-      throw new RangeError(`The given content type of "${options.contentType}" was not a HTML or XML content type`);
-    }
-
-    transformed.windowOptions.contentType = mimeType.essence;
-    transformed.windowOptions.parsingMode = mimeType.isHTML() ? "html" : "xml";
+  // options.contentType was parsed into mimeType by the caller.
+  if (!mimeType.isHTML() && !mimeType.isXML()) {
+    throw new RangeError(`The given content type of "${options.contentType}" was not a HTML or XML content type`);
   }
 
+  transformed.windowOptions.contentType = mimeType.essence;
+  transformed.windowOptions.parsingMode = mimeType.isHTML() ? "html" : "xml";
+
   if (options.url !== undefined) {
     transformed.windowOptions.url = (new URL(options.url)).href;
   }
@@ -272,16 +239,12 @@ function transformOptions(options, encoding) {
     transformed.windowOptions.referrer = (new URL(options.referrer)).href;
   }
 
-  if (options.userAgent !== undefined) {
-    transformed.windowOptions.userAgent = String(options.userAgent);
-  }
-
   if (options.includeNodeLocations) {
     if (transformed.windowOptions.parsingMode === "xml") {
       throw new TypeError("Cannot set includeNodeLocations to true with an XML content type");
     }
 
-    transformed.windowOptions.parseOptions = { locationInfo: true };
+    transformed.windowOptions.parseOptions = { sourceCodeLocationInfo: true };
   }
 
   transformed.windowOptions.cookieJar = options.cookieJar === undefined ?
@@ -292,17 +255,17 @@ function transformOptions(options, encoding) {
                                             (new VirtualConsole()).sendTo(console) :
                                             options.virtualConsole;
 
-  if (options.resources !== undefined) {
-    transformed.resources = String(options.resources);
-    if (transformed.resources !== "usable") {
-      throw new RangeError(`resources must be undefined or "usable"`);
-    }
+  if (!(transformed.windowOptions.virtualConsole instanceof VirtualConsole)) {
+    throw new TypeError("virtualConsole must be an instance of VirtualConsole");
   }
 
+  transformed.windowOptions.resourceLoader = resourcesToResourceLoader(options.resources);
+
   if (options.runScripts !== undefined) {
     transformed.windowOptions.runScripts = String(options.runScripts);
-    if (transformed.windowOptions.runScripts !== "dangerously" &&
-        transformed.windowOptions.runScripts !== "outside-only") {
+    if (transformed.windowOptions.runScripts === "dangerously") {
+      transformed.windowOptions.parseOptions.scriptingEnabled = true;
+    } else if (transformed.windowOptions.runScripts !== "outside-only") {
       throw new RangeError(`runScripts must be undefined, "dangerously", or "outside-only"`);
     }
   }
@@ -324,7 +287,7 @@ function transformOptions(options, encoding) {
   return transformed;
 }
 
-function normalizeHTML(html = "", transportLayerEncodingLabel) {
+function normalizeHTML(html = "", mimeType) {
   let encoding = "UTF-8";
 
   if (ArrayBuffer.isView(html)) {
@@ -334,7 +297,10 @@ function normalizeHTML(html = "", transportLayerEncodingLabel) {
   }
 
   if (Buffer.isBuffer(html)) {
-    encoding = sniffHTMLEncoding(html, { defaultEncoding: "windows-1252", transportLayerEncodingLabel });
+    encoding = sniffHTMLEncoding(html, {
+      defaultEncoding: mimeType.isXML() ? "UTF-8" : "windows-1252",
+      transportLayerEncodingLabel: mimeType.parameters.get("charset")
+    });
     html = whatwgEncoding.decode(html, encoding);
   } else {
     html = String(html);
@@ -343,9 +309,27 @@ function normalizeHTML(html = "", transportLayerEncodingLabel) {
   return { html, encoding };
 }
 
+function resourcesToResourceLoader(resources) {
+  switch (resources) {
+    case undefined: {
+      return new NoOpResourceLoader();
+    }
+    case "usable": {
+      return new ResourceLoader();
+    }
+    default: {
+      if (!(resources instanceof ResourceLoader)) {
+        throw new TypeError("resources must be an instance of ResourceLoader");
+      }
+      return resources;
+    }
+  }
+}
+
 exports.JSDOM = JSDOM;
 
 exports.VirtualConsole = VirtualConsole;
 exports.CookieJar = CookieJar;
+exports.ResourceLoader = ResourceLoader;
 
 exports.toughCookie = toughCookie;