Spoof navigator.webdriver to false

.dockerignore

@@ -26,3 +26,4 @@ automation/Extension/firefox/dist
 automation/Extension/firefox/openwpm.xpi
 automation/Extension/firefox/src/content.js
 automation/Extension/firefox/src/feature.js
+automation/Extension/firefox/src/spoof.js

.gitignore

@@ -94,3 +94,4 @@ automation/Extension/firefox/dist
 automation/Extension/firefox/openwpm.xpi
 automation/Extension/firefox/src/content.js
 automation/Extension/firefox/src/feature.js
+automation/Extension/firefox/src/spoof.js

LICENSE

@@ -41,6 +41,11 @@ https://github.com/redline13/selenium-jmeter
 By Richard Friedman
 Licensed GPLv3+
 
+Incorporating code from User Agent Switcher
+https://gitlab.com/ntninja/user-agent-switcher/
+Copyright © 2017 – 2019  Alexander Schlarb
+Licensed GPLv3+
+
 Text of GPLv3 License:
 ======================
                     GNU GENERAL PUBLIC LICENSE

README.md

@@ -334,6 +334,8 @@ left out of this section.
   * **NOT SUPPORTED.** See [#101](https://github.com/citp/OpenWPM/issues/101).
   * Set to `True` to enable Firefox's built-in
     [Tracking Protection](https://developer.mozilla.org/en-US/Firefox/Privacy/Tracking_Protection).
+* `hide_webdriver`
+  * Set to `True` to hide that OpenWPM uses webdriver. This option spoofs the JavaScript DOM attribute `navigator.webdriver` to be `false`.
 
 Browser Profile Support
 -----------------------

automation/DeployBrowsers/configure_firefox.py

@@ -210,3 +210,6 @@ def optimize_prefs(fo):
     # Enable legacy extensions and disable extension signing
     fo.set_preference("extensions.legacy.enabled", True)
     fo.set_preference("xpinstall.signatures.required", False)
+
+    # Enable prevention against pop-up windows/tabs (`window.open('')`)
+    fo.set_preference("dom.disable_open_during_load", True)

automation/Extension/firefox/feature.js/index.js

@@ -2,7 +2,8 @@ import {
   CookieInstrument,
   JavascriptInstrument,
   HttpInstrument,
-  NavigationInstrument
+  NavigationInstrument,
+  HideWebdriver
 } from "openwpm-webext-instrumentation";
 
 import * as loggingDB from "./loggingdb.js";
@@ -23,6 +24,7 @@ async function main() {
       js_instrument_modules:"fingerprinting",
       http_instrument:true,
       callstack_instrument:true,
+      hide_webdriver:false,
       save_content:false,
       testing:true,
       crawl_id:0
@@ -66,6 +68,12 @@ async function main() {
     let callstackInstrument = new CallstackInstrument(loggingDB);
     callstackInstrument.run(config['crawl_id']);
   }
+
+  if (config['hide_webdriver']) {
+    loggingDB.logDebug("Hide webdriver enabled");
+    let hideWebdriver = new HideWebdriver();
+    await hideWebdriver.registerContentScript();
+  }
 }
 
 main();

automation/Extension/firefox/spoof.js/index.js

@@ -0,0 +1,173 @@
+/**
+ * This code to spoof values of the `window.navigator` object using a
+ * JavaScript proxy is based on:
+ * User Agent Switcher
+ * Copyright © 2017 – 2019 Alexander Schlarb (https://gitlab.com/ntninja)
+ * For the used part see: https://gitlab.com/ntninja/user-agent-switcher/blob/6aacc15ed6651317776f7abb3a85d6f34fc1a254/content/override-navigator-data.js
+ * 
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Set of all object that we have already proxied to prevent them from being
+ * proxied twice.
+ */
+let proxiedObjects = new Set();
+
+/**
+ * Convenience wrapped around `cloneInto` that enables all possible cloning
+ * options by default.
+ */
+function cloneIntoFull(value, scope) {
+	return cloneInto(value, scope, {
+		cloneFunctions: true,
+		wrapReflectors: true
+	});
+}
+
+/**
+ * Spoof `navigator` by overwriting the `navigator` of the given
+ * (content script scope) `window` object with a proxy, if applicable.
+ */
+function spoofNavigator(window) {
+	if (!(window instanceof Window)) { // Not actually a window object
+		return window;
+	}
+
+	let origNavigator = window.navigator.wrappedJSObject; // `navigator` of the page scope
+	if (proxiedObjects.has(origNavigator)) { // Window was already shadowed
+		return window;
+	}
+
+	let spoofedGet_PageScope = cloneIntoFull({
+		get: (target, prop, receiver) => {
+			if (prop === "webdriver") {
+				return false;
+			} else {
+				let value = Reflect.get(origNavigator, prop);
+				if(typeof(value) === "function") { // Bind functions like `navigator.javaEnabled()` to the orginal object in the page scope to allow them to execute
+					let boundFunc = Function.prototype.bind.call(value, origNavigator); // `value` is used as `this` to call the `bind` function that creates a copy of `Function.prototype` that always runs in the `this` context `origiNavigator`
+					value = cloneIntoFull(boundFunc, window.wrappedJSObject);
+				}
+				return value;
+			}
+		}
+	}, window.wrappedJSObject); // The `get` function, defined in privileged code (that is here in the extension / content script), is cloned into the target scope (that is the web page / `window.wrappedJSObject`) and thus accessible there. The return value is the reference to the cloned object in the defined scope.
+
+	let origProxy = window.wrappedJSObject.Proxy;
+	let navigatorProxy = new origProxy(origNavigator, spoofedGet_PageScope);
+
+	proxiedObjects.add(origNavigator);
+
+	let returnFunc_PageScope = exportFunction(() => {
+		return navigatorProxy;
+	}, window.wrappedJSObject);
+	// Using `__defineGetter__` here our function gets assigned the correct
+	// name of `get navigator`. Additionally its property descriptor has
+	// no `set` function and will silently ignore any assigned value. – This
+	// exact configuration is not achievable using `Object.defineProperty`.
+	Object.prototype.__defineGetter__.call(window.wrappedJSObject, "navigator", returnFunc_PageScope);
+	return window;
+}
+
+/**
+ * Override `navigator` with the given data on the given page scoped `window`
+ * object if applicable
+ *
+ * This will convert the given `window` object to being content-script scoped
+ * after checking whether it can be converted at all or is just a restricted
+ * accessor that does not grant access to anything important.
+ */
+function spoofNavigatorFromPageScope(unsafeWindow) {
+	if(!(unsafeWindow instanceof Window)) {
+		return unsafeWindow; // Not actually a window object
+	}
+
+	try {
+		unsafeWindow.navigator; // This will throw if this is a cross-origin frame
+
+		let windowObj = cloneIntoFull(unsafeWindow, window);
+		return spoofNavigator(windowObj).wrappedJSObject;
+	} catch(e) {
+		if(e instanceof DOMException && e.name == "SecurityError") {
+			// Ignore error created by accessing a cross-origin frame and
+			// just return the restricted frame (`navigator` is inaccessible
+			// on these so there is nothing to patch)
+			return unsafeWindow;
+		} else {
+			throw e;
+		}
+	}
+}
+
+
+spoofNavigator(window);
+
+// Use some prototype hacking to prevent access to the original `navigator`
+// through the IFrame leak
+const IFRAME_TYPES = Object.freeze([HTMLFrameElement, HTMLIFrameElement]);
+for(let type of IFRAME_TYPES) {
+	// Get reference to contentWindow & contentDocument accessors into the
+	// content script scope
+	let contentWindowGetter = Reflect.getOwnPropertyDescriptor(
+		type.prototype.wrappedJSObject, "contentWindow"
+	).get;
+	contentWindowGetter = cloneIntoFull(contentWindowGetter, window);
+	let contentDocumentGetter = Reflect.getOwnPropertyDescriptor(
+		type.prototype.wrappedJSObject, "contentDocument"
+	).get;
+	contentDocumentGetter = cloneIntoFull(contentDocumentGetter, window);
+
+	// Export compatible accessor on the property that patches the navigator
+	// element before returning
+	Object.prototype.__defineGetter__.call(type.prototype.wrappedJSObject, "contentWindow",
+			exportFunction(function () {
+				let contentWindow = contentWindowGetter.call(this);
+				return spoofNavigatorFromPageScope(contentWindow);
+			}, window.wrappedJSObject)
+	);
+	Object.prototype.__defineGetter__.call(type.prototype.wrappedJSObject, "contentDocument",
+			exportFunction(function () {
+				let contentDocument = contentDocumentGetter.call(this);
+				if(contentDocument !== null) {
+					spoofNavigatorFromPageScope(contentDocument.defaultView);
+				}
+				return contentDocument;
+			}, window.wrappedJSObject)
+	);
+}
+
+// Asynchrously track added IFrame elements and trigger their prototype
+// properties defined above to ensure that they are patched
+// (This is a best-effort workaround for us being unable to *properly* fix the `window[0]` case.)
+let patchNodes = (nodes) => {
+	for(let node of nodes) {
+		let isNodeFrameType = false;
+		for(let type of IFRAME_TYPES) {
+			if(isNodeFrameType = (node instanceof type)){ break; }
+		}
+		if(!isNodeFrameType) {
+			continue;
+		}
+
+		node.contentWindow;
+		node.contentDocument;
+	}
+};
+let observer = new MutationObserver((mutations) => {
+	for(let mutation of mutations) {
+		patchNodes(mutation.addedNodes);
+	}
+});
+observer.observe(document.documentElement, {
+	childList: true,
+	subtree: true
+});
+patchNodes(document.querySelectorAll("frame,iframe"));
+

automation/Extension/firefox/webpack.config.js

@@ -6,6 +6,7 @@ module.exports = {
   entry: {
     feature: "./feature.js/index.js",
     content: "./content.js/index.js",
+    spoof:   "./spoof.js/index.js",
   },
   output: {
     path: path.resolve(__dirname, "src"),

automation/Extension/webext-instrumentation/src/background/hide-webdriver.ts

@@ -0,0 +1,16 @@
+export class HideWebdriver {
+  /**
+   * Dynamically register the content script to proxy
+   * `window.navigator`. The proxy object returns `false`
+   * for the `window.navigator.webdriver` attribute.
+   */
+  public async registerContentScript() {
+    return browser.contentScripts.register({
+      js: [{ file: "/spoof.js" }],
+      matches: ["<all_urls>"],
+      allFrames: true,
+      runAt: "document_start",
+      matchAboutBlank: true,
+    });
+  }
+}

automation/Extension/webext-instrumentation/src/index.ts

@@ -2,6 +2,7 @@ export * from "./background/cookie-instrument";
 export * from "./background/http-instrument";
 export * from "./background/javascript-instrument";
 export * from "./background/navigation-instrument";
+export * from "./background/hide-webdriver";
 export * from "./content/javascript-instrument-content-scope";
 export * from "./lib/http-post-parser";
 export * from "./lib/string-utils";

automation/default_browser_params.json

@@ -5,6 +5,7 @@
     "js_instrument_modules": "fingerprinting",
     "http_instrument": false,
     "navigation_instrument": false,
+    "hide_webdriver": false,
     "save_content": false,
     "callstack_instrument": false,
 

crawler.py

@@ -20,6 +20,7 @@
 COOKIE_INSTRUMENT = os.getenv('COOKIE_INSTRUMENT', '1') == '1'
 NAVIGATION_INSTRUMENT = os.getenv('NAVIGATION_INSTRUMENT', '1') == '1'
 JS_INSTRUMENT = os.getenv('JS_INSTRUMENT', '1') == '1'
+HIDE_WEBDRIVER = os.getenv('HIDE_WEBDRIVER', '1') == '1'
 JS_INSTRUMENT_MODULES = os.getenv('JS_INSTRUMENT_MODULES', None)
 SAVE_CONTENT = os.getenv('SAVE_CONTENT', '')
 PREFS = os.getenv('PREFS', None)
@@ -42,6 +43,7 @@
     browser_params[i]['cookie_instrument'] = COOKIE_INSTRUMENT
     browser_params[i]['navigation_instrument'] = NAVIGATION_INSTRUMENT
     browser_params[i]['js_instrument'] = JS_INSTRUMENT
+    browser_params[i]['hide_webdriver'] = HIDE_WEBDRIVER
     if JS_INSTRUMENT_MODULES:
         browser_params[i]['js_instrument_modules'] = JS_INSTRUMENT_MODULES
     if SAVE_CONTENT == '1':
@@ -84,6 +86,7 @@
         scope.set_tag('COOKIE_INSTRUMENT', COOKIE_INSTRUMENT)
         scope.set_tag('NAVIGATION_INSTRUMENT', NAVIGATION_INSTRUMENT)
         scope.set_tag('JS_INSTRUMENT', JS_INSTRUMENT)
+        scope.set_tag('HIDE_WEBDRIVER', HIDE_WEBDRIVER)
         scope.set_tag('JS_INSTRUMENT_MODULES', JS_INSTRUMENT)
         scope.set_tag('SAVE_CONTENT', SAVE_CONTENT)
         scope.set_tag('DWELL_TIME', DWELL_TIME)