bug 1694236: rename whitelist -> allowlist; add test for vpn allowlist (

#2065)

* Rename domain whitelist to domain allowlist

* Add a test to ensure that domain allowlisting for vpn doesn't break

scripts/test-rules.py

@@ -20,7 +20,7 @@
 
 log = logging.getLogger(__name__)
 
-WHITELISTED_DOMAINS = {
+ALLOWLISTED_DOMAINS = {
     "download.mozilla.org": ("Firefox",),
     "stage-old.mozilla.org": ("Firefox",),
     "ftp.mozilla.org": ("Firefox",),
@@ -103,7 +103,7 @@ def walkSnippets(AUS, testPath):
         testQuery["queryVersion"] = 3
         release, update_type, _ = AUS.evaluateRules(testQuery)
         if release:
-            balrog_snippets = release.createSnippets(testQuery, update_type, WHITELISTED_DOMAINS, SPECIAL_FORCE_HOSTS)
+            balrog_snippets = release.createSnippets(testQuery, update_type, ALLOWLISTED_DOMAINS, SPECIAL_FORCE_HOSTS)
         else:
             balrog_snippets = {"partial": "", "complete": ""}
 
@@ -186,7 +186,7 @@ def isValidTestDir(d):
         AUS = AUS_Class()
         dbo.setDb(dbPath)
         dbo.create()
-        dbo.setDomainWhitelist(WHITELISTED_DOMAINS)
+        dbo.setDomainAllowlist(ALLOWLISTED_DOMAINS)
         populateDB(td)
         if options.dumprules:
             log.info(

src/auslib/AUS.py

@@ -34,14 +34,14 @@ def isSpecialURL(url, specialForceHosts):
     return False
 
 
-def isForbiddenUrl(url, product, whitelistedDomains):
-    if whitelistedDomains is None:
-        whitelistedDomains = []
+def isForbiddenUrl(url, product, allowlistedDomains):
+    if allowlistedDomains is None:
+        allowlistedDomains = []
     domain = urlparse(url)[1]
-    if domain not in whitelistedDomains:
+    if domain not in allowlistedDomains:
         logging.warning("Forbidden domain: %s", domain)
         return True
-    if product not in whitelistedDomains[domain]:
+    if product not in allowlistedDomains[domain]:
         logging.warning("Forbidden domain for product %s: %s", product, domain)
         return True
     return False

src/auslib/blobs/apprelease.py

@@ -107,7 +107,7 @@ def _getFromRelease(self, patch):
     def _getAdditionalPatchAttributes(self, patch):
         return {}
 
-    def _getSpecificPatchXML(self, patchKey, patchType, patch, updateQuery, whitelistedDomains, specialForceHosts):
+    def _getSpecificPatchXML(self, patchKey, patchType, patch, updateQuery, allowlistedDomains, specialForceHosts):
         fromRelease = self._getFromRelease(patch)
         # Find all the alias' for this build target so we can look for the current platform
         # in the fromRelease
@@ -142,7 +142,7 @@ def _getSpecificPatchXML(self, patchKey, patchType, patch, updateQuery, whitelis
         # the update entirely? Right now, another patch type could still
         # return an update. Eg, the partial could contain a forbidden domain
         # but the complete could still return an update from an accepted one.
-        if isForbiddenUrl(url, updateQuery["product"], whitelistedDomains):
+        if isForbiddenUrl(url, updateQuery["product"], allowlistedDomains):
             return None
 
         patchXML = '        <patch type="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%s"' % (
@@ -159,13 +159,13 @@ def _getSpecificPatchXML(self, patchKey, patchType, patch, updateQuery, whitelis
 
         return patchXML
 
-    def getInnerHeaderXML(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
+    def getInnerHeaderXML(self, updateQuery, update_type, allowlistedDomains, specialForceHosts):
         return self._getUpdateLineXML(updateQuery, update_type)
 
-    def getInnerFooterXML(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
+    def getInnerFooterXML(self, updateQuery, update_type, allowlistedDomains, specialForceHosts):
         return "    </update>"
 
-    def getInnerXML(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
+    def getInnerXML(self, updateQuery, update_type, allowlistedDomains, specialForceHosts):
         """This method, along with getHeaderXML and getFooterXML are the entry point
         for update XML creation for all Gecko app blobs. However, the XML and
         underlying data has changed over time, so there is a lot of indirection
@@ -209,7 +209,7 @@ def getInnerXML(self, updateQuery, update_type, whitelistedDomains, specialForce
         locale = updateQuery["locale"]
         localeData = self.getLocaleData(buildTarget, locale)
 
-        patches = self._getPatchesXML(localeData, updateQuery, whitelistedDomains, specialForceHosts)
+        patches = self._getPatchesXML(localeData, updateQuery, allowlistedDomains, specialForceHosts)
         return patches
 
     def shouldServeUpdate(self, updateQuery):
@@ -233,7 +233,7 @@ def shouldServeUpdate(self, updateQuery):
 
         return True
 
-    def containsForbiddenDomain(self, product, whitelistedDomains):
+    def containsForbiddenDomain(self, product, allowlistedDomains):
         """Returns True if the blob contains any file URLs that contain a
         domain that we're not allowed to serve updates to."""
         # Check the top level URLs, if the exist.
@@ -242,24 +242,24 @@ def containsForbiddenDomain(self, product, whitelistedDomains):
             if isinstance(c, dict):
                 for from_ in c.values():
                     for url in from_.values():
-                        if isForbiddenUrl(url, product, whitelistedDomains):
+                        if isForbiddenUrl(url, product, allowlistedDomains):
                             return True
             # Old-style
             else:
-                if isForbiddenUrl(c, product, whitelistedDomains):
+                if isForbiddenUrl(c, product, allowlistedDomains):
                     return True
 
         # And also the locale-level URLs.
         for platform in self.get("platforms", {}).values():
             for locale in platform.get("locales", {}).values():
                 for type_ in ("partial", "complete"):
                     if type_ in locale and "fileUrl" in locale[type_]:
-                        if isForbiddenUrl(locale[type_]["fileUrl"], product, whitelistedDomains):
+                        if isForbiddenUrl(locale[type_]["fileUrl"], product, allowlistedDomains):
                             return True
                 for type_ in ("partials", "completes"):
                     for update in locale.get(type_, {}):
                         if "fileUrl" in update:
-                            if isForbiddenUrl(update["fileUrl"], product, whitelistedDomains):
+                            if isForbiddenUrl(update["fileUrl"], product, allowlistedDomains):
                                 return True
 
         return False
@@ -311,14 +311,14 @@ def _getUrl(self, updateQuery, patchKey, patch, specialForceHosts):
 
 
 class SingleUpdateXMLMixin(object):
-    def _getPatchesXML(self, localeData, updateQuery, whitelistedDomains, specialForceHosts):
+    def _getPatchesXML(self, localeData, updateQuery, allowlistedDomains, specialForceHosts):
         patches = []
         for patchKey in ("complete", "partial"):
             patch = localeData.get(patchKey)
             if not patch:
                 continue
 
-            xml = self._getSpecificPatchXML(patchKey, patchKey, patch, updateQuery, whitelistedDomains, specialForceHosts)
+            xml = self._getSpecificPatchXML(patchKey, patchKey, patch, updateQuery, allowlistedDomains, specialForceHosts)
             if xml:
                 patches.append(xml)
 
@@ -370,7 +370,7 @@ def getReferencedReleases(self):
 
     # TODO: kill me when aus3.m.o is dead, and snippet tests have been
     # converted to unit tests.
-    def createSnippets(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
+    def createSnippets(self, updateQuery, update_type, allowlistedDomains, specialForceHosts):
         snippets = {}
         buildTarget = updateQuery["buildTarget"]
         locale = updateQuery["locale"]
@@ -385,7 +385,7 @@ def createSnippets(self, updateQuery, update_type, whitelistedDomains, specialFo
                 continue
 
             url = self._getUrl(updateQuery, patchKey, patch, specialForceHosts)
-            if isForbiddenUrl(url, updateQuery["product"], whitelistedDomains):
+            if isForbiddenUrl(url, updateQuery["product"], allowlistedDomains):
                 break
 
             snippet = [
@@ -440,7 +440,7 @@ def _getUpdateLineXML(self, updateQuery, update_type):
 
         return updateLine
 
-    def getInnerHeaderXML(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
+    def getInnerHeaderXML(self, updateQuery, update_type, allowlistedDomains, specialForceHosts):
         """In order to update some older versions of Firefox without prompting
         them for add-on compatibility, we need to be able to modify the appVersion
         and extVersion attributes. bug 998721 and bug 1174605 have additional
@@ -449,7 +449,7 @@ def getInnerHeaderXML(self, updateQuery, update_type, whitelistedDomains, specia
         this method, but that doesn't have access to the updateQuery to lookup
         the version making the request.
         """
-        xml = super(ReleaseBlobV1, self).getInnerHeaderXML(updateQuery, update_type, whitelistedDomains, specialForceHosts)
+        xml = super(ReleaseBlobV1, self).getInnerHeaderXML(updateQuery, update_type, allowlistedDomains, specialForceHosts)
 
         if self.get("oldVersionSpecialCases"):
             query_ver = MozillaVersion(updateQuery["version"])
@@ -480,7 +480,7 @@ def getPlatformVersion(self, platform, locale):
         return self.getLocaleOrTopLevelParam(platform, locale, "platformVersion")
 
     def getApplicationVersion(self, platform, locale):
-        """ For v2 schema, appVersion really is the app version """
+        """For v2 schema, appVersion really is the app version"""
         return self.getAppVersion(platform, locale)
 
     def _getUpdateLineXML(self, updateQuery, update_type):
@@ -561,7 +561,7 @@ def __init__(self, **kwargs):
 
     # TODO: kill me when aus3.m.o is dead, and snippet tests have been
     # converted to unit tests.
-    def createSnippets(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
+    def createSnippets(self, updateQuery, update_type, allowlistedDomains, specialForceHosts):
         snippets = {}
         buildTarget = updateQuery["buildTarget"]
         locale = updateQuery["locale"]
@@ -576,7 +576,7 @@ def createSnippets(self, updateQuery, update_type, whitelistedDomains, specialFo
                 continue
 
             url = self._getUrl(updateQuery, patchKey, patch, specialForceHosts)
-            if isForbiddenUrl(url, updateQuery["product"], whitelistedDomains):
+            if isForbiddenUrl(url, updateQuery["product"], allowlistedDomains):
                 break
 
             snippet = [
@@ -629,11 +629,11 @@ def getReferencedReleases(self):
 
 
 class MultipleUpdatesXMLMixin(object):
-    def _getPatchesXML(self, localeData, updateQuery, whitelistedDomains, specialForceHosts):
+    def _getPatchesXML(self, localeData, updateQuery, allowlistedDomains, specialForceHosts):
         patches = []
         for patchKey, patchType in (("completes", "complete"), ("partials", "partial")):
             for patch in localeData.get(patchKey, {}):
-                xml = self._getSpecificPatchXML(patchKey, patchType, patch, updateQuery, whitelistedDomains, specialForceHosts)
+                xml = self._getSpecificPatchXML(patchKey, patchType, patch, updateQuery, allowlistedDomains, specialForceHosts)
                 if xml:
                     patches.append(xml)
                     break
@@ -672,7 +672,7 @@ def _getFtpFilename(self, patchKey, from_):
     def _getBouncerProduct(self, patchKey, from_):
         return self.get("bouncerProducts", {}).get(patchKey, {}).get(from_, "")
 
-    def createSnippets(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
+    def createSnippets(self, updateQuery, update_type, allowlistedDomains, specialForceHosts):
         # We have no tests that require this, probably not worthwhile to implement.
         return {}
 
@@ -1203,10 +1203,10 @@ def shouldServeUpdate(self, updateQuery):
         # desupport messages should always be returned
         return True
 
-    def getInnerHeaderXML(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
+    def getInnerHeaderXML(self, updateQuery, update_type, allowlistedDomains, specialForceHosts):
         return ""
 
-    def getInnerXML(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
+    def getInnerXML(self, updateQuery, update_type, allowlistedDomains, specialForceHosts):
         tmp_url = (
             self["detailsUrl"]
             .replace("%LOCALE%", updateQuery["locale"])
@@ -1222,10 +1222,10 @@ def getInnerXML(self, updateQuery, update_type, whitelistedDomains, specialForce
         xml.append('    <update type="%s" unsupported="true" detailsURL="%s" displayVersion="%s">' % (update_type, tmp_url, self["displayVersion"]))
         return xml
 
-    def getInnerFooterXML(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
+    def getInnerFooterXML(self, updateQuery, update_type, allowlistedDomains, specialForceHosts):
         return "</update>"
 
-    def containsForbiddenDomain(self, product, whitelistedDomains):
+    def containsForbiddenDomain(self, product, allowlistedDomains):
         # Although DesupportBlob contains a domain (detailsUrl), that attribute
-        # is not used to deliver binaries, so it is exempt from whitelist checks.
+        # is not used to deliver binaries, so it is exempt from allowlist checks.
         return False

src/auslib/blobs/base.py

@@ -145,7 +145,7 @@ def __init__(self, *args, **kwargs):
         logger_name = "{0}.{1}".format(self.__class__.__module__, self.__class__.__name__)
         self.__class__.log = logging.getLogger(logger_name)
 
-    def validate(self, product, whitelistedDomains):
+    def validate(self, product, allowlistedDomains):
         """Raises a BlobValidationError if the blob is invalid."""
         self.log.debug("Validating blob %s" % self)
         validator = jsonschema.Draft4Validator(self.getSchema(), format_checker=jsonschema.draft4_format_checker)
@@ -158,7 +158,7 @@ def validate(self, product, whitelistedDomains):
         if errors:
             raise BlobValidationError("Invalid blob! See 'errors' for details.", errors)
 
-        if self.containsForbiddenDomain(product, whitelistedDomains):
+        if self.containsForbiddenDomain(product, allowlistedDomains):
             raise ValueError("Blob contains forbidden domain(s)")
 
     def getSchema(self):
@@ -183,7 +183,7 @@ def shouldServeUpdate(self, updateQuery):
         failing open)."""
         return False
 
-    def containsForbiddenDomain(self, product, whitelistedDomains):
+    def containsForbiddenDomain(self, product, allowlistedDomains):
         raise NotImplementedError()
 
     def getReferencedReleases(self):
@@ -212,19 +212,19 @@ def getResponseBlobs(self):
         """
         return None
 
-    def getInnerHeaderXML(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
+    def getInnerHeaderXML(self, updateQuery, update_type, allowlistedDomains, specialForceHosts):
         """
         :return: Releases-specific header should be implemented for individual blobs
         """
         raise NotImplementedError()
 
-    def getInnerFooterXML(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
+    def getInnerFooterXML(self, updateQuery, update_type, allowlistedDomains, specialForceHosts):
         """
         :return: Releases-specific header should be implemented for individual blobs
         """
         raise NotImplementedError()
 
-    def getInnerXML(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
+    def getInnerXML(self, updateQuery, update_type, allowlistedDomains, specialForceHosts):
         raise NotImplementedError()
 
     def getHeaderXML(self):

src/auslib/blobs/gmp.py

@@ -12,8 +12,8 @@ def __init__(self, **kwargs):
         if "schema_version" not in self:
             self["schema_version"] = 1000
 
-    def validate(self, product, whitelistedDomains):
-        XMLBlob.validate(self, product, whitelistedDomains)
+    def validate(self, product, allowlistedDomains):
+        XMLBlob.validate(self, product, allowlistedDomains)
         for vendor in self["vendors"]:
             for platform in self["vendors"][vendor].get("platforms", {}).values():
                 if "hashValue" in platform:
@@ -50,16 +50,16 @@ def shouldServeUpdate(self, updateQuery):
         # of the client to decide whether or not any action needs to be taken.
         return True
 
-    def getInnerHeaderXML(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
+    def getInnerHeaderXML(self, updateQuery, update_type, allowlistedDomains, specialForceHosts):
         return "    <addons>"
 
-    def getInnerFooterXML(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
+    def getInnerFooterXML(self, updateQuery, update_type, allowlistedDomains, specialForceHosts):
         return "    </addons>"
 
     # Because specialForceHosts is only relevant to our own internal servers,
     # and these type of updates are always served externally, we don't process
     # them in GMP blobs.
-    def getInnerXML(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
+    def getInnerXML(self, updateQuery, update_type, allowlistedDomains, specialForceHosts):
         buildTarget = updateQuery["buildTarget"]
 
         vendorXML = []
@@ -68,7 +68,7 @@ def getInnerXML(self, updateQuery, update_type, whitelistedDomains, specialForce
             platformData = self.getPlatformData(vendor, buildTarget)
 
             url = platformData["fileUrl"]
-            if isForbiddenUrl(url, updateQuery["product"], whitelistedDomains):
+            if isForbiddenUrl(url, updateQuery["product"], allowlistedDomains):
                 continue
             vendorXML.append(
                 '        <addon id="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%s" version="%s"/>'
@@ -77,12 +77,12 @@ def getInnerXML(self, updateQuery, update_type, whitelistedDomains, specialForce
 
         return vendorXML
 
-    def containsForbiddenDomain(self, product, whitelistedDomains):
+    def containsForbiddenDomain(self, product, allowlistedDomains):
         """Returns True if the blob contains any file URLs that contain a
         domain that we're not allowed to serve updates to."""
         for vendor in self.get("vendors", {}).values():
             for platform in vendor.get("platforms", {}).values():
                 if "fileUrl" in platform:
-                    if isForbiddenUrl(platform["fileUrl"], product, whitelistedDomains):
+                    if isForbiddenUrl(platform["fileUrl"], product, allowlistedDomains):
                         return True
         return False

src/auslib/blobs/guardian.py

@@ -7,9 +7,9 @@
 class GuardianBlob(GenericBlob):
     jsonschema = "guardian.yml"
 
-    def containsForbiddenDomain(self, product, whitelistedDomains):
+    def containsForbiddenDomain(self, product, allowlistedDomains):
         urls = [p["fileUrl"] for p in self["platforms"].values()]
-        return any([isForbiddenUrl(url, product, whitelistedDomains) for url in urls])
+        return any([isForbiddenUrl(url, product, allowlistedDomains) for url in urls])
 
     def shouldServeUpdate(self, updateQuery):
         if updateQuery["buildTarget"] not in self.get("platforms", {}):
@@ -19,13 +19,13 @@ def shouldServeUpdate(self, updateQuery):
 
         return True
 
-    def getResponse(self, updateQuery, whitelistedDomains):
+    def getResponse(self, updateQuery, allowlistedDomains):
         url = self.get("platforms", {}).get(updateQuery["buildTarget"], {}).get("fileUrl")
         hashValue = self.get("platforms", {}).get(updateQuery["buildTarget"], {}).get("hashValue")
         if not url:
             return {}
 
-        if isForbiddenUrl(url, updateQuery["product"], whitelistedDomains):
+        if isForbiddenUrl(url, updateQuery["product"], allowlistedDomains):
             return {}
 
         return {"version": self["version"], "url": url, "required": self["required"], "hashFunction": self["hashFunction"], "hashValue": hashValue}