From 49ef7063b0e2f55fe2a3fafec55c189bacad70b0 Mon Sep 17 00:00:00 2001 From: olaughter Date: Thu, 7 Dec 2023 17:48:16 +0000 Subject: [PATCH 1/4] Detect sensitive terms in queries This applies existing logic that currently is applied during opensearch queries and adds it for vespa queries. --- .../resources/sensitive_query_terms.tsv | 2988 +++++++++++++++++ src/cpr_data_access/search_adaptors.py | 6 +- src/cpr_data_access/utils.py | 64 + tests/test_utils.py | 40 + 4 files changed, 3097 insertions(+), 1 deletion(-) create mode 100644 src/cpr_data_access/resources/sensitive_query_terms.tsv create mode 100644 src/cpr_data_access/utils.py create mode 100644 tests/test_utils.py diff --git a/src/cpr_data_access/resources/sensitive_query_terms.tsv b/src/cpr_data_access/resources/sensitive_query_terms.tsv new file mode 100644 index 0000000..131c101 --- /dev/null +++ b/src/cpr_data_access/resources/sensitive_query_terms.tsv @@ -0,0 +1,2988 @@ +group_name keyword +sex/gender transgender +sex/gender transgender man +sex/gender transgender woman +sex/gender non-binary +sex/gender non binary +sex/gender nonbinary +sex/gender bigender +sex/gender genderfluid +sex/gender agender +sex/gender eunuch +sex/gender transgender men +sex/gender transgender women +sex/gender men +sex/gender women +sex/gender man +sex/gender woman +sex/gender boy +sex/gender girl +nationality Abazins +nationality Abenaki +nationality Abitibiwinni First Nation +nationality Abkhazians +nationality Aboriginal Australians +nationality Abu Alkian +nationality Abydonian +nationality Acadians +nationality Aceh +nationality Acehnese +nationality Achakzai +nationality Acholi +nationality Achomi +nationality Achuar +nationality Acjachemen +nationality Acoma Pueblo tribe +nationality Adi +nationality Adivasi +nationality Adnyamathanha +nationality Aedui +nationality Afar +nationality Afghans +nationality Africa +nationality African +nationality African Americans +nationality African Brazilian +nationality African Haitian +nationality African Jamaican +nationality Afrikaners +nationality Afro Argentines +nationality Afro Caribbeans +nationality Afro Dominicans +nationality Afro Latin Americans +nationality Afro Mexicans +nationality Afro Venezuelans +nationality Afro-Barbadian +nationality Afro-Colombians +nationality Afro-Costa Rican +nationality Afro-Cuban +nationality Afro-Descendant +nationality Afro-Germans +nationality Afro-Nicaraguan +nationality Afro-Peruvian +nationality Afro-Russian +nationality Afro-Spaniard +nationality Afro-Uruguayan +nationality Ag Qoyunlu tribe +nationality Aguaruna +nationality Ahom +nationality Aiel +nationality Aimaq +nationality Ainu +nationality Akan +nationality Akan +nationality Akha +nationality Akwamu +nationality Al Bhed +nationality Alaska Natives +nationality Alaskan Athabaskans +nationality Albanians +nationality Aleut +nationality Algerians +nationality Algonquian +nationality Algonquin +nationality Algonquin +nationality Altai +nationality Alutiiq +nationality Alyawarre +nationality Amazigh +nationality Amazons +nationality American +nationality American Japanese +nationality American Japanese +nationality American Jews +nationality American-born Chinese +nationality Americans +nationality Americans +nationality Amhara +nationality Ami +nationality Amish +nationality Ammon +nationality Amorites +nationality Anatolians +nationality Ancient +nationality Ancient Celts +nationality Ancient Macedonians +nationality Ancient Pueblo +nationality Andalusian +nationality Andorian +nationality Andorrans +nationality Angles +nationality Anglo-Celtic Australians +nationality Anglo-Indian +nationality Anglo-Irish +nationality Anglo-Normans +nationality Anglo-Saxons +nationality Anishinaabe +nationality Anlo Ewe +nationality Anmatyerre +nationality Antes +nationality Antigua +nationality Antiochian Greeks +nationality Apache +nationality Apalachicola +nationality Aquitani +nationality Arab Americans +nationality Arab Canadians +nationality Arab Christians +nationality Arab citizens of Israel +nationality Arab Venezuelan +nationality Arab-Berber +nationality Arabized Berber +nationality Arabs +nationality Arain +nationality Arameans +nationality Arameans +nationality Arapaho +nationality Argentine Americans +nationality Argentines +nationality Arikara +nationality Armenian American +nationality Armenian Argentine +nationality Armenian Australian +nationality Armenian Canadians +nationality Armenians +nationality Aromanians +nationality Arrernte +nationality Asante +nationality Ashanti +nationality Ashkenazi Jews +nationality Asian +nationality Asian Americans +nationality Asian Brazilians +nationality Asian Canadians +nationality Asian New Zealanders +nationality Asmat +nationality Assamese +nationality Assiniboine +nationality Assyrian +nationality Astrakhan Tatars +nationality Asutsuare +nationality Atayal +nationality Atikamekw +nationality Augment +nationality Australia +nationality Australian American +nationality Australian Jews +nationality Australians +nationality Austrian Americans +nationality Austrian Empire +nationality Austrians +nationality Avars +nationality Awori tribe +nationality Axanar +nationality Aymara +nationality Azerbaijanis +nationality Aztecs +nationality Baga +nationality Bagale Thapa +nationality Baganda +nationality Bagobo +nationality Bahamians +nationality Bahawalpur State +nationality Bahun +nationality Bai +nationality Bailgu +nationality Bainuk +nationality Bajau +nationality Bajoran +nationality Baker Lake +nationality Bakhtiari +nationality Bakoena +nationality Balanta +nationality Balete +nationality Balinese +nationality Baloch +nationality Baloch of Iran +nationality Baltic Germans +nationality Balts +nationality Bamar +nationality Bambara +nationality Bamileke +nationality Bamun +nationality Banat Swabians +nationality Bangladeshis +nationality Bangwaketse +nationality Bani Assad +nationality Banjar +nationality Bannock +nationality Bantenese +nationality Bantu +nationality Banu Khazraj +nationality Banyankole +nationality Barbadian American +nationality Barbadians +nationality Bardings +nationality Bari +nationality Barngarla +nationality Basa +nationality Bashkir +nationality Basoga +nationality Basque +nationality Basque American +nationality Basque Colombians +nationality Bassa +nationality Batak +nationality Bedouin +nationality Beja +nationality Belarusians +nationality Belgian American +nationality Belgian Canadians +nationality Belgian Malinois +nationality Belgians +nationality Belgo-Moroccans +nationality Belizean Americans +nationality Belizean Creole +nationality Bemba +nationality Benadiri +nationality Bene Israel +nationality Benga +nationality Bengali +nationality Bengali Hindus +nationality Bengali Muslims +nationality Beornings +nationality Beothuk +nationality Berbers +nationality Bessarabian Bulgarians +nationality Bessi +nationality Beta Israel +nationality Betawi +nationality Betazoid +nationality Bhil +nationality Biblical Hittites +nationality Bibulman +nationality Bicolano +nationality Bisaya +nationality Bitterroot Salish +nationality Black +nationality Black Africans +nationality Black British +nationality Black Canadians +nationality Black Seminoles +nationality Blackfeet Nation +nationality Blackfoot Confederacy +nationality Blasians +nationality Bohemian +nationality Boholano +nationality Bolivians +nationality Bora +nationality Bosniaks +nationality Bosnians +nationality Botocudo +nationality Bouyei +nationality Brahmin +nationality Brahui +nationality Brancos +nationality Brazilian Americans +nationality Brazilian Canadians +nationality Brazilians +nationality Brazilian +nationality Bretons +nationality British +nationality British Americans +nationality British Armenians +nationality British Chinese +nationality British Indian +nationality British Iraqis +nationality British Jamaican +nationality British Jews +nationality British Nigerian +nationality British Pakistanis +nationality British Sri Lankans +nationality Brushed Ceramics Culture +nationality Bubi +nationality Buenos Aires Province +nationality Bugis +nationality Bukharan Jews +nationality Bulgarian Canadians +nationality Bulgarians +nationality Bulgars +nationality Bundjalung +nationality Bundjalung +nationality Bungandidj +nationality Burarra +nationality Burgenland Croats +nationality Burgundy +nationality Burmese Americans +nationality Buryats +nationality Butchulla +nationality Byzantine Greeks +nationality Caddo +nationality Cadusii +nationality Cagot +nationality Cahuilla +nationality Cajun +nationality Calabar +nationality Cambeba +nationality Cambodian Americans +nationality Canada +nationality Canadian Americans +nationality Canadians +nationality Canadian +nationality Canarian Americans +nationality Candoshi +nationality Cantonese +nationality Cantonese +nationality Capiznon +nationality Cappadocian Greeks +nationality Cardassian +nationality Caribbean +nationality Carinthian Slovenes +nationality Carolinian +nationality Carrier +nationality Cashibo +nationality Castilians +nationality Castizo +nationality Catalan +nationality Catawba +nationality Caucasian +nationality Cayuga +nationality Cebuano +nationality Celtic Britons +nationality Celts +nationality Chaga +nationality Chagossians +nationality Cham +nationality Chamar +nationality Chameloid +nationality Chamorro +nationality Chamorro +nationality Chechens +nationality Chemehuevi +nationality Cherokee +nationality Cherusci +nationality Cheyenne River Sioux Tribe +nationality Cheyennes +nationality Chhetri +nationality Chickasaw +nationality Chilcotin +nationality Chilean American +nationality Chilean Canadians +nationality Chileans +nationality Chin +nationality Chinantec +nationality Chinese +nationality Chinese +nationality Chinese Americans +nationality Chinese Australians +nationality Chinese Cambodian +nationality Chinese Canadians +nationality Chinese Indonesians +nationality Chinese New Zealander +nationality Chinese Singaporeans +nationality Chinese Surinamese +nationality Chinese Tatars +nationality Chinese Venezuelan +nationality Chinois +nationality Chiricahua +nationality Chitpavan +nationality Chochenyo +nationality Choctaw +nationality Choctaw Nation of Oklahoma +nationality Choros +nationality Chukchi +nationality Chuvash +nationality Ciguayos +nationality Circassians +nationality Coast Salish +nationality Cochiti +nationality Coharie +nationality Colombian Americans +nationality Colombians +nationality Coloured +nationality Colville tribe +nationality Comanche +nationality Converso +nationality Cook Islands Maori +nationality Coos +nationality Copts +nationality Cora +nationality Cornish +nationality Corsicans +nationality Costa Ricans +nationality Coushatta +nationality Cowlitz +nationality Cree +nationality Creole +nationality Creoles +nationality Crimean Karaites +nationality Crimean Tatar +nationality Crimean Tatars +nationality Croatian Americans +nationality Croatian Australian +nationality Croatian Canadians +nationality Croats +nationality Crow Nation +nationality Cuban +nationality Cuban American +nationality Cubans +nationality Cushitic +nationality Czech Americans +nationality Czech Brazilians +nationality Czech Canadians +nationality Czech-Brazilians +nationality Czechs +nationality Czech +nationality Dacians +nationality Dagbamba +nationality Dai +nationality Dakota +nationality Dalit +nationality Dalmatae +nationality Danes +nationality Danish Americans +nationality Danu +nationality Danube Swabians +nationality Dargwa +nationality Darod +nationality Daur +nationality Dayak +nationality Dene +nationality Denobulan +nationality Desano +nationality Devangar +nationality Dhuwal +nationality Di +nationality Diffa Arabs +nationality Dinka +nationality Dioula +nationality Djaru +nationality Dogrib +nationality Doliones +nationality Dom +nationality Dominican American +nationality Dominicans +nationality Don Cossacks +nationality Dorians +nationality Dothraki +nationality Doukhobors +nationality Dravida +nationality Druze +nationality Duala +nationality Dunlendings +nationality Dusun +nationality Dutch +nationality Dutch American +nationality Dzungar +nationality Easterlings +nationality Eastern Europeans +nationality Ebira +nationality Edain +nationality Edo +nationality Efik +nationality Egba +nationality Egyptian Americans +nationality Egyptian Arabic +nationality Egyptian Canadians +nationality Egyptians +nationality Egyptian +nationality Ekiti +nationality Eloi +nationality Emiratis +nationality Emishi +nationality England +nationality English +nationality English American +nationality English Australian +nationality English Canadians +nationality English River First Nation +nationality Eritreans +nationality Erzya +nationality Esan +nationality Eskimo +nationality Esselen +nationality Estonians +nationality Estonian +nationality Ethiopians +nationality Ethiopian +nationality Etruscans +nationality Eurasian +nationality European +nationality European Americans +nationality European Canadians +nationality European New Zealanders +nationality Evenks +nationality Ewe +nationality Ewe +nationality Ewondo +nationality Eyeish +nationality Ezhava +nationality Falathrim +nationality Falmari +nationality Fang +nationality Fante +nationality Faroese +nationality Ferengi +nationality Feylis +nationality Fijians +nationality Filipino +nationality Filipinos +nationality Filipino Americans +nationality Filipino Canadians +nationality Finnish Americans +nationality Finnish Kale +nationality Finnish Ukrainians +nationality Finns +nationality Fipa +nationality Fire Nation +nationality First Nations +nationality Flemings +nationality Fon +nationality Franco-Albertan +nationality Franco-Manitoban +nationality Franco-Mauritian +nationality Franco-Ontarian +nationality Franks +nationality Fremen +nationality French +nationality French Americans +nationality French Brazilian +nationality French Canadians +nationality French Trotter +nationality French-speaking Quebecer +nationality Frisians +nationality Fulani +nationality Fulbe +nationality Funj +nationality Fur +nationality Ga +nationality Ga-Adangbe +nationality Gaels +nationality Gagauz +nationality Galicians +nationality Gallaeci +nationality Gamo +nationality Garhajis +nationality Gauls +nationality Geats +nationality Gedeo +nationality Georgians +nationality Gepids +nationality German Americans +nationality German Brazilians +nationality German Canadians +nationality German diaspora +nationality German Texan +nationality German-Russians +nationality Germanic +nationality Germans +nationality German +nationality Germans of Romania +nationality Germans of Serbia +nationality Germany +nationality Gerudo +nationality Gezawa +nationality Ghanaian American +nationality Ghanaians +nationality Ghanaian +nationality Ghilzai +nationality Ghorbati +nationality Gibraltarian +nationality Gija +nationality Gisu +nationality Gitxsan +nationality Godala +nationality Gonja +nationality Gonja +nationality Gooniyandi +nationality Gorals +nationality Goths +nationality Gotlander +nationality Greek American +nationality Greek Australian +nationality Greek Canadians +nationality Greek Cypriots +nationality Greeks +nationality Greek +nationality Greenlandic Inuit +nationality Greenskins +nationality Grenadians +nationality Groningers +nationality Gros Ventre +nationality Guanches +nationality Gujarati +nationality Gujaratis +nationality Gujarati Americans +nationality Gullah +nationality Gumbaynggirr +nationality Guna +nationality Gunai +nationality Gunditjmara +nationality Gunwinggu +nationality Gurage +nationality Gurbeti +nationality Gurene +nationality Gurindji +nationality Gurjar +nationality Gurung +nationality Gushiegu +nationality Guugu Yimithirr +nationality Guyanese +nationality Gwong +nationality Habr Awal +nationality Hadiya +nationality Haida +nationality Haida +nationality Haisla +nationality Haitian +nationality Haitians +nationality Haitian American +nationality Haitians +nationality Hakka +nationality Han Chinese +nationality Hani +nationality Hapa +nationality Harakmbut +nationality Hashemites +nationality Haudenosaunee Confederacy +nationality Hausa +nationality Hausa +nationality Hayato +nationality Hazaras +nationality Hebrews +nationality Heiltsuk +nationality Hellenes +nationality Helvetii +nationality Herero +nationality Hesquiaht First Nation +nationality Hevelli +nationality Hidatsa +nationality Hiligaynon +nationality Hindustani ethnics +nationality Hispanic +nationality Hittites +nationality Hivite +nationality Hlai +nationality Hmong +nationality Hmong American +nationality Ho +nationality Ho-Chunk +nationality Hoa +nationality Hobbit +nationality Hoklo +nationality Hopi +nationality Houara +nationality House of Hador +nationality House of Haleth +nationality Hualapai +nationality Huaxia +nationality Huaxia +nationality Huguenot +nationality Hui +nationality Huilliche +nationality Hungarian +nationality Hungarian Americans +nationality Hungarians +nationality Hunkpapa +nationality Huns +nationality Huron +nationality Hutsuls +nationality Hutu +nationality Iapyges +nationality Iberians +nationality Ibibio +nationality Icelanders +nationality Icelandic Americans +nationality Icelandic Canadians +nationality Idoma +nationality Igbo +nationality Igbo +nationality Igorot +nationality Ijaw +nationality Ijebu +nationality Illyrians +nationality Ilocano +nationality Imperial Germans +nationality Inari Sami +nationality Inca +nationality Indian Americans +nationality Indian Australian +nationality Indian Mexicans +nationality Indian New Zealander +nationality Indian Singaporeans +nationality Indian South Africans +nationality Indians +nationality Indigenous Australians +nationality Indo Caribbeans +nationality Indo-Aryan +nationality Indo-Canadians +nationality Indo-Iranians +nationality Indo-Surinamese +nationality Indonesian Americans +nationality Indonesians +nationality Ingush +nationality Innu +nationality Inuit +nationality Inupiat +nationality Inuvialuit +nationality Iowa +nationality Iran +nationality Iranian +nationality Iranian American +nationality Iranian Arabs +nationality Iranian Armenians +nationality Iranian Azerbaijanis +nationality Iranian Georgians +nationality Iranian Turkmen +nationality Iranians +nationality Iraqi Americans +nationality Iraqi Canadians +nationality Iraqi Turkmens +nationality Iraqis +nationality Irish +nationality Irish Americans +nationality Irish Argentine +nationality Irish Australian +nationality Irish Canadians +nationality Irish New Zealanders +nationality Irish Quebecers +nationality Isleta +nationality Isoko +nationality Israel +nationality Israeli Jews +nationality Israelis +nationality Israelites +nationality Issei +nationality Italian Americans +nationality Italian Argentines +nationality Italian Brazilian +nationality Italian Brazilians +nationality Italian Canadians +nationality Italian Jews +nationality Italian New Zealanders +nationality Italian Scots +nationality Italians +nationality Italians of Romania +nationality Italic +nationality Italy +nationality Itsekiri +nationality Ivatan +nationality Iwaidja +nationality Ixil +nationality Izumo zoku +nationality Jamaica +nationality Jamaican American +nationality Jamaican Maroons +nationality Jamaican +nationality Jamaicans +nationality Japanese +nationality Japanese American +nationality Japanese Australian +nationality Japanese Brazilians +nationality Japanese Canadians +nationality Japanese Venezuelan +nationality Jaredites +nationality Javanese +nationality Jemez Puebloans +nationality Jew +nationality Jews +nationality Jewish +nationality Jicarilla Apache +nationality Jingpo +nationality Joketsuzoku +nationality Judaism +nationality Juif +nationality Jukun +nationality Jurchen +nationality Kabarday +nationality Kabyle +nationality Kadazan-Dusun +nationality Kadu +nationality Kafficho +nationality Kahlan +nationality Kaiadilt +nationality Kaingang +nationality Kakwa +nationality Kalaallit +nationality Kalapuya +nationality Kalbelia +nationality Kalderash +nationality Kale +nationality Kalenjin +nationality Kalinago +nationality Kalmyks +nationality Kam +nationality Kamba +nationality Kamchadals +nationality Kami +nationality Kamilaroi +nationality Kamoro +nationality Kanak +nationality Kankalis +nationality Kannada +nationality Kanuri +nationality Kapampangan +nationality Kaqchikel +nationality Karakalpaks +nationality Karay-a +nationality Karen +nationality Karenni +nationality Karo +nationality Karuk +nationality Kashmiri +nationality Kashmiri Pandit +nationality Kashubians +nationality Kaskaskia +nationality Kaurna +nationality Kayan +nationality Kayapo +nationality Kayastha +nationality Kazakhs +nationality Kelantanese Malay +nationality Kenite +nationality Kenyan Americans +nationality Keres +nationality Kewa Pueblo +nationality Khanty +nationality Khas +nationality Khasas +nationality Khazars +nationality Khitan +nationality Khmer +nationality Khond +nationality Khorchin Mongols +nationality Kickapoo +nationality Kikuyu +nationality Kilamiut +nationality Kildin Saami +nationality Kiowa +nationality Kipchaks +nationality Kisii +nationality Kist +nationality Kitigan Zibi Anishinabeg +nationality Klemantan +nationality Kluane First Nation +nationality Koibal +nationality Kokama +nationality Kokang +nationality Kombe +nationality Komi +nationality Kongo +nationality Kongu Vellalar +nationality Konkomba +nationality Korea +nationality Korean American +nationality Korean Australian +nationality Korean Brazilian +nationality Korean Canadians +nationality Korean +nationality Koreans +nationality Koryo-saram +nationality Kpando +nationality Krahn +nationality Krobo +nationality Kryptonian +nationality Kshatriya +nationality Kubu +nationality Kukatja +nationality Kumawat +nationality Kumawu +nationality Kumeyaay +nationality Kumyks +nationality Kurdish +nationality Kurdish American +nationality Kurds +nationality Kushwaha +nationality Kusunda +nationality Kven +nationality Kyrgyz +nationality Lac Seul First Nation +nationality Ladin +nationality Laguna +nationality Lahu +nationality Lake Babine Nation +nationality Lakota +nationality Langha +nationality Langi +nationality Lao +nationality Latin Americans +nationality Latino +nationality Latvian American +nationality Latvians +nationality Laz +nationality Lebanese +nationality Lebanese American +nationality Lebanese Australian +nationality Lebanese Brazilians +nationality Lebanese Canadians +nationality Lebanese Colombian +nationality Lebanese Venezuelan +nationality Lechites +nationality Lemkos +nationality Lenape +nationality Lenca +nationality Lezgian +nationality Liberian American +nationality Libu +nationality Ligures +nationality Limbu +nationality Lipan Apache +nationality Lipka Tatars +nationality Lisu +nationality Lithuanian American +nationality Lithuanian Jews +nationality Lithuanian +nationality Lithuanians +nationality Liu +nationality Lombards +nationality Lotud +nationality Louisiana Creole +nationality Lovari +nationality Lower Brule Sioux Tribe +nationality Luba +nationality Lucenses +nationality Lucumi +nationality Lugbara +nationality Luguru +nationality Luhya +nationality Lumbee +nationality Lummi +nationality Lunda +nationality Luo +nationality Luritja +nationality Lurs +nationality Lutici +nationality Luxembourgers +nationality Maasai +nationality Macedonians +nationality Macorix +nationality Macushi +nationality Madurese +nationality Magadha +nationality Magars +nationality Maghrebi Jews +nationality Maidu +nationality Malawian +nationality Malawians +nationality Malay Singaporeans +nationality Malayali +nationality Malays +nationality Malaysian +nationality Malaysian Americans +nationality Malaysian Chinese +nationality Malaysian Indian +nationality Malaysian Malays +nationality Maltese +nationality Maltese American +nationality Mam +nationality Mamprusi +nationality Manchu +nationality Mandaeans +nationality Mandalorians +nationality Mandan +nationality Mandinka +nationality Manganiar +nationality Mangbetu +nationality Mangue +nationality Manobo +nationality Mapuches +nationality Maranao +nationality Marathi +nationality Marehan +nationality Mari +nationality Maropa +nationality Marsi +nationality Marwari +nationality Mashpee Wampanoag Tribe +nationality Masmuda +nationality Massachusett +nationality Masurians +nationality Mauri +nationality Maya +nationality Maya civilization +nationality Mbundu +nationality Mdewakanton +nationality Medes +nationality Mediterranean race +nationality Meghwal +nationality Meherrin +nationality Meitei +nationality Meknes +nationality Melanesians +nationality Memon +nationality Mende +nationality Mennonites +nationality Menominee +nationality Meriam +nationality Meru +nationality Mescalero +nationality Meskhetian Turks +nationality Meskwaki +nationality Mestizo Colombian +nationality Mexican American +nationality Mexicans +nationality Mhallami +nationality Miami +nationality Miao +nationality Miccosukee +nationality Micronesians +nationality Midian +nationality Midianites +nationality Miji +nationality Mijikenda +nationality Milagro-Quevedo culture +nationality Miluk +nationality Minangkabau +nationality Miniconjou +nationality Minnesota Chippewa Tribe +nationality Miskitos +nationality Mission Indians +nationality Mississaugas +nationality Missouria +nationality Miwok +nationality Mixed +nationality Mixtec +nationality Mizrahi Jews +nationality Moab +nationality Modh +nationality Modoc +nationality Mohawk +nationality Mohegan +nationality Mokshas +nationality Moldovans +nationality Moluccans +nationality Mon +nationality Mongo +nationality Mongolian American +nationality Mongolic +nationality Mongols +nationality Monguor +nationality Monpa +nationality Montenegrins +nationality Montserrat +nationality Moors +nationality Moors +nationality Moravians +nationality Mordvins +nationality Morisco +nationality Moroccan Dutch +nationality Moroccan Jews +nationality Moroccans +nationality Mosuo +nationality Mountain Mari +nationality Mozarab +nationality Mpongwe +nationality Muhajir +nationality Multiracial American +nationality Munchkin +nationality Munduruku +nationality Munsee +nationality Mununjali clan +nationality Muong +nationality Murri +nationality Murut +nationality Muruwari +nationality Muscogee +nationality Muslim +nationality Muslims +nationality Musulamii +nationality Muthi Muthi +nationality Myene +nationality Myrmidons +nationality Nabataeans +nationality Naga +nationality Nahuas +nationality Naimans +nationality Nair +nationality Nakhi +nationality Nakota +nationality Nanai +nationality Nanda tribe +nationality Nandor +nationality Narentines +nationality Narungga +nationality Naskapi +nationality Native Hawaiians +nationality Navajo +nationality Nchumbulu +nationality Ndyuka +nationality Negev Bedouin +nationality Negrito +nationality Neimoidians +nationality Nenets +nationality Nepalese Americans +nationality Nephites +nationality New Christian +nationality New Zealand American +nationality New Zealanders +nationality Newar +nationality Nez Perce +nationality Ngadjuri +nationality Ngaiawang +nationality Ngarigo +nationality Ngarinjin +nationality Ngarluma +nationality Ngbaka +nationality Ngunnawal +nationality Ni-Vanuatu +nationality Nibblonians +nationality Nicaraguans +nationality Nigerian +nationality Nigerian American +nationality Nigerian Canadians +nationality Nigerians +nationality Nipissing First Nation +nationality Noldor +nationality Noongar +nationality Normans +nationality Norsemen +nationality North Germanic +nationality North Korean +nationality North Koreans +nationality Northern Cheyenne Tribe +nationality Northern Irish +nationality Northern Ndebele +nationality Norwegian Americans +nationality Norwegian +nationality Norwegians +nationality Norzai +nationality Novgorod Slavs +nationality Nsawkaw +nationality Nuba +nationality Nubian +nationality Nuer +nationality Nung Rawang +nationality Nuristani +nationality Nyakyusa +nationality Nyamwezi +nationality Nyanga +nationality Nzakara +nationality Obotrites +nationality Ocampa +nationality Occaneechi +nationality Occitans +nationality Odawa +nationality Odia +nationality Ogan +nationality Oghuz Turks +nationality Oglala Lakota +nationality Oglala Sioux Tribe +nationality Ogoni +nationality Ohkay Owingeh +nationality Ohkay Owingeh +nationality Ohlone +nationality Oirats +nationality Oji-Cree +nationality Ojibwe +nationality Okanagan +nationality Okinawan +nationality Old Order Amish +nationality Old Prussians +nationality Omaha +nationality Oneida +nationality Onondaga +nationality Orang Asli +nationality Organian +nationality Orion +nationality Oromo +nationality Osage Nation +nationality Ossetians +nationality Ostrogoths +nationality Otoe tribe +nationality Otomi +nationality Ottoman Greeks +nationality Ottoman Turks +nationality Ovambo +nationality Ovimbundu +nationality Ozbek +nationality Pacific Islanders +nationality Paez +nationality Paiute +nationality Paiwan +nationality Pakistani American +nationality Pakistanis +nationality Palaung +nationality Palembangese +nationality Palestinian American +nationality Palestinian Jews +nationality Palestinian +nationality Palestinians +nationality Pamiri +nationality Pamunkey +nationality Panamanian American +nationality Panamanians +nationality Pangasinan +nationality Pannonians +nationality Papel +nationality Papuan +nationality Pardo Brazilians +nationality Pare +nationality Parsi +nationality Parthians +nationality Pashayi +nationality Pashtuns +nationality Patamona +nationality Patawomeck +nationality Pawnee +nationality Pazeh +nationality Pemon +nationality Pennsylvania Dutch +nationality Peoria tribe +nationality Peranakan +nationality Persians +nationality Peru American +nationality Peruvian Australian +nationality Peruvians +nationality Petty Dwarves +nationality Phanariotes +nationality Philistines +nationality Piapoco +nationality Piaroa +nationality Picts +nationality Picts +nationality Pied-Noir +nationality Piegan Blackfeet +nationality Pima +nationality Pima Bajo +nationality Pintupi +nationality Piro Pueblos +nationality Pitcairn Islanders +nationality Plains Apache +nationality Plains Cree +nationality Plains Indians +nationality Polabian Slavs +nationality Polans +nationality Polans +nationality Poles +nationality Polish Americans +nationality Polish Australian +nationality Polish Brazilian +nationality Polish Canadians +nationality Polynesians +nationality Pomo +nationality Ponca +nationality Pondo +nationality Porcko +nationality Portuguese +nationality Portuguese Africans +nationality Portuguese Americans +nationality Portuguese Brazilians +nationality Portuguese Canadians +nationality Portuguese Venezuelan +nationality Potawatomi +nationality Potiguara +nationality Powhatan +nationality Proto-Indo-Europeans +nationality Puebloan +nationality Puerto Ricans +nationality Pumi +nationality Punjabi +nationality Punjabis +nationality Puyallup +nationality Puyuma +nationality Qiang +nationality Qiang +nationality Qibi +nationality Qizilbash +nationality Quapaw +nationality Quebeckers +nationality Quechan +nationality Quechua +nationality Quinault +nationality Qulla +nationality Quraysh +nationality Rai +nationality Rajasthani +nationality Rakhine +nationality Rapa Iti +nationality Rapa Nui +nationality Rappahannock Tribe +nationality Rawas +nationality Remans +nationality Rigellian +nationality Rincon Reservation +nationality Rito +nationality River Severn +nationality Rohingya +nationality Rohirrim +nationality Romani +nationality Romani +nationality Romani Americans +nationality Romanian Americans +nationality Romanian Argentines +nationality Romanian Canadians +nationality Romanian diaspora +nationality Romanian Jews +nationality Romanians +nationality Romanians of Serbia +nationality Romanichal +nationality Romaniote Jews +nationality Romulan +nationality Rongowhakaata +nationality Rosebud Sioux Tribe +nationality Rotuma +nationality Rotuman +nationality Russia +nationality Russian Americans +nationality Russian Brazilians +nationality Russian Canadians +nationality Russian Jews +nationality Russian Mennonite +nationality Russian +nationality Russians +nationality Rusyns +nationality Ruthenians +nationality Rutul +nationality Rutuli +nationality Ryukyuan +nationality Sabines +nationality Sac and Fox Nation +nationality Sahrawi +nationality Saint Petersburg +nationality Saiyan +nationality Sakhalin Koreans +nationality Sakizaya +nationality Salentini +nationality Salian Franks +nationality Salish +nationality Salvadoran Americans +nationality Samantha Kshatriya +nationality Samaritan +nationality Samnites +nationality Samo +nationality Samoan American +nationality Samoan New Zealander +nationality Samoans +nationality San +nationality San Carlos Apache Tribe +nationality San Ildefonso Pueblo +nationality Sanhaja +nationality Sankethi +nationality Sans Arc +nationality Santal +nationality Santee tribe +nationality Saracen +nationality Saraiki +nationality Sarakatsani +nationality Sarki +nationality Sarmatians +nationality Sasak +nationality Satedan +nationality Sauk +nationality Saulteaux +nationality Savoyard +nationality Sawa +nationality Saxons +nationality Sayisi Dene +nationality Sayyid +nationality Scandinavian Americans +nationality Scandinavians +nationality Scotch-Irish Americans +nationality Scotch-Irish Canadians +nationality Scotland +nationality Scottish +nationality Scots +nationality Scottish American +nationality Scottish Australian +nationality Scottish Canadians +nationality Sechelt +nationality Seediq +nationality Semai +nationality Seminole +nationality Seminole Tribe of Florida +nationality Seneca +nationality Senegalese +nationality Sengwer +nationality Senufo +nationality Sephardi Jews +nationality Serbian +nationality Serbian Americans +nationality Serbs +nationality Serbs of Croatia +nationality Serbs of Montenegro +nationality Seri +nationality Serrano +nationality Servitka Roma +nationality Shabak +nationality Shambala +nationality Shan +nationality Shawnee +nationality Shilha +nationality Shinnecock Indian Nation +nationality Shona +nationality Shoshone +nationality Siberian +nationality Siberians +nationality Sicels +nationality Sicilian Americans +nationality Sicilians +nationality Sidama +nationality Siddi +nationality Sihasapa +nationality Sikh +nationality Sikhs +nationality Siksika Nation +nationality Silesians +nationality Silvan Elves +nationality Sindar +nationality Sindhi Rajput +nationality Sindhis +nationality Singaporean Australians +nationality Singaporeans +nationality Sinhalese +nationality Sinti +nationality Sioux +nationality Sisseton Wahpeton Oyate +nationality Sitka Tribe of Alaska +nationality Skagit tribes +nationality Skolts +nationality Skownan First Nation +nationality Slavs +nationality Slovak Americans +nationality Slovak +nationality Slovaks +nationality Slovene American +nationality Slovenes +nationality Slovenian Germans +nationality Snohomish tribe +nationality Sogdo +nationality Somali American +nationality Somali +nationality Somalis +nationality Songhai +nationality Sorbs +nationality Sotho +nationality South African New Zealander +nationality South Africans +nationality South Asia +nationality South Asian Americans +nationality South Korean +nationality South Koreans +nationality South Sea Islander +nationality South Sudanese Canadians +nationality Soviet +nationality Spaniards +nationality Spanish American +nationality Spanish Argentines +nationality Speakers of Wu Chinese +nationality Spokane +nationality Squamish +nationality Sri Lankan Americans +nationality Sri Lankan Tamils +nationality Standing Rock Sioux Tribe +nationality Stateside Puerto Ricans +nationality Stoors +nationality Sudanese +nationality Sudanese American +nationality Sudanese Arabs +nationality Sudeten Germans +nationality Suebi +nationality Sumba +nationality Sundanese +nationality Surinamers +nationality Swabians +nationality Swahili +nationality Swedes +nationality Swedish Americans +nationality Swiss +nationality Swiss Americans +nationality Syrian Americans +nationality Syrian Canadians +nationality Syrian Jews +nationality Syrian Turkmen +nationality Syrians +nationality Tagalog +nationality Tagbanwa +nationality Tagish +nationality Tahitians +nationality Tahltan +nationality Tai +nationality Tainui +nationality Taiwan Japanese +nationality Taiwanese +nationality Taiwanese Americans +nationality Taiwanese Hakka +nationality Taiwanese Plains Aborigines +nationality Tajiks +nationality Talaxian +nationality Talysh +nationality Tamang +nationality Tamiang +nationality Tamil +nationality Tamils +nationality Tamil Americans +nationality Tamil Canadians +nationality Tangale +nationality Tangut +nationality Taos Pueblo +nationality Tarahumara +nationality Tatar +nationality Tatars +nationality Tausug +nationality Tay +nationality Te Arawa +nationality Te Rarawa +nationality Te Roroa +nationality Tejano +nationality Teke +nationality Teleri +nationality Temne +nationality Temuan +nationality Tenharim +nationality Tenino +nationality Teptjars +nationality Ter Sami +nationality Terena +nationality Teso +nationality Tesuque +nationality Tewa +nationality Thadou +nationality Thai +nationality Thai Americans +nationality Thai Britons +nationality Thai Chinese +nationality Thakuri +nationality Tharawal +nationality Tharu +nationality Thracians +nationality Tibetan +nationality Tigray +nationality Tiv +nationality Tiwi +nationality Tlahuica +nationality Tlingit +nationality Toba Batak +nationality Togolese +nationality Tokelauan +nationality Tolai +nationality Toli +nationality Tolupan +nationality Tonga +nationality Tongan American +nationality Tongans +nationality Tongva +nationality Tonkawa +nationality Toro +nationality Torres Strait Islanders +nationality Toucouleur +nationality Transylvanian Saxons +nationality Tribe of Asher +nationality Tribe of Benjamin +nationality Tribe of Dan +nationality Tribe of Levi +nationality Trill +nationality Trinidadians and Tobagonians +nationality Trinovantes +nationality Trojans +nationality Truku +nationality Tsimshian +nationality Tswana +nationality Tuareg +nationality Tucano +nationality Tujia +nationality Tukkers +nationality Tungusic +nationality Tunisians +nationality Tunumiit +nationality Tupuri +nationality Turco-Mongol +nationality Turkic +nationality Turkish Americans +nationality Turkish Cypriots +nationality Turkmens +nationality Turkish +nationality Turks +nationality Turks of Romania +nationality Tuscarora +nationality Tutsi +nationality Tuvans +nationality Tuyuca +nationality Udmurt +nationality Uganda +nationality Ugandan +nationality Ukrainian +nationality Ukrainian Americans +nationality Ukrainian Argentine +nationality Ukrainian Canadians +nationality Ukrainian Jews +nationality Ukrainians +nationality Umatilla +nationality Ume Saami +nationality United States of America +nationality Urhobo +nationality Urhobo +nationality Uruguayan +nationality Uruguayans +nationality Ute +nationality Uyghur +nationality Uyghurs +nationality Uz +nationality Uzbek +nationality Vanara +nationality Vandals +nationality Vanyar +nationality Varciani +nationality Vellalar +nationality Venda +nationality Venezuelan +nationality Venezuelans +nationality Vietnamese +nationality Vietnamese Americans +nationality Vietnamese Australians +nationality Vietnamese Korean +nationality Vietnamese New Zealanders +nationality Vikings +nationality Visayans +nationality Visigoths +nationality Vlachs +nationality Volga Tatars +nationality Vorta +nationality Vulcan +nationality Wa +nationality Waikato Tainui +nationality Waitaha +nationality Wakka Wakka +nationality Wallachian Roma +nationality Wallisians and Futunians +nationality Walloons +nationality Wampanoag +nationality Waorani +nationality Wapishana +nationality Wappo +nationality Waray +nationality Wari +nationality Warlpiri +nationality Warmian +nationality Washo +nationality Water Tribe +nationality Watjarri +nationality Wayampi +nationality Wayana +nationality Wea +nationality Weequays +nationality Welayta +nationality Welsh +nationality Welsh American +nationality Welsh Australian +nationality Wemba-Wemba +nationality Wenatchi +nationality West Frisians +nationality West Slavs +nationality White Angolans +nationality White Australians +nationality White British +nationality White Colombian +nationality White Dominican +nationality White Mexicans +nationality White South Africans +nationality Wichita +nationality Winnebago Tribe of Nebraska +nationality Wintu +nationality Wintun +nationality Wiradjuri +nationality Wirangu +nationality Wixarika +nationality Wiyot +nationality Woodland Cree +nationality Woppaburra +nationality Worimi +nationality Woyo +nationality Wurundjeri +nationality Wyandot +nationality Xavante +nationality Xhosa +nationality Xianbei +nationality Yadav +nationality Yaghan +nationality Yakama +nationality Yakuts +nationality Yamatji +nationality Yamato +nationality Yankton Sioux Tribe +nationality Yanktonai +nationality Yanomamis +nationality Yao +nationality Yao +nationality Yaqui +nationality Yaqui +nationality Yawalapiti +nationality Yawuru +nationality Yaygir +nationality Yazidis +nationality Yemenite Jews +nationality Yeniche +nationality Yi +nationality Yorta Yorta +nationality Yoruba +nationality Yoruba +nationality Yucatec Maya +nationality Yuchi +nationality Yugambeh +nationality Yugoslavs +nationality Yuhup +nationality Yuin +nationality Yupik +nationality Yurok +nationality Yuwaalaraay +nationality Yuzzum +nationality Zafimaniry +nationality Zaghawa +nationality Zande +nationality Zapotec +nationality Zarma +nationality Zarubintsy culture +nationality Zaza +nationality Zenata +nationality Zhuang +nationality Zia +nationality Zimbabwean +nationality Zimbabweans +nationality Zoque +nationality Zulu +nationality Zuni +religion A Moslem +religion Abakuá +religion Abenaki mythology +religion Abkhaz neopaganism +religion Abraham +religion Abrahamic +religion Acacians +religion Āḏar Kayvānī +religion Advaita Vedanta +religion Adventista +religion Agon Shu +religion Ahl al-Hadith +religion Ahmadiyya +religion Ājīvika +religion Akan +religion Akhbari +religion Akkadian mythology +religion Alawites +religion Alevism +religion Alexandrian Wicca +religion American Jews +religion Amish +religion Ananaikyo +religion anarchism +religion Ancient +religion Anglican Communion +religion Anglicanism +religion Anglo-Catholicism +religion Anglo-Saxon Christianity +religion Anglo-Saxon paganism +religion Anti-Hinduism +religion Antoinism +religion Antonianism +religion Apollinarism +religion Apostolic Brethren +religion Apostolic Church +religion Arab Christians +religion Arabian mythology +religion Arabian polytheism +religion Ari Buddhism +religion Arianism +religion Arianism visigothic +religion Armenian mythology +religion Armenian Rite +religion Arminianism +religion Arya Samaj +religion Ásatrúarfélagið +religion Ash'ari +religion Ashkenazi Jews +religion Ashurism +religion Atea +religion Atea +religion Ateizm Derneği +religion Athari +religion atheist +religion Atheists +religion Augsburg Confession +religion Augustinians +religion Aum Shinrikyo +religion Avenna +religion Awakening +religion Azerbaijani +religion Azhaliism +religion Azraqites +religion Aztec +religion Baalism +religion Bábism +religion Babylonian +religion Baghdadi Jews +religion Baháʼí Faith +religion Balinese Hinduism +religion Baltic +religion Baltic mythology +religion Baptists +religion Barnabites +religion Basilian monks +religion Basque mythology +religion Bathouism +religion Bayhasiyyah +religion Bear worship +religion Bektashi Order +religion Bell Church +religion Belokrinitskaya Hierarchy +religion Benedictines +religion Bengali +religion Bengali Christians +religion Benten-shū +religion Benzhuism +religion Beta Israel +religion Bezpopovtsy +religion Bhakti +religion Big Drum +religion Bimoism +religion Black church +religion Blavatskian theosophy +religion Bocheon-gyo +religion Bodong +religion Bogomilism +religion Bon +religion Bosnian Church +religion Brahmanism +religion Brahmin +religion Brahmo +religion Brahmo Samaj +religion Brahmoism +religion Branch Davidians +religion Brethren +religion Bridgettines +religion Broad church +religion Buchanites +religion Buddhism +religion Buddhism +religion Buddhist +religion Budha +religion Burkhanism +religion Bwiti +religion Byzantine Church +religion Byzantine Rite +religion Calvinism +religion Campbellite +religion Canaanite +religion Candomblé Bantu +religion Candomblé Ketu +religion Cao Đài +religion Caodaism +religion Catalan Jews +religion Catharism +religion Catholic +religion Catholic +religion Catholic +religion Catholic +religion Catholic cemetery +religion Catholic Church +religion Catholicism +religion Catolico +religion Celtic Christianity +religion Celtic polytheism +religion Chabad Lubavitch +religion Chalcedonian Christianity +religion Chan Buddhism +religion Chardal +religion Charismatic Christianity +religion Cheondoism +religion Chilote mythology +religion Chinese Buddhism +religion Chinese shamanism +religion Chinzei +religion Chod +religion Christ Embassy +religion Christadelphians +religion Christain +religion Christendom +religion Christian +religion Christian +religion Christian atheism +religion Christian Church +religion Christian Church +religion Christian Connection +religion Christian denomination +religion Christian fundamentalism +religion Christian Identity +religion Christian radicalism +religion Christian Science +religion Christian socialism +religion Christianism +religion Christianity +religion Christianity +religion Christianity +religion Circassian paganism +religion Církev bratrská +religion Cistercians +religion communism +religion Confessing Church +religion Confessio Bohemica +religion Confucianism +religion Congregational Church +religion Congregationalism +religion Congregationalist polity +religion Conservative Friends +religion Conservative Judaism +religion Conservative Laestadianism +religion Conservative Mennonites +religion Constitutional Church +religion Converso +religion Copts +religion Creativity +religion Cristiana +religion Cristiano +religion Crypto-Judaism +religion Cuban Vodú +religion Dacians +religion Daejonggyo +religion Dagpo Kagyu +religion Daheshism +religion Daijokyo +religion Daitoku-ji school +religion Darqawa +religion Daruma-shū +religion Destiny Church +religion Diablo +religion Dievturība +religion Digambara Terapanth +religion Din-e Ilahi +religion Discordianism +religion Dominican Order +religion Dominican Vudú +religion Donatism +religion Dönmeh +religion Doukhobors +religion Dravidar Kazhagam +religion Drikung Kagyu +religion Drukpa Lineage +religion Druze +religion Druze-Israeli +religion Druzism +religion Dudeism +religion Early Christianity +religion Eastern Christianity +religion Eastern Orthodoxy +religion Eckankar +religion Ecumenic community +religion Edinoverie +religion Egyptian mythology +religion EL BICHO +religion Endtime Ministries +religion Episcopal Church +religion Episcopal Church +religion Esoteric Christianity +religion Esoteric Nazism +religion Esperanto +religion Espiritismo +religion Essenes +religion Etruscan mythology +religion Evangelical +religion Evangelical Anglicanism +religion Evangelical Association +religion Evangelical Christians-Baptists +religion Evangelical Friends +religion Evangelicalism +religion Evangelisch-lutherische Kirche +religion evangelism +religion evangelism +religion Evanglische Kirche +religion Falun Gong +religion Fedoseevtsy +religion Feri Tradition +religion Finno-Ugric mythology +religion First Secession +religion Five-Percent Nation +religion Folk Catholicism +religion Folk Orthodoxy +religion Foundationism +religion Franciscan Church +religion Franciscan spirituality +religion Franciscans +religion Frankism +religion freemason +religion Freireligiöse Bewegung +religion Fuji lineage +religion Fuji worship +religion Fujikō +religion Fuju-fuse +religion Fuke-shū +religion Fukkoshinto +religion Gallicanism +religion Gaudiya Vaishnavism +religion Gelug +religion General Baptists +religion Georgian mythology +religion German Catholics +religion Germanic paganism +religion Ghost Dance +religion Ghulat +religion Gion Faith +religion Giudaismo +religion Glasite +religion God-fearer +religion Godianism +religion Goryō faith +religion Gothic art +religion Gothic Christianity +religion Gottgläubig +religion Greco-Roman +religion Greek mythology +religion Grundtvigianism +religion Guanche mythology +religion Guiyidao +religion Gurneyite Friends +religion Haitian Vodou +religion Hakusan cult +religion Handsome Lake +religion Hangui +religion Hanif +religion Haredi Judaism +religion Hasidism +religion Hattic mythology +religion Haugean +religion Hawaiian +religion Heathenry +religion Heaven worship +religion Hellenism +religion Helvetic Confessions +religion Hermetic Qabalah +religion Hervormde kerk +religion Hetanism +religion Hicksite Friends +religion Higashi Honganji-ha +religion High church +religion Hikawa shinkō +religion Hillsong Church +religion Hindu +religion Hindu +religion Hindu +religion Hindu temple +religion Hinduism +religion Hittite +religion Hittite mythology +religion Hòa Hảo +religion Hokke-shū +religion Hokke-shū Honmon-ryū +religion Hokke-shū Jinmon-ryū +religion Hokkeshū Shinmonryū +religion Holy Spirit +religion Homaranismo +religion Honganji-ha +religion Honmon Butsuryū-shū +religion Honzan Shugen-shū +religion Hoodoo +religion Huayan school +religion Huguenot +religion Humanistic Buddhism +religion Humanistic Judaism +religion Hungarian mythology +religion Hussites +religion Hutterite +religion Hyper-Calvinism +religion Ibandla lamaNazaretha +religion Iconodule +religion Ietsism +religion Ifá +religion Ignatian spirituality +religion Imamiyyah +religion Immersion baptism +religion Inayati Order +religion Inca mythology +religion Independent Baptist +religion Independent Catholicism +religion Independents +religion India +religion Indian +religion Inghamites +religion Interspirituality +religion Ise Shintō +religion Islam +religion Islam +religion Islam +religion Islam +religion Islamic architecture +religion Islamic extremism +religion Islamic state +religion Izumo-taishakyo +religion Jabriyah +religion Jainism +religion Jariri +religion Jaririya +religion Jedi +religion Jediism +religion Jehovah's Witnesses +religion Jesuism +religion Jesuit +religion Jewish +religion Jewish +religion Jewish atheism +religion Jewish Buddhist +religion Jewish cemetery +religion Jewish Christian +religion Jewish Renewal +religion Jewish secularism +religion Ji-shū +religion Jingming Taoism +religion Jinja Honkyō +religion Jōdo Shinshū +religion Jodo shu +religion Jogye Order +religion Jonang +religion Jordruk +religion Judaism +religion Judaism +religion Judio +religion Kabbalah +religion Kadam +religion Kagyu +religion Kalachakra +religion Kalash +religion Kalmyks +religion Kannada +religion Karaite +religion Karaite Judaism +religion Karma Kagyu +religion Kaumaram +religion Kegon +religion Kejawen +religion Kélé +religion Kemetism +religion Kempon Hokke +religion Khalwati order +religion Kharijites +religion Khatmiyya +religion Kimbanguism +religion Kinomiya faith +religion Kirchenkreis Gladbach-Neuss +religion Kirchenkreis Niederlausitz +religion Kito +religion Konkokyo +religion Korean Buddhism +religion Korean shamanism +religion Kōshin +religion Kotohira Honkyō +religion Kōyasan Shingon-shū +religion Kubrawiya +religion Kuksu +religion Kumina +religion Kurama Kōkyō +religion Kurozumikyō +religion Kusha-shū +religion Kyiv Metropolis +religion Labadists +religion Laestadianism +religion Landmarkism +religion Lapsed Catholic +religion Latin Church +religion Latter Rain +religion LaVeyan Satanism +religion Laypeople +religion Legio Maria +religion Liberal Catholicism +religion Lingayatism +religion Linji school +religion Lisu Church +religion Lithuanian mythology +religion Livets Ord +religion Living Church +religion Loco +religion Lollardy +religion Longhouse +religion Louisiana Voodoo +religion Lutheran +religion Lutheran Churches +religion Lutheranism +religion Luxor +religion Madhwas +religion Madkhalism +religion Mahanubhava +religion Mahāyāna +religion Mainline Protestant +religion Malagasy mythology +religion Malkia +religion Mandaeism +religion Manichaeism +religion Mapuche +religion Marapu +religion Mariavite Church +religion Marist Brothers +religion Maronite Church +religion Maronites +religion Marrano +religion Martinism +religion Martsang Kagyü +religion Masortim +religion Maturidi +religion Maya +religion Meiteism +religion Mennonites +religion Messianic Judaism +religion Methodism +religion Methodism +religion Miaphysitism +religion Middle Orthodoxy +religion Midewiwin +religion Millerism +religion Millî Görüş +religion Missionary Baptists +religion Mithraic mysteries +religion Mizrahi Jews +religion Mo +religion Mohéli +religion Moloch +religion Molokans +religion Mongolian shamanism +religion Monophysitism +religion monotheism +religion Monothelitism +religion Montanism +religion Moravian Church +religion Moravian Church +religion Mormon +religion Mormon fundamentalism +religion Mormonism +religion Mormons +religion Mu'tazila +religion Muisca +religion Murayama Shugen +religion Muridism +religion Murji'ah +religion Muslim +religion Musulman +religion Musulman +religion Mwari +religion Myōken-shū +religion Myōshin-ji sect +religion mysticism +religion Namdhari +religion Naqshbandi +religion Narodniks +religion Native faith +religion Navayana +religion Nazarene +religion Nedo Kagyü +religion Neo-Confucianism +religion Neolog Judaism +religion Nestorianism +religion New Age +religion New Atheism +religion New Christian +religion New Thought +religion Nicene Christianity +religion Nichiren Buddhism +religion Nichiren Shōshū +religion Nichiren Shū +religion Nichirenshū Fuju-fuse-ha +religion Nizari Isma'ilism +religion Nonconformists +religion Nondenominational Christianity +religion None +religion Nonjuring schism +religion nontheism +religion Norse mythology +religion Nuoism +religion Nuwaubian Nation +religion Nyingma +religion Ōbaku +religion Obeah +religion Odinism +religion Old Believers +religion Old Calendarism +religion Omuro school +religion Oneness Pentecostalism +religion Onmyōdō +religion Oomoto +religion Open Brethren +religion Orthodox Christian +religion Orthodox Church +religion Orthodox Friends +religion Orthodox Judaism +religion orthodoxy +religion Ōtani-ha +religion Pajero +religion Pallottines +religion pantheism +religion Paradesi Jews +religion Parsi +religion Pastafarianism +religion Paulicianism +religion Pene +religion Pentecostalism +religion Persian Jews +religion Petite Église +religion Phagdru Kagyu +religion Pharisees +religion Philippine mythology +religion Philosotology +religion Phoenician mythology +religion Pietism +religion Pietistic Reformed +religion PL Kyodan +religion Plymouth Brethren +religion Polish Brethren +religion Polish Catholicism +religion Polynesian mythology +religion polytheism +religion Pomo +religion Pre-sectarian Buddhism +religion Presbyterianism +religion Primitive Baptists +religion Primitive Methodism +religion Priscillianism +religion Progressive Christianity +religion Protestant +religion Protestant +religion Protestant Ascendancy +religion Protestant theology +religion Protestantism +religion Proto-Indo-European +religion Providence +religion Prussian mythology +religion Punic +religion Puritanism +religion Qadariyya +religion Qadiriyya +religion Qarmatians +religion Qibla +religion Quaker +religion Quaker +religion Quakers +religion Quanzhen School +religion Quimbanda +religion Quranism +religion Qutbism +religion Rabbinic Judaism +religion Radha Soami +religion Radical Pietism +religion Raelism +religion Rahmaniyya +religion Rātana +religion Reconstructionist Judaism +religion Reform Judaism +religion Reformation +religion Reformed Baptists +religion Reformed Church +religion Reformed Church +religion Reiyūkai +religion Remonstrants +religion Restorationism +religion Rien +religion Rifaʽi +religion Ringatū +religion Rinzai school +religion Risshū +religion Rodnovery +religion Rodzima Wiara +religion Roman Catholic +religion Roman Catholics +religion Roman Rite +religion Romani mythology +religion Romaniote Jews +religion Roshaniyya +religion Rosicrucianism +religion Ruri Kyōkai +religion Russian Mennonite +religion Ryukyuan +religion Sabbateans +religion Sahaja Yoga +religion Saiva +religion Saivam +religion Sakya +religion Salafi jihadism +religion Salafism +religion Salvita Decorte +religion Samaniyya +religion Samaritan +religion Samaritanism +religion Samoan mythology +religion Sanamahism +religion Sanātana Dharma +religion Sanbo Kyodan +religion Sannō +religion Sannō Shintō +religion Sanron +religion Sant Mat +religion Santería +religion Santo Daime +religion Satanism +religion Satanism +religion Satmar +religion Schwarzenau Brethren +religion Schwenkfelder Church +religion Scientology +religion Sect Shinto +religion Secular +religion secularism +religion Sedeprivationism +religion Seicho-no-Ie +religion Seizan Jōdo-shū +religion Sephardi Jews +religion Sephardic Haredim +religion Serer +religion Seventh-day Adventism +religion Shadhili +religion Shaiva Siddhanta +religion Shaivism +religion Shakers +religion shamanism +religion Shambhala Buddhism +religion Shangpa Kagyu +religion Shaykhism +religion Shia Islam +religion Shigisan Shingon-shū +religion Shinbutsu-shūgō +religion Shingon Buddhism +religion Shingon Risshu +religion Shingon-shu Buzan-ha +religion Shingon-shū Chizan-ha +religion Shingon-shū Daigo-ha +religion Shingon-shū Daikakuji-ha +religion Shingon-shū Sennyūji-ha +religion Shingon-shū Tōji-ha +religion Shinji Shumeikai +religion Shinnyo Sanmaya-ryū +religion Shinshū Bukkōji-ha +religion Shinshū Jōshōjiha +religion Shinshū Kōshō-ha +religion Shinto +religion Shōtoku-shū +religion Shramana +religion Shri Vidya +religion Shrine Shinto +religion Shtundists +religion Shugendō +religion Sikh +religion Sikhs +religion Sikhism +religion Sith +religion Skoptsy +religion Slavic +religion Slavic Christianity +religion Slavic mythology +religion Socinianism +religion Sofia +religion Soka Gakkai +religion Sōtō +religion Spiritual Baptist +religion Spiritual Christianity +religion State Shinto +religion Stregheria +religion Subud +religion Sufi +religion Sufism +religion Sufri +religion Suika Shinto +religion Sukyo Mahikari +religion Sumerian +religion Sunda Wiwitan +religion Sunni Islam +religion Śvētāmbara +religion Swaminarayan Hinduism +religion Swedenborgian Church +religion Swiss Brethren +religion Syntheism +religion Syro-Malabar Church +religion Szekler Sabbatarians +religion Taklung Kagyu +religion Talmont +religion Tanritsu +religion Tantric Buddhism +religion Taoism +religion Tateyama Shugen +religion Telugu Christian +religion Tendai +religion Tendaijimon Sect +religion Tengri +religion Tengrism +religion Tenjin +religion Tenjin faith +religion Tenrikyo +religion Tetragrammaton +religion Tewahedo Church +religion Thelema +religion Theophilanthropy +religion Theravāda +religion Third Convention +religion Thomism +religion Thracian mythology +religion Three teachings +religion Tiandism +religion Tiantai +religion Tibetan Buddhism +religion Tōji Shingon-shū +religion Traditionalist School +religion Transcendental Meditation +religion Trekkie +religion Tsalpa Kagyü +religion Tsarebozhiye +religion Tsushima faith +religion Twelver Shiism +religion Tymora +religion Umbanda +religion Umbanda +religion Unification Church +religion Unitarian Christianity +religion Unitarian Universalism +religion Unitarianism +religion United Order +religion Unity Church +religion Urantia Foundation +religion Usuli +religion Utraquism +religion Uttaradi Math +religion Uwaisi +religion Vaishnavism +religion Vajrayana +religion Vedanta +religion Vedda +religion Veerashaiva +religion Venezuelan mythology +religion Vibhajyavāda +religion Victory Outreach +religion Viracocha +religion Vizhnitz +religion Vrouwenpartij +religion Wahhabism +religion Waldensians +religion Walloon church +religion Watra +religion Wesleyan Church +religion Wesleyanism +religion Western Christianity +religion Western esotericism +religion White Lotus +religion Wicca +religion WinningJahrian +religion Winti +religion Won Buddhism +religion Yahweh +religion Yahwism +religion Yao Taoism +religion Yarsanism +religion Yasawiyya +religion Yazdânism +religion Yazidis +religion Yazidism +religion Yemeni Judaism +religion Yiguandao +religion Yogacara +religion Yoruba +religion Yoruba +religion Yoruba +religion Yoshida Shintō +religion Yoshikawa Shintō +religion Yugyō-ha +religion Yuzu Nembutsu +religion Zealot +religion Zen +religion Zen +religion Zen Peacemakers +religion Zhalu +religion Zhengyi Dao +religion Zionism +religion Zionist Churches +religion Zoroastrianism +religion Zwinglians +country Chad +country Cuba +country Fiji +country Iran +country Iraq +country Laos +country Mali +country Oman +country Peru +country Togo +country Benin +country Chile +country China +country Congo +country Egypt +country Gabon +country Ghana +country Haiti +country India +country Italy +country Japan +country Kenya +country Libya +country Malta +country Nauru +country Nepal +country Niger +country Palau +country Qatar +country Samoa +country Spain +country Sudan +country Syria +country Tonga +country Yemen +country Angola +country Belize +country Bhutan +country Brazil +country Brunei +country Canada +country Cyprus +country France +country Greece +country Guinea +country Guyana +country Israel +country Jordan +country Kosovo +country Kuwait +country Latvia +country Malawi +country Mexico +country Monaco +country Norway +country Panama +country Poland +country Russia +country Rwanda +country Serbia +country Sweden +country Turkey +country Tuvalu +country Uganda +country Zambia +country Albania +country Algeria +country Andorra +country Armenia +country Austria +country Bahrain +country Belarus +country Belgium +country Bolivia +country Burundi +country Comoros +country Croatia +country Czechia +country Denmark +country Ecuador +country Eritrea +country Estonia +country Finland +country Georgia +country Germany +country Grenada +country Hungary +country Iceland +country Ireland +country Jamaica +country Lebanon +country Lesotho +country Liberia +country Moldova +country Morocco +country Namibia +country Nigeria +country Romania +country Senegal +country Somalia +country Tunisia +country Ukraine +country Uruguay +country Vanuatu +country Vietnam +country Barbados +country Botswana +country Bulgaria +country Cambodia +country Cameroon +country Colombia +country Djibouti +country Dominica +country Eswatini +country Ethiopia +country Honduras +country Kiribati +country Malaysia +country Maldives +country Mongolia +country Pakistan +country Paraguay +country Portugal +country Slovakia +country Slovenia +country St Lucia +country Suriname +country Tanzania +country Thailand +country Zimbabwe +country Argentina +country Australia +country Guatemala +country Indonesia +country Lithuania +country Mauritius +country Nicaragua +country Singapore +country Sri Lanka +country Venezuela +country Azerbaijan +country Bangladesh +country Cape Verde +country Costa Rica +country East Timor +country Kazakhstan +country Kyrgyzstan +country Luxembourg +country Madagascar +country Mauritania +country Micronesia +country Montenegro +country Mozambique +country San Marino +country Seychelles +country St Vincent +country Tajikistan +country Uzbekistan +country Afghanistan +country El Salvador +country The Gambia +country Gambia +country Ivory Coast +country Netherlands +country New Zealand +country North Korea +country Philippines +country Saint Lucia +country South Korea +country South Sudan +country Switzerland +country The Bahamas +country Bahamas +country Burkina Faso +country Saudi Arabia +country Sierra Leone +country South Africa +country Turkmenistan +country Vatican City +country Guinea-Bissau +country Liechtenstein +country United States +country Czech Republic +country State of Libya +country State of Qatar +country United Kingdom +country French Republic +country Kyrgyz Republic +country Myanmar (Burma) +country North Macedonia +country Slovak Republic +country Solomon Islands +country State of Israel +country State of Kuwait +country Italian Republic +country Kingdom of Spain +country Kingdom of Tonga +country Marshall Islands +country Papua New Guinea +country Republic of Chad +country Republic of Cuba +country Republic of Fiji +country Republic of Iraq +country Republic of Mali +country Republic of Peru +country State of Eritrea +country Brunei Darussalam +country Equatorial Guinea +country Gabonese Republic +country Hellenic Republic +country Kingdom of Bhutan +country Kingdom of Norway +country Kingdom of Sweden +country Lebanese Republic +country Republic of Benin +country Republic of Chile +country Republic of Ghana +country Republic of Haiti +country Republic of India +country Republic of Kenya +country Republic of Korea +country Republic of Malta +country Republic of Nauru +country Republic of Niger +country Republic of Palau +country Republic of Yemen +country Sultanate of Oman +country Togolese Republic +country Argentine Republic +country Dominican Republic +country Kingdom of Bahrain +country Kingdom of Belgium +country Kingdom of Denmark +country Kingdom of Lesotho +country Kingdom of Morocco +country Republic of Angola +country Republic of Cyprus +country Republic of Guinea +country Republic of Kosovo +country Republic of Latvia +country Republic of Malawi +country Republic of Panama +country Republic of Poland +country Republic of Rwanda +country Republic of Serbia +country Republic of Uganda +country Republic of Zambia +country Russian Federation +country St Kitts and Nevis +country Antigua and Barbuda +country Kingdom of Cambodia +country Kingdom of Eswatini +country Kingdom of Thailand +country Portuguese Republic +country Republic of Albania +country Republic of Armenia +country Republic of Austria +country Republic of Belarus +country Republic of Burundi +country Republic of Croatia +country Republic of Ecuador +country Republic of Estonia +country Republic of Finland +country Republic of Iceland +country Republic of Liberia +country Republic of Moldova +country Republic of Namibia +country Republic of Senegal +country Republic of Tunisia +country Republic of Türkiye +country Republic of Vanuatu +country Swiss Confederation +country Trinidad and Tobago +country Republic of Botswana +country Republic of Bulgaria +country Republic of Cameroon +country Republic of Colombia +country Republic of Djibouti +country Republic of Honduras +country Republic of Kiribati +country Republic of Maldives +country Republic of Paraguay +country Republic of Slovenia +country Republic of Suriname +country Republic of Zimbabwe +country Syrian Arab Republic +country Union of the Comoros +country United Arab Emirates +country Republic of Guatemala +country Republic of Indonesia +country Republic of Lithuania +country Republic of Mauritius +country Republic of Nicaragua +country Republic of Singapore +country Republic of the Congo +country Republic of the Sudan +country Sao Tome and Principe +country United Mexican States +country Arab Republic of Egypt +country Bosnia and Herzegovina +country Principality of Monaco +country Republic of Azerbaijan +country Republic of Cabo Verde +country Republic of Costa Rica +country Republic of Kazakhstan +country Republic of Madagascar +country Republic of Mozambique +country Republic of San Marino +country Republic of Seychelles +country Republic of Tajikistan +country Republic of The Gambia +country Republic of Uzbekistan +country Kingdom of Saudi Arabia +country Principality of Andorra +country Republic of El Salvador +country Republic of South Sudan +country Central African Republic +country Commonwealth of Dominica +country Islamic Republic of Iran +country Republic of Sierra Leone +country Republic of South Africa +country United States of America +country Commonwealth of Australia +country Grand Duchy of Luxembourg +country Republic of Côte d’Ivoire +country Republic of Guinea-Bissau +country Independent State of Samoa +country Kingdom of the Netherlands +country People’s Republic of China +country Commonwealth of The Bahamas +country Congo (Democratic Republic) +country Federal Republic of Germany +country Federal Republic of Nigeria +country Federal Republic of Somalia +country Hashemite Kingdom of Jordan +country Republic of North Macedonia +country Republic of the Philippines +country United Republic of Tanzania +country Islamic Republic of Pakistan +country Oriental Republic of Uruguay +country Federative Republic of Brazil +country Principality of Liechtenstein +country Republic of Equatorial Guinea +country Federated States of Micronesia +country Islamic Republic of Mauritania +country Plurinational State of Bolivia +country Socialist Republic of Viet Nam +country Co-operative Republic of Guyana +country Islamic Republic of Afghanistan +country People’s Republic of Bangladesh +country Republic of Trinidad and Tobago +country Bolivarian Republic of Venezuela +country Democratic Republic of the Congo +country Lao People’s Democratic Republic +country Republic of the Marshall Islands +country Republic of the Union of Myanmar +country Saint Vincent and the Grenadines +country Democratic Republic of Timor-Leste +country Democratic Peopleís Republic of Korea +country Independent State of Papua New Guinea +country Federal Democratic Republic of Ethiopia +country People’s Democratic Republic of Algeria +country Federation of Saint Christopher and Nevis +country Democratic Socialist Republic of Sri Lanka +country Democratic Republic of Sao Tome and Principe +country United Kingdom of Great Britain and Northern Ireland +country Vatican City State +country Swaziland +sexual orientation gay +sexual orientation homosexual +sexual orientation homosexuality +sexual orientation bisexuality +sexual orientation lesbian +sexual orientation lesbianism +sexual orientation pansexual +sexual orientation pansexuality +sexual orientation heterosexuality +sexual orientation heterosexual +sexual orientation asexual +sexual orientation asexuality +sexual orientation monosexual +sexual orientation monosexuality +sexual orientation demisexual +sexual orientation demisexuality +sexual orientation sapphic +sexual orientation sapphism +sexual orientation bisexual +sexual orientation bisexual man +sexual orientation bisexual woman +sexual orientation gays +sexual orientation lesbians +misc fascism +misc prostitution +misc abortion +misc extremism +misc ISIS +misc nazi +misc nazis +misc quran +misc bible +misc synagogue +misc genocide +misc islamic extremists +misc extremists +misc antisemitism +misc anti-semitism +misc antizionism +misc anti-zionism +misc racism +misc rape +misc ku klux klan +misc bombing +misc bombings +misc nazism +misc neo-nazism +misc neonazism diff --git a/src/cpr_data_access/search_adaptors.py b/src/cpr_data_access/search_adaptors.py index f298700..a190a96 100644 --- a/src/cpr_data_access/search_adaptors.py +++ b/src/cpr_data_access/search_adaptors.py @@ -8,6 +8,7 @@ from cpr_data_access.embedding import Embedder from cpr_data_access.exceptions import DocumentNotFoundError, FetchError, QueryError from cpr_data_access.models.search import Hit, SearchParameters, SearchResponse +from cpr_data_access.utils import is_sensitive_query, load_sensitive_query_terms from cpr_data_access.vespa import ( build_yql, find_vespa_cert_paths, @@ -20,6 +21,7 @@ from vespa.exceptions import VespaError LOGGER = logging.getLogger(__name__) +SENSITIVE_QUERY_TERMS = load_sensitive_query_terms() class SearchAdapter(ABC): @@ -86,7 +88,9 @@ def search(self, parameters: SearchParameters) -> SearchResponse: "timeout": "20", "ranking.softtimeout.factor": "0.7", } - if parameters.exact_match: + if parameters.exact_match or is_sensitive_query( + parameters.query_string, SENSITIVE_QUERY_TERMS + ): vespa_request_body["ranking.profile"] = "exact" else: vespa_request_body["ranking.profile"] = "hybrid" diff --git a/src/cpr_data_access/utils.py b/src/cpr_data_access/utils.py new file mode 100644 index 0000000..67340b5 --- /dev/null +++ b/src/cpr_data_access/utils.py @@ -0,0 +1,64 @@ +import csv +from pathlib import Path + + +def is_sensitive_query(text: str, sensitive_terms: set) -> bool: + """ + Scans text to determine if the query should be considered sensitive + + It does this by evaluating it against a set of predefined sensitive terms. + These, as well as the specific logic are reproduced from previous work, which + stated that "If the query contains any sensitive terms, and the length of the + shortest sensitive term is >=50% of the length of the query by number of words..." + then it is considered sensitive. Further details on the original can be found here: + https://github.com/climatepolicyradar/navigator/pull/815 + + This updated version builds on the above to avoid a loophole where a string of + sensitive terms can be incorrectly flagged as not being sensitive. This happens + because it is the shortest sensitive term that is compared to the rest of the + query, and the rest of the query at that point can contain other sensitive terms. + + """ + sensitive_terms_in_query = [ + term for term in sensitive_terms if term in text.lower() + ] + + if sensitive_terms_in_query: + shortest_sensitive_term = min(sensitive_terms_in_query, key=len) + shortest_sensitive_word_count = len(shortest_sensitive_term.split(" ")) + + remaining_sensitive_word_count = sum( + [ + len(term.split()) + for term in sensitive_terms_in_query + if term != shortest_sensitive_term + ] + ) + + query_word_count = len(text.split()) + remaining_query_word_count = query_word_count - remaining_sensitive_word_count + proportion_sensitive = ( + shortest_sensitive_word_count / remaining_query_word_count + ) + + if proportion_sensitive >= 0.5: + return True + + return False + + +def load_sensitive_query_terms() -> set[str]: + """ + Return sensitive query terms from the first column of a TSV file. + + Outputs are lowercased for case-insensitive matching. + + :return [set[str]]: sensitive query terms + """ + tsv_path = Path(__file__).parent / "resources" / "sensitive_query_terms.tsv" + with open(tsv_path, "r") as tsv_file: + reader = csv.DictReader(tsv_file, delimiter="\t") + + sensitive_terms = set([row["keyword"].lower().strip() for row in reader]) + + return sensitive_terms diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..bf2a507 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,40 @@ +import pytest + +from cpr_data_access.utils import is_sensitive_query, load_sensitive_query_terms + + +TEST_SENSITIVE_QUERY_TERMS = ( + "word", + "test term", + "another phrase example", +) + + +@pytest.mark.parametrize( + argnames="expected, text", + argvalues=( + [False, ""], + [False, "ordinary query"], + [False, "word but outnumbered"], + [False, "word another phrase example but with many other items"], + [True, "word"], + [True, "wordle"], + [True, "test term"], + [True, "test term word"], + [True, "test term and"], + [True, "test term and some"], + [True, "another phrase example"], + [True, "another phrase example word short"], + [True, "another phrase example with other items"], + ), +) +def test_is_sensitive_query(expected, text): + assert ( + is_sensitive_query(text, sensitive_terms=TEST_SENSITIVE_QUERY_TERMS) == expected + ) + + +def test_load_sensitive_query_terms(): + terms = load_sensitive_query_terms() + assert terms + assert len(terms) > 2000 From 4a34448205ebdbbb64d3b6331c96b2e4d5f65981 Mon Sep 17 00:00:00 2001 From: olaughter Date: Tue, 12 Dec 2023 12:46:28 +0000 Subject: [PATCH 2/4] Add query profile for sensitive search --- src/cpr_data_access/search_adaptors.py | 13 +++++++---- src/cpr_data_access/vespa.py | 11 +++++++++- tests/test_search_requests.py | 30 ++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/src/cpr_data_access/search_adaptors.py b/src/cpr_data_access/search_adaptors.py index a190a96..35348d5 100644 --- a/src/cpr_data_access/search_adaptors.py +++ b/src/cpr_data_access/search_adaptors.py @@ -82,16 +82,21 @@ def search(self, parameters: SearchParameters) -> SearchResponse: :return SearchResponse: a list of families, with response metadata """ total_time_start = time.time() + sensitive = is_sensitive_query(parameters.query_string, SENSITIVE_QUERY_TERMS) vespa_request_body: dict[str, Any] = { - "yql": build_yql(parameters), + "yql": build_yql(parameters, sensitive), "timeout": "20", "ranking.softtimeout.factor": "0.7", } - if parameters.exact_match or is_sensitive_query( - parameters.query_string, SENSITIVE_QUERY_TERMS - ): + + if parameters.exact_match: vespa_request_body["ranking.profile"] = "exact" + elif sensitive: + vespa_request_body["ranking.profile"] = "hybrid_no_closeness" + embedding = self.embedder.embed( + parameters.query_string, normalize=False, show_progress_bar=False + ) else: vespa_request_body["ranking.profile"] = "hybrid" embedding = self.embedder.embed( diff --git a/src/cpr_data_access/vespa.py b/src/cpr_data_access/vespa.py index 346fc5b..2c0e79a 100644 --- a/src/cpr_data_access/vespa.py +++ b/src/cpr_data_access/vespa.py @@ -91,7 +91,7 @@ def sanitize(user_input: str) -> str: return user_input -def build_yql(request: SearchParameters) -> str: +def build_yql(request: SearchParameters, sensitive: bool = False) -> str: """ Build a YQL string for retrieving relevant, filtered, sorted results from vespa @@ -109,6 +109,15 @@ def build_yql(request: SearchParameters) -> str: (text_block contains ({{stem: false}}"{request.query_string}")) ) """ + elif sensitive: + rendered_query_string_match = f""" + where ( + {{"targetHits": 1000}} weakAnd( + family_name contains "{ request.query_string }", + family_description contains "{ request.query_string }", + text_block contains "{ request.query_string }" + ) + """ else: rendered_query_string_match = f""" where (( diff --git a/tests/test_search_requests.py b/tests/test_search_requests.py index 1a6a53d..bc03d51 100644 --- a/tests/test_search_requests.py +++ b/tests/test_search_requests.py @@ -148,3 +148,33 @@ def test_vespa_error_details(): err = VespaError(err_object) details = VespaErrorDetails(err) assert not details.is_invalid_query_parameter + + +def test_filter_profiles_return_different_queries(): + exact_yql = build_yql( + request=SearchParameters( + query_string="test", year_range=(2000, 2023), exact_match=True + ), + sensitive=False, + ) + assert "stem: false" in exact_yql + assert "nearestNeighbor" not in exact_yql + + hybrid_yql = build_yql( + request=SearchParameters( + query_string="test", year_range=(2000, 2023), exact_match=False + ), + sensitive=False, + ) + assert "nearestNeighbor" in hybrid_yql + + sensitive_yql = build_yql( + request=SearchParameters( + query_string="test", year_range=(2000, 2023), exact_match=False + ), + sensitive=True, + ) + assert "nearestNeighbor" not in sensitive_yql + + queries = [exact_yql, hybrid_yql, sensitive_yql] + assert len(queries) == len(set(queries)) From f309525bac3da2f1126ebb4b275e64532c0430fa Mon Sep 17 00:00:00 2001 From: olaughter Date: Tue, 12 Dec 2023 13:18:20 +0000 Subject: [PATCH 3/4] Handle rare occurences of nested sensitive term If a sensitive term contains sensitive terms it can break this calculation by causing a division by zero, this ensures we handle this scenario --- src/cpr_data_access/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/cpr_data_access/utils.py b/src/cpr_data_access/utils.py index 67340b5..c79ef82 100644 --- a/src/cpr_data_access/utils.py +++ b/src/cpr_data_access/utils.py @@ -37,10 +37,13 @@ def is_sensitive_query(text: str, sensitive_terms: set) -> bool: query_word_count = len(text.split()) remaining_query_word_count = query_word_count - remaining_sensitive_word_count + + if remaining_query_word_count <= 0: + return True + proportion_sensitive = ( shortest_sensitive_word_count / remaining_query_word_count ) - if proportion_sensitive >= 0.5: return True From 871aafb771a03fec394bfc1feb592e1ebcfd1d2c Mon Sep 17 00:00:00 2001 From: olaughter Date: Tue, 12 Dec 2023 16:01:46 +0000 Subject: [PATCH 4/4] Fix query typo --- src/cpr_data_access/vespa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpr_data_access/vespa.py b/src/cpr_data_access/vespa.py index 2c0e79a..f1b2474 100644 --- a/src/cpr_data_access/vespa.py +++ b/src/cpr_data_access/vespa.py @@ -116,7 +116,7 @@ def build_yql(request: SearchParameters, sensitive: bool = False) -> str: family_name contains "{ request.query_string }", family_description contains "{ request.query_string }", text_block contains "{ request.query_string }" - ) + )) """ else: rendered_query_string_match = f"""