redhat-products-cloudproviders-ner.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will see how to use PhraseMatcher component to build domain specific Product Entity Recognition and also add custom attributes to our entity."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "#The below list of products are obtained from the Red Hat Product Life\n",
    "#Cycle API\n",
    "# curl -vk \"https://access.redhat.com/labs/plccapi/lifecycle.json\" | jq '.[].name'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = '''\"Red Hat Enterprise Linux\"\n",
    "\"Red Hat OpenStack Platform\"\n",
    "\"Openshift Container Platform\"\n",
    "\"Red Hat CloudForms\"\n",
    "\"Red Hat Certificate System\"\n",
    "\"Red Hat Directory Server\"\n",
    "\"Red Hat Developer Toolset\"\n",
    "\"Red Hat Virtualization\"\n",
    "\"Red Hat Satellite Server\"\n",
    "\"Red Hat JBoss Enterprise Application Platform\"\n",
    "\"Red Hat JBoss Web Server\"\n",
    "\"Red Hat Developer Studio\"\n",
    "\"Red Hat JBoss Fuse Service Works\"\n",
    "\"Red Hat JBoss Data Virtualization\"\n",
    "\"Red Hat Process Automation Manager\"\n",
    "\"Red Hat Decision Manager\"\n",
    "\"Red Hat JBoss Operations Network\"\n",
    "\"Red Hat Data Grid\"\n",
    "\"Red Hat Fuse\"\n",
    "\"Red Hat AMQ\"\n",
    "\"Red Hat Gluster Storage\"\n",
    "\"Red Hat Ceph Storage\"\n",
    "\"Red Hat 3scale API Management Platform\"\n",
    "\"Red Hat Software Collections\"\n",
    "\"Red Hat Container Development Kit\"\n",
    "\"Red Hat Single Sign-On\"\n",
    "\".Net Core\"\n",
    "\"Apache HTTP Server\"\n",
    "\"Ansible Tower\"\n",
    "\"OpenJDK\"\n",
    "\"Red Hat CodeReady Workspaces\"'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = re.sub(\"\\\"\", \" \", text)\n",
    "raw_products = text.split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Red Hat Enterprise Linux',\n",
       " 'Red Hat OpenStack Platform',\n",
       " 'Openshift Container Platform',\n",
       " 'Red Hat CloudForms',\n",
       " 'Red Hat Certificate System',\n",
       " 'Red Hat Directory Server',\n",
       " 'Red Hat Developer Toolset',\n",
       " 'Red Hat Virtualization',\n",
       " 'Red Hat Satellite Server',\n",
       " 'Red Hat JBoss Enterprise Application Platform',\n",
       " 'Red Hat JBoss Web Server',\n",
       " 'Red Hat Developer Studio',\n",
       " 'Red Hat JBoss Fuse Service Works',\n",
       " 'Red Hat JBoss Data Virtualization',\n",
       " 'Red Hat Process Automation Manager',\n",
       " 'Red Hat Decision Manager',\n",
       " 'Red Hat JBoss Operations Network',\n",
       " 'Red Hat Data Grid',\n",
       " 'Red Hat Fuse',\n",
       " 'Red Hat AMQ',\n",
       " 'Red Hat Gluster Storage',\n",
       " 'Red Hat Ceph Storage',\n",
       " 'Red Hat 3scale API Management Platform',\n",
       " 'Red Hat Software Collections',\n",
       " 'Red Hat Container Development Kit',\n",
       " 'Red Hat Single Sign-On',\n",
       " '.Net Core',\n",
       " 'Apache HTTP Server',\n",
       " 'Ansible Tower',\n",
       " 'OpenJDK',\n",
       " 'Red Hat CodeReady Workspaces']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PRODUCTS = [product.strip() for product in raw_products];products_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "CANONICALS = {\n",
    "    'Red Hat Enterprise Linux': 'rhel',\n",
    "    'Red Hat OpenStack Platform': 'openstack',\n",
    "    'Openshift Container Platform': 'openshift',\n",
    "    'Red Hat CloudForms': 'cloudforms',\n",
    "    'Red Hat Certificate System': 'certificate',\n",
    "    'Red Hat Directory Server': 'directory_server',\n",
    "    'Red Hat Developer Toolset': 'developer_toolset',\n",
    "    'Red Hat Virtualization': 'rhv',\n",
    "    'Red Hat Satellite Server': 'satallite',\n",
    "    'Red Hat JBoss Enterprise Application Platform': 'eap',\n",
    "    'Red Hat JBoss Web Server': 'jboss_web_server',\n",
    "    'Red Hat Developer Studio': 'rhds',\n",
    "    'Red Hat JBoss Fuse Service Works': 'fuse_service',\n",
    "    'Red Hat JBoss Data Virtualization': 'datavirt',\n",
    "    'Red Hat Process Automation Manager': 'pam',\n",
    "    'Red Hat Decision Manager': 'rhdm',\n",
    "    'Red Hat JBoss Operations Network': 'jon',\n",
    "    'Red Hat Data Grid': 'datagrid',\n",
    "    'Red Hat Fuse': 'fuse',\n",
    "    'Red Hat AMQ': 'amq',\n",
    "    'Red Hat Gluster Storage': 'gluster',\n",
    "    'Red Hat Ceph Storage': 'ceph',\n",
    "    'Red Hat 3scale API Management Platform': '3scale',\n",
    "    'Red Hat Software Collections': 'rhscl',\n",
    "    'Red Hat Single Sign-On': 'sso',\n",
    "    '.Net Core': '.net',\n",
    "    'Apache HTTP Server': 'apache_http',\n",
    "    'Ansible Tower': 'tower',\n",
    "    'OpenJDK': 'openjdk',\n",
    "    'Red Hat CodeReady Workspaces': 'codeready'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "from spacy.tokens import Span\n",
    "from spacy.matcher import PhraseMatcher"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_lg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('Red Hat Enterprise Linux', 'ORG', 'rhel'), ('Red Hat Inc', 'ORG', None)]\n"
     ]
    }
   ],
   "source": [
    "def products_component(doc):\n",
    "    matcher = PhraseMatcher(nlp.vocab)\n",
    "    patterns = list(nlp.pipe(PRODUCTS))\n",
    "    matcher.add('PRODUCT', None, *patterns)\n",
    "    # Create an entity Span with the label 'PRODUCT' for all matches\n",
    "    doc.ents = [Span(doc, start, end, label='PRODUCT') \n",
    "                 for match_id, start, end in matcher(doc)]\n",
    "    return doc\n",
    "\n",
    "#Getter that looks up the span text in the dictionary of product canonical\n",
    "get_canonical = lambda span: CANONICALS.get(span.text)\n",
    "\n",
    "#Register the Span extension attribute 'canonical' with the getter get_canonical\n",
    "Span.set_extension('canonical', getter=get_canonical, force=True)\n",
    "\n",
    "doc = nlp(\"Red Hat Enterprise Linux is the flagship product of Red Hat Inc\"\n",
    "         )\n",
    "\n",
    "print([(ent.text, ent.label_, ent._.canonical)for ent in doc.ents])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's use the rule based matching approach provided by Spacy to create CLOUD_PROVIDER entity."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "from spacy.pipeline import EntityRuler\n",
    "\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "ruler = EntityRuler(nlp).from_disk(\"./cloud_providers_patterns.jsonl\")\n",
    "#print(ruler.patterns)\n",
    "# Example patterns file - list of dictionaries with one token per dictonary\n",
    "'''\n",
    "{\"label\": \"CLOUD_PROVIDER\", \"pattern\": [{\"lower\": \"aws\"}]}\n",
    "{\"label\": \"CLOUD_PROVIDER\", \"pattern\": [{\"lower\": \"amazon\"}, {\"lower\": \"web\"}, {\"lower\": \"services\"}]}\n",
    "{\"label\": \"CLOUD_PROVIDER\", \"pattern\": [{\"lower\": \"azure\"}]}\n",
    "{\"label\": \"CLOUD_PROVIDER\", \"pattern\": [{\"lower\": \"microsoft\"}, {\"lower\": \"azure\"}]}\n",
    "{\"label\": \"CLOUD_PROVIDER\", \"pattern\": [{\"lower\": \"gcp\"}]}\n",
    "{\"label\": \"CLOUD_PROVIDER\", \"pattern\": [{\"lower\": \"google\"},{\"lower\": \"cloud\"}, {\"lower\": \"platform\"}]}\n",
    "{\"label\": \"CLOUD_PROVIDER\", \"pattern\": [{\"lower\": \"vsphere\"}]}\n",
    "{\"label\": \"CLOUD_PROVIDER\", \"pattern\": [{\"lower\": \"vmware\"}, {\"lower\": \"vsphere\"}]}\n",
    "'''\n",
    "nlp.add_pipe(ruler, before=\"ner\")\n",
    "#print(nlp.pipe_names)\n",
    "#prints => ['tagger', 'parser', 'entity_ruler', 'ner']\n",
    "\n",
    "doc = nlp(u'AWS is the largest cloud provider than Azure or Google Cloud Platform or even vSphere')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('AWS', 'CLOUD_PROVIDER'), ('Azure', 'CLOUD_PROVIDER'), ('Google Cloud Platform', 'CLOUD_PROVIDER'), ('vSphere', 'CLOUD_PROVIDER')]\n"
     ]
    }
   ],
   "source": [
    "print([(ent.text, ent.label_) for ent in doc.ents])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}