DTN_100G_monitor.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tuning 100G Network "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 0. Parameter Setting "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import dtnmonitor_v2 as dtn\n",
    "import subprocess,datetime,sys\n",
    "\n",
    "def parse_mlnx_log(key_word_t):\n",
    "    fr = open(\"/tmp/mlnx_tune_jupyper.log\",'r')\n",
    "    lock=True\n",
    "    lo=True\n",
    "    count=0\n",
    "    while lock:\n",
    "    #for line in fr:\n",
    "        line = fr.readline()\n",
    "        if 'mlnx-en' in line:\n",
    "            print(line.strip())\n",
    "            while lo:\n",
    "                txt=fr.readline()\n",
    "                for ky in key_word_t:\n",
    "                    if ky in txt:\n",
    "                        print((txt).strip())\n",
    "                        #print(\"count=%s, txt=%s\",(count, txt))\n",
    "\n",
    "                count+=1\n",
    "                if count > 40:\n",
    "                    lock=False\n",
    "                    lo=False\n",
    "    fr.close()\n",
    "\n",
    "\n",
    "def exec_print(command):\n",
    "  with open(logfile, 'a') as f:\n",
    "      process = subprocess.Popen([command], stdout=subprocess.PIPE,stderr=subprocess.STDOUT, shell=True)\n",
    "      f.write(\"\\n# command: \" + command + \"(time:\" + datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\") + \")\\n\")\n",
    "      for line in iter(process.stdout.readline, b''):\n",
    "          # system print\n",
    "          sys.stdout.write(line)\n",
    "          # log\n",
    "          f.write(line.decode(sys.stdout.encoding) )\n",
    "\n",
    "\n",
    "            \n",
    "key_word_t=(\"numa:\",\"pci_slot:\",\"mtu:\",\"rx_queue_size:\",\"tx_queue_size:\",\"name:\")            \n",
    "\n",
    "logfile='/tmp/test2.log' # log file\n",
    "\n",
    "graph_mode=1  #0 all, 1 100pt\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 1. run  mlnx_tune for getting more information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### 1.1 run mlnx_tune\n",
    "#$ sudo mlnx_tune\n",
    "# set it and checking information\n",
    "exec_print(\"sudo /usr/sbin/mlnx_tune -i /tmp/mlnx_tune_jupyper.log\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### 1.2 get system information\n",
    "# be note that following procedure will use these information\n",
    "parse_mlnx_log(key_word_t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 2. 100G DTN System Checking Instruction Guild"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1. To change TCP parameters, edit /etc/sysctl.conf as:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### change TCP parameters, \n",
    "# vim /etc/sysctl.conf as below : \n",
    "# =======================\n",
    "# net.core.default_qdisc = fq\n",
    "# net.core.netdev_max_backlog=250000\n",
    "# net.ipv4.tcp_no_metrics_save=1\n",
    "# net.ipv4.tcp_congestion_control=htcp\n",
    "# net.ipv4.tcp_mtu_probing=1\n",
    "# net.core.rmem_max=671088640\n",
    "# net.core.wmem_max=671088640\n",
    "# net.core.rmem_default=335544320\n",
    "# net.core.wmem_default=335544320\n",
    "# net.core.optmem_max=335544320\n",
    "# net.ipv4.tcp_mem=33554432 335544320 2355443200\n",
    "# net.ipv4.tcp_rmem = 409600 100873800 2147483647\n",
    "# net.ipv4.tcp_wmem = 409600 100873800 2147483647\n",
    "# ======================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "### check :\n",
    "# show your system config\n",
    "exec_print(\"sysctl -a |grep 'net.core\\|net.ipv4.tcp'\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 update Mellanox driver to the most recent one (4.1) and tuning script package.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Run the tuning script in priority: \n",
    "# 2.2 how to set\n",
    "# sudo mlnx_tune\n",
    "# you have run it in step 1\n",
    "# "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.3 Bind NIC interrupt with the correct numa cores using:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### bind NIC interrupt with the correct numa cores\n",
    "# 2.3 how to set : <numa> & <name> can refer step1.2 \n",
    "#$  /user/sbin/set_irq_affinity_bynode.sh <numa> <name>\n",
    "# ie : /user/sbin/set_irq_affinity_bynode.sh 1 ens4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### check :\n",
    "# no check command"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.4 Check PCIe Width, Speed, Max Payload Size and Max Read Request"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### detail ref = lspci: https://community.mellanox.com/docs/DOC-2496\n",
    "# set the PCIe speed, such as \n",
    "#$ sudo /usr/bin/setpci -s <pci_slot> 68.W=5xxx\n",
    "# but some information we should get"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "### check :\n",
    "# you should change xx as the refered step1.2 information <pci_slot>\n",
    "# ie : $ sudo /usr/bin/setpci -s 83:00.0 68.W\n",
    "exec_print(\"sudo setpci -s 83:00.0 68.w\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# it should be the 5xxx | ie : 5820\n",
    "# 2.4 how to set : (if xxx=820) than\n",
    "#$ sudo /usr/bin/setpci -s 83:00.0 68.w=5820"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.5. Increase txqueuelen to 10000:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Increase 100G ethernet txqueuelen to 10000\n",
    "# 2.5 how to set : \n",
    "#$ ifconfig <name> txqueuelen 10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### check :\n",
    "exec_print('ifconfig ens4 |grep txqueuelen')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.6 Set CPU gonvener setting to performance:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### tune the CPU performance\n",
    "# 2.6 how to set : \n",
    "#$ sudo cpupower frequency-set -g performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "exec_print('cpupower frequency-info -g')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.7. Turn off irqbalance and tuned:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### turn off irqbalance \n",
    "# 2.7 how to set : \n",
    "#$ service  irqbalance stop\n",
    "### turn off tuned\n",
    "# 2.7 how to set : \n",
    "### service  tuned stop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exec_print('service irqbalance status ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exec_print('service tuned status ')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 3. Start monitor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "## download and monitor\n",
    "### select network interface ,  [all | <net_interface> ]  ###\n",
    "## if you don't know the interface name, you can use 'all'\n",
    "## make sure the interface is correct, error_naming or without network flow will get empty grath\n",
    "#dtn.interface = 'eth0.1301'\n",
    "#dtn.interface = 'eth0.2038' \n",
    "#dtn.interface = 'all'\n",
    "dtn.interface = 'ens4.2038' \n",
    "#dtn.interface = 'eth1'\n",
    "dtn.exec_command(\"python callmonitor_v2.py\",graph_mode)\n",
    "\n",
    "## you can stop monitoring anytime when you press \"interrupt kernel\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}