From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by dpdk.org (Postfix) with ESMTP id 50338AFD0 for ; Tue, 27 May 2014 08:27:31 +0200 (CEST) Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga102.fm.intel.com with ESMTP; 26 May 2014 23:27:41 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="4.98,917,1392192000"; d="scan'208,217";a="545336722" Received: from fmsmsx107.amr.corp.intel.com ([10.19.9.54]) by fmsmga002.fm.intel.com with ESMTP; 26 May 2014 23:27:40 -0700 Received: from FMSMSX109.amr.corp.intel.com (10.18.116.9) by FMSMSX107.amr.corp.intel.com (10.19.9.54) with Microsoft SMTP Server (TLS) id 14.3.123.3; Mon, 26 May 2014 23:27:40 -0700 Received: from shsmsx102.ccr.corp.intel.com (10.239.4.154) by fmsmsx109.amr.corp.intel.com (10.18.116.9) with Microsoft SMTP Server (TLS) id 14.3.123.3; Mon, 26 May 2014 23:27:39 -0700 Received: from shsmsx101.ccr.corp.intel.com ([169.254.1.7]) by shsmsx102.ccr.corp.intel.com ([169.254.2.190]) with mapi id 14.03.0123.003; Tue, 27 May 2014 14:27:37 +0800 From: "Xu, HuilongX" To: "dev@dpdk.org" , "Burakov, Anatoly" , Thomas Monjalon Thread-Topic: RE: [dpdk-dev] [PATCH v2 16/16] Adding support for VFIO to setup.sh Thread-Index: Ac95dMC7JHKW7nReQNSZLxKTZIIOnw== Date: Tue, 27 May 2014 06:27:37 +0000 Message-ID: Accept-Language: en-US Content-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: x-originating-ip: [10.239.127.40] MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable X-Content-Filtered-By: Mailman/MimeDel 2.1.15 Subject: Re: [dpdk-dev] [PATCH v2 16/16] Adding support for VFIO to setup.sh X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 27 May 2014 06:27:33 -0000 Support for loading/unloading VFIO drivers, binding/unbinding devices to/from VFIO, also setting up correct userspace permissions. Signed-off-by: Anatoly Burakov Test-by: HuilongX Xu huilongx.xu@intel.com Based-commit: 356cb732d5381140f42ef8b55492339579854986 Compile patch: pass Function test(include setup.sh test): pass >>Compile OS: FC20 X86_64 >>Kernel version: 3.13.6-200. >>GCC version: 4.8.2 >>CPU: Intel(R) Xeon(R) CPU E5-2680 v2 @ 2.80GHz >>NIC: Niantic 82599 --- tools/setup.sh | 168 ++++++++++++++++++++++++++++++++++++++++++++++++-----= --- 1 files changed, 145 insertions(+), 23 deletions(-) diff --git a/tools/setup.sh b/tools/setup.sh index 39be8fc..2ffa55a 100755 --- a/tools/setup.sh +++ b/tools/setup.sh @@ -187,6 +187,52 @@ load_igb_uio_module() } # +# Unloads VFIO modules. +# +remove_vfio_module() +{ + echo "Unloading any existing VFIO module" + /sbin/lsmod | grep -s vfio > /dev/null + if [ $? -eq 0 ] ; then + sudo /sbin/rmmod vfio-pci + sudo /sbin/rmmod vfio_iommu_type1 + sudo /sbin/rmmod vfio + fi +} + +# +# Loads new vfio-pci (and vfio module if needed). +# +load_vfio_module() +{ + remove_vfio_module + + echo "Loading VFIO module" + /sbin/lsmod | grep -s vfio_pci > /dev/null + if [ $? -ne 0 ] ; then + if [ -f /lib/modules/$(uname -r)/kernel/drive= rs/vfio/pci/vfio-pci.ko ] ; then + sudo /sbin/modprobe vfio-pci + fi + fi + + # make sure regular users can read /dev/vfio + echo "chmod /dev/vfio" + sudo /usr/bin/chmod a+x /dev/vfio + if [ $? -ne 0 ] ; then + echo "FAIL" + quit + fi + echo "OK" + + # check if /dev/vfio/vfio exists - that way we + # know we either loaded the module, or it was + # compiled into the kernel + if [ ! -e /dev/vfio/vfio ] ; then + echo "## ERROR: VFIO not found!" + fi +} + +# # Unloads the rte_kni.ko module. # remove_kni_module() @@ -223,6 +269,53 @@ load_kni_module() } # +# Sets appropriate permissions on /dev/vfio/* files +# +set_vfio_permissions() +{ + # make sure regular users can read /dev/vfio + echo "chmod /dev/vfio" + sudo /usr/bin/chmod a+x /dev/vfio + if [ $? -ne 0 ] ; then + echo "FAIL" + quit + fi + echo "OK" + + # make sure regular user can access everything inside /dev/vf= io + echo "chmod /dev/vfio/*" + sudo /usr/bin/chmod 0666 /dev/vfio/* + if [ $? -ne 0 ] ; then + echo "FAIL" + quit + fi + echo "OK" + + # since permissions are only to be set when running as + # regular user, we only check ulimit here + # + # warn if regular user is only allowed + # to memlock <64M of memory + MEMLOCK_AMNT=3D`ulimit -l` + + if [ "$MEMLOCK_AMNT" !=3D "unlimited" ] ; then + MEMLOCK_MB=3D`expr $MEMLOCK_AMNT / 1024` + echo "" + echo "Current user memlock limit: ${MEMLOCK_M= B} MB" + echo "" + echo "This is the maximum amount of memory yo= u will be" + echo "able to use with DPDK and VFIO if run a= s current user." + echo "To change this, please adjust limits.co= nf memlock limit for current user." + + if [ $MEMLOCK_AMNT -lt 65536 ] ; then + echo "" + echo "## WARNING: memlock lim= it is less than 64MB" + echo "## DPDK with VFIO may n= ot be able to initialize if run as current user." + fi + fi +} + +# # Removes all reserved hugepages. # clear_huge_pages() @@ -324,13 +417,13 @@ grep_meminfo() } # -# Calls igb_uio_bind.py --status to show the NIC and what they +# Calls dpdk_nic_bind.py --status to show the NIC and what they # are all bound to, in terms of drivers. # show_nics() { if /sbin/lsmod | grep -q igb_uio ; then - ${RTE_SDK}/tools/igb_uio_bind.py --status + ${RTE_SDK}/tools/dpdk_nic_bind.py --status else echo "# Please load the 'igb_uio' kernel mo= dule before querying or " echo "# adjusting NIC device bindings" @@ -338,16 +431,33 @@ show_nics() } # -# Uses igb_uio_bind.py to move devices to work with igb_uio +# Uses dpdk_nic_bind.py to move devices to work with igb_uio # -bind_nics() +bind_nics_to_vfio() +{ + if /sbin/lsmod | grep -q vfio_pci ; then + ${RTE_SDK}/tools/dpdk_nic_bind.py --status + echo "" + echo -n "Enter PCI address of device to bind = to VFIO driver: " + read PCI_PATH + sudo ${RTE_SDK}/tools/dpdk_nic_bind.py -b vfi= o-pci $PCI_PATH && echo "OK" + else + echo "# Please load the 'vfio-pci' kernel mod= ule before querying or " + echo "# adjusting NIC device bindings" + fi +} + +# +# Uses dpdk_nic_bind.py to move devices to work with igb_uio +# +bind_nics_to_igb_uio() { if /sbin/lsmod | grep -q igb_uio ; then - ${RTE_SDK}/tools/igb_uio_bind.py --status + ${RTE_SDK}/tools/dpdk_nic_bind.py --status echo "" echo -n "Enter PCI address of device to bind= to IGB UIO driver: " read PCI_PATH - sudo ${RTE_SDK}/tools/igb_uio_bind.py -b igb= _uio $PCI_PATH && echo "OK" + sudo ${RTE_SDK}/tools/dpdk_nic_bind.py -b igb= _uio $PCI_PATH && echo "OK" else echo "# Please load the 'igb_uio' kernel mo= dule before querying or " echo "# adjusting NIC device bindings" @@ -355,18 +465,18 @@ bind_nics() } # -# Uses igb_uio_bind.py to move devices to work with kernel drivers again +# Uses dpdk_nic_bind.py to move devices to work with kernel drivers again # unbind_nics() { - ${RTE_SDK}/tools/igb_uio_bind.py --status + ${RTE_SDK}/tools/dpdk_nic_bind.py --status echo "" echo -n "Enter PCI address of device to bind to IGB UIO driv= er: " read PCI_PATH echo "" echo -n "Enter name of kernel driver to bind the device to: = " read DRV - sudo ${RTE_SDK}/tools/igb_uio_bind.py -b $DRV $PCI_PATH && e= cho "OK" + sudo ${RTE_SDK}/tools/dpdk_nic_bind.py -b $DRV $PCI_PATH && e= cho "OK" } # @@ -396,21 +506,30 @@ step2_func() TEXT[1]=3D"Insert IGB UIO module" FUNC[1]=3D"load_igb_uio_module" + + TEXT[2]=3D"Insert VFIO module" + FUNC[2]=3D"load_vfio_module" - TEXT[2]=3D"Insert KNI module" - FUNC[2]=3D"load_kni_module" + TEXT[3]=3D"Insert KNI module" + FUNC[3]=3D"load_kni_module" - TEXT[3]=3D"Setup hugepage mappings for non-NUMA systems" - FUNC[3]=3D"set_non_numa_pages" + TEXT[4]=3D"Setup hugepage mappings for non-NUMA systems" + FUNC[4]=3D"set_non_numa_pages" - TEXT[4]=3D"Setup hugepage mappings for NUMA systems" - FUNC[4]=3D"set_numa_pages" + TEXT[5]=3D"Setup hugepage mappings for NUMA systems" + FUNC[5]=3D"set_numa_pages" - TEXT[5]=3D"Display current Ethernet device settings" - FUNC[5]=3D"show_nics" + TEXT[6]=3D"Display current Ethernet device settings" + FUNC[6]=3D"show_nics" - TEXT[6]=3D"Bind Ethernet device to IGB UIO module" - FUNC[6]=3D"bind_nics" + TEXT[7]=3D"Bind Ethernet device to IGB UIO module" + FUNC[7]=3D"bind_nics_to_igb_uio" + + TEXT[8]=3D"Bind Ethernet device to VFIO module" + FUNC[8]=3D"bind_nics_to_vfio" + + TEXT[9]=3D"Setup VFIO permissions" + FUNC[9]=3D"set_vfio_permissions" } # @@ -455,11 +574,14 @@ step5_func() TEXT[3]=3D"Remove IGB UIO module" FUNC[3]=3D"remove_igb_uio_module" - TEXT[4]=3D"Remove KNI module" - FUNC[4]=3D"remove_kni_module" + TEXT[4]=3D"Remove VFIO module" + FUNC[4]=3D"remove_vfio_module" + + TEXT[5]=3D"Remove KNI module" + FUNC[5]=3D"remove_kni_module" - TEXT[5]=3D"Remove hugepage mappings" - FUNC[5]=3D"clear_huge_pages" + TEXT[6]=3D"Remove hugepage mappings" + FUNC[6]=3D"clear_huge_pages" } STEPS[1]=3D"step1_func" -- 1.7.0.7