lkml.org 
[lkml]   [2016]   [Sep]   [19]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH 4/8] samples: move mic/mpssd example code from Documentation
    Date
    Move mic/mpssd examples to samples and remove it from Documentation
    Makefile. Create a new Makefile to build mic/mpssd. It can be built
    from top level directory or from mic/mpssd directory:

    Run make -C samples/mic/mpssd or cd samples/mic/mpssd; make

    Signed-off-by: Shuah Khan <shuahkh@osg.samsung.com>
    ---
    Documentation/Makefile | 2 +-
    Documentation/mic/Makefile | 1 -
    Documentation/mic/mpssd/.gitignore | 1 -
    Documentation/mic/mpssd/Makefile | 21 -
    Documentation/mic/mpssd/micctrl | 173 ----
    Documentation/mic/mpssd/mpss | 200 ----
    Documentation/mic/mpssd/mpssd.c | 1826 ------------------------------------
    Documentation/mic/mpssd/mpssd.h | 103 --
    Documentation/mic/mpssd/sysfs.c | 102 --
    samples/mic/mpssd/.gitignore | 1 +
    samples/mic/mpssd/Makefile | 27 +
    samples/mic/mpssd/micctrl | 173 ++++
    samples/mic/mpssd/mpss | 200 ++++
    samples/mic/mpssd/mpssd.c | 1826 ++++++++++++++++++++++++++++++++++++
    samples/mic/mpssd/mpssd.h | 103 ++
    samples/mic/mpssd/sysfs.c | 102 ++
    16 files changed, 2433 insertions(+), 2428 deletions(-)
    delete mode 100644 Documentation/mic/Makefile
    delete mode 100644 Documentation/mic/mpssd/.gitignore
    delete mode 100644 Documentation/mic/mpssd/Makefile
    delete mode 100755 Documentation/mic/mpssd/micctrl
    delete mode 100755 Documentation/mic/mpssd/mpss
    delete mode 100644 Documentation/mic/mpssd/mpssd.c
    delete mode 100644 Documentation/mic/mpssd/mpssd.h
    delete mode 100644 Documentation/mic/mpssd/sysfs.c
    create mode 100644 samples/mic/mpssd/.gitignore
    create mode 100644 samples/mic/mpssd/Makefile
    create mode 100755 samples/mic/mpssd/micctrl
    create mode 100755 samples/mic/mpssd/mpss
    create mode 100644 samples/mic/mpssd/mpssd.c
    create mode 100644 samples/mic/mpssd/mpssd.h
    create mode 100644 samples/mic/mpssd/sysfs.c

    diff --git a/Documentation/Makefile b/Documentation/Makefile
    index 539120b..90271bc 100644
    --- a/Documentation/Makefile
    +++ b/Documentation/Makefile
    @@ -1 +1 @@
    -subdir-y := blackfin mic misc-devices pcmcia timers watchdog
    +subdir-y := blackfin misc-devices pcmcia timers watchdog
    diff --git a/Documentation/mic/Makefile b/Documentation/mic/Makefile
    deleted file mode 100644
    index a191d45..0000000
    --- a/Documentation/mic/Makefile
    +++ /dev/null
    @@ -1 +0,0 @@
    -subdir-y := mpssd
    diff --git a/Documentation/mic/mpssd/.gitignore b/Documentation/mic/mpssd/.gitignore
    deleted file mode 100644
    index 8b7c72f..0000000
    --- a/Documentation/mic/mpssd/.gitignore
    +++ /dev/null
    @@ -1 +0,0 @@
    -mpssd
    diff --git a/Documentation/mic/mpssd/Makefile b/Documentation/mic/mpssd/Makefile
    deleted file mode 100644
    index 06871b0..0000000
    --- a/Documentation/mic/mpssd/Makefile
    +++ /dev/null
    @@ -1,21 +0,0 @@
    -ifndef CROSS_COMPILE
    -# List of programs to build
    -hostprogs-$(CONFIG_X86_64) := mpssd
    -
    -mpssd-objs := mpssd.o sysfs.o
    -
    -# Tell kbuild to always build the programs
    -always := $(hostprogs-y)
    -
    -HOSTCFLAGS += -I$(objtree)/usr/include -I$(srctree)/tools/include
    -
    -ifdef DEBUG
    -HOSTCFLAGS += -DDEBUG=$(DEBUG)
    -endif
    -
    -HOSTLOADLIBES_mpssd := -lpthread
    -
    -install:
    - install mpssd /usr/sbin/mpssd
    - install micctrl /usr/sbin/micctrl
    -endif
    diff --git a/Documentation/mic/mpssd/micctrl b/Documentation/mic/mpssd/micctrl
    deleted file mode 100755
    index 8f2629b..0000000
    --- a/Documentation/mic/mpssd/micctrl
    +++ /dev/null
    @@ -1,173 +0,0 @@
    -#!/bin/bash
    -# Intel MIC Platform Software Stack (MPSS)
    -#
    -# Copyright(c) 2013 Intel Corporation.
    -#
    -# This program is free software; you can redistribute it and/or modify
    -# it under the terms of the GNU General Public License, version 2, as
    -# published by the Free Software Foundation.
    -#
    -# This program is distributed in the hope that it will be useful, but
    -# WITHOUT ANY WARRANTY; without even the implied warranty of
    -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
    -# General Public License for more details.
    -#
    -# The full GNU General Public License is included in this distribution in
    -# the file called "COPYING".
    -#
    -# Intel MIC User Space Tools.
    -#
    -# micctrl - Controls MIC boot/start/stop.
    -#
    -# chkconfig: 2345 95 05
    -# description: start MPSS stack processing.
    -#
    -### BEGIN INIT INFO
    -# Provides: micctrl
    -### END INIT INFO
    -
    -# Source function library.
    -. /etc/init.d/functions
    -
    -sysfs="/sys/class/mic"
    -
    -_status()
    -{
    - f=$sysfs/$1
    - echo -e $1 state: "`cat $f/state`" shutdown_status: "`cat $f/shutdown_status`"
    -}
    -
    -status()
    -{
    - if [ "`echo $1 | head -c3`" == "mic" ]; then
    - _status $1
    - return $?
    - fi
    - for f in $sysfs/*
    - do
    - _status `basename $f`
    - RETVAL=$?
    - [ $RETVAL -ne 0 ] && return $RETVAL
    - done
    - return 0
    -}
    -
    -_reset()
    -{
    - f=$sysfs/$1
    - echo reset > $f/state
    -}
    -
    -reset()
    -{
    - if [ "`echo $1 | head -c3`" == "mic" ]; then
    - _reset $1
    - return $?
    - fi
    - for f in $sysfs/*
    - do
    - _reset `basename $f`
    - RETVAL=$?
    - [ $RETVAL -ne 0 ] && return $RETVAL
    - done
    - return 0
    -}
    -
    -_boot()
    -{
    - f=$sysfs/$1
    - echo "linux" > $f/bootmode
    - echo "mic/uos.img" > $f/firmware
    - echo "mic/$1.image" > $f/ramdisk
    - echo "boot" > $f/state
    -}
    -
    -boot()
    -{
    - if [ "`echo $1 | head -c3`" == "mic" ]; then
    - _boot $1
    - return $?
    - fi
    - for f in $sysfs/*
    - do
    - _boot `basename $f`
    - RETVAL=$?
    - [ $RETVAL -ne 0 ] && return $RETVAL
    - done
    - return 0
    -}
    -
    -_shutdown()
    -{
    - f=$sysfs/$1
    - echo shutdown > $f/state
    -}
    -
    -shutdown()
    -{
    - if [ "`echo $1 | head -c3`" == "mic" ]; then
    - _shutdown $1
    - return $?
    - fi
    - for f in $sysfs/*
    - do
    - _shutdown `basename $f`
    - RETVAL=$?
    - [ $RETVAL -ne 0 ] && return $RETVAL
    - done
    - return 0
    -}
    -
    -_wait()
    -{
    - f=$sysfs/$1
    - while [ "`cat $f/state`" != "offline" -a "`cat $f/state`" != "online" ]
    - do
    - sleep 1
    - echo -e "Waiting for $1 to go offline"
    - done
    -}
    -
    -wait()
    -{
    - if [ "`echo $1 | head -c3`" == "mic" ]; then
    - _wait $1
    - return $?
    - fi
    - # Wait for the cards to go offline
    - for f in $sysfs/*
    - do
    - _wait `basename $f`
    - RETVAL=$?
    - [ $RETVAL -ne 0 ] && return $RETVAL
    - done
    - return 0
    -}
    -
    -if [ ! -d "$sysfs" ]; then
    - echo -e $"Module unloaded "
    - exit 3
    -fi
    -
    -case $1 in
    - -s)
    - status $2
    - ;;
    - -r)
    - reset $2
    - ;;
    - -b)
    - boot $2
    - ;;
    - -S)
    - shutdown $2
    - ;;
    - -w)
    - wait $2
    - ;;
    - *)
    - echo $"Usage: $0 {-s (status) |-r (reset) |-b (boot) |-S (shutdown) |-w (wait)}"
    - exit 2
    -esac
    -
    -exit $?
    diff --git a/Documentation/mic/mpssd/mpss b/Documentation/mic/mpssd/mpss
    deleted file mode 100755
    index 5fcf9fa..0000000
    --- a/Documentation/mic/mpssd/mpss
    +++ /dev/null
    @@ -1,200 +0,0 @@
    -#!/bin/bash
    -# Intel MIC Platform Software Stack (MPSS)
    -#
    -# Copyright(c) 2013 Intel Corporation.
    -#
    -# This program is free software; you can redistribute it and/or modify
    -# it under the terms of the GNU General Public License, version 2, as
    -# published by the Free Software Foundation.
    -#
    -# This program is distributed in the hope that it will be useful, but
    -# WITHOUT ANY WARRANTY; without even the implied warranty of
    -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
    -# General Public License for more details.
    -#
    -# The full GNU General Public License is included in this distribution in
    -# the file called "COPYING".
    -#
    -# Intel MIC User Space Tools.
    -#
    -# mpss Start mpssd.
    -#
    -# chkconfig: 2345 95 05
    -# description: start MPSS stack processing.
    -#
    -### BEGIN INIT INFO
    -# Provides: mpss
    -# Required-Start:
    -# Required-Stop:
    -# Short-Description: MPSS stack control
    -# Description: MPSS stack control
    -### END INIT INFO
    -
    -# Source function library.
    -. /etc/init.d/functions
    -
    -exec=/usr/sbin/mpssd
    -sysfs="/sys/class/mic"
    -mic_modules="mic_host mic_x100_dma scif vop"
    -
    -start()
    -{
    - [ -x $exec ] || exit 5
    -
    - if [ "`ps -e | awk '{print $4}' | grep mpssd | head -1`" = "mpssd" ]; then
    - echo -e $"MPSSD already running! "
    - success
    - echo
    - return 0
    - fi
    -
    - echo -e $"Starting MPSS Stack"
    - echo -e $"Loading MIC drivers:" $mic_modules
    -
    - modprobe -a $mic_modules
    - RETVAL=$?
    - if [ $RETVAL -ne 0 ]; then
    - failure
    - echo
    - return $RETVAL
    - fi
    -
    - # Start the daemon
    - echo -n $"Starting MPSSD "
    - $exec
    - RETVAL=$?
    - if [ $RETVAL -ne 0 ]; then
    - failure
    - echo
    - return $RETVAL
    - fi
    - success
    - echo
    -
    - sleep 5
    -
    - # Boot the cards
    - micctrl -b
    -
    - # Wait till ping works
    - for f in $sysfs/*
    - do
    - count=100
    - ipaddr=`cat $f/cmdline`
    - ipaddr=${ipaddr#*address,}
    - ipaddr=`echo $ipaddr | cut -d, -f1 | cut -d\; -f1`
    - while [ $count -ge 0 ]
    - do
    - echo -e "Pinging "`basename $f`" "
    - ping -c 1 $ipaddr &> /dev/null
    - RETVAL=$?
    - if [ $RETVAL -eq 0 ]; then
    - success
    - break
    - fi
    - sleep 1
    - count=`expr $count - 1`
    - done
    - [ $RETVAL -ne 0 ] && failure || success
    - echo
    - done
    - return $RETVAL
    -}
    -
    -stop()
    -{
    - echo -e $"Shutting down MPSS Stack: "
    -
    - # Bail out if module is unloaded
    - if [ ! -d "$sysfs" ]; then
    - echo -n $"Module unloaded "
    - success
    - echo
    - return 0
    - fi
    -
    - # Shut down the cards.
    - micctrl -S
    -
    - # Wait for the cards to go offline
    - for f in $sysfs/*
    - do
    - while [ "`cat $f/state`" != "ready" ]
    - do
    - sleep 1
    - echo -e "Waiting for "`basename $f`" to become ready"
    - done
    - done
    -
    - # Display the status of the cards
    - micctrl -s
    -
    - # Kill MPSSD now
    - echo -n $"Killing MPSSD"
    - killall -9 mpssd 2>/dev/null
    - RETVAL=$?
    - [ $RETVAL -ne 0 ] && failure || success
    - echo
    - return $RETVAL
    -}
    -
    -restart()
    -{
    - stop
    - sleep 5
    - start
    -}
    -
    -status()
    -{
    - micctrl -s
    - if [ "`ps -e | awk '{print $4}' | grep mpssd | head -n 1`" = "mpssd" ]; then
    - echo "mpssd is running"
    - else
    - echo "mpssd is stopped"
    - fi
    - return 0
    -}
    -
    -unload()
    -{
    - if [ ! -d "$sysfs" ]; then
    - echo -n $"No MIC_HOST Module: "
    - success
    - echo
    - return
    - fi
    -
    - stop
    -
    - sleep 5
    - echo -n $"Removing MIC drivers:" $mic_modules
    - modprobe -r $mic_modules
    - RETVAL=$?
    - [ $RETVAL -ne 0 ] && failure || success
    - echo
    - return $RETVAL
    -}
    -
    -case $1 in
    - start)
    - start
    - ;;
    - stop)
    - stop
    - ;;
    - restart)
    - restart
    - ;;
    - status)
    - status
    - ;;
    - unload)
    - unload
    - ;;
    - *)
    - echo $"Usage: $0 {start|stop|restart|status|unload}"
    - exit 2
    -esac
    -
    -exit $?
    diff --git a/Documentation/mic/mpssd/mpssd.c b/Documentation/mic/mpssd/mpssd.c
    deleted file mode 100644
    index 49db1de..0000000
    --- a/Documentation/mic/mpssd/mpssd.c
    +++ /dev/null
    @@ -1,1826 +0,0 @@
    -/*
    - * Intel MIC Platform Software Stack (MPSS)
    - *
    - * Copyright(c) 2013 Intel Corporation.
    - *
    - * This program is free software; you can redistribute it and/or modify
    - * it under the terms of the GNU General Public License, version 2, as
    - * published by the Free Software Foundation.
    - *
    - * This program is distributed in the hope that it will be useful, but
    - * WITHOUT ANY WARRANTY; without even the implied warranty of
    - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
    - * General Public License for more details.
    - *
    - * The full GNU General Public License is included in this distribution in
    - * the file called "COPYING".
    - *
    - * Intel MIC User Space Tools.
    - */
    -
    -#define _GNU_SOURCE
    -
    -#include <stdlib.h>
    -#include <fcntl.h>
    -#include <getopt.h>
    -#include <assert.h>
    -#include <unistd.h>
    -#include <stdbool.h>
    -#include <signal.h>
    -#include <poll.h>
    -#include <features.h>
    -#include <sys/types.h>
    -#include <sys/stat.h>
    -#include <sys/mman.h>
    -#include <sys/socket.h>
    -#include <linux/virtio_ring.h>
    -#include <linux/virtio_net.h>
    -#include <linux/virtio_console.h>
    -#include <linux/virtio_blk.h>
    -#include <linux/version.h>
    -#include "mpssd.h"
    -#include <linux/mic_ioctl.h>
    -#include <linux/mic_common.h>
    -#include <tools/endian.h>
    -
    -static void *init_mic(void *arg);
    -
    -static FILE *logfp;
    -static struct mic_info mic_list;
    -
    -#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
    -
    -#define min_t(type, x, y) ({ \
    - type __min1 = (x); \
    - type __min2 = (y); \
    - __min1 < __min2 ? __min1 : __min2; })
    -
    -/* align addr on a size boundary - adjust address up/down if needed */
    -#define _ALIGN_DOWN(addr, size) ((addr)&(~((size)-1)))
    -#define _ALIGN_UP(addr, size) _ALIGN_DOWN(addr + size - 1, size)
    -
    -/* align addr on a size boundary - adjust address up if needed */
    -#define _ALIGN(addr, size) _ALIGN_UP(addr, size)
    -
    -/* to align the pointer to the (next) page boundary */
    -#define PAGE_ALIGN(addr) _ALIGN(addr, PAGE_SIZE)
    -
    -#define ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x))
    -
    -#define GSO_ENABLED 1
    -#define MAX_GSO_SIZE (64 * 1024)
    -#define ETH_H_LEN 14
    -#define MAX_NET_PKT_SIZE (_ALIGN_UP(MAX_GSO_SIZE + ETH_H_LEN, 64))
    -#define MIC_DEVICE_PAGE_END 0x1000
    -
    -#ifndef VIRTIO_NET_HDR_F_DATA_VALID
    -#define VIRTIO_NET_HDR_F_DATA_VALID 2 /* Csum is valid */
    -#endif
    -
    -static struct {
    - struct mic_device_desc dd;
    - struct mic_vqconfig vqconfig[2];
    - __u32 host_features, guest_acknowledgements;
    - struct virtio_console_config cons_config;
    -} virtcons_dev_page = {
    - .dd = {
    - .type = VIRTIO_ID_CONSOLE,
    - .num_vq = ARRAY_SIZE(virtcons_dev_page.vqconfig),
    - .feature_len = sizeof(virtcons_dev_page.host_features),
    - .config_len = sizeof(virtcons_dev_page.cons_config),
    - },
    - .vqconfig[0] = {
    - .num = htole16(MIC_VRING_ENTRIES),
    - },
    - .vqconfig[1] = {
    - .num = htole16(MIC_VRING_ENTRIES),
    - },
    -};
    -
    -static struct {
    - struct mic_device_desc dd;
    - struct mic_vqconfig vqconfig[2];
    - __u32 host_features, guest_acknowledgements;
    - struct virtio_net_config net_config;
    -} virtnet_dev_page = {
    - .dd = {
    - .type = VIRTIO_ID_NET,
    - .num_vq = ARRAY_SIZE(virtnet_dev_page.vqconfig),
    - .feature_len = sizeof(virtnet_dev_page.host_features),
    - .config_len = sizeof(virtnet_dev_page.net_config),
    - },
    - .vqconfig[0] = {
    - .num = htole16(MIC_VRING_ENTRIES),
    - },
    - .vqconfig[1] = {
    - .num = htole16(MIC_VRING_ENTRIES),
    - },
    -#if GSO_ENABLED
    - .host_features = htole32(
    - 1 << VIRTIO_NET_F_CSUM |
    - 1 << VIRTIO_NET_F_GSO |
    - 1 << VIRTIO_NET_F_GUEST_TSO4 |
    - 1 << VIRTIO_NET_F_GUEST_TSO6 |
    - 1 << VIRTIO_NET_F_GUEST_ECN),
    -#else
    - .host_features = 0,
    -#endif
    -};
    -
    -static const char *mic_config_dir = "/etc/mpss";
    -static const char *virtblk_backend = "VIRTBLK_BACKEND";
    -static struct {
    - struct mic_device_desc dd;
    - struct mic_vqconfig vqconfig[1];
    - __u32 host_features, guest_acknowledgements;
    - struct virtio_blk_config blk_config;
    -} virtblk_dev_page = {
    - .dd = {
    - .type = VIRTIO_ID_BLOCK,
    - .num_vq = ARRAY_SIZE(virtblk_dev_page.vqconfig),
    - .feature_len = sizeof(virtblk_dev_page.host_features),
    - .config_len = sizeof(virtblk_dev_page.blk_config),
    - },
    - .vqconfig[0] = {
    - .num = htole16(MIC_VRING_ENTRIES),
    - },
    - .host_features =
    - htole32(1<<VIRTIO_BLK_F_SEG_MAX),
    - .blk_config = {
    - .seg_max = htole32(MIC_VRING_ENTRIES - 2),
    - .capacity = htole64(0),
    - }
    -};
    -
    -static char *myname;
    -
    -static int
    -tap_configure(struct mic_info *mic, char *dev)
    -{
    - pid_t pid;
    - char *ifargv[7];
    - char ipaddr[IFNAMSIZ];
    - int ret = 0;
    -
    - pid = fork();
    - if (pid == 0) {
    - ifargv[0] = "ip";
    - ifargv[1] = "link";
    - ifargv[2] = "set";
    - ifargv[3] = dev;
    - ifargv[4] = "up";
    - ifargv[5] = NULL;
    - mpsslog("Configuring %s\n", dev);
    - ret = execvp("ip", ifargv);
    - if (ret < 0) {
    - mpsslog("%s execvp failed errno %s\n",
    - mic->name, strerror(errno));
    - return ret;
    - }
    - }
    - if (pid < 0) {
    - mpsslog("%s fork failed errno %s\n",
    - mic->name, strerror(errno));
    - return ret;
    - }
    -
    - ret = waitpid(pid, NULL, 0);
    - if (ret < 0) {
    - mpsslog("%s waitpid failed errno %s\n",
    - mic->name, strerror(errno));
    - return ret;
    - }
    -
    - snprintf(ipaddr, IFNAMSIZ, "172.31.%d.254/24", mic->id + 1);
    -
    - pid = fork();
    - if (pid == 0) {
    - ifargv[0] = "ip";
    - ifargv[1] = "addr";
    - ifargv[2] = "add";
    - ifargv[3] = ipaddr;
    - ifargv[4] = "dev";
    - ifargv[5] = dev;
    - ifargv[6] = NULL;
    - mpsslog("Configuring %s ipaddr %s\n", dev, ipaddr);
    - ret = execvp("ip", ifargv);
    - if (ret < 0) {
    - mpsslog("%s execvp failed errno %s\n",
    - mic->name, strerror(errno));
    - return ret;
    - }
    - }
    - if (pid < 0) {
    - mpsslog("%s fork failed errno %s\n",
    - mic->name, strerror(errno));
    - return ret;
    - }
    -
    - ret = waitpid(pid, NULL, 0);
    - if (ret < 0) {
    - mpsslog("%s waitpid failed errno %s\n",
    - mic->name, strerror(errno));
    - return ret;
    - }
    - mpsslog("MIC name %s %s %d DONE!\n",
    - mic->name, __func__, __LINE__);
    - return 0;
    -}
    -
    -static int tun_alloc(struct mic_info *mic, char *dev)
    -{
    - struct ifreq ifr;
    - int fd, err;
    -#if GSO_ENABLED
    - unsigned offload;
    -#endif
    - fd = open("/dev/net/tun", O_RDWR);
    - if (fd < 0) {
    - mpsslog("Could not open /dev/net/tun %s\n", strerror(errno));
    - goto done;
    - }
    -
    - memset(&ifr, 0, sizeof(ifr));
    -
    - ifr.ifr_flags = IFF_TAP | IFF_NO_PI | IFF_VNET_HDR;
    - if (*dev)
    - strncpy(ifr.ifr_name, dev, IFNAMSIZ);
    -
    - err = ioctl(fd, TUNSETIFF, (void *)&ifr);
    - if (err < 0) {
    - mpsslog("%s %s %d TUNSETIFF failed %s\n",
    - mic->name, __func__, __LINE__, strerror(errno));
    - close(fd);
    - return err;
    - }
    -#if GSO_ENABLED
    - offload = TUN_F_CSUM | TUN_F_TSO4 | TUN_F_TSO6 | TUN_F_TSO_ECN;
    -
    - err = ioctl(fd, TUNSETOFFLOAD, offload);
    - if (err < 0) {
    - mpsslog("%s %s %d TUNSETOFFLOAD failed %s\n",
    - mic->name, __func__, __LINE__, strerror(errno));
    - close(fd);
    - return err;
    - }
    -#endif
    - strcpy(dev, ifr.ifr_name);
    - mpsslog("Created TAP %s\n", dev);
    -done:
    - return fd;
    -}
    -
    -#define NET_FD_VIRTIO_NET 0
    -#define NET_FD_TUN 1
    -#define MAX_NET_FD 2
    -
    -static void set_dp(struct mic_info *mic, int type, void *dp)
    -{
    - switch (type) {
    - case VIRTIO_ID_CONSOLE:
    - mic->mic_console.console_dp = dp;
    - return;
    - case VIRTIO_ID_NET:
    - mic->mic_net.net_dp = dp;
    - return;
    - case VIRTIO_ID_BLOCK:
    - mic->mic_virtblk.block_dp = dp;
    - return;
    - }
    - mpsslog("%s %s %d not found\n", mic->name, __func__, type);
    - assert(0);
    -}
    -
    -static void *get_dp(struct mic_info *mic, int type)
    -{
    - switch (type) {
    - case VIRTIO_ID_CONSOLE:
    - return mic->mic_console.console_dp;
    - case VIRTIO_ID_NET:
    - return mic->mic_net.net_dp;
    - case VIRTIO_ID_BLOCK:
    - return mic->mic_virtblk.block_dp;
    - }
    - mpsslog("%s %s %d not found\n", mic->name, __func__, type);
    - assert(0);
    - return NULL;
    -}
    -
    -static struct mic_device_desc *get_device_desc(struct mic_info *mic, int type)
    -{
    - struct mic_device_desc *d;
    - int i;
    - void *dp = get_dp(mic, type);
    -
    - for (i = sizeof(struct mic_bootparam); i < PAGE_SIZE;
    - i += mic_total_desc_size(d)) {
    - d = dp + i;
    -
    - /* End of list */
    - if (d->type == 0)
    - break;
    -
    - if (d->type == -1)
    - continue;
    -
    - mpsslog("%s %s d-> type %d d %p\n",
    - mic->name, __func__, d->type, d);
    -
    - if (d->type == (__u8)type)
    - return d;
    - }
    - mpsslog("%s %s %d not found\n", mic->name, __func__, type);
    - return NULL;
    -}
    -
    -/* See comments in vhost.c for explanation of next_desc() */
    -static unsigned next_desc(struct vring_desc *desc)
    -{
    - unsigned int next;
    -
    - if (!(le16toh(desc->flags) & VRING_DESC_F_NEXT))
    - return -1U;
    - next = le16toh(desc->next);
    - return next;
    -}
    -
    -/* Sum up all the IOVEC length */
    -static ssize_t
    -sum_iovec_len(struct mic_copy_desc *copy)
    -{
    - ssize_t sum = 0;
    - unsigned int i;
    -
    - for (i = 0; i < copy->iovcnt; i++)
    - sum += copy->iov[i].iov_len;
    - return sum;
    -}
    -
    -static inline void verify_out_len(struct mic_info *mic,
    - struct mic_copy_desc *copy)
    -{
    - if (copy->out_len != sum_iovec_len(copy)) {
    - mpsslog("%s %s %d BUG copy->out_len 0x%x len 0x%zx\n",
    - mic->name, __func__, __LINE__,
    - copy->out_len, sum_iovec_len(copy));
    - assert(copy->out_len == sum_iovec_len(copy));
    - }
    -}
    -
    -/* Display an iovec */
    -static void
    -disp_iovec(struct mic_info *mic, struct mic_copy_desc *copy,
    - const char *s, int line)
    -{
    - unsigned int i;
    -
    - for (i = 0; i < copy->iovcnt; i++)
    - mpsslog("%s %s %d copy->iov[%d] addr %p len 0x%zx\n",
    - mic->name, s, line, i,
    - copy->iov[i].iov_base, copy->iov[i].iov_len);
    -}
    -
    -static inline __u16 read_avail_idx(struct mic_vring *vr)
    -{
    - return ACCESS_ONCE(vr->info->avail_idx);
    -}
    -
    -static inline void txrx_prepare(int type, bool tx, struct mic_vring *vr,
    - struct mic_copy_desc *copy, ssize_t len)
    -{
    - copy->vr_idx = tx ? 0 : 1;
    - copy->update_used = true;
    - if (type == VIRTIO_ID_NET)
    - copy->iov[1].iov_len = len - sizeof(struct virtio_net_hdr);
    - else
    - copy->iov[0].iov_len = len;
    -}
    -
    -/* Central API which triggers the copies */
    -static int
    -mic_virtio_copy(struct mic_info *mic, int fd,
    - struct mic_vring *vr, struct mic_copy_desc *copy)
    -{
    - int ret;
    -
    - ret = ioctl(fd, MIC_VIRTIO_COPY_DESC, copy);
    - if (ret) {
    - mpsslog("%s %s %d errno %s ret %d\n",
    - mic->name, __func__, __LINE__,
    - strerror(errno), ret);
    - }
    - return ret;
    -}
    -
    -static inline unsigned _vring_size(unsigned int num, unsigned long align)
    -{
    - return ((sizeof(struct vring_desc) * num + sizeof(__u16) * (3 + num)
    - + align - 1) & ~(align - 1))
    - + sizeof(__u16) * 3 + sizeof(struct vring_used_elem) * num;
    -}
    -
    -/*
    - * This initialization routine requires at least one
    - * vring i.e. vr0. vr1 is optional.
    - */
    -static void *
    -init_vr(struct mic_info *mic, int fd, int type,
    - struct mic_vring *vr0, struct mic_vring *vr1, int num_vq)
    -{
    - int vr_size;
    - char *va;
    -
    - vr_size = PAGE_ALIGN(_vring_size(MIC_VRING_ENTRIES,
    - MIC_VIRTIO_RING_ALIGN) +
    - sizeof(struct _mic_vring_info));
    - va = mmap(NULL, MIC_DEVICE_PAGE_END + vr_size * num_vq,
    - PROT_READ, MAP_SHARED, fd, 0);
    - if (MAP_FAILED == va) {
    - mpsslog("%s %s %d mmap failed errno %s\n",
    - mic->name, __func__, __LINE__,
    - strerror(errno));
    - goto done;
    - }
    - set_dp(mic, type, va);
    - vr0->va = (struct mic_vring *)&va[MIC_DEVICE_PAGE_END];
    - vr0->info = vr0->va +
    - _vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN);
    - vring_init(&vr0->vr,
    - MIC_VRING_ENTRIES, vr0->va, MIC_VIRTIO_RING_ALIGN);
    - mpsslog("%s %s vr0 %p vr0->info %p vr_size 0x%x vring 0x%x ",
    - __func__, mic->name, vr0->va, vr0->info, vr_size,
    - _vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN));
    - mpsslog("magic 0x%x expected 0x%x\n",
    - le32toh(vr0->info->magic), MIC_MAGIC + type);
    - assert(le32toh(vr0->info->magic) == MIC_MAGIC + type);
    - if (vr1) {
    - vr1->va = (struct mic_vring *)
    - &va[MIC_DEVICE_PAGE_END + vr_size];
    - vr1->info = vr1->va + _vring_size(MIC_VRING_ENTRIES,
    - MIC_VIRTIO_RING_ALIGN);
    - vring_init(&vr1->vr,
    - MIC_VRING_ENTRIES, vr1->va, MIC_VIRTIO_RING_ALIGN);
    - mpsslog("%s %s vr1 %p vr1->info %p vr_size 0x%x vring 0x%x ",
    - __func__, mic->name, vr1->va, vr1->info, vr_size,
    - _vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN));
    - mpsslog("magic 0x%x expected 0x%x\n",
    - le32toh(vr1->info->magic), MIC_MAGIC + type + 1);
    - assert(le32toh(vr1->info->magic) == MIC_MAGIC + type + 1);
    - }
    -done:
    - return va;
    -}
    -
    -static int
    -wait_for_card_driver(struct mic_info *mic, int fd, int type)
    -{
    - struct pollfd pollfd;
    - int err;
    - struct mic_device_desc *desc = get_device_desc(mic, type);
    - __u8 prev_status;
    -
    - if (!desc)
    - return -ENODEV;
    - prev_status = desc->status;
    - pollfd.fd = fd;
    - mpsslog("%s %s Waiting .... desc-> type %d status 0x%x\n",
    - mic->name, __func__, type, desc->status);
    -
    - while (1) {
    - pollfd.events = POLLIN;
    - pollfd.revents = 0;
    - err = poll(&pollfd, 1, -1);
    - if (err < 0) {
    - mpsslog("%s %s poll failed %s\n",
    - mic->name, __func__, strerror(errno));
    - continue;
    - }
    -
    - if (pollfd.revents) {
    - if (desc->status != prev_status) {
    - mpsslog("%s %s Waiting... desc-> type %d "
    - "status 0x%x\n",
    - mic->name, __func__, type,
    - desc->status);
    - prev_status = desc->status;
    - }
    - if (desc->status & VIRTIO_CONFIG_S_DRIVER_OK) {
    - mpsslog("%s %s poll.revents %d\n",
    - mic->name, __func__, pollfd.revents);
    - mpsslog("%s %s desc-> type %d status 0x%x\n",
    - mic->name, __func__, type,
    - desc->status);
    - break;
    - }
    - }
    - }
    - return 0;
    -}
    -
    -/* Spin till we have some descriptors */
    -static void
    -spin_for_descriptors(struct mic_info *mic, struct mic_vring *vr)
    -{
    - __u16 avail_idx = read_avail_idx(vr);
    -
    - while (avail_idx == le16toh(ACCESS_ONCE(vr->vr.avail->idx))) {
    -#ifdef DEBUG
    - mpsslog("%s %s waiting for desc avail %d info_avail %d\n",
    - mic->name, __func__,
    - le16toh(vr->vr.avail->idx), vr->info->avail_idx);
    -#endif
    - sched_yield();
    - }
    -}
    -
    -static void *
    -virtio_net(void *arg)
    -{
    - static __u8 vnet_hdr[2][sizeof(struct virtio_net_hdr)];
    - static __u8 vnet_buf[2][MAX_NET_PKT_SIZE] __attribute__ ((aligned(64)));
    - struct iovec vnet_iov[2][2] = {
    - { { .iov_base = vnet_hdr[0], .iov_len = sizeof(vnet_hdr[0]) },
    - { .iov_base = vnet_buf[0], .iov_len = sizeof(vnet_buf[0]) } },
    - { { .iov_base = vnet_hdr[1], .iov_len = sizeof(vnet_hdr[1]) },
    - { .iov_base = vnet_buf[1], .iov_len = sizeof(vnet_buf[1]) } },
    - };
    - struct iovec *iov0 = vnet_iov[0], *iov1 = vnet_iov[1];
    - struct mic_info *mic = (struct mic_info *)arg;
    - char if_name[IFNAMSIZ];
    - struct pollfd net_poll[MAX_NET_FD];
    - struct mic_vring tx_vr, rx_vr;
    - struct mic_copy_desc copy;
    - struct mic_device_desc *desc;
    - int err;
    -
    - snprintf(if_name, IFNAMSIZ, "mic%d", mic->id);
    - mic->mic_net.tap_fd = tun_alloc(mic, if_name);
    - if (mic->mic_net.tap_fd < 0)
    - goto done;
    -
    - if (tap_configure(mic, if_name))
    - goto done;
    - mpsslog("MIC name %s id %d\n", mic->name, mic->id);
    -
    - net_poll[NET_FD_VIRTIO_NET].fd = mic->mic_net.virtio_net_fd;
    - net_poll[NET_FD_VIRTIO_NET].events = POLLIN;
    - net_poll[NET_FD_TUN].fd = mic->mic_net.tap_fd;
    - net_poll[NET_FD_TUN].events = POLLIN;
    -
    - if (MAP_FAILED == init_vr(mic, mic->mic_net.virtio_net_fd,
    - VIRTIO_ID_NET, &tx_vr, &rx_vr,
    - virtnet_dev_page.dd.num_vq)) {
    - mpsslog("%s init_vr failed %s\n",
    - mic->name, strerror(errno));
    - goto done;
    - }
    -
    - copy.iovcnt = 2;
    - desc = get_device_desc(mic, VIRTIO_ID_NET);
    -
    - while (1) {
    - ssize_t len;
    -
    - net_poll[NET_FD_VIRTIO_NET].revents = 0;
    - net_poll[NET_FD_TUN].revents = 0;
    -
    - /* Start polling for data from tap and virtio net */
    - err = poll(net_poll, 2, -1);
    - if (err < 0) {
    - mpsslog("%s poll failed %s\n",
    - __func__, strerror(errno));
    - continue;
    - }
    - if (!(desc->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
    - err = wait_for_card_driver(mic,
    - mic->mic_net.virtio_net_fd,
    - VIRTIO_ID_NET);
    - if (err) {
    - mpsslog("%s %s %d Exiting...\n",
    - mic->name, __func__, __LINE__);
    - break;
    - }
    - }
    - /*
    - * Check if there is data to be read from TUN and write to
    - * virtio net fd if there is.
    - */
    - if (net_poll[NET_FD_TUN].revents & POLLIN) {
    - copy.iov = iov0;
    - len = readv(net_poll[NET_FD_TUN].fd,
    - copy.iov, copy.iovcnt);
    - if (len > 0) {
    - struct virtio_net_hdr *hdr
    - = (struct virtio_net_hdr *)vnet_hdr[0];
    -
    - /* Disable checksums on the card since we are on
    - a reliable PCIe link */
    - hdr->flags |= VIRTIO_NET_HDR_F_DATA_VALID;
    -#ifdef DEBUG
    - mpsslog("%s %s %d hdr->flags 0x%x ", mic->name,
    - __func__, __LINE__, hdr->flags);
    - mpsslog("copy.out_len %d hdr->gso_type 0x%x\n",
    - copy.out_len, hdr->gso_type);
    -#endif
    -#ifdef DEBUG
    - disp_iovec(mic, copy, __func__, __LINE__);
    - mpsslog("%s %s %d read from tap 0x%lx\n",
    - mic->name, __func__, __LINE__,
    - len);
    -#endif
    - spin_for_descriptors(mic, &tx_vr);
    - txrx_prepare(VIRTIO_ID_NET, 1, &tx_vr, &copy,
    - len);
    -
    - err = mic_virtio_copy(mic,
    - mic->mic_net.virtio_net_fd, &tx_vr,
    - &copy);
    - if (err < 0) {
    - mpsslog("%s %s %d mic_virtio_copy %s\n",
    - mic->name, __func__, __LINE__,
    - strerror(errno));
    - }
    - if (!err)
    - verify_out_len(mic, &copy);
    -#ifdef DEBUG
    - disp_iovec(mic, copy, __func__, __LINE__);
    - mpsslog("%s %s %d wrote to net 0x%lx\n",
    - mic->name, __func__, __LINE__,
    - sum_iovec_len(&copy));
    -#endif
    - /* Reinitialize IOV for next run */
    - iov0[1].iov_len = MAX_NET_PKT_SIZE;
    - } else if (len < 0) {
    - disp_iovec(mic, &copy, __func__, __LINE__);
    - mpsslog("%s %s %d read failed %s ", mic->name,
    - __func__, __LINE__, strerror(errno));
    - mpsslog("cnt %d sum %zd\n",
    - copy.iovcnt, sum_iovec_len(&copy));
    - }
    - }
    -
    - /*
    - * Check if there is data to be read from virtio net and
    - * write to TUN if there is.
    - */
    - if (net_poll[NET_FD_VIRTIO_NET].revents & POLLIN) {
    - while (rx_vr.info->avail_idx !=
    - le16toh(rx_vr.vr.avail->idx)) {
    - copy.iov = iov1;
    - txrx_prepare(VIRTIO_ID_NET, 0, &rx_vr, &copy,
    - MAX_NET_PKT_SIZE
    - + sizeof(struct virtio_net_hdr));
    -
    - err = mic_virtio_copy(mic,
    - mic->mic_net.virtio_net_fd, &rx_vr,
    - &copy);
    - if (!err) {
    -#ifdef DEBUG
    - struct virtio_net_hdr *hdr
    - = (struct virtio_net_hdr *)
    - vnet_hdr[1];
    -
    - mpsslog("%s %s %d hdr->flags 0x%x, ",
    - mic->name, __func__, __LINE__,
    - hdr->flags);
    - mpsslog("out_len %d gso_type 0x%x\n",
    - copy.out_len,
    - hdr->gso_type);
    -#endif
    - /* Set the correct output iov_len */
    - iov1[1].iov_len = copy.out_len -
    - sizeof(struct virtio_net_hdr);
    - verify_out_len(mic, &copy);
    -#ifdef DEBUG
    - disp_iovec(mic, copy, __func__,
    - __LINE__);
    - mpsslog("%s %s %d ",
    - mic->name, __func__, __LINE__);
    - mpsslog("read from net 0x%lx\n",
    - sum_iovec_len(copy));
    -#endif
    - len = writev(net_poll[NET_FD_TUN].fd,
    - copy.iov, copy.iovcnt);
    - if (len != sum_iovec_len(&copy)) {
    - mpsslog("Tun write failed %s ",
    - strerror(errno));
    - mpsslog("len 0x%zx ", len);
    - mpsslog("read_len 0x%zx\n",
    - sum_iovec_len(&copy));
    - } else {
    -#ifdef DEBUG
    - disp_iovec(mic, &copy, __func__,
    - __LINE__);
    - mpsslog("%s %s %d ",
    - mic->name, __func__,
    - __LINE__);
    - mpsslog("wrote to tap 0x%lx\n",
    - len);
    -#endif
    - }
    - } else {
    - mpsslog("%s %s %d mic_virtio_copy %s\n",
    - mic->name, __func__, __LINE__,
    - strerror(errno));
    - break;
    - }
    - }
    - }
    - if (net_poll[NET_FD_VIRTIO_NET].revents & POLLERR)
    - mpsslog("%s: %s: POLLERR\n", __func__, mic->name);
    - }
    -done:
    - pthread_exit(NULL);
    -}
    -
    -/* virtio_console */
    -#define VIRTIO_CONSOLE_FD 0
    -#define MONITOR_FD (VIRTIO_CONSOLE_FD + 1)
    -#define MAX_CONSOLE_FD (MONITOR_FD + 1) /* must be the last one + 1 */
    -#define MAX_BUFFER_SIZE PAGE_SIZE
    -
    -static void *
    -virtio_console(void *arg)
    -{
    - static __u8 vcons_buf[2][PAGE_SIZE];
    - struct iovec vcons_iov[2] = {
    - { .iov_base = vcons_buf[0], .iov_len = sizeof(vcons_buf[0]) },
    - { .iov_base = vcons_buf[1], .iov_len = sizeof(vcons_buf[1]) },
    - };
    - struct iovec *iov0 = &vcons_iov[0], *iov1 = &vcons_iov[1];
    - struct mic_info *mic = (struct mic_info *)arg;
    - int err;
    - struct pollfd console_poll[MAX_CONSOLE_FD];
    - int pty_fd;
    - char *pts_name;
    - ssize_t len;
    - struct mic_vring tx_vr, rx_vr;
    - struct mic_copy_desc copy;
    - struct mic_device_desc *desc;
    -
    - pty_fd = posix_openpt(O_RDWR);
    - if (pty_fd < 0) {
    - mpsslog("can't open a pseudoterminal master device: %s\n",
    - strerror(errno));
    - goto _return;
    - }
    - pts_name = ptsname(pty_fd);
    - if (pts_name == NULL) {
    - mpsslog("can't get pts name\n");
    - goto _close_pty;
    - }
    - printf("%s console message goes to %s\n", mic->name, pts_name);
    - mpsslog("%s console message goes to %s\n", mic->name, pts_name);
    - err = grantpt(pty_fd);
    - if (err < 0) {
    - mpsslog("can't grant access: %s %s\n",
    - pts_name, strerror(errno));
    - goto _close_pty;
    - }
    - err = unlockpt(pty_fd);
    - if (err < 0) {
    - mpsslog("can't unlock a pseudoterminal: %s %s\n",
    - pts_name, strerror(errno));
    - goto _close_pty;
    - }
    - console_poll[MONITOR_FD].fd = pty_fd;
    - console_poll[MONITOR_FD].events = POLLIN;
    -
    - console_poll[VIRTIO_CONSOLE_FD].fd = mic->mic_console.virtio_console_fd;
    - console_poll[VIRTIO_CONSOLE_FD].events = POLLIN;
    -
    - if (MAP_FAILED == init_vr(mic, mic->mic_console.virtio_console_fd,
    - VIRTIO_ID_CONSOLE, &tx_vr, &rx_vr,
    - virtcons_dev_page.dd.num_vq)) {
    - mpsslog("%s init_vr failed %s\n",
    - mic->name, strerror(errno));
    - goto _close_pty;
    - }
    -
    - copy.iovcnt = 1;
    - desc = get_device_desc(mic, VIRTIO_ID_CONSOLE);
    -
    - for (;;) {
    - console_poll[MONITOR_FD].revents = 0;
    - console_poll[VIRTIO_CONSOLE_FD].revents = 0;
    - err = poll(console_poll, MAX_CONSOLE_FD, -1);
    - if (err < 0) {
    - mpsslog("%s %d: poll failed: %s\n", __func__, __LINE__,
    - strerror(errno));
    - continue;
    - }
    - if (!(desc->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
    - err = wait_for_card_driver(mic,
    - mic->mic_console.virtio_console_fd,
    - VIRTIO_ID_CONSOLE);
    - if (err) {
    - mpsslog("%s %s %d Exiting...\n",
    - mic->name, __func__, __LINE__);
    - break;
    - }
    - }
    -
    - if (console_poll[MONITOR_FD].revents & POLLIN) {
    - copy.iov = iov0;
    - len = readv(pty_fd, copy.iov, copy.iovcnt);
    - if (len > 0) {
    -#ifdef DEBUG
    - disp_iovec(mic, copy, __func__, __LINE__);
    - mpsslog("%s %s %d read from tap 0x%lx\n",
    - mic->name, __func__, __LINE__,
    - len);
    -#endif
    - spin_for_descriptors(mic, &tx_vr);
    - txrx_prepare(VIRTIO_ID_CONSOLE, 1, &tx_vr,
    - &copy, len);
    -
    - err = mic_virtio_copy(mic,
    - mic->mic_console.virtio_console_fd,
    - &tx_vr, &copy);
    - if (err < 0) {
    - mpsslog("%s %s %d mic_virtio_copy %s\n",
    - mic->name, __func__, __LINE__,
    - strerror(errno));
    - }
    - if (!err)
    - verify_out_len(mic, &copy);
    -#ifdef DEBUG
    - disp_iovec(mic, copy, __func__, __LINE__);
    - mpsslog("%s %s %d wrote to net 0x%lx\n",
    - mic->name, __func__, __LINE__,
    - sum_iovec_len(copy));
    -#endif
    - /* Reinitialize IOV for next run */
    - iov0->iov_len = PAGE_SIZE;
    - } else if (len < 0) {
    - disp_iovec(mic, &copy, __func__, __LINE__);
    - mpsslog("%s %s %d read failed %s ",
    - mic->name, __func__, __LINE__,
    - strerror(errno));
    - mpsslog("cnt %d sum %zd\n",
    - copy.iovcnt, sum_iovec_len(&copy));
    - }
    - }
    -
    - if (console_poll[VIRTIO_CONSOLE_FD].revents & POLLIN) {
    - while (rx_vr.info->avail_idx !=
    - le16toh(rx_vr.vr.avail->idx)) {
    - copy.iov = iov1;
    - txrx_prepare(VIRTIO_ID_CONSOLE, 0, &rx_vr,
    - &copy, PAGE_SIZE);
    -
    - err = mic_virtio_copy(mic,
    - mic->mic_console.virtio_console_fd,
    - &rx_vr, &copy);
    - if (!err) {
    - /* Set the correct output iov_len */
    - iov1->iov_len = copy.out_len;
    - verify_out_len(mic, &copy);
    -#ifdef DEBUG
    - disp_iovec(mic, copy, __func__,
    - __LINE__);
    - mpsslog("%s %s %d ",
    - mic->name, __func__, __LINE__);
    - mpsslog("read from net 0x%lx\n",
    - sum_iovec_len(copy));
    -#endif
    - len = writev(pty_fd,
    - copy.iov, copy.iovcnt);
    - if (len != sum_iovec_len(&copy)) {
    - mpsslog("Tun write failed %s ",
    - strerror(errno));
    - mpsslog("len 0x%zx ", len);
    - mpsslog("read_len 0x%zx\n",
    - sum_iovec_len(&copy));
    - } else {
    -#ifdef DEBUG
    - disp_iovec(mic, copy, __func__,
    - __LINE__);
    - mpsslog("%s %s %d ",
    - mic->name, __func__,
    - __LINE__);
    - mpsslog("wrote to tap 0x%lx\n",
    - len);
    -#endif
    - }
    - } else {
    - mpsslog("%s %s %d mic_virtio_copy %s\n",
    - mic->name, __func__, __LINE__,
    - strerror(errno));
    - break;
    - }
    - }
    - }
    - if (console_poll[NET_FD_VIRTIO_NET].revents & POLLERR)
    - mpsslog("%s: %s: POLLERR\n", __func__, mic->name);
    - }
    -_close_pty:
    - close(pty_fd);
    -_return:
    - pthread_exit(NULL);
    -}
    -
    -static void
    -add_virtio_device(struct mic_info *mic, struct mic_device_desc *dd)
    -{
    - char path[PATH_MAX];
    - int fd, err;
    -
    - snprintf(path, PATH_MAX, "/dev/vop_virtio%d", mic->id);
    - fd = open(path, O_RDWR);
    - if (fd < 0) {
    - mpsslog("Could not open %s %s\n", path, strerror(errno));
    - return;
    - }
    -
    - err = ioctl(fd, MIC_VIRTIO_ADD_DEVICE, dd);
    - if (err < 0) {
    - mpsslog("Could not add %d %s\n", dd->type, strerror(errno));
    - close(fd);
    - return;
    - }
    - switch (dd->type) {
    - case VIRTIO_ID_NET:
    - mic->mic_net.virtio_net_fd = fd;
    - mpsslog("Added VIRTIO_ID_NET for %s\n", mic->name);
    - break;
    - case VIRTIO_ID_CONSOLE:
    - mic->mic_console.virtio_console_fd = fd;
    - mpsslog("Added VIRTIO_ID_CONSOLE for %s\n", mic->name);
    - break;
    - case VIRTIO_ID_BLOCK:
    - mic->mic_virtblk.virtio_block_fd = fd;
    - mpsslog("Added VIRTIO_ID_BLOCK for %s\n", mic->name);
    - break;
    - }
    -}
    -
    -static bool
    -set_backend_file(struct mic_info *mic)
    -{
    - FILE *config;
    - char buff[PATH_MAX], *line, *evv, *p;
    -
    - snprintf(buff, PATH_MAX, "%s/mpssd%03d.conf", mic_config_dir, mic->id);
    - config = fopen(buff, "r");
    - if (config == NULL)
    - return false;
    - do { /* look for "virtblk_backend=XXXX" */
    - line = fgets(buff, PATH_MAX, config);
    - if (line == NULL)
    - break;
    - if (*line == '#')
    - continue;
    - p = strchr(line, '\n');
    - if (p)
    - *p = '\0';
    - } while (strncmp(line, virtblk_backend, strlen(virtblk_backend)) != 0);
    - fclose(config);
    - if (line == NULL)
    - return false;
    - evv = strchr(line, '=');
    - if (evv == NULL)
    - return false;
    - mic->mic_virtblk.backend_file = malloc(strlen(evv) + 1);
    - if (mic->mic_virtblk.backend_file == NULL) {
    - mpsslog("%s %d can't allocate memory\n", mic->name, mic->id);
    - return false;
    - }
    - strcpy(mic->mic_virtblk.backend_file, evv + 1);
    - return true;
    -}
    -
    -#define SECTOR_SIZE 512
    -static bool
    -set_backend_size(struct mic_info *mic)
    -{
    - mic->mic_virtblk.backend_size = lseek(mic->mic_virtblk.backend, 0,
    - SEEK_END);
    - if (mic->mic_virtblk.backend_size < 0) {
    - mpsslog("%s: can't seek: %s\n",
    - mic->name, mic->mic_virtblk.backend_file);
    - return false;
    - }
    - virtblk_dev_page.blk_config.capacity =
    - mic->mic_virtblk.backend_size / SECTOR_SIZE;
    - if ((mic->mic_virtblk.backend_size % SECTOR_SIZE) != 0)
    - virtblk_dev_page.blk_config.capacity++;
    -
    - virtblk_dev_page.blk_config.capacity =
    - htole64(virtblk_dev_page.blk_config.capacity);
    -
    - return true;
    -}
    -
    -static bool
    -open_backend(struct mic_info *mic)
    -{
    - if (!set_backend_file(mic))
    - goto _error_exit;
    - mic->mic_virtblk.backend = open(mic->mic_virtblk.backend_file, O_RDWR);
    - if (mic->mic_virtblk.backend < 0) {
    - mpsslog("%s: can't open: %s\n", mic->name,
    - mic->mic_virtblk.backend_file);
    - goto _error_free;
    - }
    - if (!set_backend_size(mic))
    - goto _error_close;
    - mic->mic_virtblk.backend_addr = mmap(NULL,
    - mic->mic_virtblk.backend_size,
    - PROT_READ|PROT_WRITE, MAP_SHARED,
    - mic->mic_virtblk.backend, 0L);
    - if (mic->mic_virtblk.backend_addr == MAP_FAILED) {
    - mpsslog("%s: can't map: %s %s\n",
    - mic->name, mic->mic_virtblk.backend_file,
    - strerror(errno));
    - goto _error_close;
    - }
    - return true;
    -
    - _error_close:
    - close(mic->mic_virtblk.backend);
    - _error_free:
    - free(mic->mic_virtblk.backend_file);
    - _error_exit:
    - return false;
    -}
    -
    -static void
    -close_backend(struct mic_info *mic)
    -{
    - munmap(mic->mic_virtblk.backend_addr, mic->mic_virtblk.backend_size);
    - close(mic->mic_virtblk.backend);
    - free(mic->mic_virtblk.backend_file);
    -}
    -
    -static bool
    -start_virtblk(struct mic_info *mic, struct mic_vring *vring)
    -{
    - if (((unsigned long)&virtblk_dev_page.blk_config % 8) != 0) {
    - mpsslog("%s: blk_config is not 8 byte aligned.\n",
    - mic->name);
    - return false;
    - }
    - add_virtio_device(mic, &virtblk_dev_page.dd);
    - if (MAP_FAILED == init_vr(mic, mic->mic_virtblk.virtio_block_fd,
    - VIRTIO_ID_BLOCK, vring, NULL,
    - virtblk_dev_page.dd.num_vq)) {
    - mpsslog("%s init_vr failed %s\n",
    - mic->name, strerror(errno));
    - return false;
    - }
    - return true;
    -}
    -
    -static void
    -stop_virtblk(struct mic_info *mic)
    -{
    - int vr_size, ret;
    -
    - vr_size = PAGE_ALIGN(_vring_size(MIC_VRING_ENTRIES,
    - MIC_VIRTIO_RING_ALIGN) +
    - sizeof(struct _mic_vring_info));
    - ret = munmap(mic->mic_virtblk.block_dp,
    - MIC_DEVICE_PAGE_END + vr_size * virtblk_dev_page.dd.num_vq);
    - if (ret < 0)
    - mpsslog("%s munmap errno %d\n", mic->name, errno);
    - close(mic->mic_virtblk.virtio_block_fd);
    -}
    -
    -static __u8
    -header_error_check(struct vring_desc *desc)
    -{
    - if (le32toh(desc->len) != sizeof(struct virtio_blk_outhdr)) {
    - mpsslog("%s() %d: length is not sizeof(virtio_blk_outhd)\n",
    - __func__, __LINE__);
    - return -EIO;
    - }
    - if (!(le16toh(desc->flags) & VRING_DESC_F_NEXT)) {
    - mpsslog("%s() %d: alone\n",
    - __func__, __LINE__);
    - return -EIO;
    - }
    - if (le16toh(desc->flags) & VRING_DESC_F_WRITE) {
    - mpsslog("%s() %d: not read\n",
    - __func__, __LINE__);
    - return -EIO;
    - }
    - return 0;
    -}
    -
    -static int
    -read_header(int fd, struct virtio_blk_outhdr *hdr, __u32 desc_idx)
    -{
    - struct iovec iovec;
    - struct mic_copy_desc copy;
    -
    - iovec.iov_len = sizeof(*hdr);
    - iovec.iov_base = hdr;
    - copy.iov = &iovec;
    - copy.iovcnt = 1;
    - copy.vr_idx = 0; /* only one vring on virtio_block */
    - copy.update_used = false; /* do not update used index */
    - return ioctl(fd, MIC_VIRTIO_COPY_DESC, &copy);
    -}
    -
    -static int
    -transfer_blocks(int fd, struct iovec *iovec, __u32 iovcnt)
    -{
    - struct mic_copy_desc copy;
    -
    - copy.iov = iovec;
    - copy.iovcnt = iovcnt;
    - copy.vr_idx = 0; /* only one vring on virtio_block */
    - copy.update_used = false; /* do not update used index */
    - return ioctl(fd, MIC_VIRTIO_COPY_DESC, &copy);
    -}
    -
    -static __u8
    -status_error_check(struct vring_desc *desc)
    -{
    - if (le32toh(desc->len) != sizeof(__u8)) {
    - mpsslog("%s() %d: length is not sizeof(status)\n",
    - __func__, __LINE__);
    - return -EIO;
    - }
    - return 0;
    -}
    -
    -static int
    -write_status(int fd, __u8 *status)
    -{
    - struct iovec iovec;
    - struct mic_copy_desc copy;
    -
    - iovec.iov_base = status;
    - iovec.iov_len = sizeof(*status);
    - copy.iov = &iovec;
    - copy.iovcnt = 1;
    - copy.vr_idx = 0; /* only one vring on virtio_block */
    - copy.update_used = true; /* Update used index */
    - return ioctl(fd, MIC_VIRTIO_COPY_DESC, &copy);
    -}
    -
    -#ifndef VIRTIO_BLK_T_GET_ID
    -#define VIRTIO_BLK_T_GET_ID 8
    -#endif
    -
    -static void *
    -virtio_block(void *arg)
    -{
    - struct mic_info *mic = (struct mic_info *)arg;
    - int ret;
    - struct pollfd block_poll;
    - struct mic_vring vring;
    - __u16 avail_idx;
    - __u32 desc_idx;
    - struct vring_desc *desc;
    - struct iovec *iovec, *piov;
    - __u8 status;
    - __u32 buffer_desc_idx;
    - struct virtio_blk_outhdr hdr;
    - void *fos;
    -
    - for (;;) { /* forever */
    - if (!open_backend(mic)) { /* No virtblk */
    - for (mic->mic_virtblk.signaled = 0;
    - !mic->mic_virtblk.signaled;)
    - sleep(1);
    - continue;
    - }
    -
    - /* backend file is specified. */
    - if (!start_virtblk(mic, &vring))
    - goto _close_backend;
    - iovec = malloc(sizeof(*iovec) *
    - le32toh(virtblk_dev_page.blk_config.seg_max));
    - if (!iovec) {
    - mpsslog("%s: can't alloc iovec: %s\n",
    - mic->name, strerror(ENOMEM));
    - goto _stop_virtblk;
    - }
    -
    - block_poll.fd = mic->mic_virtblk.virtio_block_fd;
    - block_poll.events = POLLIN;
    - for (mic->mic_virtblk.signaled = 0;
    - !mic->mic_virtblk.signaled;) {
    - block_poll.revents = 0;
    - /* timeout in 1 sec to see signaled */
    - ret = poll(&block_poll, 1, 1000);
    - if (ret < 0) {
    - mpsslog("%s %d: poll failed: %s\n",
    - __func__, __LINE__,
    - strerror(errno));
    - continue;
    - }
    -
    - if (!(block_poll.revents & POLLIN)) {
    -#ifdef DEBUG
    - mpsslog("%s %d: block_poll.revents=0x%x\n",
    - __func__, __LINE__, block_poll.revents);
    -#endif
    - continue;
    - }
    -
    - /* POLLIN */
    - while (vring.info->avail_idx !=
    - le16toh(vring.vr.avail->idx)) {
    - /* read header element */
    - avail_idx =
    - vring.info->avail_idx &
    - (vring.vr.num - 1);
    - desc_idx = le16toh(
    - vring.vr.avail->ring[avail_idx]);
    - desc = &vring.vr.desc[desc_idx];
    -#ifdef DEBUG
    - mpsslog("%s() %d: avail_idx=%d ",
    - __func__, __LINE__,
    - vring.info->avail_idx);
    - mpsslog("vring.vr.num=%d desc=%p\n",
    - vring.vr.num, desc);
    -#endif
    - status = header_error_check(desc);
    - ret = read_header(
    - mic->mic_virtblk.virtio_block_fd,
    - &hdr, desc_idx);
    - if (ret < 0) {
    - mpsslog("%s() %d %s: ret=%d %s\n",
    - __func__, __LINE__,
    - mic->name, ret,
    - strerror(errno));
    - break;
    - }
    - /* buffer element */
    - piov = iovec;
    - status = 0;
    - fos = mic->mic_virtblk.backend_addr +
    - (hdr.sector * SECTOR_SIZE);
    - buffer_desc_idx = next_desc(desc);
    - desc_idx = buffer_desc_idx;
    - for (desc = &vring.vr.desc[buffer_desc_idx];
    - desc->flags & VRING_DESC_F_NEXT;
    - desc_idx = next_desc(desc),
    - desc = &vring.vr.desc[desc_idx]) {
    - piov->iov_len = desc->len;
    - piov->iov_base = fos;
    - piov++;
    - fos += desc->len;
    - }
    - /* Returning NULLs for VIRTIO_BLK_T_GET_ID. */
    - if (hdr.type & ~(VIRTIO_BLK_T_OUT |
    - VIRTIO_BLK_T_GET_ID)) {
    - /*
    - VIRTIO_BLK_T_IN - does not do
    - anything. Probably for documenting.
    - VIRTIO_BLK_T_SCSI_CMD - for
    - virtio_scsi.
    - VIRTIO_BLK_T_FLUSH - turned off in
    - config space.
    - VIRTIO_BLK_T_BARRIER - defined but not
    - used in anywhere.
    - */
    - mpsslog("%s() %d: type %x ",
    - __func__, __LINE__,
    - hdr.type);
    - mpsslog("is not supported\n");
    - status = -ENOTSUP;
    -
    - } else {
    - ret = transfer_blocks(
    - mic->mic_virtblk.virtio_block_fd,
    - iovec,
    - piov - iovec);
    - if (ret < 0 &&
    - status != 0)
    - status = ret;
    - }
    - /* write status and update used pointer */
    - if (status != 0)
    - status = status_error_check(desc);
    - ret = write_status(
    - mic->mic_virtblk.virtio_block_fd,
    - &status);
    -#ifdef DEBUG
    - mpsslog("%s() %d: write status=%d on desc=%p\n",
    - __func__, __LINE__,
    - status, desc);
    -#endif
    - }
    - }
    - free(iovec);
    -_stop_virtblk:
    - stop_virtblk(mic);
    -_close_backend:
    - close_backend(mic);
    - } /* forever */
    -
    - pthread_exit(NULL);
    -}
    -
    -static void
    -reset(struct mic_info *mic)
    -{
    -#define RESET_TIMEOUT 120
    - int i = RESET_TIMEOUT;
    - setsysfs(mic->name, "state", "reset");
    - while (i) {
    - char *state;
    - state = readsysfs(mic->name, "state");
    - if (!state)
    - goto retry;
    - mpsslog("%s: %s %d state %s\n",
    - mic->name, __func__, __LINE__, state);
    -
    - if (!strcmp(state, "ready")) {
    - free(state);
    - break;
    - }
    - free(state);
    -retry:
    - sleep(1);
    - i--;
    - }
    -}
    -
    -static int
    -get_mic_shutdown_status(struct mic_info *mic, char *shutdown_status)
    -{
    - if (!strcmp(shutdown_status, "nop"))
    - return MIC_NOP;
    - if (!strcmp(shutdown_status, "crashed"))
    - return MIC_CRASHED;
    - if (!strcmp(shutdown_status, "halted"))
    - return MIC_HALTED;
    - if (!strcmp(shutdown_status, "poweroff"))
    - return MIC_POWER_OFF;
    - if (!strcmp(shutdown_status, "restart"))
    - return MIC_RESTART;
    - mpsslog("%s: BUG invalid status %s\n", mic->name, shutdown_status);
    - /* Invalid state */
    - assert(0);
    -};
    -
    -static int get_mic_state(struct mic_info *mic)
    -{
    - char *state = NULL;
    - enum mic_states mic_state;
    -
    - while (!state) {
    - state = readsysfs(mic->name, "state");
    - sleep(1);
    - }
    - mpsslog("%s: %s %d state %s\n",
    - mic->name, __func__, __LINE__, state);
    -
    - if (!strcmp(state, "ready")) {
    - mic_state = MIC_READY;
    - } else if (!strcmp(state, "booting")) {
    - mic_state = MIC_BOOTING;
    - } else if (!strcmp(state, "online")) {
    - mic_state = MIC_ONLINE;
    - } else if (!strcmp(state, "shutting_down")) {
    - mic_state = MIC_SHUTTING_DOWN;
    - } else if (!strcmp(state, "reset_failed")) {
    - mic_state = MIC_RESET_FAILED;
    - } else if (!strcmp(state, "resetting")) {
    - mic_state = MIC_RESETTING;
    - } else {
    - mpsslog("%s: BUG invalid state %s\n", mic->name, state);
    - assert(0);
    - }
    -
    - free(state);
    - return mic_state;
    -};
    -
    -static void mic_handle_shutdown(struct mic_info *mic)
    -{
    -#define SHUTDOWN_TIMEOUT 60
    - int i = SHUTDOWN_TIMEOUT;
    - char *shutdown_status;
    - while (i) {
    - shutdown_status = readsysfs(mic->name, "shutdown_status");
    - if (!shutdown_status) {
    - sleep(1);
    - continue;
    - }
    - mpsslog("%s: %s %d shutdown_status %s\n",
    - mic->name, __func__, __LINE__, shutdown_status);
    - switch (get_mic_shutdown_status(mic, shutdown_status)) {
    - case MIC_RESTART:
    - mic->restart = 1;
    - case MIC_HALTED:
    - case MIC_POWER_OFF:
    - case MIC_CRASHED:
    - free(shutdown_status);
    - goto reset;
    - default:
    - break;
    - }
    - free(shutdown_status);
    - sleep(1);
    - i--;
    - }
    -reset:
    - if (!i)
    - mpsslog("%s: %s %d timing out waiting for shutdown_status %s\n",
    - mic->name, __func__, __LINE__, shutdown_status);
    - reset(mic);
    -}
    -
    -static int open_state_fd(struct mic_info *mic)
    -{
    - char pathname[PATH_MAX];
    - int fd;
    -
    - snprintf(pathname, PATH_MAX - 1, "%s/%s/%s",
    - MICSYSFSDIR, mic->name, "state");
    -
    - fd = open(pathname, O_RDONLY);
    - if (fd < 0)
    - mpsslog("%s: opening file %s failed %s\n",
    - mic->name, pathname, strerror(errno));
    - return fd;
    -}
    -
    -static int block_till_state_change(int fd, struct mic_info *mic)
    -{
    - struct pollfd ufds[1];
    - char value[PAGE_SIZE];
    - int ret;
    -
    - ufds[0].fd = fd;
    - ufds[0].events = POLLERR | POLLPRI;
    - ret = poll(ufds, 1, -1);
    - if (ret < 0) {
    - mpsslog("%s: %s %d poll failed %s\n",
    - mic->name, __func__, __LINE__, strerror(errno));
    - return ret;
    - }
    -
    - ret = lseek(fd, 0, SEEK_SET);
    - if (ret < 0) {
    - mpsslog("%s: %s %d Failed to seek to 0: %s\n",
    - mic->name, __func__, __LINE__, strerror(errno));
    - return ret;
    - }
    -
    - ret = read(fd, value, sizeof(value));
    - if (ret < 0) {
    - mpsslog("%s: %s %d Failed to read sysfs entry: %s\n",
    - mic->name, __func__, __LINE__, strerror(errno));
    - return ret;
    - }
    -
    - return 0;
    -}
    -
    -static void *
    -mic_config(void *arg)
    -{
    - struct mic_info *mic = (struct mic_info *)arg;
    - int fd, ret, stat = 0;
    -
    - fd = open_state_fd(mic);
    - if (fd < 0) {
    - mpsslog("%s: %s %d open state fd failed %s\n",
    - mic->name, __func__, __LINE__, strerror(errno));
    - goto exit;
    - }
    -
    - do {
    - ret = block_till_state_change(fd, mic);
    - if (ret < 0) {
    - mpsslog("%s: %s %d block_till_state_change error %s\n",
    - mic->name, __func__, __LINE__, strerror(errno));
    - goto close_exit;
    - }
    -
    - switch (get_mic_state(mic)) {
    - case MIC_SHUTTING_DOWN:
    - mic_handle_shutdown(mic);
    - break;
    - case MIC_READY:
    - case MIC_RESET_FAILED:
    - ret = kill(mic->pid, SIGTERM);
    - mpsslog("%s: %s %d kill pid %d ret %d\n",
    - mic->name, __func__, __LINE__,
    - mic->pid, ret);
    - if (!ret) {
    - ret = waitpid(mic->pid, &stat,
    - WIFSIGNALED(stat));
    - mpsslog("%s: %s %d waitpid ret %d pid %d\n",
    - mic->name, __func__, __LINE__,
    - ret, mic->pid);
    - }
    - if (mic->boot_on_resume) {
    - setsysfs(mic->name, "state", "boot");
    - mic->boot_on_resume = 0;
    - }
    - goto close_exit;
    - default:
    - break;
    - }
    - } while (1);
    -
    -close_exit:
    - close(fd);
    -exit:
    - init_mic(mic);
    - pthread_exit(NULL);
    -}
    -
    -static void
    -set_cmdline(struct mic_info *mic)
    -{
    - char buffer[PATH_MAX];
    - int len;
    -
    - len = snprintf(buffer, PATH_MAX,
    - "clocksource=tsc highres=off nohz=off ");
    - len += snprintf(buffer + len, PATH_MAX - len,
    - "cpufreq_on;corec6_off;pc3_off;pc6_off ");
    - len += snprintf(buffer + len, PATH_MAX - len,
    - "ifcfg=static;address,172.31.%d.1;netmask,255.255.255.0",
    - mic->id + 1);
    -
    - setsysfs(mic->name, "cmdline", buffer);
    - mpsslog("%s: Command line: \"%s\"\n", mic->name, buffer);
    - snprintf(buffer, PATH_MAX, "172.31.%d.1", mic->id + 1);
    - mpsslog("%s: IPADDR: \"%s\"\n", mic->name, buffer);
    -}
    -
    -static void
    -set_log_buf_info(struct mic_info *mic)
    -{
    - int fd;
    - off_t len;
    - char system_map[] = "/lib/firmware/mic/System.map";
    - char *map, *temp, log_buf[17] = {'\0'};
    -
    - fd = open(system_map, O_RDONLY);
    - if (fd < 0) {
    - mpsslog("%s: Opening System.map failed: %d\n",
    - mic->name, errno);
    - return;
    - }
    - len = lseek(fd, 0, SEEK_END);
    - if (len < 0) {
    - mpsslog("%s: Reading System.map size failed: %d\n",
    - mic->name, errno);
    - close(fd);
    - return;
    - }
    - map = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0);
    - if (map == MAP_FAILED) {
    - mpsslog("%s: mmap of System.map failed: %d\n",
    - mic->name, errno);
    - close(fd);
    - return;
    - }
    - temp = strstr(map, "__log_buf");
    - if (!temp) {
    - mpsslog("%s: __log_buf not found: %d\n", mic->name, errno);
    - munmap(map, len);
    - close(fd);
    - return;
    - }
    - strncpy(log_buf, temp - 19, 16);
    - setsysfs(mic->name, "log_buf_addr", log_buf);
    - mpsslog("%s: log_buf_addr: %s\n", mic->name, log_buf);
    - temp = strstr(map, "log_buf_len");
    - if (!temp) {
    - mpsslog("%s: log_buf_len not found: %d\n", mic->name, errno);
    - munmap(map, len);
    - close(fd);
    - return;
    - }
    - strncpy(log_buf, temp - 19, 16);
    - setsysfs(mic->name, "log_buf_len", log_buf);
    - mpsslog("%s: log_buf_len: %s\n", mic->name, log_buf);
    - munmap(map, len);
    - close(fd);
    -}
    -
    -static void
    -change_virtblk_backend(int x, siginfo_t *siginfo, void *p)
    -{
    - struct mic_info *mic;
    -
    - for (mic = mic_list.next; mic != NULL; mic = mic->next)
    - mic->mic_virtblk.signaled = 1/* true */;
    -}
    -
    -static void
    -set_mic_boot_params(struct mic_info *mic)
    -{
    - set_log_buf_info(mic);
    - set_cmdline(mic);
    -}
    -
    -static void *
    -init_mic(void *arg)
    -{
    - struct mic_info *mic = (struct mic_info *)arg;
    - struct sigaction ignore = {
    - .sa_flags = 0,
    - .sa_handler = SIG_IGN
    - };
    - struct sigaction act = {
    - .sa_flags = SA_SIGINFO,
    - .sa_sigaction = change_virtblk_backend,
    - };
    - char buffer[PATH_MAX];
    - int err, fd;
    -
    - /*
    - * Currently, one virtio block device is supported for each MIC card
    - * at a time. Any user (or test) can send a SIGUSR1 to the MIC daemon.
    - * The signal informs the virtio block backend about a change in the
    - * configuration file which specifies the virtio backend file name on
    - * the host. Virtio block backend then re-reads the configuration file
    - * and switches to the new block device. This signalling mechanism may
    - * not be required once multiple virtio block devices are supported by
    - * the MIC daemon.
    - */
    - sigaction(SIGUSR1, &ignore, NULL);
    -retry:
    - fd = open_state_fd(mic);
    - if (fd < 0) {
    - mpsslog("%s: %s %d open state fd failed %s\n",
    - mic->name, __func__, __LINE__, strerror(errno));
    - sleep(2);
    - goto retry;
    - }
    -
    - if (mic->restart) {
    - snprintf(buffer, PATH_MAX, "boot");
    - setsysfs(mic->name, "state", buffer);
    - mpsslog("%s restarting mic %d\n",
    - mic->name, mic->restart);
    - mic->restart = 0;
    - }
    -
    - while (1) {
    - while (block_till_state_change(fd, mic)) {
    - mpsslog("%s: %s %d block_till_state_change error %s\n",
    - mic->name, __func__, __LINE__, strerror(errno));
    - sleep(2);
    - continue;
    - }
    -
    - if (get_mic_state(mic) == MIC_BOOTING)
    - break;
    - }
    -
    - mic->pid = fork();
    - switch (mic->pid) {
    - case 0:
    - add_virtio_device(mic, &virtcons_dev_page.dd);
    - add_virtio_device(mic, &virtnet_dev_page.dd);
    - err = pthread_create(&mic->mic_console.console_thread, NULL,
    - virtio_console, mic);
    - if (err)
    - mpsslog("%s virtcons pthread_create failed %s\n",
    - mic->name, strerror(err));
    - err = pthread_create(&mic->mic_net.net_thread, NULL,
    - virtio_net, mic);
    - if (err)
    - mpsslog("%s virtnet pthread_create failed %s\n",
    - mic->name, strerror(err));
    - err = pthread_create(&mic->mic_virtblk.block_thread, NULL,
    - virtio_block, mic);
    - if (err)
    - mpsslog("%s virtblk pthread_create failed %s\n",
    - mic->name, strerror(err));
    - sigemptyset(&act.sa_mask);
    - err = sigaction(SIGUSR1, &act, NULL);
    - if (err)
    - mpsslog("%s sigaction SIGUSR1 failed %s\n",
    - mic->name, strerror(errno));
    - while (1)
    - sleep(60);
    - case -1:
    - mpsslog("fork failed MIC name %s id %d errno %d\n",
    - mic->name, mic->id, errno);
    - break;
    - default:
    - err = pthread_create(&mic->config_thread, NULL,
    - mic_config, mic);
    - if (err)
    - mpsslog("%s mic_config pthread_create failed %s\n",
    - mic->name, strerror(err));
    - }
    -
    - return NULL;
    -}
    -
    -static void
    -start_daemon(void)
    -{
    - struct mic_info *mic;
    - int err;
    -
    - for (mic = mic_list.next; mic; mic = mic->next) {
    - set_mic_boot_params(mic);
    - err = pthread_create(&mic->init_thread, NULL, init_mic, mic);
    - if (err)
    - mpsslog("%s init_mic pthread_create failed %s\n",
    - mic->name, strerror(err));
    - }
    -
    - while (1)
    - sleep(60);
    -}
    -
    -static int
    -init_mic_list(void)
    -{
    - struct mic_info *mic = &mic_list;
    - struct dirent *file;
    - DIR *dp;
    - int cnt = 0;
    -
    - dp = opendir(MICSYSFSDIR);
    - if (!dp)
    - return 0;
    -
    - while ((file = readdir(dp)) != NULL) {
    - if (!strncmp(file->d_name, "mic", 3)) {
    - mic->next = calloc(1, sizeof(struct mic_info));
    - if (mic->next) {
    - mic = mic->next;
    - mic->id = atoi(&file->d_name[3]);
    - mic->name = malloc(strlen(file->d_name) + 16);
    - if (mic->name)
    - strcpy(mic->name, file->d_name);
    - mpsslog("MIC name %s id %d\n", mic->name,
    - mic->id);
    - cnt++;
    - }
    - }
    - }
    -
    - closedir(dp);
    - return cnt;
    -}
    -
    -void
    -mpsslog(char *format, ...)
    -{
    - va_list args;
    - char buffer[4096];
    - char ts[52], *ts1;
    - time_t t;
    -
    - if (logfp == NULL)
    - return;
    -
    - va_start(args, format);
    - vsprintf(buffer, format, args);
    - va_end(args);
    -
    - time(&t);
    - ts1 = ctime_r(&t, ts);
    - ts1[strlen(ts1) - 1] = '\0';
    - fprintf(logfp, "%s: %s", ts1, buffer);
    -
    - fflush(logfp);
    -}
    -
    -int
    -main(int argc, char *argv[])
    -{
    - int cnt;
    - pid_t pid;
    -
    - myname = argv[0];
    -
    - logfp = fopen(LOGFILE_NAME, "a+");
    - if (!logfp) {
    - fprintf(stderr, "cannot open logfile '%s'\n", LOGFILE_NAME);
    - exit(1);
    - }
    - pid = fork();
    - switch (pid) {
    - case 0:
    - break;
    - case -1:
    - exit(2);
    - default:
    - exit(0);
    - }
    -
    - mpsslog("MIC Daemon start\n");
    -
    - cnt = init_mic_list();
    - if (cnt == 0) {
    - mpsslog("MIC module not loaded\n");
    - exit(3);
    - }
    - mpsslog("MIC found %d devices\n", cnt);
    -
    - start_daemon();
    -
    - exit(0);
    -}
    diff --git a/Documentation/mic/mpssd/mpssd.h b/Documentation/mic/mpssd/mpssd.h
    deleted file mode 100644
    index 8bd6494..0000000
    --- a/Documentation/mic/mpssd/mpssd.h
    +++ /dev/null
    @@ -1,103 +0,0 @@
    -/*
    - * Intel MIC Platform Software Stack (MPSS)
    - *
    - * Copyright(c) 2013 Intel Corporation.
    - *
    - * This program is free software; you can redistribute it and/or modify
    - * it under the terms of the GNU General Public License, version 2, as
    - * published by the Free Software Foundation.
    - *
    - * This program is distributed in the hope that it will be useful, but
    - * WITHOUT ANY WARRANTY; without even the implied warranty of
    - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
    - * General Public License for more details.
    - *
    - * The full GNU General Public License is included in this distribution in
    - * the file called "COPYING".
    - *
    - * Intel MIC User Space Tools.
    - */
    -#ifndef _MPSSD_H_
    -#define _MPSSD_H_
    -
    -#include <stdio.h>
    -#include <stdlib.h>
    -#include <string.h>
    -#include <fcntl.h>
    -#include <unistd.h>
    -#include <dirent.h>
    -#include <libgen.h>
    -#include <pthread.h>
    -#include <stdarg.h>
    -#include <time.h>
    -#include <errno.h>
    -#include <sys/dir.h>
    -#include <sys/ioctl.h>
    -#include <sys/poll.h>
    -#include <sys/types.h>
    -#include <sys/socket.h>
    -#include <sys/stat.h>
    -#include <sys/types.h>
    -#include <sys/mman.h>
    -#include <sys/utsname.h>
    -#include <sys/wait.h>
    -#include <netinet/in.h>
    -#include <arpa/inet.h>
    -#include <netdb.h>
    -#include <pthread.h>
    -#include <signal.h>
    -#include <limits.h>
    -#include <syslog.h>
    -#include <getopt.h>
    -#include <net/if.h>
    -#include <linux/if_tun.h>
    -#include <linux/if_tun.h>
    -#include <linux/virtio_ids.h>
    -
    -#define MICSYSFSDIR "/sys/class/mic"
    -#define LOGFILE_NAME "/var/log/mpssd"
    -#define PAGE_SIZE 4096
    -
    -struct mic_console_info {
    - pthread_t console_thread;
    - int virtio_console_fd;
    - void *console_dp;
    -};
    -
    -struct mic_net_info {
    - pthread_t net_thread;
    - int virtio_net_fd;
    - int tap_fd;
    - void *net_dp;
    -};
    -
    -struct mic_virtblk_info {
    - pthread_t block_thread;
    - int virtio_block_fd;
    - void *block_dp;
    - volatile sig_atomic_t signaled;
    - char *backend_file;
    - int backend;
    - void *backend_addr;
    - long backend_size;
    -};
    -
    -struct mic_info {
    - int id;
    - char *name;
    - pthread_t config_thread;
    - pthread_t init_thread;
    - pid_t pid;
    - struct mic_console_info mic_console;
    - struct mic_net_info mic_net;
    - struct mic_virtblk_info mic_virtblk;
    - int restart;
    - int boot_on_resume;
    - struct mic_info *next;
    -};
    -
    -__attribute__((format(printf, 1, 2)))
    -void mpsslog(char *format, ...);
    -char *readsysfs(char *dir, char *entry);
    -int setsysfs(char *dir, char *entry, char *value);
    -#endif
    diff --git a/Documentation/mic/mpssd/sysfs.c b/Documentation/mic/mpssd/sysfs.c
    deleted file mode 100644
    index 8dd3269..0000000
    --- a/Documentation/mic/mpssd/sysfs.c
    +++ /dev/null
    @@ -1,102 +0,0 @@
    -/*
    - * Intel MIC Platform Software Stack (MPSS)
    - *
    - * Copyright(c) 2013 Intel Corporation.
    - *
    - * This program is free software; you can redistribute it and/or modify
    - * it under the terms of the GNU General Public License, version 2, as
    - * published by the Free Software Foundation.
    - *
    - * This program is distributed in the hope that it will be useful, but
    - * WITHOUT ANY WARRANTY; without even the implied warranty of
    - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
    - * General Public License for more details.
    - *
    - * The full GNU General Public License is included in this distribution in
    - * the file called "COPYING".
    - *
    - * Intel MIC User Space Tools.
    - */
    -
    -#include "mpssd.h"
    -
    -#define PAGE_SIZE 4096
    -
    -char *
    -readsysfs(char *dir, char *entry)
    -{
    - char filename[PATH_MAX];
    - char value[PAGE_SIZE];
    - char *string = NULL;
    - int fd;
    - int len;
    -
    - if (dir == NULL)
    - snprintf(filename, PATH_MAX, "%s/%s", MICSYSFSDIR, entry);
    - else
    - snprintf(filename, PATH_MAX,
    - "%s/%s/%s", MICSYSFSDIR, dir, entry);
    -
    - fd = open(filename, O_RDONLY);
    - if (fd < 0) {
    - mpsslog("Failed to open sysfs entry '%s': %s\n",
    - filename, strerror(errno));
    - return NULL;
    - }
    -
    - len = read(fd, value, sizeof(value));
    - if (len < 0) {
    - mpsslog("Failed to read sysfs entry '%s': %s\n",
    - filename, strerror(errno));
    - goto readsys_ret;
    - }
    - if (len == 0)
    - goto readsys_ret;
    -
    - value[len - 1] = '\0';
    -
    - string = malloc(strlen(value) + 1);
    - if (string)
    - strcpy(string, value);
    -
    -readsys_ret:
    - close(fd);
    - return string;
    -}
    -
    -int
    -setsysfs(char *dir, char *entry, char *value)
    -{
    - char filename[PATH_MAX];
    - char *oldvalue;
    - int fd, ret = 0;
    -
    - if (dir == NULL)
    - snprintf(filename, PATH_MAX, "%s/%s", MICSYSFSDIR, entry);
    - else
    - snprintf(filename, PATH_MAX, "%s/%s/%s",
    - MICSYSFSDIR, dir, entry);
    -
    - oldvalue = readsysfs(dir, entry);
    -
    - fd = open(filename, O_RDWR);
    - if (fd < 0) {
    - ret = errno;
    - mpsslog("Failed to open sysfs entry '%s': %s\n",
    - filename, strerror(errno));
    - goto done;
    - }
    -
    - if (!oldvalue || strcmp(value, oldvalue)) {
    - if (write(fd, value, strlen(value)) < 0) {
    - ret = errno;
    - mpsslog("Failed to write new sysfs entry '%s': %s\n",
    - filename, strerror(errno));
    - }
    - }
    - close(fd);
    -done:
    - if (oldvalue)
    - free(oldvalue);
    - return ret;
    -}
    diff --git a/samples/mic/mpssd/.gitignore b/samples/mic/mpssd/.gitignore
    new file mode 100644
    index 0000000..8b7c72f
    --- /dev/null
    +++ b/samples/mic/mpssd/.gitignore
    @@ -0,0 +1 @@
    +mpssd
    diff --git a/samples/mic/mpssd/Makefile b/samples/mic/mpssd/Makefile
    new file mode 100644
    index 0000000..3e3ef91
    --- /dev/null
    +++ b/samples/mic/mpssd/Makefile
    @@ -0,0 +1,27 @@
    +ifndef CROSS_COMPILE
    +uname_M := $(shell uname -m 2>/dev/null || echo not)
    +ARCH ?= $(shell echo $(uname_M) | sed -e s/i.86/x86/ -e s/x86_64/x86/)
    +
    +ifeq ($(ARCH),x86)
    +
    +PROGS := mpssd
    +CC = $(CROSS_COMPILE)gcc
    +CFLAGS := -I../../../usr/include -I../../../tools/include
    +
    +ifdef DEBUG
    +CFLAGS += -DDEBUG=$(DEBUG)
    +endif
    +
    +all: $(PROGS)
    +mpssd: mpssd.c sysfs.c
    + $(CC) $(CFLAGS) mpssd.c sysfs.c -o mpssd -lpthread
    +
    +install:
    + install mpssd /usr/sbin/mpssd
    + install micctrl /usr/sbin/micctrl
    +
    +clean:
    + rm -fr $(PROGS)
    +
    +endif
    +endif
    diff --git a/samples/mic/mpssd/micctrl b/samples/mic/mpssd/micctrl
    new file mode 100755
    index 0000000..8f2629b
    --- /dev/null
    +++ b/samples/mic/mpssd/micctrl
    @@ -0,0 +1,173 @@
    +#!/bin/bash
    +# Intel MIC Platform Software Stack (MPSS)
    +#
    +# Copyright(c) 2013 Intel Corporation.
    +#
    +# This program is free software; you can redistribute it and/or modify
    +# it under the terms of the GNU General Public License, version 2, as
    +# published by the Free Software Foundation.
    +#
    +# This program is distributed in the hope that it will be useful, but
    +# WITHOUT ANY WARRANTY; without even the implied warranty of
    +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
    +# General Public License for more details.
    +#
    +# The full GNU General Public License is included in this distribution in
    +# the file called "COPYING".
    +#
    +# Intel MIC User Space Tools.
    +#
    +# micctrl - Controls MIC boot/start/stop.
    +#
    +# chkconfig: 2345 95 05
    +# description: start MPSS stack processing.
    +#
    +### BEGIN INIT INFO
    +# Provides: micctrl
    +### END INIT INFO
    +
    +# Source function library.
    +. /etc/init.d/functions
    +
    +sysfs="/sys/class/mic"
    +
    +_status()
    +{
    + f=$sysfs/$1
    + echo -e $1 state: "`cat $f/state`" shutdown_status: "`cat $f/shutdown_status`"
    +}
    +
    +status()
    +{
    + if [ "`echo $1 | head -c3`" == "mic" ]; then
    + _status $1
    + return $?
    + fi
    + for f in $sysfs/*
    + do
    + _status `basename $f`
    + RETVAL=$?
    + [ $RETVAL -ne 0 ] && return $RETVAL
    + done
    + return 0
    +}
    +
    +_reset()
    +{
    + f=$sysfs/$1
    + echo reset > $f/state
    +}
    +
    +reset()
    +{
    + if [ "`echo $1 | head -c3`" == "mic" ]; then
    + _reset $1
    + return $?
    + fi
    + for f in $sysfs/*
    + do
    + _reset `basename $f`
    + RETVAL=$?
    + [ $RETVAL -ne 0 ] && return $RETVAL
    + done
    + return 0
    +}
    +
    +_boot()
    +{
    + f=$sysfs/$1
    + echo "linux" > $f/bootmode
    + echo "mic/uos.img" > $f/firmware
    + echo "mic/$1.image" > $f/ramdisk
    + echo "boot" > $f/state
    +}
    +
    +boot()
    +{
    + if [ "`echo $1 | head -c3`" == "mic" ]; then
    + _boot $1
    + return $?
    + fi
    + for f in $sysfs/*
    + do
    + _boot `basename $f`
    + RETVAL=$?
    + [ $RETVAL -ne 0 ] && return $RETVAL
    + done
    + return 0
    +}
    +
    +_shutdown()
    +{
    + f=$sysfs/$1
    + echo shutdown > $f/state
    +}
    +
    +shutdown()
    +{
    + if [ "`echo $1 | head -c3`" == "mic" ]; then
    + _shutdown $1
    + return $?
    + fi
    + for f in $sysfs/*
    + do
    + _shutdown `basename $f`
    + RETVAL=$?
    + [ $RETVAL -ne 0 ] && return $RETVAL
    + done
    + return 0
    +}
    +
    +_wait()
    +{
    + f=$sysfs/$1
    + while [ "`cat $f/state`" != "offline" -a "`cat $f/state`" != "online" ]
    + do
    + sleep 1
    + echo -e "Waiting for $1 to go offline"
    + done
    +}
    +
    +wait()
    +{
    + if [ "`echo $1 | head -c3`" == "mic" ]; then
    + _wait $1
    + return $?
    + fi
    + # Wait for the cards to go offline
    + for f in $sysfs/*
    + do
    + _wait `basename $f`
    + RETVAL=$?
    + [ $RETVAL -ne 0 ] && return $RETVAL
    + done
    + return 0
    +}
    +
    +if [ ! -d "$sysfs" ]; then
    + echo -e $"Module unloaded "
    + exit 3
    +fi
    +
    +case $1 in
    + -s)
    + status $2
    + ;;
    + -r)
    + reset $2
    + ;;
    + -b)
    + boot $2
    + ;;
    + -S)
    + shutdown $2
    + ;;
    + -w)
    + wait $2
    + ;;
    + *)
    + echo $"Usage: $0 {-s (status) |-r (reset) |-b (boot) |-S (shutdown) |-w (wait)}"
    + exit 2
    +esac
    +
    +exit $?
    diff --git a/samples/mic/mpssd/mpss b/samples/mic/mpssd/mpss
    new file mode 100755
    index 0000000..5fcf9fa
    --- /dev/null
    +++ b/samples/mic/mpssd/mpss
    @@ -0,0 +1,200 @@
    +#!/bin/bash
    +# Intel MIC Platform Software Stack (MPSS)
    +#
    +# Copyright(c) 2013 Intel Corporation.
    +#
    +# This program is free software; you can redistribute it and/or modify
    +# it under the terms of the GNU General Public License, version 2, as
    +# published by the Free Software Foundation.
    +#
    +# This program is distributed in the hope that it will be useful, but
    +# WITHOUT ANY WARRANTY; without even the implied warranty of
    +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
    +# General Public License for more details.
    +#
    +# The full GNU General Public License is included in this distribution in
    +# the file called "COPYING".
    +#
    +# Intel MIC User Space Tools.
    +#
    +# mpss Start mpssd.
    +#
    +# chkconfig: 2345 95 05
    +# description: start MPSS stack processing.
    +#
    +### BEGIN INIT INFO
    +# Provides: mpss
    +# Required-Start:
    +# Required-Stop:
    +# Short-Description: MPSS stack control
    +# Description: MPSS stack control
    +### END INIT INFO
    +
    +# Source function library.
    +. /etc/init.d/functions
    +
    +exec=/usr/sbin/mpssd
    +sysfs="/sys/class/mic"
    +mic_modules="mic_host mic_x100_dma scif vop"
    +
    +start()
    +{
    + [ -x $exec ] || exit 5
    +
    + if [ "`ps -e | awk '{print $4}' | grep mpssd | head -1`" = "mpssd" ]; then
    + echo -e $"MPSSD already running! "
    + success
    + echo
    + return 0
    + fi
    +
    + echo -e $"Starting MPSS Stack"
    + echo -e $"Loading MIC drivers:" $mic_modules
    +
    + modprobe -a $mic_modules
    + RETVAL=$?
    + if [ $RETVAL -ne 0 ]; then
    + failure
    + echo
    + return $RETVAL
    + fi
    +
    + # Start the daemon
    + echo -n $"Starting MPSSD "
    + $exec
    + RETVAL=$?
    + if [ $RETVAL -ne 0 ]; then
    + failure
    + echo
    + return $RETVAL
    + fi
    + success
    + echo
    +
    + sleep 5
    +
    + # Boot the cards
    + micctrl -b
    +
    + # Wait till ping works
    + for f in $sysfs/*
    + do
    + count=100
    + ipaddr=`cat $f/cmdline`
    + ipaddr=${ipaddr#*address,}
    + ipaddr=`echo $ipaddr | cut -d, -f1 | cut -d\; -f1`
    + while [ $count -ge 0 ]
    + do
    + echo -e "Pinging "`basename $f`" "
    + ping -c 1 $ipaddr &> /dev/null
    + RETVAL=$?
    + if [ $RETVAL -eq 0 ]; then
    + success
    + break
    + fi
    + sleep 1
    + count=`expr $count - 1`
    + done
    + [ $RETVAL -ne 0 ] && failure || success
    + echo
    + done
    + return $RETVAL
    +}
    +
    +stop()
    +{
    + echo -e $"Shutting down MPSS Stack: "
    +
    + # Bail out if module is unloaded
    + if [ ! -d "$sysfs" ]; then
    + echo -n $"Module unloaded "
    + success
    + echo
    + return 0
    + fi
    +
    + # Shut down the cards.
    + micctrl -S
    +
    + # Wait for the cards to go offline
    + for f in $sysfs/*
    + do
    + while [ "`cat $f/state`" != "ready" ]
    + do
    + sleep 1
    + echo -e "Waiting for "`basename $f`" to become ready"
    + done
    + done
    +
    + # Display the status of the cards
    + micctrl -s
    +
    + # Kill MPSSD now
    + echo -n $"Killing MPSSD"
    + killall -9 mpssd 2>/dev/null
    + RETVAL=$?
    + [ $RETVAL -ne 0 ] && failure || success
    + echo
    + return $RETVAL
    +}
    +
    +restart()
    +{
    + stop
    + sleep 5
    + start
    +}
    +
    +status()
    +{
    + micctrl -s
    + if [ "`ps -e | awk '{print $4}' | grep mpssd | head -n 1`" = "mpssd" ]; then
    + echo "mpssd is running"
    + else
    + echo "mpssd is stopped"
    + fi
    + return 0
    +}
    +
    +unload()
    +{
    + if [ ! -d "$sysfs" ]; then
    + echo -n $"No MIC_HOST Module: "
    + success
    + echo
    + return
    + fi
    +
    + stop
    +
    + sleep 5
    + echo -n $"Removing MIC drivers:" $mic_modules
    + modprobe -r $mic_modules
    + RETVAL=$?
    + [ $RETVAL -ne 0 ] && failure || success
    + echo
    + return $RETVAL
    +}
    +
    +case $1 in
    + start)
    + start
    + ;;
    + stop)
    + stop
    + ;;
    + restart)
    + restart
    + ;;
    + status)
    + status
    + ;;
    + unload)
    + unload
    + ;;
    + *)
    + echo $"Usage: $0 {start|stop|restart|status|unload}"
    + exit 2
    +esac
    +
    +exit $?
    diff --git a/samples/mic/mpssd/mpssd.c b/samples/mic/mpssd/mpssd.c
    new file mode 100644
    index 0000000..49db1de
    --- /dev/null
    +++ b/samples/mic/mpssd/mpssd.c
    @@ -0,0 +1,1826 @@
    +/*
    + * Intel MIC Platform Software Stack (MPSS)
    + *
    + * Copyright(c) 2013 Intel Corporation.
    + *
    + * This program is free software; you can redistribute it and/or modify
    + * it under the terms of the GNU General Public License, version 2, as
    + * published by the Free Software Foundation.
    + *
    + * This program is distributed in the hope that it will be useful, but
    + * WITHOUT ANY WARRANTY; without even the implied warranty of
    + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
    + * General Public License for more details.
    + *
    + * The full GNU General Public License is included in this distribution in
    + * the file called "COPYING".
    + *
    + * Intel MIC User Space Tools.
    + */
    +
    +#define _GNU_SOURCE
    +
    +#include <stdlib.h>
    +#include <fcntl.h>
    +#include <getopt.h>
    +#include <assert.h>
    +#include <unistd.h>
    +#include <stdbool.h>
    +#include <signal.h>
    +#include <poll.h>
    +#include <features.h>
    +#include <sys/types.h>
    +#include <sys/stat.h>
    +#include <sys/mman.h>
    +#include <sys/socket.h>
    +#include <linux/virtio_ring.h>
    +#include <linux/virtio_net.h>
    +#include <linux/virtio_console.h>
    +#include <linux/virtio_blk.h>
    +#include <linux/version.h>
    +#include "mpssd.h"
    +#include <linux/mic_ioctl.h>
    +#include <linux/mic_common.h>
    +#include <tools/endian.h>
    +
    +static void *init_mic(void *arg);
    +
    +static FILE *logfp;
    +static struct mic_info mic_list;
    +
    +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
    +
    +#define min_t(type, x, y) ({ \
    + type __min1 = (x); \
    + type __min2 = (y); \
    + __min1 < __min2 ? __min1 : __min2; })
    +
    +/* align addr on a size boundary - adjust address up/down if needed */
    +#define _ALIGN_DOWN(addr, size) ((addr)&(~((size)-1)))
    +#define _ALIGN_UP(addr, size) _ALIGN_DOWN(addr + size - 1, size)
    +
    +/* align addr on a size boundary - adjust address up if needed */
    +#define _ALIGN(addr, size) _ALIGN_UP(addr, size)
    +
    +/* to align the pointer to the (next) page boundary */
    +#define PAGE_ALIGN(addr) _ALIGN(addr, PAGE_SIZE)
    +
    +#define ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x))
    +
    +#define GSO_ENABLED 1
    +#define MAX_GSO_SIZE (64 * 1024)
    +#define ETH_H_LEN 14
    +#define MAX_NET_PKT_SIZE (_ALIGN_UP(MAX_GSO_SIZE + ETH_H_LEN, 64))
    +#define MIC_DEVICE_PAGE_END 0x1000
    +
    +#ifndef VIRTIO_NET_HDR_F_DATA_VALID
    +#define VIRTIO_NET_HDR_F_DATA_VALID 2 /* Csum is valid */
    +#endif
    +
    +static struct {
    + struct mic_device_desc dd;
    + struct mic_vqconfig vqconfig[2];
    + __u32 host_features, guest_acknowledgements;
    + struct virtio_console_config cons_config;
    +} virtcons_dev_page = {
    + .dd = {
    + .type = VIRTIO_ID_CONSOLE,
    + .num_vq = ARRAY_SIZE(virtcons_dev_page.vqconfig),
    + .feature_len = sizeof(virtcons_dev_page.host_features),
    + .config_len = sizeof(virtcons_dev_page.cons_config),
    + },
    + .vqconfig[0] = {
    + .num = htole16(MIC_VRING_ENTRIES),
    + },
    + .vqconfig[1] = {
    + .num = htole16(MIC_VRING_ENTRIES),
    + },
    +};
    +
    +static struct {
    + struct mic_device_desc dd;
    + struct mic_vqconfig vqconfig[2];
    + __u32 host_features, guest_acknowledgements;
    + struct virtio_net_config net_config;
    +} virtnet_dev_page = {
    + .dd = {
    + .type = VIRTIO_ID_NET,
    + .num_vq = ARRAY_SIZE(virtnet_dev_page.vqconfig),
    + .feature_len = sizeof(virtnet_dev_page.host_features),
    + .config_len = sizeof(virtnet_dev_page.net_config),
    + },
    + .vqconfig[0] = {
    + .num = htole16(MIC_VRING_ENTRIES),
    + },
    + .vqconfig[1] = {
    + .num = htole16(MIC_VRING_ENTRIES),
    + },
    +#if GSO_ENABLED
    + .host_features = htole32(
    + 1 << VIRTIO_NET_F_CSUM |
    + 1 << VIRTIO_NET_F_GSO |
    + 1 << VIRTIO_NET_F_GUEST_TSO4 |
    + 1 << VIRTIO_NET_F_GUEST_TSO6 |
    + 1 << VIRTIO_NET_F_GUEST_ECN),
    +#else
    + .host_features = 0,
    +#endif
    +};
    +
    +static const char *mic_config_dir = "/etc/mpss";
    +static const char *virtblk_backend = "VIRTBLK_BACKEND";
    +static struct {
    + struct mic_device_desc dd;
    + struct mic_vqconfig vqconfig[1];
    + __u32 host_features, guest_acknowledgements;
    + struct virtio_blk_config blk_config;
    +} virtblk_dev_page = {
    + .dd = {
    + .type = VIRTIO_ID_BLOCK,
    + .num_vq = ARRAY_SIZE(virtblk_dev_page.vqconfig),
    + .feature_len = sizeof(virtblk_dev_page.host_features),
    + .config_len = sizeof(virtblk_dev_page.blk_config),
    + },
    + .vqconfig[0] = {
    + .num = htole16(MIC_VRING_ENTRIES),
    + },
    + .host_features =
    + htole32(1<<VIRTIO_BLK_F_SEG_MAX),
    + .blk_config = {
    + .seg_max = htole32(MIC_VRING_ENTRIES - 2),
    + .capacity = htole64(0),
    + }
    +};
    +
    +static char *myname;
    +
    +static int
    +tap_configure(struct mic_info *mic, char *dev)
    +{
    + pid_t pid;
    + char *ifargv[7];
    + char ipaddr[IFNAMSIZ];
    + int ret = 0;
    +
    + pid = fork();
    + if (pid == 0) {
    + ifargv[0] = "ip";
    + ifargv[1] = "link";
    + ifargv[2] = "set";
    + ifargv[3] = dev;
    + ifargv[4] = "up";
    + ifargv[5] = NULL;
    + mpsslog("Configuring %s\n", dev);
    + ret = execvp("ip", ifargv);
    + if (ret < 0) {
    + mpsslog("%s execvp failed errno %s\n",
    + mic->name, strerror(errno));
    + return ret;
    + }
    + }
    + if (pid < 0) {
    + mpsslog("%s fork failed errno %s\n",
    + mic->name, strerror(errno));
    + return ret;
    + }
    +
    + ret = waitpid(pid, NULL, 0);
    + if (ret < 0) {
    + mpsslog("%s waitpid failed errno %s\n",
    + mic->name, strerror(errno));
    + return ret;
    + }
    +
    + snprintf(ipaddr, IFNAMSIZ, "172.31.%d.254/24", mic->id + 1);
    +
    + pid = fork();
    + if (pid == 0) {
    + ifargv[0] = "ip";
    + ifargv[1] = "addr";
    + ifargv[2] = "add";
    + ifargv[3] = ipaddr;
    + ifargv[4] = "dev";
    + ifargv[5] = dev;
    + ifargv[6] = NULL;
    + mpsslog("Configuring %s ipaddr %s\n", dev, ipaddr);
    + ret = execvp("ip", ifargv);
    + if (ret < 0) {
    + mpsslog("%s execvp failed errno %s\n",
    + mic->name, strerror(errno));
    + return ret;
    + }
    + }
    + if (pid < 0) {
    + mpsslog("%s fork failed errno %s\n",
    + mic->name, strerror(errno));
    + return ret;
    + }
    +
    + ret = waitpid(pid, NULL, 0);
    + if (ret < 0) {
    + mpsslog("%s waitpid failed errno %s\n",
    + mic->name, strerror(errno));
    + return ret;
    + }
    + mpsslog("MIC name %s %s %d DONE!\n",
    + mic->name, __func__, __LINE__);
    + return 0;
    +}
    +
    +static int tun_alloc(struct mic_info *mic, char *dev)
    +{
    + struct ifreq ifr;
    + int fd, err;
    +#if GSO_ENABLED
    + unsigned offload;
    +#endif
    + fd = open("/dev/net/tun", O_RDWR);
    + if (fd < 0) {
    + mpsslog("Could not open /dev/net/tun %s\n", strerror(errno));
    + goto done;
    + }
    +
    + memset(&ifr, 0, sizeof(ifr));
    +
    + ifr.ifr_flags = IFF_TAP | IFF_NO_PI | IFF_VNET_HDR;
    + if (*dev)
    + strncpy(ifr.ifr_name, dev, IFNAMSIZ);
    +
    + err = ioctl(fd, TUNSETIFF, (void *)&ifr);
    + if (err < 0) {
    + mpsslog("%s %s %d TUNSETIFF failed %s\n",
    + mic->name, __func__, __LINE__, strerror(errno));
    + close(fd);
    + return err;
    + }
    +#if GSO_ENABLED
    + offload = TUN_F_CSUM | TUN_F_TSO4 | TUN_F_TSO6 | TUN_F_TSO_ECN;
    +
    + err = ioctl(fd, TUNSETOFFLOAD, offload);
    + if (err < 0) {
    + mpsslog("%s %s %d TUNSETOFFLOAD failed %s\n",
    + mic->name, __func__, __LINE__, strerror(errno));
    + close(fd);
    + return err;
    + }
    +#endif
    + strcpy(dev, ifr.ifr_name);
    + mpsslog("Created TAP %s\n", dev);
    +done:
    + return fd;
    +}
    +
    +#define NET_FD_VIRTIO_NET 0
    +#define NET_FD_TUN 1
    +#define MAX_NET_FD 2
    +
    +static void set_dp(struct mic_info *mic, int type, void *dp)
    +{
    + switch (type) {
    + case VIRTIO_ID_CONSOLE:
    + mic->mic_console.console_dp = dp;
    + return;
    + case VIRTIO_ID_NET:
    + mic->mic_net.net_dp = dp;
    + return;
    + case VIRTIO_ID_BLOCK:
    + mic->mic_virtblk.block_dp = dp;
    + return;
    + }
    + mpsslog("%s %s %d not found\n", mic->name, __func__, type);
    + assert(0);
    +}
    +
    +static void *get_dp(struct mic_info *mic, int type)
    +{
    + switch (type) {
    + case VIRTIO_ID_CONSOLE:
    + return mic->mic_console.console_dp;
    + case VIRTIO_ID_NET:
    + return mic->mic_net.net_dp;
    + case VIRTIO_ID_BLOCK:
    + return mic->mic_virtblk.block_dp;
    + }
    + mpsslog("%s %s %d not found\n", mic->name, __func__, type);
    + assert(0);
    + return NULL;
    +}
    +
    +static struct mic_device_desc *get_device_desc(struct mic_info *mic, int type)
    +{
    + struct mic_device_desc *d;
    + int i;
    + void *dp = get_dp(mic, type);
    +
    + for (i = sizeof(struct mic_bootparam); i < PAGE_SIZE;
    + i += mic_total_desc_size(d)) {
    + d = dp + i;
    +
    + /* End of list */
    + if (d->type == 0)
    + break;
    +
    + if (d->type == -1)
    + continue;
    +
    + mpsslog("%s %s d-> type %d d %p\n",
    + mic->name, __func__, d->type, d);
    +
    + if (d->type == (__u8)type)
    + return d;
    + }
    + mpsslog("%s %s %d not found\n", mic->name, __func__, type);
    + return NULL;
    +}
    +
    +/* See comments in vhost.c for explanation of next_desc() */
    +static unsigned next_desc(struct vring_desc *desc)
    +{
    + unsigned int next;
    +
    + if (!(le16toh(desc->flags) & VRING_DESC_F_NEXT))
    + return -1U;
    + next = le16toh(desc->next);
    + return next;
    +}
    +
    +/* Sum up all the IOVEC length */
    +static ssize_t
    +sum_iovec_len(struct mic_copy_desc *copy)
    +{
    + ssize_t sum = 0;
    + unsigned int i;
    +
    + for (i = 0; i < copy->iovcnt; i++)
    + sum += copy->iov[i].iov_len;
    + return sum;
    +}
    +
    +static inline void verify_out_len(struct mic_info *mic,
    + struct mic_copy_desc *copy)
    +{
    + if (copy->out_len != sum_iovec_len(copy)) {
    + mpsslog("%s %s %d BUG copy->out_len 0x%x len 0x%zx\n",
    + mic->name, __func__, __LINE__,
    + copy->out_len, sum_iovec_len(copy));
    + assert(copy->out_len == sum_iovec_len(copy));
    + }
    +}
    +
    +/* Display an iovec */
    +static void
    +disp_iovec(struct mic_info *mic, struct mic_copy_desc *copy,
    + const char *s, int line)
    +{
    + unsigned int i;
    +
    + for (i = 0; i < copy->iovcnt; i++)
    + mpsslog("%s %s %d copy->iov[%d] addr %p len 0x%zx\n",
    + mic->name, s, line, i,
    + copy->iov[i].iov_base, copy->iov[i].iov_len);
    +}
    +
    +static inline __u16 read_avail_idx(struct mic_vring *vr)
    +{
    + return ACCESS_ONCE(vr->info->avail_idx);
    +}
    +
    +static inline void txrx_prepare(int type, bool tx, struct mic_vring *vr,
    + struct mic_copy_desc *copy, ssize_t len)
    +{
    + copy->vr_idx = tx ? 0 : 1;
    + copy->update_used = true;
    + if (type == VIRTIO_ID_NET)
    + copy->iov[1].iov_len = len - sizeof(struct virtio_net_hdr);
    + else
    + copy->iov[0].iov_len = len;
    +}
    +
    +/* Central API which triggers the copies */
    +static int
    +mic_virtio_copy(struct mic_info *mic, int fd,
    + struct mic_vring *vr, struct mic_copy_desc *copy)
    +{
    + int ret;
    +
    + ret = ioctl(fd, MIC_VIRTIO_COPY_DESC, copy);
    + if (ret) {
    + mpsslog("%s %s %d errno %s ret %d\n",
    + mic->name, __func__, __LINE__,
    + strerror(errno), ret);
    + }
    + return ret;
    +}
    +
    +static inline unsigned _vring_size(unsigned int num, unsigned long align)
    +{
    + return ((sizeof(struct vring_desc) * num + sizeof(__u16) * (3 + num)
    + + align - 1) & ~(align - 1))
    + + sizeof(__u16) * 3 + sizeof(struct vring_used_elem) * num;
    +}
    +
    +/*
    + * This initialization routine requires at least one
    + * vring i.e. vr0. vr1 is optional.
    + */
    +static void *
    +init_vr(struct mic_info *mic, int fd, int type,
    + struct mic_vring *vr0, struct mic_vring *vr1, int num_vq)
    +{
    + int vr_size;
    + char *va;
    +
    + vr_size = PAGE_ALIGN(_vring_size(MIC_VRING_ENTRIES,
    + MIC_VIRTIO_RING_ALIGN) +
    + sizeof(struct _mic_vring_info));
    + va = mmap(NULL, MIC_DEVICE_PAGE_END + vr_size * num_vq,
    + PROT_READ, MAP_SHARED, fd, 0);
    + if (MAP_FAILED == va) {
    + mpsslog("%s %s %d mmap failed errno %s\n",
    + mic->name, __func__, __LINE__,
    + strerror(errno));
    + goto done;
    + }
    + set_dp(mic, type, va);
    + vr0->va = (struct mic_vring *)&va[MIC_DEVICE_PAGE_END];
    + vr0->info = vr0->va +
    + _vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN);
    + vring_init(&vr0->vr,
    + MIC_VRING_ENTRIES, vr0->va, MIC_VIRTIO_RING_ALIGN);
    + mpsslog("%s %s vr0 %p vr0->info %p vr_size 0x%x vring 0x%x ",
    + __func__, mic->name, vr0->va, vr0->info, vr_size,
    + _vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN));
    + mpsslog("magic 0x%x expected 0x%x\n",
    + le32toh(vr0->info->magic), MIC_MAGIC + type);
    + assert(le32toh(vr0->info->magic) == MIC_MAGIC + type);
    + if (vr1) {
    + vr1->va = (struct mic_vring *)
    + &va[MIC_DEVICE_PAGE_END + vr_size];
    + vr1->info = vr1->va + _vring_size(MIC_VRING_ENTRIES,
    + MIC_VIRTIO_RING_ALIGN);
    + vring_init(&vr1->vr,
    + MIC_VRING_ENTRIES, vr1->va, MIC_VIRTIO_RING_ALIGN);
    + mpsslog("%s %s vr1 %p vr1->info %p vr_size 0x%x vring 0x%x ",
    + __func__, mic->name, vr1->va, vr1->info, vr_size,
    + _vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN));
    + mpsslog("magic 0x%x expected 0x%x\n",
    + le32toh(vr1->info->magic), MIC_MAGIC + type + 1);
    + assert(le32toh(vr1->info->magic) == MIC_MAGIC + type + 1);
    + }
    +done:
    + return va;
    +}
    +
    +static int
    +wait_for_card_driver(struct mic_info *mic, int fd, int type)
    +{
    + struct pollfd pollfd;
    + int err;
    + struct mic_device_desc *desc = get_device_desc(mic, type);
    + __u8 prev_status;
    +
    + if (!desc)
    + return -ENODEV;
    + prev_status = desc->status;
    + pollfd.fd = fd;
    + mpsslog("%s %s Waiting .... desc-> type %d status 0x%x\n",
    + mic->name, __func__, type, desc->status);
    +
    + while (1) {
    + pollfd.events = POLLIN;
    + pollfd.revents = 0;
    + err = poll(&pollfd, 1, -1);
    + if (err < 0) {
    + mpsslog("%s %s poll failed %s\n",
    + mic->name, __func__, strerror(errno));
    + continue;
    + }
    +
    + if (pollfd.revents) {
    + if (desc->status != prev_status) {
    + mpsslog("%s %s Waiting... desc-> type %d "
    + "status 0x%x\n",
    + mic->name, __func__, type,
    + desc->status);
    + prev_status = desc->status;
    + }
    + if (desc->status & VIRTIO_CONFIG_S_DRIVER_OK) {
    + mpsslog("%s %s poll.revents %d\n",
    + mic->name, __func__, pollfd.revents);
    + mpsslog("%s %s desc-> type %d status 0x%x\n",
    + mic->name, __func__, type,
    + desc->status);
    + break;
    + }
    + }
    + }
    + return 0;
    +}
    +
    +/* Spin till we have some descriptors */
    +static void
    +spin_for_descriptors(struct mic_info *mic, struct mic_vring *vr)
    +{
    + __u16 avail_idx = read_avail_idx(vr);
    +
    + while (avail_idx == le16toh(ACCESS_ONCE(vr->vr.avail->idx))) {
    +#ifdef DEBUG
    + mpsslog("%s %s waiting for desc avail %d info_avail %d\n",
    + mic->name, __func__,
    + le16toh(vr->vr.avail->idx), vr->info->avail_idx);
    +#endif
    + sched_yield();
    + }
    +}
    +
    +static void *
    +virtio_net(void *arg)
    +{
    + static __u8 vnet_hdr[2][sizeof(struct virtio_net_hdr)];
    + static __u8 vnet_buf[2][MAX_NET_PKT_SIZE] __attribute__ ((aligned(64)));
    + struct iovec vnet_iov[2][2] = {
    + { { .iov_base = vnet_hdr[0], .iov_len = sizeof(vnet_hdr[0]) },
    + { .iov_base = vnet_buf[0], .iov_len = sizeof(vnet_buf[0]) } },
    + { { .iov_base = vnet_hdr[1], .iov_len = sizeof(vnet_hdr[1]) },
    + { .iov_base = vnet_buf[1], .iov_len = sizeof(vnet_buf[1]) } },
    + };
    + struct iovec *iov0 = vnet_iov[0], *iov1 = vnet_iov[1];
    + struct mic_info *mic = (struct mic_info *)arg;
    + char if_name[IFNAMSIZ];
    + struct pollfd net_poll[MAX_NET_FD];
    + struct mic_vring tx_vr, rx_vr;
    + struct mic_copy_desc copy;
    + struct mic_device_desc *desc;
    + int err;
    +
    + snprintf(if_name, IFNAMSIZ, "mic%d", mic->id);
    + mic->mic_net.tap_fd = tun_alloc(mic, if_name);
    + if (mic->mic_net.tap_fd < 0)
    + goto done;
    +
    + if (tap_configure(mic, if_name))
    + goto done;
    + mpsslog("MIC name %s id %d\n", mic->name, mic->id);
    +
    + net_poll[NET_FD_VIRTIO_NET].fd = mic->mic_net.virtio_net_fd;
    + net_poll[NET_FD_VIRTIO_NET].events = POLLIN;
    + net_poll[NET_FD_TUN].fd = mic->mic_net.tap_fd;
    + net_poll[NET_FD_TUN].events = POLLIN;
    +
    + if (MAP_FAILED == init_vr(mic, mic->mic_net.virtio_net_fd,
    + VIRTIO_ID_NET, &tx_vr, &rx_vr,
    + virtnet_dev_page.dd.num_vq)) {
    + mpsslog("%s init_vr failed %s\n",
    + mic->name, strerror(errno));
    + goto done;
    + }
    +
    + copy.iovcnt = 2;
    + desc = get_device_desc(mic, VIRTIO_ID_NET);
    +
    + while (1) {
    + ssize_t len;
    +
    + net_poll[NET_FD_VIRTIO_NET].revents = 0;
    + net_poll[NET_FD_TUN].revents = 0;
    +
    + /* Start polling for data from tap and virtio net */
    + err = poll(net_poll, 2, -1);
    + if (err < 0) {
    + mpsslog("%s poll failed %s\n",
    + __func__, strerror(errno));
    + continue;
    + }
    + if (!(desc->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
    + err = wait_for_card_driver(mic,
    + mic->mic_net.virtio_net_fd,
    + VIRTIO_ID_NET);
    + if (err) {
    + mpsslog("%s %s %d Exiting...\n",
    + mic->name, __func__, __LINE__);
    + break;
    + }
    + }
    + /*
    + * Check if there is data to be read from TUN and write to
    + * virtio net fd if there is.
    + */
    + if (net_poll[NET_FD_TUN].revents & POLLIN) {
    + copy.iov = iov0;
    + len = readv(net_poll[NET_FD_TUN].fd,
    + copy.iov, copy.iovcnt);
    + if (len > 0) {
    + struct virtio_net_hdr *hdr
    + = (struct virtio_net_hdr *)vnet_hdr[0];
    +
    + /* Disable checksums on the card since we are on
    + a reliable PCIe link */
    + hdr->flags |= VIRTIO_NET_HDR_F_DATA_VALID;
    +#ifdef DEBUG
    + mpsslog("%s %s %d hdr->flags 0x%x ", mic->name,
    + __func__, __LINE__, hdr->flags);
    + mpsslog("copy.out_len %d hdr->gso_type 0x%x\n",
    + copy.out_len, hdr->gso_type);
    +#endif
    +#ifdef DEBUG
    + disp_iovec(mic, copy, __func__, __LINE__);
    + mpsslog("%s %s %d read from tap 0x%lx\n",
    + mic->name, __func__, __LINE__,
    + len);
    +#endif
    + spin_for_descriptors(mic, &tx_vr);
    + txrx_prepare(VIRTIO_ID_NET, 1, &tx_vr, &copy,
    + len);
    +
    + err = mic_virtio_copy(mic,
    + mic->mic_net.virtio_net_fd, &tx_vr,
    + &copy);
    + if (err < 0) {
    + mpsslog("%s %s %d mic_virtio_copy %s\n",
    + mic->name, __func__, __LINE__,
    + strerror(errno));
    + }
    + if (!err)
    + verify_out_len(mic, &copy);
    +#ifdef DEBUG
    + disp_iovec(mic, copy, __func__, __LINE__);
    + mpsslog("%s %s %d wrote to net 0x%lx\n",
    + mic->name, __func__, __LINE__,
    + sum_iovec_len(&copy));
    +#endif
    + /* Reinitialize IOV for next run */
    + iov0[1].iov_len = MAX_NET_PKT_SIZE;
    + } else if (len < 0) {
    + disp_iovec(mic, &copy, __func__, __LINE__);
    + mpsslog("%s %s %d read failed %s ", mic->name,
    + __func__, __LINE__, strerror(errno));
    + mpsslog("cnt %d sum %zd\n",
    + copy.iovcnt, sum_iovec_len(&copy));
    + }
    + }
    +
    + /*
    + * Check if there is data to be read from virtio net and
    + * write to TUN if there is.
    + */
    + if (net_poll[NET_FD_VIRTIO_NET].revents & POLLIN) {
    + while (rx_vr.info->avail_idx !=
    + le16toh(rx_vr.vr.avail->idx)) {
    + copy.iov = iov1;
    + txrx_prepare(VIRTIO_ID_NET, 0, &rx_vr, &copy,
    + MAX_NET_PKT_SIZE
    + + sizeof(struct virtio_net_hdr));
    +
    + err = mic_virtio_copy(mic,
    + mic->mic_net.virtio_net_fd, &rx_vr,
    + &copy);
    + if (!err) {
    +#ifdef DEBUG
    + struct virtio_net_hdr *hdr
    + = (struct virtio_net_hdr *)
    + vnet_hdr[1];
    +
    + mpsslog("%s %s %d hdr->flags 0x%x, ",
    + mic->name, __func__, __LINE__,
    + hdr->flags);
    + mpsslog("out_len %d gso_type 0x%x\n",
    + copy.out_len,
    + hdr->gso_type);
    +#endif
    + /* Set the correct output iov_len */
    + iov1[1].iov_len = copy.out_len -
    + sizeof(struct virtio_net_hdr);
    + verify_out_len(mic, &copy);
    +#ifdef DEBUG
    + disp_iovec(mic, copy, __func__,
    + __LINE__);
    + mpsslog("%s %s %d ",
    + mic->name, __func__, __LINE__);
    + mpsslog("read from net 0x%lx\n",
    + sum_iovec_len(copy));
    +#endif
    + len = writev(net_poll[NET_FD_TUN].fd,
    + copy.iov, copy.iovcnt);
    + if (len != sum_iovec_len(&copy)) {
    + mpsslog("Tun write failed %s ",
    + strerror(errno));
    + mpsslog("len 0x%zx ", len);
    + mpsslog("read_len 0x%zx\n",
    + sum_iovec_len(&copy));
    + } else {
    +#ifdef DEBUG
    + disp_iovec(mic, &copy, __func__,
    + __LINE__);
    + mpsslog("%s %s %d ",
    + mic->name, __func__,
    + __LINE__);
    + mpsslog("wrote to tap 0x%lx\n",
    + len);
    +#endif
    + }
    + } else {
    + mpsslog("%s %s %d mic_virtio_copy %s\n",
    + mic->name, __func__, __LINE__,
    + strerror(errno));
    + break;
    + }
    + }
    + }
    + if (net_poll[NET_FD_VIRTIO_NET].revents & POLLERR)
    + mpsslog("%s: %s: POLLERR\n", __func__, mic->name);
    + }
    +done:
    + pthread_exit(NULL);
    +}
    +
    +/* virtio_console */
    +#define VIRTIO_CONSOLE_FD 0
    +#define MONITOR_FD (VIRTIO_CONSOLE_FD + 1)
    +#define MAX_CONSOLE_FD (MONITOR_FD + 1) /* must be the last one + 1 */
    +#define MAX_BUFFER_SIZE PAGE_SIZE
    +
    +static void *
    +virtio_console(void *arg)
    +{
    + static __u8 vcons_buf[2][PAGE_SIZE];
    + struct iovec vcons_iov[2] = {
    + { .iov_base = vcons_buf[0], .iov_len = sizeof(vcons_buf[0]) },
    + { .iov_base = vcons_buf[1], .iov_len = sizeof(vcons_buf[1]) },
    + };
    + struct iovec *iov0 = &vcons_iov[0], *iov1 = &vcons_iov[1];
    + struct mic_info *mic = (struct mic_info *)arg;
    + int err;
    + struct pollfd console_poll[MAX_CONSOLE_FD];
    + int pty_fd;
    + char *pts_name;
    + ssize_t len;
    + struct mic_vring tx_vr, rx_vr;
    + struct mic_copy_desc copy;
    + struct mic_device_desc *desc;
    +
    + pty_fd = posix_openpt(O_RDWR);
    + if (pty_fd < 0) {
    + mpsslog("can't open a pseudoterminal master device: %s\n",
    + strerror(errno));
    + goto _return;
    + }
    + pts_name = ptsname(pty_fd);
    + if (pts_name == NULL) {
    + mpsslog("can't get pts name\n");
    + goto _close_pty;
    + }
    + printf("%s console message goes to %s\n", mic->name, pts_name);
    + mpsslog("%s console message goes to %s\n", mic->name, pts_name);
    + err = grantpt(pty_fd);
    + if (err < 0) {
    + mpsslog("can't grant access: %s %s\n",
    + pts_name, strerror(errno));
    + goto _close_pty;
    + }
    + err = unlockpt(pty_fd);
    + if (err < 0) {
    + mpsslog("can't unlock a pseudoterminal: %s %s\n",
    + pts_name, strerror(errno));
    + goto _close_pty;
    + }
    + console_poll[MONITOR_FD].fd = pty_fd;
    + console_poll[MONITOR_FD].events = POLLIN;
    +
    + console_poll[VIRTIO_CONSOLE_FD].fd = mic->mic_console.virtio_console_fd;
    + console_poll[VIRTIO_CONSOLE_FD].events = POLLIN;
    +
    + if (MAP_FAILED == init_vr(mic, mic->mic_console.virtio_console_fd,
    + VIRTIO_ID_CONSOLE, &tx_vr, &rx_vr,
    + virtcons_dev_page.dd.num_vq)) {
    + mpsslog("%s init_vr failed %s\n",
    + mic->name, strerror(errno));
    + goto _close_pty;
    + }
    +
    + copy.iovcnt = 1;
    + desc = get_device_desc(mic, VIRTIO_ID_CONSOLE);
    +
    + for (;;) {
    + console_poll[MONITOR_FD].revents = 0;
    + console_poll[VIRTIO_CONSOLE_FD].revents = 0;
    + err = poll(console_poll, MAX_CONSOLE_FD, -1);
    + if (err < 0) {
    + mpsslog("%s %d: poll failed: %s\n", __func__, __LINE__,
    + strerror(errno));
    + continue;
    + }
    + if (!(desc->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
    + err = wait_for_card_driver(mic,
    + mic->mic_console.virtio_console_fd,
    + VIRTIO_ID_CONSOLE);
    + if (err) {
    + mpsslog("%s %s %d Exiting...\n",
    + mic->name, __func__, __LINE__);
    + break;
    + }
    + }
    +
    + if (console_poll[MONITOR_FD].revents & POLLIN) {
    + copy.iov = iov0;
    + len = readv(pty_fd, copy.iov, copy.iovcnt);
    + if (len > 0) {
    +#ifdef DEBUG
    + disp_iovec(mic, copy, __func__, __LINE__);
    + mpsslog("%s %s %d read from tap 0x%lx\n",
    + mic->name, __func__, __LINE__,
    + len);
    +#endif
    + spin_for_descriptors(mic, &tx_vr);
    + txrx_prepare(VIRTIO_ID_CONSOLE, 1, &tx_vr,
    + &copy, len);
    +
    + err = mic_virtio_copy(mic,
    + mic->mic_console.virtio_console_fd,
    + &tx_vr, &copy);
    + if (err < 0) {
    + mpsslog("%s %s %d mic_virtio_copy %s\n",
    + mic->name, __func__, __LINE__,
    + strerror(errno));
    + }
    + if (!err)
    + verify_out_len(mic, &copy);
    +#ifdef DEBUG
    + disp_iovec(mic, copy, __func__, __LINE__);
    + mpsslog("%s %s %d wrote to net 0x%lx\n",
    + mic->name, __func__, __LINE__,
    + sum_iovec_len(copy));
    +#endif
    + /* Reinitialize IOV for next run */
    + iov0->iov_len = PAGE_SIZE;
    + } else if (len < 0) {
    + disp_iovec(mic, &copy, __func__, __LINE__);
    + mpsslog("%s %s %d read failed %s ",
    + mic->name, __func__, __LINE__,
    + strerror(errno));
    + mpsslog("cnt %d sum %zd\n",
    + copy.iovcnt, sum_iovec_len(&copy));
    + }
    + }
    +
    + if (console_poll[VIRTIO_CONSOLE_FD].revents & POLLIN) {
    + while (rx_vr.info->avail_idx !=
    + le16toh(rx_vr.vr.avail->idx)) {
    + copy.iov = iov1;
    + txrx_prepare(VIRTIO_ID_CONSOLE, 0, &rx_vr,
    + &copy, PAGE_SIZE);
    +
    + err = mic_virtio_copy(mic,
    + mic->mic_console.virtio_console_fd,
    + &rx_vr, &copy);
    + if (!err) {
    + /* Set the correct output iov_len */
    + iov1->iov_len = copy.out_len;
    + verify_out_len(mic, &copy);
    +#ifdef DEBUG
    + disp_iovec(mic, copy, __func__,
    + __LINE__);
    + mpsslog("%s %s %d ",
    + mic->name, __func__, __LINE__);
    + mpsslog("read from net 0x%lx\n",
    + sum_iovec_len(copy));
    +#endif
    + len = writev(pty_fd,
    + copy.iov, copy.iovcnt);
    + if (len != sum_iovec_len(&copy)) {
    + mpsslog("Tun write failed %s ",
    + strerror(errno));
    + mpsslog("len 0x%zx ", len);
    + mpsslog("read_len 0x%zx\n",
    + sum_iovec_len(&copy));
    + } else {
    +#ifdef DEBUG
    + disp_iovec(mic, copy, __func__,
    + __LINE__);
    + mpsslog("%s %s %d ",
    + mic->name, __func__,
    + __LINE__);
    + mpsslog("wrote to tap 0x%lx\n",
    + len);
    +#endif
    + }
    + } else {
    + mpsslog("%s %s %d mic_virtio_copy %s\n",
    + mic->name, __func__, __LINE__,
    + strerror(errno));
    + break;
    + }
    + }
    + }
    + if (console_poll[NET_FD_VIRTIO_NET].revents & POLLERR)
    + mpsslog("%s: %s: POLLERR\n", __func__, mic->name);
    + }
    +_close_pty:
    + close(pty_fd);
    +_return:
    + pthread_exit(NULL);
    +}
    +
    +static void
    +add_virtio_device(struct mic_info *mic, struct mic_device_desc *dd)
    +{
    + char path[PATH_MAX];
    + int fd, err;
    +
    + snprintf(path, PATH_MAX, "/dev/vop_virtio%d", mic->id);
    + fd = open(path, O_RDWR);
    + if (fd < 0) {
    + mpsslog("Could not open %s %s\n", path, strerror(errno));
    + return;
    + }
    +
    + err = ioctl(fd, MIC_VIRTIO_ADD_DEVICE, dd);
    + if (err < 0) {
    + mpsslog("Could not add %d %s\n", dd->type, strerror(errno));
    + close(fd);
    + return;
    + }
    + switch (dd->type) {
    + case VIRTIO_ID_NET:
    + mic->mic_net.virtio_net_fd = fd;
    + mpsslog("Added VIRTIO_ID_NET for %s\n", mic->name);
    + break;
    + case VIRTIO_ID_CONSOLE:
    + mic->mic_console.virtio_console_fd = fd;
    + mpsslog("Added VIRTIO_ID_CONSOLE for %s\n", mic->name);
    + break;
    + case VIRTIO_ID_BLOCK:
    + mic->mic_virtblk.virtio_block_fd = fd;
    + mpsslog("Added VIRTIO_ID_BLOCK for %s\n", mic->name);
    + break;
    + }
    +}
    +
    +static bool
    +set_backend_file(struct mic_info *mic)
    +{
    + FILE *config;
    + char buff[PATH_MAX], *line, *evv, *p;
    +
    + snprintf(buff, PATH_MAX, "%s/mpssd%03d.conf", mic_config_dir, mic->id);
    + config = fopen(buff, "r");
    + if (config == NULL)
    + return false;
    + do { /* look for "virtblk_backend=XXXX" */
    + line = fgets(buff, PATH_MAX, config);
    + if (line == NULL)
    + break;
    + if (*line == '#')
    + continue;
    + p = strchr(line, '\n');
    + if (p)
    + *p = '\0';
    + } while (strncmp(line, virtblk_backend, strlen(virtblk_backend)) != 0);
    + fclose(config);
    + if (line == NULL)
    + return false;
    + evv = strchr(line, '=');
    + if (evv == NULL)
    + return false;
    + mic->mic_virtblk.backend_file = malloc(strlen(evv) + 1);
    + if (mic->mic_virtblk.backend_file == NULL) {
    + mpsslog("%s %d can't allocate memory\n", mic->name, mic->id);
    + return false;
    + }
    + strcpy(mic->mic_virtblk.backend_file, evv + 1);
    + return true;
    +}
    +
    +#define SECTOR_SIZE 512
    +static bool
    +set_backend_size(struct mic_info *mic)
    +{
    + mic->mic_virtblk.backend_size = lseek(mic->mic_virtblk.backend, 0,
    + SEEK_END);
    + if (mic->mic_virtblk.backend_size < 0) {
    + mpsslog("%s: can't seek: %s\n",
    + mic->name, mic->mic_virtblk.backend_file);
    + return false;
    + }
    + virtblk_dev_page.blk_config.capacity =
    + mic->mic_virtblk.backend_size / SECTOR_SIZE;
    + if ((mic->mic_virtblk.backend_size % SECTOR_SIZE) != 0)
    + virtblk_dev_page.blk_config.capacity++;
    +
    + virtblk_dev_page.blk_config.capacity =
    + htole64(virtblk_dev_page.blk_config.capacity);
    +
    + return true;
    +}
    +
    +static bool
    +open_backend(struct mic_info *mic)
    +{
    + if (!set_backend_file(mic))
    + goto _error_exit;
    + mic->mic_virtblk.backend = open(mic->mic_virtblk.backend_file, O_RDWR);
    + if (mic->mic_virtblk.backend < 0) {
    + mpsslog("%s: can't open: %s\n", mic->name,
    + mic->mic_virtblk.backend_file);
    + goto _error_free;
    + }
    + if (!set_backend_size(mic))
    + goto _error_close;
    + mic->mic_virtblk.backend_addr = mmap(NULL,
    + mic->mic_virtblk.backend_size,
    + PROT_READ|PROT_WRITE, MAP_SHARED,
    + mic->mic_virtblk.backend, 0L);
    + if (mic->mic_virtblk.backend_addr == MAP_FAILED) {
    + mpsslog("%s: can't map: %s %s\n",
    + mic->name, mic->mic_virtblk.backend_file,
    + strerror(errno));
    + goto _error_close;
    + }
    + return true;
    +
    + _error_close:
    + close(mic->mic_virtblk.backend);
    + _error_free:
    + free(mic->mic_virtblk.backend_file);
    + _error_exit:
    + return false;
    +}
    +
    +static void
    +close_backend(struct mic_info *mic)
    +{
    + munmap(mic->mic_virtblk.backend_addr, mic->mic_virtblk.backend_size);
    + close(mic->mic_virtblk.backend);
    + free(mic->mic_virtblk.backend_file);
    +}
    +
    +static bool
    +start_virtblk(struct mic_info *mic, struct mic_vring *vring)
    +{
    + if (((unsigned long)&virtblk_dev_page.blk_config % 8) != 0) {
    + mpsslog("%s: blk_config is not 8 byte aligned.\n",
    + mic->name);
    + return false;
    + }
    + add_virtio_device(mic, &virtblk_dev_page.dd);
    + if (MAP_FAILED == init_vr(mic, mic->mic_virtblk.virtio_block_fd,
    + VIRTIO_ID_BLOCK, vring, NULL,
    + virtblk_dev_page.dd.num_vq)) {
    + mpsslog("%s init_vr failed %s\n",
    + mic->name, strerror(errno));
    + return false;
    + }
    + return true;
    +}
    +
    +static void
    +stop_virtblk(struct mic_info *mic)
    +{
    + int vr_size, ret;
    +
    + vr_size = PAGE_ALIGN(_vring_size(MIC_VRING_ENTRIES,
    + MIC_VIRTIO_RING_ALIGN) +
    + sizeof(struct _mic_vring_info));
    + ret = munmap(mic->mic_virtblk.block_dp,
    + MIC_DEVICE_PAGE_END + vr_size * virtblk_dev_page.dd.num_vq);
    + if (ret < 0)
    + mpsslog("%s munmap errno %d\n", mic->name, errno);
    + close(mic->mic_virtblk.virtio_block_fd);
    +}
    +
    +static __u8
    +header_error_check(struct vring_desc *desc)
    +{
    + if (le32toh(desc->len) != sizeof(struct virtio_blk_outhdr)) {
    + mpsslog("%s() %d: length is not sizeof(virtio_blk_outhd)\n",
    + __func__, __LINE__);
    + return -EIO;
    + }
    + if (!(le16toh(desc->flags) & VRING_DESC_F_NEXT)) {
    + mpsslog("%s() %d: alone\n",
    + __func__, __LINE__);
    + return -EIO;
    + }
    + if (le16toh(desc->flags) & VRING_DESC_F_WRITE) {
    + mpsslog("%s() %d: not read\n",
    + __func__, __LINE__);
    + return -EIO;
    + }
    + return 0;
    +}
    +
    +static int
    +read_header(int fd, struct virtio_blk_outhdr *hdr, __u32 desc_idx)
    +{
    + struct iovec iovec;
    + struct mic_copy_desc copy;
    +
    + iovec.iov_len = sizeof(*hdr);
    + iovec.iov_base = hdr;
    + copy.iov = &iovec;
    + copy.iovcnt = 1;
    + copy.vr_idx = 0; /* only one vring on virtio_block */
    + copy.update_used = false; /* do not update used index */
    + return ioctl(fd, MIC_VIRTIO_COPY_DESC, &copy);
    +}
    +
    +static int
    +transfer_blocks(int fd, struct iovec *iovec, __u32 iovcnt)
    +{
    + struct mic_copy_desc copy;
    +
    + copy.iov = iovec;
    + copy.iovcnt = iovcnt;
    + copy.vr_idx = 0; /* only one vring on virtio_block */
    + copy.update_used = false; /* do not update used index */
    + return ioctl(fd, MIC_VIRTIO_COPY_DESC, &copy);
    +}
    +
    +static __u8
    +status_error_check(struct vring_desc *desc)
    +{
    + if (le32toh(desc->len) != sizeof(__u8)) {
    + mpsslog("%s() %d: length is not sizeof(status)\n",
    + __func__, __LINE__);
    + return -EIO;
    + }
    + return 0;
    +}
    +
    +static int
    +write_status(int fd, __u8 *status)
    +{
    + struct iovec iovec;
    + struct mic_copy_desc copy;
    +
    + iovec.iov_base = status;
    + iovec.iov_len = sizeof(*status);
    + copy.iov = &iovec;
    + copy.iovcnt = 1;
    + copy.vr_idx = 0; /* only one vring on virtio_block */
    + copy.update_used = true; /* Update used index */
    + return ioctl(fd, MIC_VIRTIO_COPY_DESC, &copy);
    +}
    +
    +#ifndef VIRTIO_BLK_T_GET_ID
    +#define VIRTIO_BLK_T_GET_ID 8
    +#endif
    +
    +static void *
    +virtio_block(void *arg)
    +{
    + struct mic_info *mic = (struct mic_info *)arg;
    + int ret;
    + struct pollfd block_poll;
    + struct mic_vring vring;
    + __u16 avail_idx;
    + __u32 desc_idx;
    + struct vring_desc *desc;
    + struct iovec *iovec, *piov;
    + __u8 status;
    + __u32 buffer_desc_idx;
    + struct virtio_blk_outhdr hdr;
    + void *fos;
    +
    + for (;;) { /* forever */
    + if (!open_backend(mic)) { /* No virtblk */
    + for (mic->mic_virtblk.signaled = 0;
    + !mic->mic_virtblk.signaled;)
    + sleep(1);
    + continue;
    + }
    +
    + /* backend file is specified. */
    + if (!start_virtblk(mic, &vring))
    + goto _close_backend;
    + iovec = malloc(sizeof(*iovec) *
    + le32toh(virtblk_dev_page.blk_config.seg_max));
    + if (!iovec) {
    + mpsslog("%s: can't alloc iovec: %s\n",
    + mic->name, strerror(ENOMEM));
    + goto _stop_virtblk;
    + }
    +
    + block_poll.fd = mic->mic_virtblk.virtio_block_fd;
    + block_poll.events = POLLIN;
    + for (mic->mic_virtblk.signaled = 0;
    + !mic->mic_virtblk.signaled;) {
    + block_poll.revents = 0;
    + /* timeout in 1 sec to see signaled */
    + ret = poll(&block_poll, 1, 1000);
    + if (ret < 0) {
    + mpsslog("%s %d: poll failed: %s\n",
    + __func__, __LINE__,
    + strerror(errno));
    + continue;
    + }
    +
    + if (!(block_poll.revents & POLLIN)) {
    +#ifdef DEBUG
    + mpsslog("%s %d: block_poll.revents=0x%x\n",
    + __func__, __LINE__, block_poll.revents);
    +#endif
    + continue;
    + }
    +
    + /* POLLIN */
    + while (vring.info->avail_idx !=
    + le16toh(vring.vr.avail->idx)) {
    + /* read header element */
    + avail_idx =
    + vring.info->avail_idx &
    + (vring.vr.num - 1);
    + desc_idx = le16toh(
    + vring.vr.avail->ring[avail_idx]);
    + desc = &vring.vr.desc[desc_idx];
    +#ifdef DEBUG
    + mpsslog("%s() %d: avail_idx=%d ",
    + __func__, __LINE__,
    + vring.info->avail_idx);
    + mpsslog("vring.vr.num=%d desc=%p\n",
    + vring.vr.num, desc);
    +#endif
    + status = header_error_check(desc);
    + ret = read_header(
    + mic->mic_virtblk.virtio_block_fd,
    + &hdr, desc_idx);
    + if (ret < 0) {
    + mpsslog("%s() %d %s: ret=%d %s\n",
    + __func__, __LINE__,
    + mic->name, ret,
    + strerror(errno));
    + break;
    + }
    + /* buffer element */
    + piov = iovec;
    + status = 0;
    + fos = mic->mic_virtblk.backend_addr +
    + (hdr.sector * SECTOR_SIZE);
    + buffer_desc_idx = next_desc(desc);
    + desc_idx = buffer_desc_idx;
    + for (desc = &vring.vr.desc[buffer_desc_idx];
    + desc->flags & VRING_DESC_F_NEXT;
    + desc_idx = next_desc(desc),
    + desc = &vring.vr.desc[desc_idx]) {
    + piov->iov_len = desc->len;
    + piov->iov_base = fos;
    + piov++;
    + fos += desc->len;
    + }
    + /* Returning NULLs for VIRTIO_BLK_T_GET_ID. */
    + if (hdr.type & ~(VIRTIO_BLK_T_OUT |
    + VIRTIO_BLK_T_GET_ID)) {
    + /*
    + VIRTIO_BLK_T_IN - does not do
    + anything. Probably for documenting.
    + VIRTIO_BLK_T_SCSI_CMD - for
    + virtio_scsi.
    + VIRTIO_BLK_T_FLUSH - turned off in
    + config space.
    + VIRTIO_BLK_T_BARRIER - defined but not
    + used in anywhere.
    + */
    + mpsslog("%s() %d: type %x ",
    + __func__, __LINE__,
    + hdr.type);
    + mpsslog("is not supported\n");
    + status = -ENOTSUP;
    +
    + } else {
    + ret = transfer_blocks(
    + mic->mic_virtblk.virtio_block_fd,
    + iovec,
    + piov - iovec);
    + if (ret < 0 &&
    + status != 0)
    + status = ret;
    + }
    + /* write status and update used pointer */
    + if (status != 0)
    + status = status_error_check(desc);
    + ret = write_status(
    + mic->mic_virtblk.virtio_block_fd,
    + &status);
    +#ifdef DEBUG
    + mpsslog("%s() %d: write status=%d on desc=%p\n",
    + __func__, __LINE__,
    + status, desc);
    +#endif
    + }
    + }
    + free(iovec);
    +_stop_virtblk:
    + stop_virtblk(mic);
    +_close_backend:
    + close_backend(mic);
    + } /* forever */
    +
    + pthread_exit(NULL);
    +}
    +
    +static void
    +reset(struct mic_info *mic)
    +{
    +#define RESET_TIMEOUT 120
    + int i = RESET_TIMEOUT;
    + setsysfs(mic->name, "state", "reset");
    + while (i) {
    + char *state;
    + state = readsysfs(mic->name, "state");
    + if (!state)
    + goto retry;
    + mpsslog("%s: %s %d state %s\n",
    + mic->name, __func__, __LINE__, state);
    +
    + if (!strcmp(state, "ready")) {
    + free(state);
    + break;
    + }
    + free(state);
    +retry:
    + sleep(1);
    + i--;
    + }
    +}
    +
    +static int
    +get_mic_shutdown_status(struct mic_info *mic, char *shutdown_status)
    +{
    + if (!strcmp(shutdown_status, "nop"))
    + return MIC_NOP;
    + if (!strcmp(shutdown_status, "crashed"))
    + return MIC_CRASHED;
    + if (!strcmp(shutdown_status, "halted"))
    + return MIC_HALTED;
    + if (!strcmp(shutdown_status, "poweroff"))
    + return MIC_POWER_OFF;
    + if (!strcmp(shutdown_status, "restart"))
    + return MIC_RESTART;
    + mpsslog("%s: BUG invalid status %s\n", mic->name, shutdown_status);
    + /* Invalid state */
    + assert(0);
    +};
    +
    +static int get_mic_state(struct mic_info *mic)
    +{
    + char *state = NULL;
    + enum mic_states mic_state;
    +
    + while (!state) {
    + state = readsysfs(mic->name, "state");
    + sleep(1);
    + }
    + mpsslog("%s: %s %d state %s\n",
    + mic->name, __func__, __LINE__, state);
    +
    + if (!strcmp(state, "ready")) {
    + mic_state = MIC_READY;
    + } else if (!strcmp(state, "booting")) {
    + mic_state = MIC_BOOTING;
    + } else if (!strcmp(state, "online")) {
    + mic_state = MIC_ONLINE;
    + } else if (!strcmp(state, "shutting_down")) {
    + mic_state = MIC_SHUTTING_DOWN;
    + } else if (!strcmp(state, "reset_failed")) {
    + mic_state = MIC_RESET_FAILED;
    + } else if (!strcmp(state, "resetting")) {
    + mic_state = MIC_RESETTING;
    + } else {
    + mpsslog("%s: BUG invalid state %s\n", mic->name, state);
    + assert(0);
    + }
    +
    + free(state);
    + return mic_state;
    +};
    +
    +static void mic_handle_shutdown(struct mic_info *mic)
    +{
    +#define SHUTDOWN_TIMEOUT 60
    + int i = SHUTDOWN_TIMEOUT;
    + char *shutdown_status;
    + while (i) {
    + shutdown_status = readsysfs(mic->name, "shutdown_status");
    + if (!shutdown_status) {
    + sleep(1);
    + continue;
    + }
    + mpsslog("%s: %s %d shutdown_status %s\n",
    + mic->name, __func__, __LINE__, shutdown_status);
    + switch (get_mic_shutdown_status(mic, shutdown_status)) {
    + case MIC_RESTART:
    + mic->restart = 1;
    + case MIC_HALTED:
    + case MIC_POWER_OFF:
    + case MIC_CRASHED:
    + free(shutdown_status);
    + goto reset;
    + default:
    + break;
    + }
    + free(shutdown_status);
    + sleep(1);
    + i--;
    + }
    +reset:
    + if (!i)
    + mpsslog("%s: %s %d timing out waiting for shutdown_status %s\n",
    + mic->name, __func__, __LINE__, shutdown_status);
    + reset(mic);
    +}
    +
    +static int open_state_fd(struct mic_info *mic)
    +{
    + char pathname[PATH_MAX];
    + int fd;
    +
    + snprintf(pathname, PATH_MAX - 1, "%s/%s/%s",
    + MICSYSFSDIR, mic->name, "state");
    +
    + fd = open(pathname, O_RDONLY);
    + if (fd < 0)
    + mpsslog("%s: opening file %s failed %s\n",
    + mic->name, pathname, strerror(errno));
    + return fd;
    +}
    +
    +static int block_till_state_change(int fd, struct mic_info *mic)
    +{
    + struct pollfd ufds[1];
    + char value[PAGE_SIZE];
    + int ret;
    +
    + ufds[0].fd = fd;
    + ufds[0].events = POLLERR | POLLPRI;
    + ret = poll(ufds, 1, -1);
    + if (ret < 0) {
    + mpsslog("%s: %s %d poll failed %s\n",
    + mic->name, __func__, __LINE__, strerror(errno));
    + return ret;
    + }
    +
    + ret = lseek(fd, 0, SEEK_SET);
    + if (ret < 0) {
    + mpsslog("%s: %s %d Failed to seek to 0: %s\n",
    + mic->name, __func__, __LINE__, strerror(errno));
    + return ret;
    + }
    +
    + ret = read(fd, value, sizeof(value));
    + if (ret < 0) {
    + mpsslog("%s: %s %d Failed to read sysfs entry: %s\n",
    + mic->name, __func__, __LINE__, strerror(errno));
    + return ret;
    + }
    +
    + return 0;
    +}
    +
    +static void *
    +mic_config(void *arg)
    +{
    + struct mic_info *mic = (struct mic_info *)arg;
    + int fd, ret, stat = 0;
    +
    + fd = open_state_fd(mic);
    + if (fd < 0) {
    + mpsslog("%s: %s %d open state fd failed %s\n",
    + mic->name, __func__, __LINE__, strerror(errno));
    + goto exit;
    + }
    +
    + do {
    + ret = block_till_state_change(fd, mic);
    + if (ret < 0) {
    + mpsslog("%s: %s %d block_till_state_change error %s\n",
    + mic->name, __func__, __LINE__, strerror(errno));
    + goto close_exit;
    + }
    +
    + switch (get_mic_state(mic)) {
    + case MIC_SHUTTING_DOWN:
    + mic_handle_shutdown(mic);
    + break;
    + case MIC_READY:
    + case MIC_RESET_FAILED:
    + ret = kill(mic->pid, SIGTERM);
    + mpsslog("%s: %s %d kill pid %d ret %d\n",
    + mic->name, __func__, __LINE__,
    + mic->pid, ret);
    + if (!ret) {
    + ret = waitpid(mic->pid, &stat,
    + WIFSIGNALED(stat));
    + mpsslog("%s: %s %d waitpid ret %d pid %d\n",
    + mic->name, __func__, __LINE__,
    + ret, mic->pid);
    + }
    + if (mic->boot_on_resume) {
    + setsysfs(mic->name, "state", "boot");
    + mic->boot_on_resume = 0;
    + }
    + goto close_exit;
    + default:
    + break;
    + }
    + } while (1);
    +
    +close_exit:
    + close(fd);
    +exit:
    + init_mic(mic);
    + pthread_exit(NULL);
    +}
    +
    +static void
    +set_cmdline(struct mic_info *mic)
    +{
    + char buffer[PATH_MAX];
    + int len;
    +
    + len = snprintf(buffer, PATH_MAX,
    + "clocksource=tsc highres=off nohz=off ");
    + len += snprintf(buffer + len, PATH_MAX - len,
    + "cpufreq_on;corec6_off;pc3_off;pc6_off ");
    + len += snprintf(buffer + len, PATH_MAX - len,
    + "ifcfg=static;address,172.31.%d.1;netmask,255.255.255.0",
    + mic->id + 1);
    +
    + setsysfs(mic->name, "cmdline", buffer);
    + mpsslog("%s: Command line: \"%s\"\n", mic->name, buffer);
    + snprintf(buffer, PATH_MAX, "172.31.%d.1", mic->id + 1);
    + mpsslog("%s: IPADDR: \"%s\"\n", mic->name, buffer);
    +}
    +
    +static void
    +set_log_buf_info(struct mic_info *mic)
    +{
    + int fd;
    + off_t len;
    + char system_map[] = "/lib/firmware/mic/System.map";
    + char *map, *temp, log_buf[17] = {'\0'};
    +
    + fd = open(system_map, O_RDONLY);
    + if (fd < 0) {
    + mpsslog("%s: Opening System.map failed: %d\n",
    + mic->name, errno);
    + return;
    + }
    + len = lseek(fd, 0, SEEK_END);
    + if (len < 0) {
    + mpsslog("%s: Reading System.map size failed: %d\n",
    + mic->name, errno);
    + close(fd);
    + return;
    + }
    + map = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0);
    + if (map == MAP_FAILED) {
    + mpsslog("%s: mmap of System.map failed: %d\n",
    + mic->name, errno);
    + close(fd);
    + return;
    + }
    + temp = strstr(map, "__log_buf");
    + if (!temp) {
    + mpsslog("%s: __log_buf not found: %d\n", mic->name, errno);
    + munmap(map, len);
    + close(fd);
    + return;
    + }
    + strncpy(log_buf, temp - 19, 16);
    + setsysfs(mic->name, "log_buf_addr", log_buf);
    + mpsslog("%s: log_buf_addr: %s\n", mic->name, log_buf);
    + temp = strstr(map, "log_buf_len");
    + if (!temp) {
    + mpsslog("%s: log_buf_len not found: %d\n", mic->name, errno);
    + munmap(map, len);
    + close(fd);
    + return;
    + }
    + strncpy(log_buf, temp - 19, 16);
    + setsysfs(mic->name, "log_buf_len", log_buf);
    + mpsslog("%s: log_buf_len: %s\n", mic->name, log_buf);
    + munmap(map, len);
    + close(fd);
    +}
    +
    +static void
    +change_virtblk_backend(int x, siginfo_t *siginfo, void *p)
    +{
    + struct mic_info *mic;
    +
    + for (mic = mic_list.next; mic != NULL; mic = mic->next)
    + mic->mic_virtblk.signaled = 1/* true */;
    +}
    +
    +static void
    +set_mic_boot_params(struct mic_info *mic)
    +{
    + set_log_buf_info(mic);
    + set_cmdline(mic);
    +}
    +
    +static void *
    +init_mic(void *arg)
    +{
    + struct mic_info *mic = (struct mic_info *)arg;
    + struct sigaction ignore = {
    + .sa_flags = 0,
    + .sa_handler = SIG_IGN
    + };
    + struct sigaction act = {
    + .sa_flags = SA_SIGINFO,
    + .sa_sigaction = change_virtblk_backend,
    + };
    + char buffer[PATH_MAX];
    + int err, fd;
    +
    + /*
    + * Currently, one virtio block device is supported for each MIC card
    + * at a time. Any user (or test) can send a SIGUSR1 to the MIC daemon.
    + * The signal informs the virtio block backend about a change in the
    + * configuration file which specifies the virtio backend file name on
    + * the host. Virtio block backend then re-reads the configuration file
    + * and switches to the new block device. This signalling mechanism may
    + * not be required once multiple virtio block devices are supported by
    + * the MIC daemon.
    + */
    + sigaction(SIGUSR1, &ignore, NULL);
    +retry:
    + fd = open_state_fd(mic);
    + if (fd < 0) {
    + mpsslog("%s: %s %d open state fd failed %s\n",
    + mic->name, __func__, __LINE__, strerror(errno));
    + sleep(2);
    + goto retry;
    + }
    +
    + if (mic->restart) {
    + snprintf(buffer, PATH_MAX, "boot");
    + setsysfs(mic->name, "state", buffer);
    + mpsslog("%s restarting mic %d\n",
    + mic->name, mic->restart);
    + mic->restart = 0;
    + }
    +
    + while (1) {
    + while (block_till_state_change(fd, mic)) {
    + mpsslog("%s: %s %d block_till_state_change error %s\n",
    + mic->name, __func__, __LINE__, strerror(errno));
    + sleep(2);
    + continue;
    + }
    +
    + if (get_mic_state(mic) == MIC_BOOTING)
    + break;
    + }
    +
    + mic->pid = fork();
    + switch (mic->pid) {
    + case 0:
    + add_virtio_device(mic, &virtcons_dev_page.dd);
    + add_virtio_device(mic, &virtnet_dev_page.dd);
    + err = pthread_create(&mic->mic_console.console_thread, NULL,
    + virtio_console, mic);
    + if (err)
    + mpsslog("%s virtcons pthread_create failed %s\n",
    + mic->name, strerror(err));
    + err = pthread_create(&mic->mic_net.net_thread, NULL,
    + virtio_net, mic);
    + if (err)
    + mpsslog("%s virtnet pthread_create failed %s\n",
    + mic->name, strerror(err));
    + err = pthread_create(&mic->mic_virtblk.block_thread, NULL,
    + virtio_block, mic);
    + if (err)
    + mpsslog("%s virtblk pthread_create failed %s\n",
    + mic->name, strerror(err));
    + sigemptyset(&act.sa_mask);
    + err = sigaction(SIGUSR1, &act, NULL);
    + if (err)
    + mpsslog("%s sigaction SIGUSR1 failed %s\n",
    + mic->name, strerror(errno));
    + while (1)
    + sleep(60);
    + case -1:
    + mpsslog("fork failed MIC name %s id %d errno %d\n",
    + mic->name, mic->id, errno);
    + break;
    + default:
    + err = pthread_create(&mic->config_thread, NULL,
    + mic_config, mic);
    + if (err)
    + mpsslog("%s mic_config pthread_create failed %s\n",
    + mic->name, strerror(err));
    + }
    +
    + return NULL;
    +}
    +
    +static void
    +start_daemon(void)
    +{
    + struct mic_info *mic;
    + int err;
    +
    + for (mic = mic_list.next; mic; mic = mic->next) {
    + set_mic_boot_params(mic);
    + err = pthread_create(&mic->init_thread, NULL, init_mic, mic);
    + if (err)
    + mpsslog("%s init_mic pthread_create failed %s\n",
    + mic->name, strerror(err));
    + }
    +
    + while (1)
    + sleep(60);
    +}
    +
    +static int
    +init_mic_list(void)
    +{
    + struct mic_info *mic = &mic_list;
    + struct dirent *file;
    + DIR *dp;
    + int cnt = 0;
    +
    + dp = opendir(MICSYSFSDIR);
    + if (!dp)
    + return 0;
    +
    + while ((file = readdir(dp)) != NULL) {
    + if (!strncmp(file->d_name, "mic", 3)) {
    + mic->next = calloc(1, sizeof(struct mic_info));
    + if (mic->next) {
    + mic = mic->next;
    + mic->id = atoi(&file->d_name[3]);
    + mic->name = malloc(strlen(file->d_name) + 16);
    + if (mic->name)
    + strcpy(mic->name, file->d_name);
    + mpsslog("MIC name %s id %d\n", mic->name,
    + mic->id);
    + cnt++;
    + }
    + }
    + }
    +
    + closedir(dp);
    + return cnt;
    +}
    +
    +void
    +mpsslog(char *format, ...)
    +{
    + va_list args;
    + char buffer[4096];
    + char ts[52], *ts1;
    + time_t t;
    +
    + if (logfp == NULL)
    + return;
    +
    + va_start(args, format);
    + vsprintf(buffer, format, args);
    + va_end(args);
    +
    + time(&t);
    + ts1 = ctime_r(&t, ts);
    + ts1[strlen(ts1) - 1] = '\0';
    + fprintf(logfp, "%s: %s", ts1, buffer);
    +
    + fflush(logfp);
    +}
    +
    +int
    +main(int argc, char *argv[])
    +{
    + int cnt;
    + pid_t pid;
    +
    + myname = argv[0];
    +
    + logfp = fopen(LOGFILE_NAME, "a+");
    + if (!logfp) {
    + fprintf(stderr, "cannot open logfile '%s'\n", LOGFILE_NAME);
    + exit(1);
    + }
    + pid = fork();
    + switch (pid) {
    + case 0:
    + break;
    + case -1:
    + exit(2);
    + default:
    + exit(0);
    + }
    +
    + mpsslog("MIC Daemon start\n");
    +
    + cnt = init_mic_list();
    + if (cnt == 0) {
    + mpsslog("MIC module not loaded\n");
    + exit(3);
    + }
    + mpsslog("MIC found %d devices\n", cnt);
    +
    + start_daemon();
    +
    + exit(0);
    +}
    diff --git a/samples/mic/mpssd/mpssd.h b/samples/mic/mpssd/mpssd.h
    new file mode 100644
    index 0000000..8bd6494
    --- /dev/null
    +++ b/samples/mic/mpssd/mpssd.h
    @@ -0,0 +1,103 @@
    +/*
    + * Intel MIC Platform Software Stack (MPSS)
    + *
    + * Copyright(c) 2013 Intel Corporation.
    + *
    + * This program is free software; you can redistribute it and/or modify
    + * it under the terms of the GNU General Public License, version 2, as
    + * published by the Free Software Foundation.
    + *
    + * This program is distributed in the hope that it will be useful, but
    + * WITHOUT ANY WARRANTY; without even the implied warranty of
    + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
    + * General Public License for more details.
    + *
    + * The full GNU General Public License is included in this distribution in
    + * the file called "COPYING".
    + *
    + * Intel MIC User Space Tools.
    + */
    +#ifndef _MPSSD_H_
    +#define _MPSSD_H_
    +
    +#include <stdio.h>
    +#include <stdlib.h>
    +#include <string.h>
    +#include <fcntl.h>
    +#include <unistd.h>
    +#include <dirent.h>
    +#include <libgen.h>
    +#include <pthread.h>
    +#include <stdarg.h>
    +#include <time.h>
    +#include <errno.h>
    +#include <sys/dir.h>
    +#include <sys/ioctl.h>
    +#include <sys/poll.h>
    +#include <sys/types.h>
    +#include <sys/socket.h>
    +#include <sys/stat.h>
    +#include <sys/types.h>
    +#include <sys/mman.h>
    +#include <sys/utsname.h>
    +#include <sys/wait.h>
    +#include <netinet/in.h>
    +#include <arpa/inet.h>
    +#include <netdb.h>
    +#include <pthread.h>
    +#include <signal.h>
    +#include <limits.h>
    +#include <syslog.h>
    +#include <getopt.h>
    +#include <net/if.h>
    +#include <linux/if_tun.h>
    +#include <linux/if_tun.h>
    +#include <linux/virtio_ids.h>
    +
    +#define MICSYSFSDIR "/sys/class/mic"
    +#define LOGFILE_NAME "/var/log/mpssd"
    +#define PAGE_SIZE 4096
    +
    +struct mic_console_info {
    + pthread_t console_thread;
    + int virtio_console_fd;
    + void *console_dp;
    +};
    +
    +struct mic_net_info {
    + pthread_t net_thread;
    + int virtio_net_fd;
    + int tap_fd;
    + void *net_dp;
    +};
    +
    +struct mic_virtblk_info {
    + pthread_t block_thread;
    + int virtio_block_fd;
    + void *block_dp;
    + volatile sig_atomic_t signaled;
    + char *backend_file;
    + int backend;
    + void *backend_addr;
    + long backend_size;
    +};
    +
    +struct mic_info {
    + int id;
    + char *name;
    + pthread_t config_thread;
    + pthread_t init_thread;
    + pid_t pid;
    + struct mic_console_info mic_console;
    + struct mic_net_info mic_net;
    + struct mic_virtblk_info mic_virtblk;
    + int restart;
    + int boot_on_resume;
    + struct mic_info *next;
    +};
    +
    +__attribute__((format(printf, 1, 2)))
    +void mpsslog(char *format, ...);
    +char *readsysfs(char *dir, char *entry);
    +int setsysfs(char *dir, char *entry, char *value);
    +#endif
    diff --git a/samples/mic/mpssd/sysfs.c b/samples/mic/mpssd/sysfs.c
    new file mode 100644
    index 0000000..8dd3269
    --- /dev/null
    +++ b/samples/mic/mpssd/sysfs.c
    @@ -0,0 +1,102 @@
    +/*
    + * Intel MIC Platform Software Stack (MPSS)
    + *
    + * Copyright(c) 2013 Intel Corporation.
    + *
    + * This program is free software; you can redistribute it and/or modify
    + * it under the terms of the GNU General Public License, version 2, as
    + * published by the Free Software Foundation.
    + *
    + * This program is distributed in the hope that it will be useful, but
    + * WITHOUT ANY WARRANTY; without even the implied warranty of
    + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
    + * General Public License for more details.
    + *
    + * The full GNU General Public License is included in this distribution in
    + * the file called "COPYING".
    + *
    + * Intel MIC User Space Tools.
    + */
    +
    +#include "mpssd.h"
    +
    +#define PAGE_SIZE 4096
    +
    +char *
    +readsysfs(char *dir, char *entry)
    +{
    + char filename[PATH_MAX];
    + char value[PAGE_SIZE];
    + char *string = NULL;
    + int fd;
    + int len;
    +
    + if (dir == NULL)
    + snprintf(filename, PATH_MAX, "%s/%s", MICSYSFSDIR, entry);
    + else
    + snprintf(filename, PATH_MAX,
    + "%s/%s/%s", MICSYSFSDIR, dir, entry);
    +
    + fd = open(filename, O_RDONLY);
    + if (fd < 0) {
    + mpsslog("Failed to open sysfs entry '%s': %s\n",
    + filename, strerror(errno));
    + return NULL;
    + }
    +
    + len = read(fd, value, sizeof(value));
    + if (len < 0) {
    + mpsslog("Failed to read sysfs entry '%s': %s\n",
    + filename, strerror(errno));
    + goto readsys_ret;
    + }
    + if (len == 0)
    + goto readsys_ret;
    +
    + value[len - 1] = '\0';
    +
    + string = malloc(strlen(value) + 1);
    + if (string)
    + strcpy(string, value);
    +
    +readsys_ret:
    + close(fd);
    + return string;
    +}
    +
    +int
    +setsysfs(char *dir, char *entry, char *value)
    +{
    + char filename[PATH_MAX];
    + char *oldvalue;
    + int fd, ret = 0;
    +
    + if (dir == NULL)
    + snprintf(filename, PATH_MAX, "%s/%s", MICSYSFSDIR, entry);
    + else
    + snprintf(filename, PATH_MAX, "%s/%s/%s",
    + MICSYSFSDIR, dir, entry);
    +
    + oldvalue = readsysfs(dir, entry);
    +
    + fd = open(filename, O_RDWR);
    + if (fd < 0) {
    + ret = errno;
    + mpsslog("Failed to open sysfs entry '%s': %s\n",
    + filename, strerror(errno));
    + goto done;
    + }
    +
    + if (!oldvalue || strcmp(value, oldvalue)) {
    + if (write(fd, value, strlen(value)) < 0) {
    + ret = errno;
    + mpsslog("Failed to write new sysfs entry '%s': %s\n",
    + filename, strerror(errno));
    + }
    + }
    + close(fd);
    +done:
    + if (oldvalue)
    + free(oldvalue);
    + return ret;
    +}
    --
    2.7.4
    \
     
     \ /
      Last update: 2016-09-19 16:53    [W:3.401 / U:0.680 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site