From ba0b093ec377b6dc861b5676b70e96505a678f39 Mon Sep 17 00:00:00 2001
From: Benajmin Li <benli@broadcom.com>
Date: Tue, 20 Jul 2010 22:26:25 -0700
Subject: [PATCH 1/4] Because in the current iface file doesn't allow one to specify a VLAN
tag, the VLAN tag will be taking from the path request given by the CNIC.
This will also allow uIP to close and reopen devices properly.
---
brcm_iscsi_uio/src/unix/iscsid_ipc.c | 40 +++++++++++++++++++-
brcm_iscsi_uio/src/unix/libs/bnx2.c | 14 -------
brcm_iscsi_uio/src/unix/libs/bnx2x.c | 14 -------
brcm_iscsi_uio/src/unix/nic.c | 66 ++++++++++++++++++++------------
brcm_iscsi_uio/src/unix/nic.h | 16 +++++++-
brcm_iscsi_uio/src/unix/nic_nl.c | 56 ++++++++++++++++++++++++---
brcm_iscsi_uio/src/unix/nic_utils.c | 70 ++++++++++++++++++++++++++++++++--
brcm_iscsi_uio/src/unix/nic_utils.h | 6 ++-
brcm_iscsi_uio/src/unix/uevent.c | 2 +-
9 files changed, 217 insertions(+), 67 deletions(-)
diff --git a/brcm_iscsi_uio/src/unix/iscsid_ipc.c b/brcm_iscsi_uio/src/unix/iscsid_ipc.c
index b06100e..4c00ef2 100644
--- a/brcm_iscsi_uio/src/unix/iscsid_ipc.c
+++ b/brcm_iscsi_uio/src/unix/iscsid_ipc.c
@@ -131,6 +131,42 @@ static int parse_iface(void * arg)
data->u.iface_rec.rec.netdev);
}
+ if (nic->flags & NIC_GOING_DOWN) {
+ rc = -EIO;
+ goto done;
+ }
+
+ /* If we retry too many times allow iscsid to to timeout */
+ if (nic->pending_count > 1000) {
+ LOG_WARN(PFX "%s: pending count excceded 1000",
+ nic->log_name);
+
+ pthread_mutex_lock(&nic->nic_mutex);
+ nic->pending_count = 0;
+ nic->flags &= ~NIC_ENABLED_PENDING;
+ pthread_mutex_unlock(&nic->nic_mutex);
+
+ rc = 0;
+ goto done;
+ }
+
+ if (nic->flags & NIC_ENABLED_PENDING) {
+ struct timespec sleep_req, sleep_rem;
+
+ sleep_req.tv_sec = 0;
+ sleep_req.tv_nsec = 100000;
+ nanosleep(&sleep_req, &sleep_rem);
+
+ pthread_mutex_lock(&nic->nic_mutex);
+ nic->pending_count++;
+ pthread_mutex_unlock(&nic->nic_mutex);
+
+ LOG_INFO(PFX "%s: enabled pending",
+ nic->log_name);
+ rc = -EAGAIN;
+ goto done;
+ }
+
prepare_nic(nic);
/* Sanity Check to ensure the transport names are the same */
@@ -210,7 +246,9 @@ static int parse_iface(void * arg)
goto enable_nic;
} else {
/* Disable the NIC */
- nic_disable(nic);
+ nic_disable(nic, 0);
+
+ persist_all_nic_iface(nic);
}
/* Check to see if this is using DHCP or if this is
diff --git a/brcm_iscsi_uio/src/unix/libs/bnx2.c b/brcm_iscsi_uio/src/unix/libs/bnx2.c
index 400fd43..2b103b1 100644
--- a/brcm_iscsi_uio/src/unix/libs/bnx2.c
+++ b/brcm_iscsi_uio/src/unix/libs/bnx2.c
@@ -788,20 +788,6 @@ static int bnx2_close(nic_t *nic, NIC_SHUTDOWN_T graceful)
bnx2_uio_close_resources(nic, graceful);
- /* Free any named strings we might be holding onto */
- if(nic->flags & NIC_CONFIG_NAME_MALLOC) {
- free(nic->config_device_name);
- nic->flags &= ~NIC_CONFIG_NAME_MALLOC;
- }
- nic->config_device_name = NULL;
-
- if(nic->flags & NIC_UIO_NAME_MALLOC) {
- free(nic->uio_device_name);
- nic->uio_device_name = NULL;
-
- nic->flags &= ~NIC_UIO_NAME_MALLOC;
- }
-
return 0;
}
diff --git a/brcm_iscsi_uio/src/unix/libs/bnx2x.c b/brcm_iscsi_uio/src/unix/libs/bnx2x.c
index a3f4fb2..ce71109 100644
--- a/brcm_iscsi_uio/src/unix/libs/bnx2x.c
+++ b/brcm_iscsi_uio/src/unix/libs/bnx2x.c
@@ -985,20 +985,6 @@ static int bnx2x_close(nic_t *nic, NIC_SHUTDOWN_T graceful)
bnx2x_uio_close_resources(nic, graceful);
- /* Free any named strings we might be holding onto */
- if(nic->flags & NIC_CONFIG_NAME_MALLOC) {
- free(nic->config_device_name);
- nic->flags &= ~NIC_CONFIG_NAME_MALLOC;
- }
- nic->config_device_name = NULL;
-
- if(nic->flags & NIC_UIO_NAME_MALLOC) {
- free(nic->uio_device_name);
- nic->uio_device_name = NULL;
-
- nic->flags &= ~NIC_UIO_NAME_MALLOC;
- }
-
return 0;
}
diff --git a/brcm_iscsi_uio/src/unix/nic.c b/brcm_iscsi_uio/src/unix/nic.c
index da83e2b..2b267f2 100644
--- a/brcm_iscsi_uio/src/unix/nic.c
+++ b/brcm_iscsi_uio/src/unix/nic.c
@@ -400,7 +400,7 @@ nic_t *nic_init()
return nic;
}
-int nic_remove(nic_t *nic, int locked)
+int nic_remove(nic_t *nic)
{
int rc;
nic_t *prev, *current;
@@ -422,9 +422,6 @@ int nic_remove(nic_t *nic, int locked)
nic->thread = INVALID_THREAD;
- if(!locked)
- pthread_mutex_lock(&nic_list_mutex);
-
current = prev = nic_list;
while(current != NULL) {
if(current == nic)
@@ -445,9 +442,6 @@ int nic_remove(nic_t *nic, int locked)
LOG_ERR(PFX "%s: Coudln't find nic", nic->log_name);
}
- if(!locked)
- pthread_mutex_unlock(&nic_list_mutex);
-
return 0;
}
@@ -459,16 +453,14 @@ int nic_remove(nic_t *nic, int locked)
* before proceeding to close()
* FORCE_SHUTDOWN will force the nic to close()
* reguardless of the state
+ * @param clean - this will free the proper strings assoicated
+ * with the NIC
*/
-void nic_close(nic_t *nic, NIC_SHUTDOWN_T graceful)
+void nic_close(nic_t *nic, NIC_SHUTDOWN_T graceful, int clean)
{
int rc;
nic_interface_t *nic_iface;
- if((nic->flags & NIC_DISABLED) &&
- (graceful == ALLOW_GRACEFUL_SHUTDOWN))
- return;
-
/* The NIC could be configured by the uIP config file
* but not assoicated with a hardware library just yet
* we will need to check for this */
@@ -491,10 +483,36 @@ void nic_close(nic_t *nic, NIC_SHUTDOWN_T graceful)
while(nic_iface != NULL)
{
nic_iface->state = NIC_IFACE_STOPPED;
- uip_reset(&nic_iface->ustack);
+ if (!((nic_iface->state & NIC_IFACE_PERSIST) ==
+ NIC_IFACE_PERSIST))
+ uip_reset(&nic_iface->ustack);
nic_iface = nic_iface->next;
}
+ /* The NIC must be destroyed and init'ed once again,
+ * POSIX defines that the mutex will be undefined it
+ * init'ed twice without a destroy */
+ pthread_mutex_destroy(&nic->xmit_mutex);
+ pthread_mutex_init(&nic->xmit_mutex, NULL);
+
+ if (clean & FREE_CONFIG_NAME) {
+ /* Free any named strings we might be holding onto */
+ if(nic->flags & NIC_CONFIG_NAME_MALLOC) {
+ free(nic->config_device_name);
+ nic->flags &= ~NIC_CONFIG_NAME_MALLOC;
+ }
+ nic->config_device_name = NULL;
+ }
+
+ if (clean & FREE_UIO_NAME) {
+ if(nic->flags & NIC_UIO_NAME_MALLOC) {
+ free(nic->uio_device_name);
+ nic->uio_device_name = NULL;
+
+ nic->flags &= ~NIC_UIO_NAME_MALLOC;
+ }
+ }
+ LOG_ERR(PFX "%s: nic closed", nic->log_name);
error:
return;
}
@@ -1138,6 +1156,8 @@ void *nic_loop(void *arg)
pthread_mutex_lock(&nic->nic_mutex);
if (rc) {
+ LOG_ERR(PFX "%s: DHCP failed",
+ nic->log_name);
pthread_mutex_unlock(&nic->nic_mutex);
goto dev_close;
}
@@ -1164,6 +1184,9 @@ void *nic_loop(void *arg)
goto dev_close;
}
+ pthread_mutex_lock(&nic->nic_mutex);
+ unpersist_all_nic_iface(nic);
+
/* This is when we start the processing of packets */
nic->start_time = time(NULL);
nic->flags &= ~NIC_UNITIALIZED;
@@ -1171,8 +1194,9 @@ void *nic_loop(void *arg)
nic->state &= ~NIC_STOPPED;
nic->state |= NIC_RUNNING;
+ nic->flags &= ~NIC_ENABLED_PENDING;
+
/* Signal that the device enable is done */
- pthread_mutex_lock(&nic->nic_mutex);
pthread_cond_broadcast(&nic->enable_done_cond);
pthread_mutex_unlock(&nic->nic_mutex);
@@ -1197,17 +1221,11 @@ void *nic_loop(void *arg)
dev_close:
pthread_mutex_lock(&nic->nic_mutex);
- /* Ensure that the IP configuration is cleared */
- nic_iface = nic->nic_iface;
- while(nic_iface != NULL)
- {
- nic_iface->ustack.ip_config =
- (IPV4_CONFIG_OFF | IPV6_CONFIG_OFF);
- nic_iface = nic_iface->next;
- }
-
nic->state = NIC_STOPPED;
- nic_close(nic, 1);
+ nic_close(nic, 1, FREE_NO_STRINGS);
+
+ nic->flags |= NIC_UNITIALIZED;
+ nic->flags &= ~NIC_INITIALIZED;
/* Signal we are done closing CNIC/UIO device */
pthread_cond_broadcast(&nic->disable_wait_cond);
diff --git a/brcm_iscsi_uio/src/unix/nic.h b/brcm_iscsi_uio/src/unix/nic.h
index 097230a..7636948 100644
--- a/brcm_iscsi_uio/src/unix/nic.h
+++ b/brcm_iscsi_uio/src/unix/nic.h
@@ -45,6 +45,11 @@ extern struct nic *nic_list;
#define MAX_PCI_DEVICE_ENTRIES 64 /* Maxium number of pci_device_id
entries a hw library may contain */
+#define FREE_CONFIG_NAME 0x0001
+#define FREE_UIO_NAME 0x0002
+#define FREE_ALL_STRINGS FREE_CONFIG_NAME | FREE_UIO_NAME
+#define FREE_NO_STRINGS 0x0000
+
/******************************************************************************
* Enumerations
******************************************************************************/
@@ -89,6 +94,7 @@ typedef struct nic_interface {
uint16_t protocol;
uint16_t flags;
+#define NIC_IFACE_PERSIST 0x0001
uint16_t state;
#define NIC_IFACE_STOPPED 0x0001
#define NIC_IFACE_RUNNING 0x0002
@@ -173,9 +179,12 @@ typedef struct nic {
#define NIC_VLAN_STRIP_ENABLED 0x0100
#define NIC_MSIX_ENABLED 0x0200
#define NIC_TX_HAS_SENT 0x0400
+#define NIC_ENABLED_PENDING 0x0800
#define NIC_UIO_NAME_MALLOC 0x1000
#define NIC_CONFIG_NAME_MALLOC 0x2000
+#define NIC_EXIT_MAIN_LOOP 0x4000
+#define NIC_GOING_DOWN 0x8000
uint16_t state;
#define NIC_STOPPED 0x0001
@@ -236,6 +245,9 @@ typedef struct nic {
time_t start_time;
struct nic_stats stats;
+ /* Number of retrys from iscsid*/
+ uint32_t pending_count;
+
#define DEFAULT_RX_POLL_USEC 100 /* usec */
/* options enabled by the user */
uint32_t rx_poll_usec;
@@ -271,7 +283,7 @@ typedef struct nic {
int load_all_nic_libraries();
nic_t *nic_init();
-int nic_remove(nic_t * nic, int locked);
+int nic_remove(nic_t * nic);
int nic_add_nic_iface(nic_t *nic,
nic_interface_t *nic_iface);
@@ -298,7 +310,7 @@ void put_packet_in_free_queue(struct packet *pkt,
nic_t *nic);
int unload_all_nic_libraries();
-void nic_close(nic_t *nic, NIC_SHUTDOWN_T graceful);
+void nic_close(nic_t *nic, NIC_SHUTDOWN_T graceful, int clean);
/* Use this function to fill in minor number and uio, and eth names */
int nic_fill_name(nic_t *nic);
diff --git a/brcm_iscsi_uio/src/unix/nic_nl.c b/brcm_iscsi_uio/src/unix/nic_nl.c
index fa4033a..c624689 100644
--- a/brcm_iscsi_uio/src/unix/nic_nl.c
+++ b/brcm_iscsi_uio/src/unix/nic_nl.c
@@ -347,11 +347,51 @@ static int ctldev_handle()
nic_iface = nic_find_nic_iface_protocol(nic, path->vlan_id,
ip_type);
if (nic_iface == NULL) {
- LOG_WARN(PFX "%s: Couldn't find nic_iface "
- "vlan: %d ip_addr_len",
- nic->log_name,
- path->vlan_id, path->ip_addr_len);
- goto error;
+ nic_interface_t *default_iface;
+ default_iface = nic_find_nic_iface_protocol(nic,
+ 0,
+ ip_type);
+ if (default_iface == NULL)
+ {
+ LOG_ERR(PFX "%s: Couldn't find default iface "
+ "vlan: %d ip_type: %d "
+ "ip_addr_len: %d to clone",
+ nic->log_name, path->vlan_id, ip_type,
+ path->ip_addr_len);
+ goto error;
+ }
+
+ nic_iface = nic_iface_init();
+ if (nic_iface == NULL)
+ {
+ LOG_ERR(PFX "%s: Couldn't allocate space for "
+ "vlan: %d ip_type: %d "
+ "ip_addr_len: %d",
+ nic->log_name, path->vlan_id, ip_type,
+ path->ip_addr_len);
+
+ goto error;
+ }
+
+ nic_iface->protocol = ip_type;
+ nic_iface->vlan_id = path->vlan_id;
+ nic_add_nic_iface(nic, nic_iface);
+
+ /* TODO: When VLAN support is placed in the iface file
+ * revisit this code */
+ nic_iface->ustack.ip_config = default_iface->ustack.ip_config;
+ memcpy(&nic_iface->ustack.hostaddr,
+ &default_iface->ustack.hostaddr,
+ sizeof(nic_iface->ustack.hostaddr));
+ memcpy(&nic_iface->ustack.netmask,
+ &default_iface->ustack.netmask,
+ sizeof(nic_iface->ustack.netmask));
+ memcpy(&nic_iface->ustack.hostaddr6,
+ &default_iface->ustack.hostaddr6,
+ sizeof(nic_iface->ustack.hostaddr6));
+
+ nic_disable(nic, 0);
+ nic_enable(nic);
}
/* Ensure that the NIC is RUNNING */
@@ -386,7 +426,11 @@ static int ctldev_handle()
}
break;
case ISCSI_KEVENT_IF_DOWN:
- nic_remove(nic, 0);
+
+ pthread_mutex_lock(&nic_list_mutex);
+ nic_disable(nic, 1);
+ nic_remove(nic);
+ pthread_mutex_unlock(&nic_list_mutex);
break;
}
diff --git a/brcm_iscsi_uio/src/unix/nic_utils.c b/brcm_iscsi_uio/src/unix/nic_utils.c
index 0e36a67..eabb474 100644
--- a/brcm_iscsi_uio/src/unix/nic_utils.c
+++ b/brcm_iscsi_uio/src/unix/nic_utils.c
@@ -787,6 +787,7 @@ int nic_enable(nic_t *nic)
nic->flags &= ~NIC_DISABLED;
nic->flags |= NIC_ENABLED;
+ nic->flags |= NIC_ENABLED_PENDING;
pthread_mutex_unlock(&nic->nic_mutex);
@@ -811,6 +812,8 @@ int nic_enable(nic_t *nic)
nic->log_name, strerror(rc));
return rc;
}
+
+ nic->flags &= ~NIC_ENABLED_PENDING;
}
else
{
@@ -826,11 +829,15 @@ int nic_enable(nic_t *nic)
* @param nic - NIC to disble
* @return 0 on success, <0 on failure
*/
-int nic_disable(nic_t *nic)
+int nic_disable(nic_t *nic, int going_down)
{
if( (nic->flags & NIC_ENABLED) &&
(nic->state & NIC_RUNNING))
{
+ struct timespec ts;
+ struct timeval tp;
+ int rc;
+
/* Wait for the device to be disabled */
pthread_mutex_lock(&nic->nic_mutex);
@@ -839,8 +846,23 @@ int nic_disable(nic_t *nic)
nic->state &= ~NIC_RUNNING;
nic->state |= NIC_STOPPED;
- pthread_cond_wait(&nic->disable_wait_cond,
- &nic->nic_mutex);
+ if (going_down) {
+ nic->flags |= NIC_GOING_DOWN;
+
+ if (nic->thread != INVALID_THREAD) {
+ pthread_cancel(nic->thread);
+ }
+ }
+
+ /* Convert from timeval to timespec */
+ rc = gettimeofday(&tp, NULL);
+ ts.tv_sec = tp.tv_sec;
+ ts.tv_nsec = tp.tv_usec * 1000;
+ ts.tv_sec += 2; /* TODO: hardcoded wait for 2 seconds */
+
+ /* Wait for the device to be disabled */
+ rc = pthread_cond_timedwait(&nic->disable_wait_cond,
+ &nic->nic_mutex, &ts);
pthread_mutex_unlock(&nic->nic_mutex);
LOG_DEBUG(PFX "%s: device disabled", nic->log_name);
@@ -865,7 +887,7 @@ void nic_close_all()
nic = nic_list;
while (nic != NULL) {
pthread_mutex_lock(&nic->nic_mutex);
- nic_close(nic, 1);
+ nic_close(nic, 1, FREE_ALL_STRINGS);
pthread_mutex_unlock(&nic->nic_mutex);
nic = nic->next;
@@ -1224,6 +1246,46 @@ nic_interface_t * nic_find_nic_iface_protocol(nic_t *nic,
return NULL;
}
+/**
+ * unpersist_all_nic_iface() - Mark all the NIC interfaces as non-persistant
+ * Note: nic->nic_mutex must be taken
+ * @param nic - NIC to mark as non-persistant
+ */
+void unpersist_all_nic_iface(nic_t *nic)
+{
+ nic_interface_t *current;
+
+ current = nic->nic_iface;
+ while (current != NULL)
+ {
+ current->flags &= ~NIC_IFACE_PERSIST;
+
+ current = current->next;
+ }
+}
+
+/**
+ * persist_all_nic_iface() - Mark all the NIC interfaces as persistant
+ * Note: nic->nic_mutex must be taken
+ * @param nic - NIC to mark as persistant
+ */
+void persist_all_nic_iface(nic_t *nic)
+{
+ nic_interface_t *current;
+
+ pthread_mutex_lock(&nic->nic_mutex);
+
+ current = nic->nic_iface;
+ while (current != NULL)
+ {
+ current->flags |= NIC_IFACE_PERSIST;
+
+ current = current->next;
+ }
+
+ pthread_mutex_unlock(&nic->nic_mutex);
+}
+
/*******************************************************************************
* Packet management utility functions
******************************************************************************/
diff --git a/brcm_iscsi_uio/src/unix/nic_utils.h b/brcm_iscsi_uio/src/unix/nic_utils.h
index 5034006..283e324 100644
--- a/brcm_iscsi_uio/src/unix/nic_utils.h
+++ b/brcm_iscsi_uio/src/unix/nic_utils.h
@@ -46,6 +46,10 @@ void nic_fill_ethernet_header(nic_interface_t *nic_iface,
nic_interface_t * nic_find_nic_iface(nic_t *nic,
uint16_t vlan_id);
+
+void unpersist_all_nic_iface(nic_t *nic);
+void persist_all_nic_iface(nic_t *nic);
+
int add_vlan_interfaces(nic_t *nic);
int nic_verify_uio_sysfs_name(nic_t *nic);
@@ -56,7 +60,7 @@ uint32_t calculate_default_netmask(uint32_t ip_addr);
void prepare_nic(nic_t *nic);
int nic_enable(nic_t *nic);
-int nic_disable(nic_t *nic);
+int nic_disable(nic_t *nic, int);
void dump_packet_to_log(struct nic_interface *iface,
uint8_t *buf, uint16_t buf_len);
diff --git a/brcm_iscsi_uio/src/unix/uevent.c b/brcm_iscsi_uio/src/unix/uevent.c
index 2019c64..4ff7e04 100644
--- a/brcm_iscsi_uio/src/unix/uevent.c
+++ b/brcm_iscsi_uio/src/unix/uevent.c
@@ -333,7 +333,7 @@ static int close_cnic_dev(struct parsed_uevent *event)
nic = nic_list;
while (nic != NULL) {
if (nic->uio_minor == minor) {
- nic_remove(nic, 1);
+ nic_remove(nic);
rc = 0;
break;
--
1.7.1