IoT Hub: Cannot send while disconnected

Hello all. I have started seeing this error on my devices for the past few days. I have 2 SenseNode 003 devices running connected to the Azure Hub. The devices are not transmitting any reads to the IoT Hub at all, but I can see the reads still happening in my log. Below is what I have in one of the logs:

2017-09-02 07:51:57 -05:00 [Agent] { “readings”: [ { “Humidity”: 48.5371, “EnvReadingTime”: 1504356715, “AirPressure”: 1254.02, “Temperature”: 20.3413 } ] }
2017-09-02 07:51:57 -05:00 [Agent] Sending message: { “Row”: 1504356717, “Location”: { “Latitude”: 42.0269, “Longitude”: -88.0329 }, “DeviceID”: “30000c2a690cc75b”, “Readings”: [ { “Humidity”: 48.5371, “EnvReadingTime”: “02-09-2017, 12:51:55”, “AirPressure”: 1254.02, “Temperature”: 20.3413 } ], “Activity”: “Reading” }
2017-09-02 07:51:57 -05:00 [Agent] ERROR: Failed to send message to Azure IoT Hub: Cannot send while disconnected.
2017-09-02 07:51:57 -05:00 [Device] Sleeping until 2017-09-02 13:21:55Z
2017-09-02 08:21:59 -05:00 [Status] Device disconnected
2017-09-02 08:21:59 -05:00 [Status] Device connected
2017-09-02 08:21:59 -05:00 [Device] Readings Done
2017-09-02 08:21:59 -05:00 [Device] Sleeping until 2017-09-02 13:51:57Z
2017-09-02 08:21:59 -05:00 [Agent] { “readings”: [ { “Humidity”: 48.9856, “EnvReadingTime”: 1504358517, “AirPressure”: 1255.6, “Temperature”: 20.4145 } ] }
2017-09-02 08:21:59 -05:00 [Agent] Sending message: { “Row”: 1504358519, “Location”: { “Latitude”: 42.0269, “Longitude”: -88.0329 }, “DeviceID”: “30000c2a690cc75b”, “Readings”: [ { “Humidity”: 48.9856, “EnvReadingTime”: “02-09-2017, 13:21:57”, “AirPressure”: 1255.6, “Temperature”: 20.4145 } ], “Activity”: “Reading” }
2017-09-02 08:21:59 -05:00 [Agent] ERROR: Failed to send message to Azure IoT Hub: Cannot send while disconnected.
2017-09-02 08:52:01 -05:00 [Status] Device disconnected
2017-09-02 08:52:01 -05:00 [Agent] { “readings”: [ { “Humidity”: 48.7505, “EnvReadingTime”: 1504360319, “AirPressure”: 1255.71, “Temperature”: 20.5426 } ] }
2017-09-02 08:52:01 -05:00 [Agent] Sending message: { “Row”: 1504360321, “Location”: { “Latitude”: 42.0269, “Longitude”: -88.0329 }, “DeviceID”: “30000c2a690cc75b”, “Readings”: [ { “Humidity”: 48.7505, “EnvReadingTime”: “02-09-2017, 13:51:59”, “AirPressure”: 1255.71, “Temperature”: 20.5426 } ], “Activity”: “Reading” }
2017-09-02 08:52:01 -05:00 [Agent] ERROR: Failed to send message to Azure IoT Hub: Cannot send while disconnected.
2017-09-02 08:52:01 -05:00 [Status] Device connected
2017-09-02 08:52:01 -05:00 [Device] Readings Done
2017-09-02 08:52:01 -05:00 [Device] Sleeping until 2017-09-02 14:21:59Z

Below is the Device Code:
// Remote Monitoring Application With Interrupt Device Code
// --------------------------------------------------------

// SENSOR LIBRARIES
// --------------------------------------------------------
// Libraries must be required before all other code

// Accelerometer Library
#require “LIS3DH.device.lib.nut:2.0.0”
// Temperature Humidity sensor Library
#require “HTS221.device.lib.nut:2.0.1”
// Air Pressure sensor Library
#require “LPS22HB.device.lib.nut:2.0.0”
// Library to help with asynchonous programming
#require “promise.class.nut:3.0.1”
// Library to manage agent/device communication
#require “MessageManager.lib.nut:2.0.0”

// HARDWARE ABSTRACTION LAYER
// --------------------------------------------------------
// HAL’s are tables that map human readable names to
// the hardware objects used in the application.

// Sensor Node HAL
SensorNode_003 <- {
“LED_BLUE” : hardware.pinP,
“LED_GREEN” : hardware.pinU,
“SENSOR_I2C” : hardware.i2cAB,
“TEMP_HUMID_I2C_ADDR” : 0xBE,
“ACCEL_I2C_ADDR” : 0x32,
“PRESSURE_I2C_ADDR” : 0xB8,
“RJ12_ENABLE_PIN” : hardware.pinS,
“ONEWIRE_BUS_UART” : hardware.uartDM,
“RJ12_I2C” : hardware.i2cFG,
“RJ12_UART” : hardware.uartFG,
“WAKE_PIN” : hardware.pinW,
“ACCEL_INT_PIN” : hardware.pinT,
“PRESSURE_INT_PIN” : hardware.pinX,
“TEMP_HUMID_INT_PIN” : hardware.pinE,
“NTC_ENABLE_PIN” : hardware.pinK,
“THERMISTER_PIN” : hardware.pinJ,
“FTDI_UART” : hardware.uartQRPW,
“PWR_3v3_EN” : hardware.pinY
}

// Location Class
// --------------------------------------------------------
// Class that sends wifi scan to agent upon request
class Location {

mm = null;
wifis = null;

constructor(msgMngr) {
    mm = msgMngr;
    mm.send("wifi.networks", imp.scanwifinetworks());
}

}

// REMOTE MONITORING INTERRUPT APPLICATION CODE
// --------------------------------------------------------
// Application code, take readings from our sensors
// and send the data to the agent

class Application {

// Default time in seconds to wait between readings (30 min)
static DEFAULT_READING_INTERVAL_SEC = 1800;
// Time to wait after boot before turning off WiFi
static BOOT_TIMER_SEC = 60;
// Time (in seconds) to block motion detection interrupt after door opening
static DOOR_OPEN_TIMER_SEC = 180;
// Accelerometer data rate in Hz
static ACCEL_DATARATE = 100;
// Max number of stored readings
static MAX_NUM_STORED_DATA = 20;

// Hardware variables
i2c             = SensorNode_003.SENSOR_I2C; 
tempHumidAddr   = SensorNode_003.TEMP_HUMID_I2C_ADDR; 
pressureAddr    = SensorNode_003.PRESSURE_I2C_ADDR; 
accelAddr       = SensorNode_003.ACCEL_I2C_ADDR; 
wakePin         = SensorNode_003.WAKE_PIN; 

// Sensor variables
tempHumid = null;
pressure = null;
accel = null;

// Message Manager variable
mm = null;

// Flag to track first disconnection
_boot = false;

// Variable to store time in seconds to wait between readings
_readingInterval = null;

constructor(locEnable, readingInt = null) {
    // Power save mode will reduce power consumption when the 
    // radio is idle. This adds latency when sending data. 
    imp.setpowersave(false);

    // Change default connection policy, so our application 
    // continues to run even if the WiFi connection fails
    server.setsendtimeoutpolicy(RETURN_ON_ERROR, WAIT_TIL_SENT, 10);

    // Configure message manager for device/agent communication
    mm = MessageManager();
    // Message Manager allows us to call a function when a message  
    // has been delivered. We will use this to know when it is ok
    // to disconnect from WiFi
    mm.onAck(readingsAckHandler.bindenv(this));
    // Message Manager allows us to call a function if a message  
    // fails to be delivered. We will use this to recover data 
    mm.onFail(sendFailHandler.bindenv(this));

    // Open wifi scan listener
    if (locEnable) { Location(mm);}

    // Set a reading interval
    _readingInterval = (typeof readingInt == "integer" || typeof readingInt == "float") ? readingInt : DEFAULT_READING_INTERVAL_SEC;

    // Initialize sensors
    initializeSensors();

    // Configure different behavior based on the reason the 
    // hardware rebooted 
    checkWakeReason();
}

// Select the flow based on why we woke from sleep
function checkWakeReason() {
    // We can configure different behavior based on 
    // the reason the hardware rebooted. 
    switch (hardware.wakereason()) {
        case WAKEREASON_TIMER :
            // We woke up after sleep timer expired. 
            // Configure Sensors to take readings
            configureSensors();
            // Take readings.
            takeReadings();
            break;
        case WAKEREASON_PIN :
            // We woke up because an interrupt pin was triggerd.
            // Let's check our interrupt
            checkInterrupt(); 
            break;
        case WAKEREASON_SNOOZE : 
            // We woke up after connection timeout.
            // Check if we should take a reading or just try to connect
            if (time() >= nv.nextReadTime) {
                // Configure Sensors to take readings
                configureSensors();
                // Take readings.
                takeReadings();
            } else {
                connect();
            }
            break;
        default :
            // We pushed new code or just rebooted the device, etc. Lets
            // congigure everything. 
            server.log("Device running...");
        
            // NV can persist data when the device goes into sleep mode 
            // Set up the table with defaults - note this method will 
            // erase stored data, so we only want to call it when the
            // application is starting up.
            configureNV();

            // We want to make sure we can always blinkUp a device
            // when it is first powered on, so we do not want to
            // immediately disconnect from WiFi after boot
            // Set up first disconnect
            _boot = true;
            imp.wakeup(BOOT_TIMER_SEC, function() {
                _boot = false;
                powerDown();
            }.bindenv(this));

            // Configure Sensors to take readings
            configureSensors();
            // Take readings.
            takeReadings();
    }
}

// READING FLOW METHODS
// -------------------------------------------------------------
function takeReadings() {
    // Take readings by building an array of functions that all  
    // return promises. 
    local series = [takeTempHumidReading(), takePressureReading()];
    
    // The all method executes the series of promises in parallel 
    // and resolves when they are all done. It Returns a promise 
    // that resolves with an array of the resolved promise values.
    Promise.all(series)
        .then(function(results) {
            // Create a table to store the results from the sensor readings
            // Add a timestamp 
            local reading = {"EnvReadingTime" : time()};
            // Add all successful readings
            if ("temperature" in results[0]) reading.Temperature <- results[0].temperature;
            if ("humidity" in results[0]) reading.Humidity <- results[0].humidity;
            if ("pressure" in results[1]) reading.AirPressure <- results[1].pressure;
            // Add table to the readings array
            nv.readings.push(reading);
            
            return("Readings Done");
        }.bindenv(this))
        .finally(sendReadings.bindenv(this))
}

function takeTempHumidReading() {
    return Promise(function(resolve, reject) {
        tempHumid.read(function(result) {
            return resolve(result);
        }.bindenv(this))
    }.bindenv(this))
}

function takePressureReading() {
    return Promise(function(resolve, reject) {
        pressure.read(function(result) {
            return resolve(result);
        }.bindenv(this))
    }.bindenv(this))
}

function sendReadings(msg) {
    // Update the next reading time varaible
    setNextReadTime();
    // Connect and send stored data
    connect(msg);
}

// INTERRUPT CONFIGURATION AND FLOW METHODS
// -------------------------------------------------------------
function configureInterrupt() {
    // Update this section to modify interrupt configurations
    // Interrupt will trigger on motion
    accel.configureInterruptLatching(true);        
    accel.configureHighPassFilter(LIS3DH_HPF_AOI_INT1, LIS3DH_HPF_CUTOFF1, LIS3DH_HPF_DEFAULT_MODE);
    // Params when any axis exceeds (threshold)g for at least (duration) samples, trigger interrupt
    local enable = true;
    local threshold = 0.030;
    local duration = 100;
    accel.configureInertialInterrupt(enable, threshold, duration); 

    // Configure wake pin
    wakePin.configure(DIGITAL_IN_WAKEUP, function() {
        if (wakePin.read()) {
            checkInterrupt();
        }
    }.bindenv(this));
}

function checkInterrupt() {
    // Clear interrupt latch
    local interrupt = accel.getInterruptTable();
    // Check for interrupt condition   
    // NOTE: this logic assumes that door is closed at boot time     
    if (interrupt.xHigh || interrupt.yHigh || interrupt.zHigh) {
        // Get timestamp
        local now = time();
        // Door has not moved recently, so count this as a door open alert
        if (nv.doorOpenValid <= now) {
            // Store alert
            nv.alerts.push({"DoorOpened" : "Yes", "DoorOpenTime": now});
            // Update doorOpenValid time
            nv.doorOpenValid <- now + DOOR_OPEN_TIMER_SEC;
            // Set a log message 
            local msg = "Door Opened at " + now;
            // Try to connect and send alert
            connect(msg);
        }
    }
}

// CONNECTION HANDLER METHODS
// -------------------------------------------------------------
function connect(msg = null) {
    if (server.isconnected()) {
        // We want to log something
        if (msg) server.log(msg);
        // We are connected let's send readings
        sendData();
    } else {
        // We changed the default connection policy, so we need to 
        // use this method to connect
        server.connect(function(reason) {
            if (reason == SERVER_CONNECTED) {
                // We want to log something
                if (msg) server.log(msg);
                // We connected let's send readings
                sendData();
            } else {
                // We were not able to connect
                // Let's make sure we don't run out 
                // of meemory with our stored readings
                failHandler();
            }
        }.bindenv(this));
    }
}

function sendData() {
    local data = {};

    if (nv.readings.len() > 0) {
        data.readings <- nv.readings;
    }
    if (nv.alerts.len() > 0) {
        data.alerts <- nv.alerts;
    }

    // Send data to the agent   
    mm.send("data", data);

    // Clear readings we just sent, we can recover
    // the data if the message send fails
    nv.readings.clear();

    // Clear alerts we just sent, we can recover
    // the data if the message send fails
    nv.alerts.clear();

    // If this message is acknowleged by the agent
    // the readingsAckHandler will be triggered
    
    // If the message fails to send we will handle 
    // in the sendFailHandler handler
}

function readingsAckHandler(msg) {
    // We connected successfully & sent data

    // Reset numFailedConnects
    nv.numFailedConnects <- 0;
    
    // Disconnect from server
    powerDown();
}

function sendFailHandler(msg, error, retry) {
    // Message did not send, pass them the 
    // the connection failed handler, so they
    // can be condensed and stored
    failHandler(msg.payload.data);
}

function failHandler(data = null) {
    // We are having connection issues
    // Let's re-store the data, and delete the oldest readings
    
    // Connection failed before we could send re-store data
    if (data != null) {
        if ("readings" in data) nv.readings = data.readings;
        if ("alerts" in data) nv.alerts <- data.alerts;
    }

    local numReadings = nv.readings.len();
    local numAlerts = nv.alerts.len();

    // We have too much data and will run out of local memory
    if ((numReadings + numAlerts) > MAX_NUM_STORED_DATA) {
        // Delete oldest reading or alert if no readings
        if (numReadings > 0) {
            nv.readings.remove(0);
        } else {
            nv.alerts.remove(0);
        }
    }

    // Update the number of failed connections
    nv.numFailedConnects++;
}

// POWER DOWN METHODS
// -------------------------------------------------------------
function powerDown() {
    // Power Down sensors
    powerDownSensors();

    // Calculate how long before next reading time
    local timer = nv.nextReadTime - time();

    // Check that we did not just boot up and are 
    // not about to take a reading
    if (!_boot && timer > 2) {
        // Go to sleep
        if (server.isconnected()) {
            imp.onidle(function() {
                // This method flushes server before sleep
                server.sleepfor(timer);
            }.bindenv(this));
        } else {
            // This method just puts the device to sleep
            imp.deepsleepfor(timer);
        }
    } else {
        // Schedule next reading, but don't go to sleep
        imp.wakeup(timer, function() {
            powerUpSensors();
            takeReadings();
        }.bindenv(this));
    }
}

function powerDownSensors() {
    tempHumid.setMode(HTS221_MODE.POWER_DOWN);
}

function powerUpSensors() {
    tempHumid.setMode(HTS221_MODE.ONE_SHOT);
}

// CONFIGURATION METHODS
// -------------------------------------------------------------
function configureNV() {
    local root = getroottable();
    if (!("nv" in root)) root.nv <- {};

    setNextReadTime();
    nv.doorOpenValid <- time() - DOOR_OPEN_TIMER_SEC;
    nv.readings <- [];
    nv.alerts <- [];
    nv.numFailedConnects <- 0;
}

function setNextReadTime() {
    nv.nextReadTime <- time() + _readingInterval;
}

function initializeSensors() {
    // Configure i2c
    i2c.configure(CLOCK_SPEED_400_KHZ);

    // Initialize sensors
    tempHumid = HTS221(i2c, tempHumidAddr);
    pressure = LPS22HB(i2c, pressureAddr);
    accel = LIS3DH(i2c, accelAddr);
}

function configureSensors() {
    // Configure sensors to take readings
    tempHumid.setMode(HTS221_MODE.ONE_SHOT);
    pressure.softReset();
    pressure.enableLowCurrentMode(true);
    pressure.setMode(LPS22HB_MODE.ONE_SHOT);
    accel.reset();
    accel.setMode(LIS3DH_MODE_LOW_POWER);
    accel.setDataRate(ACCEL_DATARATE);
    accel.enable(true);
    // Configure accelerometer motion interrupt 
    configureInterrupt();
}

}

// RUNTIME
// ---------------------------------------------------
// Boolean whether to enable location listener
local ENABLE_LOCATION = true;

// Adjust how often you want to take readings
// If you don’t pass this into the application it will default to 30 min
const ENV_READING_TIME_SEC = 1800;

// Initialize application
Application(ENABLE_LOCATION, ENV_READING_TIME_SEC);

Also, as a note, I am aware that it could be a Wireless network problem. But rather than wait for an answer from my Network team I thought I would post here as well to see if anyone else has seen this and if it could be a device issue.

Hi All

Nevermind…I figured it out. Was an issue with the powersave settings. I put them to true and no issues since.