When working with limited stacks, having sendbuf on the stack can cause issues

When there is a limited stack size, having the send buffer (which everything
is built into prior to sending) on the stack can easily cause stack overflows.

dtls_mutex.h:

Create dtls_ mutex functions which are platform independent.

dtls.c:

Move sendbuf out of the functions' stack and make it a static variable
if DTLS_CONSTRAINED_STACK is defined.

Protect the usage of sendbuf by mutexes.

Signed-off-by: Jon Shallow <supjps-libcoap@jpshallow.com>

dtls.c

@@ -54,6 +54,7 @@
 #include "alert.h"
 #include "session.h"
 #include "dtls_prng.h"
+#include "dtls_mutex.h"
 
 #ifdef WITH_SHA256
 #  include "hmac.h"
@@ -1524,6 +1525,12 @@ dtls_send_handshake_msg(dtls_context_t *ctx,
      (dtls_uint16_to_int(DTLS_RECORD_HEADER(Data)->epoch > 0) ||	\
       (dtls_uint16_to_int(HANDSHAKE(Data)->message_seq) > 0)))))
 
+
+#ifdef DTLS_CONSTRAINED_STACK
+static dtls_mutex_t static_mutex = DTLS_MUTEX_INITIALIZER;
+static unsigned char sendbuf[DTLS_MAX_BUF];
+#endif /* DTLS_CONSTRAINED_STACK */
+
 /**
  * Sends the data passed in @p buf as a DTLS record of type @p type to
  * the given peer. The data will be encrypted and compressed according
@@ -1549,16 +1556,22 @@ dtls_send_multi(dtls_context_t *ctx, dtls_peer_t *peer,
    * TODO: check if we can use the receive buf here. This would mean
    * that we might not be able to handle multiple records stuffed in
    * one UDP datagram */
+#ifndef DTLS_CONSTRAINED_STACK
   unsigned char sendbuf[DTLS_MAX_BUF];
+#endif /* ! DTLS_CONSTRAINED_STACK */
   size_t len = sizeof(sendbuf);
   int res;
   unsigned int i;
   size_t overall_len = 0;
 
+#ifdef DTLS_CONSTRAINED_STACK
+  dtls_mutex_lock(&static_mutex);
+#endif /* DTLS_CONSTRAINED_STACK */
+
   res = dtls_prepare_record(peer, security, type, buf_array, buf_len_array, buf_array_len, sendbuf, &len);
 
   if (res < 0)
-    return res;
+    goto return_unlock;
 
   /* if (peer && MUST_HASH(peer, type, buf, buflen)) */
   /*   update_hs_hash(peer, buf, buflen); */
@@ -1621,6 +1634,11 @@ dtls_send_multi(dtls_context_t *ctx, dtls_peer_t *peer,
    * necessary) and initialize retransmit timer */
   res = CALL(ctx, write, session, sendbuf, len);
 
+return_unlock:
+#if DTLS_CONSTRAINED_STACK
+  dtls_mutex_unlock(&static_mutex);
+#endif /* DTLS_CONSTRAINED_STACK */
+
   /* Guess number of bytes application data actually sent:
    * dtls_prepare_record() tells us in len the number of bytes to
    * send, res will contain the bytes actually sent. */
@@ -4105,7 +4123,9 @@ dtls_retransmit(dtls_context_t *context, netq_t *node) {
 
   /* re-initialize timeout when maximum number of retransmissions are not reached yet */
   if (node->retransmit_cnt < DTLS_DEFAULT_MAX_RETRANSMIT) {
+#ifndef DTLS_CONSTRAINED_STACK
       unsigned char sendbuf[DTLS_MAX_BUF];
+#endif /* ! DTLS_CONSTRAINED_STACK */
       size_t len = sizeof(sendbuf);
       int err;
       unsigned char *data = node->data;
@@ -4113,6 +4133,10 @@ dtls_retransmit(dtls_context_t *context, netq_t *node) {
       dtls_tick_t now;
       dtls_security_parameters_t *security = dtls_security_params_epoch(node->peer, node->epoch);
 
+#ifdef DTLS_CONSTRAINED_STACK
+      dtls_mutex_lock(&static_mutex);
+#endif /* DTLS_CONSTRAINED_STACK */
+
       dtls_ticks(&now);
       node->retransmit_cnt++;
       node->t = now + (node->timeout << node->retransmit_cnt);
@@ -4131,13 +4155,18 @@ dtls_retransmit(dtls_context_t *context, netq_t *node) {
 				1, sendbuf, &len);
       if (err < 0) {
 	dtls_warn("can not retransmit packet, err: %i\n", err);
-	return;
+	goto return_unlock;
       }
       dtls_debug_hexdump("retransmit header", sendbuf,
 			 sizeof(dtls_record_header_t));
       dtls_debug_hexdump("retransmit unencrypted", node->data, node->length);
 
       (void)CALL(context, write, &node->peer->session, sendbuf, len);
+return_unlock:
+#if DTLS_CONSTRAINED_STACK
+      dtls_mutex_unlock(&static_mutex);
+#endif /* DTLS_CONSTRAINED_STACK */
+
       return;
   }
 

dtls_mutex.h

@@ -0,0 +1,57 @@
+/*******************************************************************************
+ *
+ * Copyright (c) 2019 Olaf Bergmann (TZI) and others.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * and Eclipse Distribution License v. 1.0 which accompanies this distribution.
+ *
+ * The Eclipse Public License is available at http://www.eclipse.org/legal/epl-v10.html
+ * and the Eclipse Distribution License is available at 
+ * http://www.eclipse.org/org/documents/edl-v10.php.
+ *
+ * Contributors:
+ *    Jon Shallow    - Initall add in of generic mutex support
+ *
+ *******************************************************************************/
+
+/**
+ * @file dtls_mutex.h
+ * @brief DTLS mutex mechanism wrapper
+ */
+
+#ifndef _DTLS_MUTEX_H_
+#define _DTLS_MUTEX_H_
+
+#if defined(RIOT_VERSION)
+
+#include <mutex.h>
+
+typedef mutex_t dtls_mutex_t;
+#define DTLS_MUTEX_INITIALIZER MUTEX_INIT
+#define dtls_mutex_lock(a) mutex_lock(a)
+#define dtls_mutex_trylock(a) mutex_trylock(a)
+#define dtls_mutex_unlock(a) mutex_unlock(a)
+
+#elif defined(WITH_CONTIKI)
+
+/* CONTIKI does not support mutex */
+
+typedef int dtls_mutex_t;
+#define DTLS_MUTEX_INITIALIZER 0
+#define dtls_mutex_lock(a) *(a) = 1
+#define dtls_mutex_trylock(a) *(a) = 1
+#define dtls_mutex_unlock(a) *(a) = 0
+
+#else /* ! RIOT_VERSION && ! WITH_CONTIKI */
+
+#include <pthread.h>
+
+typedef pthread_mutex_t dtls_mutex_t;
+#define DTLS_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
+#define dtls_mutex_lock(a) pthread_mutex_lock(a)
+#define dtls_mutex_trylock(a) pthread_mutex_trylock(a)
+#define dtls_mutex_unlock(a) pthread_mutex_unlock(a)
+
+#endif /* ! RIOT_VERSION && ! WITH_CONTIKI */
+
+#endif /* _DTLS_MUTEX_H_ */