rtems/rtems-libbsd
1
0
Fork 0
mirror of https://git.rtems.org/rtems-libbsd/ synced 2025-10-14 07:15:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update to FreeBSD head 2017-12-01

Browse Source 
Git mirror commit e724f51f811a4b2bd29447f8b85ab5c2f9b88266.

Update #3472.
This commit is contained in:
Sebastian Huber
2018-08-09 14:02:09 +02:00
parent e4a8065910
commit bb80d9df8b
1197 changed files with 6535 additions and 2527 deletions
Download Patch File Download Diff File Expand all files Collapse all files

314
freebsd/sys/opencrypto/crypto.c
View File

@@ -72,7 +72,9 @@ __FBSDID("$FreeBSD$");
#include <sys/malloc.h>
#include <sys/proc.h>
#include <sys/sdt.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/taskqueue.h>
#include <ddb/ddb.h>
@@ -137,26 +139,51 @@ static int crypto_drivers_num = 0;
* operations.
*/
static int crp_sleep = 0;
static TAILQ_HEAD(,cryptop) crp_q; /* request queues */
static TAILQ_HEAD(cryptop_q ,cryptop) crp_q; /* request queues */
static TAILQ_HEAD(,cryptkop) crp_kq;
static struct mtx crypto_q_mtx;
#define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx)
#define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx)
/*
* There are two queues for processing completed crypto requests; one
* for the symmetric and one for the asymmetric ops. We only need one
* but have two to avoid type futzing (cryptop vs. cryptkop). A single
* mutex is used to lock access to both queues. Note that this lock
* must be separate from the lock on request queues to insure driver
* callbacks don't generate lock order reversals.
* Taskqueue used to dispatch the crypto requests
* that have the CRYPTO_F_ASYNC flag
*/
static TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queues */
static TAILQ_HEAD(,cryptkop) crp_ret_kq;
static struct mtx crypto_ret_q_mtx;
#define CRYPTO_RETQ_LOCK() mtx_lock(&crypto_ret_q_mtx)
#define CRYPTO_RETQ_UNLOCK() mtx_unlock(&crypto_ret_q_mtx)
#define CRYPTO_RETQ_EMPTY() (TAILQ_EMPTY(&crp_ret_q) && TAILQ_EMPTY(&crp_ret_kq))
static struct taskqueue *crypto_tq;
/*
* Crypto seq numbers are operated on with modular arithmetic
*/
#define CRYPTO_SEQ_GT(a,b) ((int)((a)-(b)) > 0)
struct crypto_ret_worker {
struct mtx crypto_ret_mtx;
TAILQ_HEAD(,cryptop) crp_ordered_ret_q; /* ordered callback queue for symetric jobs */
TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queue for symetric jobs */
TAILQ_HEAD(,cryptkop) crp_ret_kq; /* callback queue for asym jobs */
u_int32_t reorder_ops; /* total ordered sym jobs received */
u_int32_t reorder_cur_seq; /* current sym job dispatched */
struct proc *cryptoretproc;
};
static struct crypto_ret_worker *crypto_ret_workers = NULL;
#define CRYPTO_RETW(i) (&crypto_ret_workers[i])
#define CRYPTO_RETW_ID(w) ((w) - crypto_ret_workers)
#define FOREACH_CRYPTO_RETW(w) \
for (w = crypto_ret_workers; w < crypto_ret_workers + crypto_workers_num; ++w)
#define CRYPTO_RETW_LOCK(w) mtx_lock(&w->crypto_ret_mtx)
#define CRYPTO_RETW_UNLOCK(w) mtx_unlock(&w->crypto_ret_mtx)
#define CRYPTO_RETW_EMPTY(w) \
(TAILQ_EMPTY(&w->crp_ret_q) && TAILQ_EMPTY(&w->crp_ret_kq) && TAILQ_EMPTY(&w->crp_ordered_ret_q))
static int crypto_workers_num = 0;
SYSCTL_INT(_kern, OID_AUTO, crypto_workers_num, CTLFLAG_RDTUN,
&crypto_workers_num, 0,
"Number of crypto workers used to dispatch crypto jobs");
static uma_zone_t cryptop_zone;
static uma_zone_t cryptodesc_zone;
@@ -174,11 +201,12 @@ MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
static void crypto_proc(void);
static struct proc *cryptoproc;
static void crypto_ret_proc(void);
static struct proc *cryptoretproc;
static void crypto_ret_proc(struct crypto_ret_worker *ret_worker);
static void crypto_destroy(void);
static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint);
static int crypto_kinvoke(struct cryptkop *krp, int flags);
static void crypto_task_invoke(void *ctx, int pending);
static void crypto_batch_enqueue(struct cryptop *crp);
static struct cryptostats cryptostats;
SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats,
@@ -224,6 +252,7 @@ struct keybuf * get_keybuf(void) {
static int
crypto_init(void)
{
struct crypto_ret_worker *ret_worker;
int error;
mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table",
@@ -233,10 +262,6 @@ crypto_init(void)
TAILQ_INIT(&crp_kq);
mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF);
TAILQ_INIT(&crp_ret_q);
TAILQ_INIT(&crp_ret_kq);
mtx_init(&crypto_ret_q_mtx, "crypto", "crypto return queues", MTX_DEF);
cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop),
0, 0, 0, 0,
UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
@@ -258,6 +283,20 @@ crypto_init(void)
goto bad;
}
if (crypto_workers_num < 1 || crypto_workers_num > mp_ncpus)
crypto_workers_num = mp_ncpus;
crypto_tq = taskqueue_create("crypto", M_WAITOK|M_ZERO,
taskqueue_thread_enqueue, &crypto_tq);
if (crypto_tq == NULL) {
printf("crypto init: cannot setup crypto taskqueue\n");
error = ENOMEM;
goto bad;
}
taskqueue_start_threads(&crypto_tq, crypto_workers_num, PRI_MIN_KERN,
"crypto");
error = kproc_create((void (*)(void *)) crypto_proc, NULL,
&cryptoproc, 0, 0, "crypto");
if (error) {
@@ -266,15 +305,35 @@ crypto_init(void)
goto bad;
}
error = kproc_create((void (*)(void *)) crypto_ret_proc, NULL,
&cryptoretproc, 0, 0, "crypto returns");
if (error) {
printf("crypto_init: cannot start cryptoret thread; error %d",
error);
crypto_ret_workers = malloc(crypto_workers_num * sizeof(struct crypto_ret_worker),
M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
if (crypto_ret_workers == NULL) {
error = ENOMEM;
printf("crypto_init: cannot allocate ret workers\n");
goto bad;
}
keybuf_init();
FOREACH_CRYPTO_RETW(ret_worker) {
TAILQ_INIT(&ret_worker->crp_ordered_ret_q);
TAILQ_INIT(&ret_worker->crp_ret_q);
TAILQ_INIT(&ret_worker->crp_ret_kq);
ret_worker->reorder_ops = 0;
ret_worker->reorder_cur_seq = 0;
mtx_init(&ret_worker->crypto_ret_mtx, "crypto", "crypto return queues", MTX_DEF);
error = kproc_create((void (*)(void *)) crypto_ret_proc, ret_worker,
&ret_worker->cryptoretproc, 0, 0, "crypto returns %td", CRYPTO_RETW_ID(ret_worker));
if (error) {
printf("crypto_init: cannot start cryptoret thread; error %d",
error);
goto bad;
}
}
keybuf_init();
return 0;
bad:
@@ -315,12 +374,17 @@ crypto_terminate(struct proc **pp, void *q)
static void
crypto_destroy(void)
{
struct crypto_ret_worker *ret_worker;
/*
* Terminate any crypto threads.
*/
if (crypto_tq != NULL)
taskqueue_drain_all(crypto_tq);
CRYPTO_DRIVER_LOCK();
crypto_terminate(&cryptoproc, &crp_q);
crypto_terminate(&cryptoretproc, &crp_ret_q);
FOREACH_CRYPTO_RETW(ret_worker)
crypto_terminate(&ret_worker->cryptoretproc, &ret_worker->crp_ret_q);
CRYPTO_DRIVER_UNLOCK();
/* XXX flush queues??? */
@@ -336,7 +400,11 @@ crypto_destroy(void)
if (cryptop_zone != NULL)
uma_zdestroy(cryptop_zone);
mtx_destroy(&crypto_q_mtx);
mtx_destroy(&crypto_ret_q_mtx);
FOREACH_CRYPTO_RETW(ret_worker)
mtx_destroy(&ret_worker->crypto_ret_mtx);
free(crypto_ret_workers, M_CRYPTO_DATA);
if (crypto_tq != NULL)
taskqueue_free(crypto_tq);
mtx_destroy(&crypto_drivers_mtx);
}
@@ -579,7 +647,7 @@ crypto_get_driverid(device_t dev, int flags)
crypto_drivers[i].cc_dev = dev;
crypto_drivers[i].cc_flags = flags;
if (bootverbose)
printf("crypto: assign %s driver id %u, flags %u\n",
printf("crypto: assign %s driver id %u, flags 0x%x\n",
device_get_nameunit(dev), i, flags);
CRYPTO_DRIVER_UNLOCK();
@@ -835,9 +903,26 @@ crypto_dispatch(struct cryptop *crp)
binuptime(&crp->crp_tstamp);
#endif
hid = CRYPTO_SESID2HID(crp->crp_sid);
if (CRYPTOP_ASYNC(crp)) {
if (crp->crp_flags & CRYPTO_F_ASYNC_KEEPORDER) {
struct crypto_ret_worker *ret_worker;
crp->crp_retw_id = crp->crp_sid % crypto_workers_num;
ret_worker = CRYPTO_RETW(crp->crp_retw_id);
CRYPTO_RETW_LOCK(ret_worker);
crp->crp_seq = ret_worker->reorder_ops++;
CRYPTO_RETW_UNLOCK(ret_worker);
}
TASK_INIT(&crp->crp_task, 0, crypto_task_invoke, crp);
taskqueue_enqueue(crypto_tq, &crp->crp_task);
return (0);
}
if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) {
hid = CRYPTO_SESID2HID(crp->crp_sid);
/*
* Caller marked the request to be processed
* immediately; dispatch it directly to the
@@ -856,12 +941,19 @@ crypto_dispatch(struct cryptop *crp)
*/
}
}
crypto_batch_enqueue(crp);
return 0;
}
void
crypto_batch_enqueue(struct cryptop *crp)
{
CRYPTO_Q_LOCK();
TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
if (crp_sleep)
wakeup_one(&crp_q);
CRYPTO_Q_UNLOCK();
return 0;
}
/*
@@ -1042,6 +1134,23 @@ crypto_tstat(struct cryptotstat *ts, struct bintime *bt)
}
#endif
static void
crypto_task_invoke(void *ctx, int pending)
{
struct cryptocap *cap;
struct cryptop *crp;
int hid, result;
crp = (struct cryptop *)ctx;
hid = CRYPTO_SESID2HID(crp->crp_sid);
cap = crypto_checkdriver(hid);
result = crypto_invoke(cap, crp, 0);
if (result == ERESTART)
crypto_batch_enqueue(crp);
}
/*
* Dispatch a crypto request to the appropriate crypto devices.
*/
@@ -1104,6 +1213,7 @@ crypto_freereq(struct cryptop *crp)
#ifdef DIAGNOSTIC
{
struct cryptop *crp2;
struct crypto_ret_worker *ret_worker;
CRYPTO_Q_LOCK();
TAILQ_FOREACH(crp2, &crp_q, crp_next) {
@@ -1112,13 +1222,16 @@ crypto_freereq(struct cryptop *crp)
crp));
}
CRYPTO_Q_UNLOCK();
CRYPTO_RETQ_LOCK();
TAILQ_FOREACH(crp2, &crp_ret_q, crp_next) {
KASSERT(crp2 != crp,
("Freeing cryptop from the return queue (%p).",
crp));
FOREACH_CRYPTO_RETW(ret_worker) {
CRYPTO_RETW_LOCK(ret_worker);
TAILQ_FOREACH(crp2, &ret_worker->crp_ret_q, crp_next) {
KASSERT(crp2 != crp,
("Freeing cryptop from the return queue (%p).",
crp));
}
CRYPTO_RETW_UNLOCK(ret_worker);
}
CRYPTO_RETQ_UNLOCK();
}
#endif
@@ -1176,9 +1289,10 @@ crypto_done(struct cryptop *crp)
* doing extraneous context switches; the latter is mostly
* used with the software crypto driver.
*/
if ((crp->crp_flags & CRYPTO_F_CBIMM) ||
if (!CRYPTOP_ASYNC_KEEPORDER(crp) &&
((crp->crp_flags & CRYPTO_F_CBIMM) ||
((crp->crp_flags & CRYPTO_F_CBIFSYNC) &&
(CRYPTO_SESID2CAPS(crp->crp_sid) & CRYPTOCAP_F_SYNC))) {
(CRYPTO_SESID2CAPS(crp->crp_sid) & CRYPTOCAP_F_SYNC)))) {
/*
* Do the callback directly. This is ok when the
* callback routine does very little (e.g. the
@@ -1199,14 +1313,45 @@ crypto_done(struct cryptop *crp)
#endif
crp->crp_callback(crp);
} else {
struct crypto_ret_worker *ret_worker;
bool wake;
ret_worker = CRYPTO_RETW(crp->crp_retw_id);
wake = false;
/*
* Normal case; queue the callback for the thread.
*/
CRYPTO_RETQ_LOCK();
if (CRYPTO_RETQ_EMPTY())
wakeup_one(&crp_ret_q); /* shared wait channel */
TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next);
CRYPTO_RETQ_UNLOCK();
CRYPTO_RETW_LOCK(ret_worker);
if (CRYPTOP_ASYNC_KEEPORDER(crp)) {
struct cryptop *tmp;
TAILQ_FOREACH_REVERSE(tmp, &ret_worker->crp_ordered_ret_q,
cryptop_q, crp_next) {
if (CRYPTO_SEQ_GT(crp->crp_seq, tmp->crp_seq)) {
TAILQ_INSERT_AFTER(&ret_worker->crp_ordered_ret_q,
tmp, crp, crp_next);
break;
}
}
if (tmp == NULL) {
TAILQ_INSERT_HEAD(&ret_worker->crp_ordered_ret_q,
crp, crp_next);
}
if (crp->crp_seq == ret_worker->reorder_cur_seq)
wake = true;
}
else {
if (CRYPTO_RETW_EMPTY(ret_worker))
wake = true;
TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp, crp_next);
}
if (wake)
wakeup_one(&ret_worker->crp_ret_q); /* shared wait channel */
CRYPTO_RETW_UNLOCK(ret_worker);
}
}
@@ -1216,6 +1361,7 @@ crypto_done(struct cryptop *crp)
void
crypto_kdone(struct cryptkop *krp)
{
struct crypto_ret_worker *ret_worker;
struct cryptocap *cap;
if (krp->krp_status != 0)
@@ -1230,11 +1376,14 @@ crypto_kdone(struct cryptkop *krp)
crypto_remove(cap);
}
CRYPTO_DRIVER_UNLOCK();
CRYPTO_RETQ_LOCK();
if (CRYPTO_RETQ_EMPTY())
wakeup_one(&crp_ret_q); /* shared wait channel */
TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next);
CRYPTO_RETQ_UNLOCK();
ret_worker = CRYPTO_RETW(0);
CRYPTO_RETW_LOCK(ret_worker);
if (CRYPTO_RETW_EMPTY(ret_worker))
wakeup_one(&ret_worker->crp_ret_q); /* shared wait channel */
TAILQ_INSERT_TAIL(&ret_worker->crp_ret_kq, krp, krp_next);
CRYPTO_RETW_UNLOCK(ret_worker);
}
int
@@ -1436,24 +1585,36 @@ crypto_proc(void)
* callbacks typically are expensive and would slow interrupt handling.
*/
static void
crypto_ret_proc(void)
crypto_ret_proc(struct crypto_ret_worker *ret_worker)
{
struct cryptop *crpt;
struct cryptkop *krpt;
CRYPTO_RETQ_LOCK();
CRYPTO_RETW_LOCK(ret_worker);
for (;;) {
/* Harvest return q's for completed ops */
crpt = TAILQ_FIRST(&crp_ret_q);
if (crpt != NULL)
TAILQ_REMOVE(&crp_ret_q, crpt, crp_next);
crpt = TAILQ_FIRST(&ret_worker->crp_ordered_ret_q);
if (crpt != NULL) {
if (crpt->crp_seq == ret_worker->reorder_cur_seq) {
TAILQ_REMOVE(&ret_worker->crp_ordered_ret_q, crpt, crp_next);
ret_worker->reorder_cur_seq++;
} else {
crpt = NULL;
}
}
krpt = TAILQ_FIRST(&crp_ret_kq);
if (crpt == NULL) {
crpt = TAILQ_FIRST(&ret_worker->crp_ret_q);
if (crpt != NULL)
TAILQ_REMOVE(&ret_worker->crp_ret_q, crpt, crp_next);
}
krpt = TAILQ_FIRST(&ret_worker->crp_ret_kq);
if (krpt != NULL)
TAILQ_REMOVE(&crp_ret_kq, krpt, krp_next);
TAILQ_REMOVE(&ret_worker->crp_ret_kq, krpt, krp_next);
if (crpt != NULL || krpt != NULL) {
CRYPTO_RETQ_UNLOCK();
CRYPTO_RETW_UNLOCK(ret_worker);
/*
* Run callbacks unlocked.
*/
@@ -1475,22 +1636,22 @@ crypto_ret_proc(void)
}
if (krpt != NULL)
krpt->krp_callback(krpt);
CRYPTO_RETQ_LOCK();
CRYPTO_RETW_LOCK(ret_worker);
} else {
/*
* Nothing more to be processed. Sleep until we're
* woken because there are more returns to process.
*/
msleep(&crp_ret_q, &crypto_ret_q_mtx, PWAIT,
msleep(&ret_worker->crp_ret_q, &ret_worker->crypto_ret_mtx, PWAIT,
"crypto_ret_wait", 0);
if (cryptoretproc == NULL)
if (ret_worker->cryptoretproc == NULL)
break;
cryptostats.cs_rets++;
}
}
CRYPTO_RETQ_UNLOCK();
CRYPTO_RETW_UNLOCK(ret_worker);
crypto_finis(&crp_ret_q);
crypto_finis(&ret_worker->crp_ret_q);
}
#ifdef DDB
@@ -1525,6 +1686,7 @@ db_show_drivers(void)
DB_SHOW_COMMAND(crypto, db_show_crypto)
{
struct cryptop *crp;
struct crypto_ret_worker *ret_worker;
db_show_drivers();
db_printf("\n");
@@ -1543,16 +1705,19 @@ DB_SHOW_COMMAND(crypto, db_show_crypto)
, crp->crp_callback
);
}
if (!TAILQ_EMPTY(&crp_ret_q)) {
db_printf("\n%4s %4s %4s %8s\n",
"HID", "Etype", "Flags", "Callback");
TAILQ_FOREACH(crp, &crp_ret_q, crp_next) {
db_printf("%4u %4u %04x %8p\n"
, (int) CRYPTO_SESID2HID(crp->crp_sid)
, crp->crp_etype
, crp->crp_flags
, crp->crp_callback
);
FOREACH_CRYPTO_RETW(ret_worker) {
db_printf("\n%8s %4s %4s %4s %8s\n",
"ret_worker", "HID", "Etype", "Flags", "Callback");
if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
TAILQ_FOREACH(crp, &ret_worker->crp_ret_q, crp_next) {
db_printf("%8td %4u %4u %04x %8p\n"
, CRYPTO_RETW_ID(ret_worker)
, (int) CRYPTO_SESID2HID(crp->crp_sid)
, crp->crp_etype
, crp->crp_flags
, crp->crp_callback
);
}
}
}
}
@@ -1560,6 +1725,7 @@ DB_SHOW_COMMAND(crypto, db_show_crypto)
DB_SHOW_COMMAND(kcrypto, db_show_kcrypto)
{
struct cryptkop *krp;
struct crypto_ret_worker *ret_worker;
db_show_drivers();
db_printf("\n");
@@ -1575,10 +1741,12 @@ DB_SHOW_COMMAND(kcrypto, db_show_kcrypto)
, krp->krp_callback
);
}
if (!TAILQ_EMPTY(&crp_ret_q)) {
ret_worker = CRYPTO_RETW(0);
if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
db_printf("%4s %5s %8s %4s %8s\n",
"Op", "Status", "CRID", "HID", "Callback");
TAILQ_FOREACH(krp, &crp_ret_kq, krp_next) {
TAILQ_FOREACH(krp, &ret_worker->crp_ret_kq, krp_next) {
db_printf("%4u %5u %08x %4u %8p\n"
, krp->krp_op
, krp->krp_status

23
freebsd/sys/opencrypto/cryptodev.h
View File

@@ -63,6 +63,7 @@
#define _CRYPTO_CRYPTO_H_
#include <sys/ioccom.h>
#include <sys/_task.h>
/* Some initial values */
#define CRYPTO_DRIVERS_INITIAL 4
@@ -391,6 +392,8 @@ struct cryptodesc {
struct cryptop {
TAILQ_ENTRY(cryptop) crp_next;
struct task crp_task;
u_int64_t crp_sid; /* Session ID */
int crp_ilen; /* Input data total length */
int crp_olen; /* Result total length */
@@ -413,6 +416,14 @@ struct cryptop {
#define CRYPTO_F_CBIMM 0x0010 /* Do callback immediately */
#define CRYPTO_F_DONE 0x0020 /* Operation completed */
#define CRYPTO_F_CBIFSYNC 0x0040 /* Do CBIMM if op is synchronous */
#define CRYPTO_F_ASYNC 0x0080 /* Dispatch crypto jobs on several threads
* if op is synchronous
*/
#define CRYPTO_F_ASYNC_KEEPORDER 0x0100 /*
* Dispatch the crypto jobs in the same
* order there are submitted. Applied only
* if CRYPTO_F_ASYNC flags is set
*/
caddr_t crp_buf; /* Data to be processed */
caddr_t crp_opaque; /* Opaque pointer, passed along */
@@ -421,8 +432,20 @@ struct cryptop {
int (*crp_callback)(struct cryptop *); /* Callback function */
struct bintime crp_tstamp; /* performance time stamp */
uint32_t crp_seq; /* used for ordered dispatch */
uint32_t crp_retw_id; /*
* the return worker to be used,
* used for ordered dispatch
*/
};
#define CRYPTOP_ASYNC(crp) \
(((crp)->crp_flags & CRYPTO_F_ASYNC) && \
CRYPTO_SESID2CAPS((crp)->crp_sid) & CRYPTOCAP_F_SYNC)
#define CRYPTOP_ASYNC_KEEPORDER(crp) \
(CRYPTOP_ASYNC(crp) && \
(crp)->crp_flags & CRYPTO_F_ASYNC_KEEPORDER)
#define CRYPTO_BUF_CONTIG 0x0
#define CRYPTO_BUF_IOV 0x1
#define CRYPTO_BUF_MBUF 0x2