1e18a033bSKonstantin Ananyev/*
2e18a033bSKonstantin Ananyev * Copyright (c) 2017  Intel Corporation.
3e18a033bSKonstantin Ananyev * All rights reserved.
4e18a033bSKonstantin Ananyev *
5e18a033bSKonstantin Ananyev * Redistribution and use in source and binary forms, with or without
6e18a033bSKonstantin Ananyev * modification, are permitted provided that the following conditions
7e18a033bSKonstantin Ananyev * are met:
8e18a033bSKonstantin Ananyev * 1. Redistributions of source code must retain the above copyright
9e18a033bSKonstantin Ananyev *    notice, this list of conditions and the following disclaimer.
10e18a033bSKonstantin Ananyev * 2. Redistributions in binary form must reproduce the above copyright
11e18a033bSKonstantin Ananyev *    notice, this list of conditions and the following disclaimer in the
12e18a033bSKonstantin Ananyev *    documentation and/or other materials provided with the distribution.
13e18a033bSKonstantin Ananyev *
14e18a033bSKonstantin Ananyev * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15e18a033bSKonstantin Ananyev * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16e18a033bSKonstantin Ananyev * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17e18a033bSKonstantin Ananyev * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18e18a033bSKonstantin Ananyev * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19e18a033bSKonstantin Ananyev * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20e18a033bSKonstantin Ananyev * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21e18a033bSKonstantin Ananyev * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22e18a033bSKonstantin Ananyev * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23e18a033bSKonstantin Ananyev * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24e18a033bSKonstantin Ananyev * SUCH DAMAGE.
25e18a033bSKonstantin Ananyev */
26e18a033bSKonstantin Ananyev
27e18a033bSKonstantin Ananyev#include <ngx_tldk.h>
28e18a033bSKonstantin Ananyev#include <tldk_sock.h>
29e18a033bSKonstantin Ananyev
30e18a033bSKonstantin Ananyev#include <rte_cycles.h>
31e18a033bSKonstantin Ananyev
32e18a033bSKonstantin Ananyev#include "be.h"
33e18a033bSKonstantin Ananyev#include "debug.h"
34e18a033bSKonstantin Ananyev
35e18a033bSKonstantin Ananyev#define	EVENT_BULK	32
36e18a033bSKonstantin Ananyev
37e18a033bSKonstantin Ananyevenum {
38e18a033bSKonstantin Ananyev	EV_ACCEPT,
39e18a033bSKonstantin Ananyev	EV_RECV,
40e18a033bSKonstantin Ananyev	EV_SEND,
41e18a033bSKonstantin Ananyev	EV_ERR,
42e18a033bSKonstantin Ananyev	EV_NUM
43e18a033bSKonstantin Ananyev};
44e18a033bSKonstantin Ananyev
45e18a033bSKonstantin Ananyevstruct tldk_event_stat {
46e18a033bSKonstantin Ananyev	uint64_t nb_get[EV_NUM];
47e18a033bSKonstantin Ananyev	uint64_t nb_post[EV_NUM];
48e18a033bSKonstantin Ananyev};
49e18a033bSKonstantin Ananyev
50e18a033bSKonstantin Ananyevstatic struct tldk_event_stat event_stat;
51e18a033bSKonstantin Ananyev
52e18a033bSKonstantin Ananyevextern ngx_event_module_t tldk_event_module;
53e18a033bSKonstantin Ananyev
54e18a033bSKonstantin Ananyev/*
55e18a033bSKonstantin Ananyev * TLDK event module implementation
56e18a033bSKonstantin Ananyev */
57e18a033bSKonstantin Ananyev
58e18a033bSKonstantin Ananyevstatic ngx_int_t
59e18a033bSKonstantin Ananyevtldk_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
60e18a033bSKonstantin Ananyev{
61e18a033bSKonstantin Ananyev	struct tldk_sock *ts;
62e18a033bSKonstantin Ananyev	ngx_connection_t *c;
63e18a033bSKonstantin Ananyev
64e18a033bSKonstantin Ananyev	c = ev->data;
65e18a033bSKonstantin Ananyev
66e18a033bSKonstantin Ananyev	FE_TRACE("%s(ev=%p,event=%#lx,flags=%#lx): fd=%d;\n",
67e18a033bSKonstantin Ananyev		__func__, ev, event, flags, c->fd);
68e18a033bSKonstantin Ananyev
69e18a033bSKonstantin Ananyev	ts = sd_to_sock(c->fd);
70e18a033bSKonstantin Ananyev	if (ts == NULL)
71e18a033bSKonstantin Ananyev		return NGX_OK;
72e18a033bSKonstantin Ananyev
73e18a033bSKonstantin Ananyev	if (event == NGX_READ_EVENT) {
74e18a033bSKonstantin Ananyev		tle_event_active(ts->rxev, TLE_SEV_DOWN);
75e18a033bSKonstantin Ananyev		tle_event_active(ts->erev, TLE_SEV_DOWN);
76e18a033bSKonstantin Ananyev		ts->rev = ev;
77e18a033bSKonstantin Ananyev	} else if (event == NGX_WRITE_EVENT) {
78e18a033bSKonstantin Ananyev		tle_event_active(ts->txev, TLE_SEV_DOWN);
79e18a033bSKonstantin Ananyev		tle_event_active(ts->erev, TLE_SEV_DOWN);
80e18a033bSKonstantin Ananyev		ts->wev = ev;
81e18a033bSKonstantin Ananyev	}
82e18a033bSKonstantin Ananyev
83e18a033bSKonstantin Ananyev	ev->active = 1;
84e18a033bSKonstantin Ananyev	return NGX_OK;
85e18a033bSKonstantin Ananyev}
86e18a033bSKonstantin Ananyev
87e18a033bSKonstantin Ananyevstatic ngx_int_t
88e18a033bSKonstantin Ananyevtldk_del_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
89e18a033bSKonstantin Ananyev{
90e18a033bSKonstantin Ananyev	struct tldk_sock *ts;
91e18a033bSKonstantin Ananyev	ngx_connection_t *c;
92e18a033bSKonstantin Ananyev
93e18a033bSKonstantin Ananyev	c = ev->data;
94e18a033bSKonstantin Ananyev
95e18a033bSKonstantin Ananyev	FE_TRACE("%s(ev=%p,event=%#lx,flags=%#lx): fd=%d;\n",
96e18a033bSKonstantin Ananyev		__func__, ev, event, flags, c->fd);
97e18a033bSKonstantin Ananyev
98e18a033bSKonstantin Ananyev	ev->active = 0;
99e18a033bSKonstantin Ananyev	if ((flags & NGX_CLOSE_EVENT) != 0)
100e18a033bSKonstantin Ananyev		return NGX_OK;
101e18a033bSKonstantin Ananyev
102e18a033bSKonstantin Ananyev	ts = sd_to_sock(c->fd);
103e18a033bSKonstantin Ananyev	if (ts == NULL)
104e18a033bSKonstantin Ananyev		return NGX_OK;
105e18a033bSKonstantin Ananyev
106e18a033bSKonstantin Ananyev	if (event == NGX_READ_EVENT) {
107e18a033bSKonstantin Ananyev		tle_event_down(ts->rxev);
108e18a033bSKonstantin Ananyev		tle_event_down(ts->erev);
109e18a033bSKonstantin Ananyev		ts->rev = NULL;
110e18a033bSKonstantin Ananyev	} else if (event == NGX_WRITE_EVENT) {
111e18a033bSKonstantin Ananyev		tle_event_down(ts->txev);
112e18a033bSKonstantin Ananyev		tle_event_down(ts->erev);
113e18a033bSKonstantin Ananyev		ts->wev = NULL;
114e18a033bSKonstantin Ananyev	}
115e18a033bSKonstantin Ananyev
116e18a033bSKonstantin Ananyev	return NGX_OK;
117e18a033bSKonstantin Ananyev}
118e18a033bSKonstantin Ananyev
119e18a033bSKonstantin Ananyevstatic inline void
120e18a033bSKonstantin Ananyevpost_event(ngx_event_t *ev, ngx_queue_t *q, ngx_uint_t flags, uint32_t type)
121e18a033bSKonstantin Ananyev{
122e18a033bSKonstantin Ananyev	if (ev != NULL && ev->active == 1) {
123e18a033bSKonstantin Ananyev		ev->ready = 1;
124e18a033bSKonstantin Ananyev		event_stat.nb_post[type]++;
125e18a033bSKonstantin Ananyev		if ((flags & NGX_POST_EVENTS) != 0) {
126e18a033bSKonstantin Ananyev			ngx_post_event(ev, q);
127e18a033bSKonstantin Ananyev		} else
128e18a033bSKonstantin Ananyev			ev->handler(ev);
129e18a033bSKonstantin Ananyev	}
130e18a033bSKonstantin Ananyev}
131e18a033bSKonstantin Ananyev
132e18a033bSKonstantin Ananyevstatic ngx_int_t
133e18a033bSKonstantin Ananyevtldk_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags)
134e18a033bSKonstantin Ananyev{
135e18a033bSKonstantin Ananyev	uint32_t i, n, ne, nr, ns, nt;
136e18a033bSKonstantin Ananyev	uint64_t tme, tms, tmw;
137e18a033bSKonstantin Ananyev	struct tldk_sock *te[EVENT_BULK];
138e18a033bSKonstantin Ananyev	struct tldk_sock *tr[EVENT_BULK];
139e18a033bSKonstantin Ananyev	struct tldk_sock *ts[EVENT_BULK];
140e18a033bSKonstantin Ananyev	struct tldk_sock *tt[EVENT_BULK];
141e18a033bSKonstantin Ananyev	struct tldk_ctx *tcx;
142e18a033bSKonstantin Ananyev
143e18a033bSKonstantin Ananyev	FE_TRACE("%s(cycle=%p,timer=%lu,flags=%#lx);\n",
144e18a033bSKonstantin Ananyev		__func__, cycle, timer, flags);
145e18a033bSKonstantin Ananyev
146e18a033bSKonstantin Ananyev	tcx =  wrk2ctx + ngx_worker;
147e18a033bSKonstantin Ananyev
148e18a033bSKonstantin Ananyev	tms = rte_get_tsc_cycles();
149e18a033bSKonstantin Ananyev	tme = (timer == NGX_TIMER_INFINITE) ? timer :
150e18a033bSKonstantin Ananyev		timer * (rte_get_tsc_hz() + MS_PER_S - 1) / MS_PER_S;
151e18a033bSKonstantin Ananyev	tmw = 0;
152e18a033bSKonstantin Ananyev	n = 0;
153e18a033bSKonstantin Ananyev
154e18a033bSKonstantin Ananyev	do {
155e18a033bSKonstantin Ananyev		if (tcx->cf->be_in_worker != 0)
156e18a033bSKonstantin Ananyev			be_lcore_tcp(tcx);
157e18a033bSKonstantin Ananyev
158e18a033bSKonstantin Ananyev		ns = tle_evq_get(stbl.syneq, (const void **)(uintptr_t)ts,
159e18a033bSKonstantin Ananyev			RTE_DIM(ts));
160e18a033bSKonstantin Ananyev		nr = tle_evq_get(stbl.rxeq, (const void **)(uintptr_t)tr,
161e18a033bSKonstantin Ananyev			RTE_DIM(tr));
162e18a033bSKonstantin Ananyev		nt = tle_evq_get(stbl.txeq, (const void **)(uintptr_t)tt,
163e18a033bSKonstantin Ananyev			RTE_DIM(tt));
164e18a033bSKonstantin Ananyev		ne = tle_evq_get(stbl.ereq, (const void **)(uintptr_t)te,
165e18a033bSKonstantin Ananyev			RTE_DIM(te));
166e18a033bSKonstantin Ananyev		n = ne + nr + ns + nt;
167e18a033bSKonstantin Ananyev
168e18a033bSKonstantin Ananyev		if (n != 0) {
169e18a033bSKonstantin Ananyev			event_stat.nb_get[EV_ACCEPT] += ns;
170e18a033bSKonstantin Ananyev			event_stat.nb_get[EV_RECV] += nr;
171e18a033bSKonstantin Ananyev			event_stat.nb_get[EV_SEND] += nt;
172e18a033bSKonstantin Ananyev			event_stat.nb_get[EV_ERR] += ne;
173e18a033bSKonstantin Ananyev			break;
174e18a033bSKonstantin Ananyev		}
175e18a033bSKonstantin Ananyev
176e18a033bSKonstantin Ananyev		if (tcx->cf->be_in_worker == 0)
177e18a033bSKonstantin Ananyev			//sched_yield();
178e18a033bSKonstantin Ananyev			rte_delay_us(1);
179e18a033bSKonstantin Ananyev
180e18a033bSKonstantin Ananyev		tmw += rte_get_tsc_cycles() - tms;
181e18a033bSKonstantin Ananyev
182e18a033bSKonstantin Ananyev	} while (tmw < tme && ngx_quit == 0 && ngx_terminate == 0);
183e18a033bSKonstantin Ananyev
184e18a033bSKonstantin Ananyev	if ((flags & NGX_UPDATE_TIME) != 0 || ngx_event_timer_alarm)
185e18a033bSKonstantin Ananyev		ngx_time_update();
186e18a033bSKonstantin Ananyev
187e18a033bSKonstantin Ananyev	if (n == 0)
188e18a033bSKonstantin Ananyev		return NGX_OK;
189e18a033bSKonstantin Ananyev
190e18a033bSKonstantin Ananyev	for (i = 0; i != ns; i++)
191e18a033bSKonstantin Ananyev		post_event(ts[i]->rev, &ngx_posted_accept_events, flags,
192e18a033bSKonstantin Ananyev			EV_ACCEPT);
193e18a033bSKonstantin Ananyev
194e18a033bSKonstantin Ananyev	for (i = 0; i != nr; i++)
195e18a033bSKonstantin Ananyev		post_event(tr[i]->rev, &ngx_posted_events, flags, EV_RECV);
196e18a033bSKonstantin Ananyev
197e18a033bSKonstantin Ananyev	for (i = 0; i != nt; i++)
198e18a033bSKonstantin Ananyev		post_event(tt[i]->wev, &ngx_posted_events, flags, EV_SEND);
199e18a033bSKonstantin Ananyev
200e18a033bSKonstantin Ananyev	for (i = 0; i != ne; i++) {
201e18a033bSKonstantin Ananyev		te[i]->posterr++;
202e18a033bSKonstantin Ananyev		post_event(te[i]->rev, &ngx_posted_events, flags, EV_ERR);
203e18a033bSKonstantin Ananyev		post_event(te[i]->wev, &ngx_posted_events, flags, EV_ERR);
204e18a033bSKonstantin Ananyev	}
205e18a033bSKonstantin Ananyev
206e18a033bSKonstantin Ananyev	return NGX_OK;
207e18a033bSKonstantin Ananyev}
208e18a033bSKonstantin Ananyev
209e18a033bSKonstantin Ananyevstatic ngx_int_t
210e18a033bSKonstantin Ananyevtldk_init_events(ngx_cycle_t *cycle, ngx_msec_t timer)
211e18a033bSKonstantin Ananyev{
212e18a033bSKonstantin Ananyev	FE_TRACE("%s(cycle=%p,timer=%lu);\n",
213e18a033bSKonstantin Ananyev		__func__, cycle, timer);
214e18a033bSKonstantin Ananyev
215e18a033bSKonstantin Ananyev	/* overwrite event actions for worker process */
216e18a033bSKonstantin Ananyev	ngx_event_actions = tldk_event_module.actions;
217e18a033bSKonstantin Ananyev	ngx_event_flags = NGX_USE_LEVEL_EVENT;
218e18a033bSKonstantin Ananyev
219e18a033bSKonstantin Ananyev	ngx_io = ngx_os_io;
220e18a033bSKonstantin Ananyev	return NGX_OK;
221e18a033bSKonstantin Ananyev}
222e18a033bSKonstantin Ananyev
223e18a033bSKonstantin Ananyevvoid
224e18a033bSKonstantin Ananyevtldk_dump_event_stats(void)
225e18a033bSKonstantin Ananyev{
226e18a033bSKonstantin Ananyev	static const char * const name[EV_NUM] = {
227e18a033bSKonstantin Ananyev		"ACCEPT",
228e18a033bSKonstantin Ananyev		"RECV",
229e18a033bSKonstantin Ananyev		"SEND",
230e18a033bSKonstantin Ananyev		"ERR",
231e18a033bSKonstantin Ananyev	};
232e18a033bSKonstantin Ananyev
233e18a033bSKonstantin Ananyev	uint32_t i;
234e18a033bSKonstantin Ananyev
235e18a033bSKonstantin Ananyev	RTE_LOG(NOTICE, USER1, "%s(worker=%lu)={\n", __func__, ngx_worker);
236e18a033bSKonstantin Ananyev	for (i = 0; i != RTE_DIM(name); i++)
237e18a033bSKonstantin Ananyev		RTE_LOG(NOTICE, USER1,
238e18a033bSKonstantin Ananyev			"%s[GET, POST]={%" PRIu64 ", %" PRIu64 "};\n",
239e18a033bSKonstantin Ananyev			name[i], event_stat.nb_get[i], event_stat.nb_post[i]);
240e18a033bSKonstantin Ananyev	RTE_LOG(NOTICE, USER1, "};\n");
241e18a033bSKonstantin Ananyev}
242e18a033bSKonstantin Ananyev
243e18a033bSKonstantin Ananyevstatic void
244e18a033bSKonstantin Ananyevtldk_done_events(ngx_cycle_t *cycle)
245e18a033bSKonstantin Ananyev{
246e18a033bSKonstantin Ananyev}
247e18a033bSKonstantin Ananyev
248e18a033bSKonstantin Ananyevstatic ngx_str_t tldk_name = ngx_string("tldk");
249e18a033bSKonstantin Ananyev
250e18a033bSKonstantin Ananyevngx_event_module_t tldk_event_module = {
251e18a033bSKonstantin Ananyev	.name = &tldk_name,
252e18a033bSKonstantin Ananyev	.actions = {
253e18a033bSKonstantin Ananyev		.add = tldk_add_event,
254e18a033bSKonstantin Ananyev		.del = tldk_del_event,
255e18a033bSKonstantin Ananyev		.enable = tldk_add_event,
256e18a033bSKonstantin Ananyev		.disable = tldk_del_event,
257e18a033bSKonstantin Ananyev		.process_events = tldk_process_events,
258e18a033bSKonstantin Ananyev		.init = tldk_init_events,
259e18a033bSKonstantin Ananyev		.done = tldk_done_events,
260e18a033bSKonstantin Ananyev	},
261e18a033bSKonstantin Ananyev};
262e18a033bSKonstantin Ananyev
263e18a033bSKonstantin Ananyevngx_module_t ngx_tldk_event_module = {
264e18a033bSKonstantin Ananyev	NGX_MODULE_V1,
265e18a033bSKonstantin Ananyev	&tldk_event_module,    /* module context */
266e18a033bSKonstantin Ananyev	NULL,                  /* module directives */
267e18a033bSKonstantin Ananyev	NGX_EVENT_MODULE,      /* module type */
268e18a033bSKonstantin Ananyev	NULL,                  /* init master */
269e18a033bSKonstantin Ananyev	NULL,                  /* init module */
270e18a033bSKonstantin Ananyev	NULL,                  /* init process */
271e18a033bSKonstantin Ananyev	NULL,                  /* init thread */
272e18a033bSKonstantin Ananyev	NULL,                  /* exit thread */
273e18a033bSKonstantin Ananyev	NULL,                  /* exit process */
274e18a033bSKonstantin Ananyev	NULL,                  /* exit master */
275e18a033bSKonstantin Ananyev	NGX_MODULE_V1_PADDING
276e18a033bSKonstantin Ananyev};
277