1/*
2 * Copyright (c) 2017  Intel Corporation.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <ngx_tldk.h>
28#include <tldk_sock.h>
29
30#include <rte_cycles.h>
31
32#include "be.h"
33#include "debug.h"
34
35#define	EVENT_BULK	32
36
37enum {
38	EV_ACCEPT,
39	EV_RECV,
40	EV_SEND,
41	EV_ERR,
42	EV_NUM
43};
44
45struct tldk_event_stat {
46	uint64_t nb_get[EV_NUM];
47	uint64_t nb_post[EV_NUM];
48};
49
50static struct tldk_event_stat event_stat;
51
52extern ngx_event_module_t tldk_event_module;
53
54/*
55 * TLDK event module implementation
56 */
57
58static ngx_int_t
59tldk_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
60{
61	struct tldk_sock *ts;
62	ngx_connection_t *c;
63
64	c = ev->data;
65
66	FE_TRACE("%s(ev=%p,event=%#lx,flags=%#lx): fd=%d;\n",
67		__func__, ev, event, flags, c->fd);
68
69	ts = sd_to_sock(c->fd);
70	if (ts == NULL)
71		return NGX_OK;
72
73	if (event == NGX_READ_EVENT) {
74		tle_event_active(ts->rxev, TLE_SEV_DOWN);
75		tle_event_active(ts->erev, TLE_SEV_DOWN);
76		ts->rev = ev;
77	} else if (event == NGX_WRITE_EVENT) {
78		tle_event_active(ts->txev, TLE_SEV_DOWN);
79		tle_event_active(ts->erev, TLE_SEV_DOWN);
80		ts->wev = ev;
81	}
82
83	ev->active = 1;
84	return NGX_OK;
85}
86
87static ngx_int_t
88tldk_del_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
89{
90	struct tldk_sock *ts;
91	ngx_connection_t *c;
92
93	c = ev->data;
94
95	FE_TRACE("%s(ev=%p,event=%#lx,flags=%#lx): fd=%d;\n",
96		__func__, ev, event, flags, c->fd);
97
98	ev->active = 0;
99	if ((flags & NGX_CLOSE_EVENT) != 0)
100		return NGX_OK;
101
102	ts = sd_to_sock(c->fd);
103	if (ts == NULL)
104		return NGX_OK;
105
106	if (event == NGX_READ_EVENT) {
107		tle_event_down(ts->rxev);
108		tle_event_down(ts->erev);
109		ts->rev = NULL;
110	} else if (event == NGX_WRITE_EVENT) {
111		tle_event_down(ts->txev);
112		tle_event_down(ts->erev);
113		ts->wev = NULL;
114	}
115
116	return NGX_OK;
117}
118
119static inline void
120post_event(ngx_event_t *ev, ngx_queue_t *q, ngx_uint_t flags, uint32_t type)
121{
122	if (ev != NULL && ev->active == 1) {
123		ev->ready = 1;
124		event_stat.nb_post[type]++;
125		if ((flags & NGX_POST_EVENTS) != 0) {
126			ngx_post_event(ev, q);
127		} else
128			ev->handler(ev);
129	}
130}
131
132static ngx_int_t
133tldk_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags)
134{
135	uint32_t i, n, ne, nr, ns, nt;
136	uint64_t tme, tms, tmw;
137	struct tldk_sock *te[EVENT_BULK];
138	struct tldk_sock *tr[EVENT_BULK];
139	struct tldk_sock *ts[EVENT_BULK];
140	struct tldk_sock *tt[EVENT_BULK];
141	struct tldk_ctx *tcx;
142
143	FE_TRACE("%s(cycle=%p,timer=%lu,flags=%#lx);\n",
144		__func__, cycle, timer, flags);
145
146	tcx =  wrk2ctx + ngx_worker;
147
148	tms = rte_get_tsc_cycles();
149	tme = (timer == NGX_TIMER_INFINITE) ? timer :
150		timer * (rte_get_tsc_hz() + MS_PER_S - 1) / MS_PER_S;
151	tmw = 0;
152	n = 0;
153
154	do {
155		if (tcx->cf->be_in_worker != 0)
156			be_lcore_tcp(tcx);
157
158		ns = tle_evq_get(stbl.syneq, (const void **)(uintptr_t)ts,
159			RTE_DIM(ts));
160		nr = tle_evq_get(stbl.rxeq, (const void **)(uintptr_t)tr,
161			RTE_DIM(tr));
162		nt = tle_evq_get(stbl.txeq, (const void **)(uintptr_t)tt,
163			RTE_DIM(tt));
164		ne = tle_evq_get(stbl.ereq, (const void **)(uintptr_t)te,
165			RTE_DIM(te));
166		n = ne + nr + ns + nt;
167
168		if (n != 0) {
169			event_stat.nb_get[EV_ACCEPT] += ns;
170			event_stat.nb_get[EV_RECV] += nr;
171			event_stat.nb_get[EV_SEND] += nt;
172			event_stat.nb_get[EV_ERR] += ne;
173			break;
174		}
175
176		if (tcx->cf->be_in_worker == 0)
177			//sched_yield();
178			rte_delay_us(1);
179
180		tmw += rte_get_tsc_cycles() - tms;
181
182	} while (tmw < tme && ngx_quit == 0 && ngx_terminate == 0);
183
184	if ((flags & NGX_UPDATE_TIME) != 0 || ngx_event_timer_alarm)
185		ngx_time_update();
186
187	if (n == 0)
188		return NGX_OK;
189
190	for (i = 0; i != ns; i++)
191		post_event(ts[i]->rev, &ngx_posted_accept_events, flags,
192			EV_ACCEPT);
193
194	for (i = 0; i != nr; i++)
195		post_event(tr[i]->rev, &ngx_posted_events, flags, EV_RECV);
196
197	for (i = 0; i != nt; i++)
198		post_event(tt[i]->wev, &ngx_posted_events, flags, EV_SEND);
199
200	for (i = 0; i != ne; i++) {
201		te[i]->posterr++;
202		post_event(te[i]->rev, &ngx_posted_events, flags, EV_ERR);
203		post_event(te[i]->wev, &ngx_posted_events, flags, EV_ERR);
204	}
205
206	return NGX_OK;
207}
208
209static ngx_int_t
210tldk_init_events(ngx_cycle_t *cycle, ngx_msec_t timer)
211{
212	FE_TRACE("%s(cycle=%p,timer=%lu);\n",
213		__func__, cycle, timer);
214
215	/* overwrite event actions for worker process */
216	ngx_event_actions = tldk_event_module.actions;
217	ngx_event_flags = NGX_USE_LEVEL_EVENT;
218
219	ngx_io = ngx_os_io;
220	return NGX_OK;
221}
222
223void
224tldk_dump_event_stats(void)
225{
226	static const char * const name[EV_NUM] = {
227		"ACCEPT",
228		"RECV",
229		"SEND",
230		"ERR",
231	};
232
233	uint32_t i;
234
235	RTE_LOG(NOTICE, USER1, "%s(worker=%lu)={\n", __func__, ngx_worker);
236	for (i = 0; i != RTE_DIM(name); i++)
237		RTE_LOG(NOTICE, USER1,
238			"%s[GET, POST]={%" PRIu64 ", %" PRIu64 "};\n",
239			name[i], event_stat.nb_get[i], event_stat.nb_post[i]);
240	RTE_LOG(NOTICE, USER1, "};\n");
241}
242
243static void
244tldk_done_events(ngx_cycle_t *cycle)
245{
246}
247
248static ngx_str_t tldk_name = ngx_string("tldk");
249
250ngx_event_module_t tldk_event_module = {
251	.name = &tldk_name,
252	.actions = {
253		.add = tldk_add_event,
254		.del = tldk_del_event,
255		.enable = tldk_add_event,
256		.disable = tldk_del_event,
257		.process_events = tldk_process_events,
258		.init = tldk_init_events,
259		.done = tldk_done_events,
260	},
261};
262
263ngx_module_t ngx_tldk_event_module = {
264	NGX_MODULE_V1,
265	&tldk_event_module,    /* module context */
266	NULL,                  /* module directives */
267	NGX_EVENT_MODULE,      /* module type */
268	NULL,                  /* init master */
269	NULL,                  /* init module */
270	NULL,                  /* init process */
271	NULL,                  /* init thread */
272	NULL,                  /* exit thread */
273	NULL,                  /* exit process */
274	NULL,                  /* exit master */
275	NGX_MODULE_V1_PADDING
276};
277